From 214e995db36098c368b0a88434b58d19a70f0637 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Wed, 24 Jun 2026 17:23:19 -0400 Subject: [PATCH] Add MariaDB major version upgrade support (OSPRH-31231) Adds spec.targetVersion to the Galera CR so the openstack-operator can signal a major version upgrade (e.g. "10.11"). When targetVersion differs from the deployed version recorded in status.clusterProperties, the controller adds TargetVersion to the ClusterProperties hash, triggering a full cluster stop via the existing StopRequired mechanism. On restart, an upgrade init container runs mysql_version_upgrade.sh before mysql_bootstrap.sh, starting mysqld in standalone mode and calling mariadb-upgrade against the node's PVC. Once the cluster bootstraps with the new image, status.clusterProperties["TargetVersion"] is advanced to match spec, removing the init container from subsequent StatefulSet reconciles. Also adds hack/Containerfile.mariadb-10.11 for building a CentOS 9 based test image with MariaDB 10.11 community packages, to be used for prototyping and validating the upgrade path until a production RHEL 10 image is available. Jira: OSPRH-32081 Co-Authored-By: Claude Sonnet 4.6 --- api/bases/mariadb.openstack.org_galeras.yaml | 8 + api/v1beta1/conditions.go | 37 +++ api/v1beta1/galera_types.go | 8 + api/v1beta1/mariadbdatabase_funcs.go | 22 +- .../bases/mariadb.openstack.org_galeras.yaml | 8 + hack/Containerfile.mariadb-10.11 | 53 ++++ internal/controller/galera_controller.go | 232 +++++++++++++++++- internal/mariadb/statefulset.go | 18 +- internal/mariadb/volumes.go | 4 + templates/galera/bin/mysql_root_auth.sh | 18 +- templates/galera/bin/mysql_version_upgrade.sh | 91 +++++++ templates/galera/config/galera.cnf.in | 16 +- test/chainsaw/common/galera-assert.yaml | 4 + .../common/galera-no-secret-assert.yaml | 4 + .../galera-assert.yaml | 4 + .../galera-topology-assert.yaml | 4 + .../missing-dbrootpassword/galera-assert.yaml | 4 + .../tests/upgrade-blocked/chainsaw-test.yaml | 63 +++++ .../upgrade-blocked-assert.yaml | 43 ++++ .../chainsaw-test.yaml | 52 ++++ .../version-mismatch-assert.yaml | 43 ++++ .../upgrade-version-probe/chainsaw-test.yaml | 40 +++ 22 files changed, 741 insertions(+), 35 deletions(-) create mode 100644 hack/Containerfile.mariadb-10.11 create mode 100755 templates/galera/bin/mysql_version_upgrade.sh create mode 100644 test/chainsaw/tests/upgrade-blocked/chainsaw-test.yaml create mode 100644 test/chainsaw/tests/upgrade-blocked/upgrade-blocked-assert.yaml create mode 100644 test/chainsaw/tests/upgrade-version-mismatch/chainsaw-test.yaml create mode 100644 test/chainsaw/tests/upgrade-version-mismatch/version-mismatch-assert.yaml create mode 100644 test/chainsaw/tests/upgrade-version-probe/chainsaw-test.yaml diff --git a/api/bases/mariadb.openstack.org_galeras.yaml b/api/bases/mariadb.openstack.org_galeras.yaml index c58c04fe..7f3c9377 100644 --- a/api/bases/mariadb.openstack.org_galeras.yaml +++ b/api/bases/mariadb.openstack.org_galeras.yaml @@ -308,6 +308,14 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + targetVersion: + description: |- + TargetVersion is the MariaDB major.minor version that the cluster should + run after an upgrade (e.g. "10.11"). When set and different from the + version recorded in the existing data files, the operator performs a full + cluster stop and runs mariadb-upgrade on each node before restarting. + Leave empty when no version upgrade is in progress. + type: string tls: description: TLS settings for MySQL service and internal Galera replication properties: diff --git a/api/v1beta1/conditions.go b/api/v1beta1/conditions.go index 01ad40a8..12834a8d 100644 --- a/api/v1beta1/conditions.go +++ b/api/v1beta1/conditions.go @@ -34,6 +34,14 @@ const ( // MariaDBServerReadyCondition Status=True condition which indicates that the MariaDB and/or // Galera server is ready for database / account create/drop operations to proceed MariaDBServerReadyCondition condition.Type = "MariaDBServerReady" + + // MariaDBServerUpgradeReadyCondition tracks the state of a MariaDB major + // version upgrade driven by Spec.TargetVersion. Status=True means no + // upgrade is pending (the deployed version matches the spec). Status=False + // with SeverityInfo means an upgrade is in progress; Status=False with + // SeverityError means an upgrade was requested but cannot start because the + // cluster is not fully available. + MariaDBServerUpgradeReadyCondition condition.Type = "MariaDBServerUpgradeReady" ) // MariaDB Reasons used by API objects. @@ -59,6 +67,17 @@ const ( // ReasonDBSync - Database sync in progress ReasonDBSync condition.Reason = "DBSync" + + // MariaDBServerUpgradeInProgressReason - a major version upgrade is running + MariaDBServerUpgradeInProgressReason condition.Reason = "UpgradeInProgress" + // MariaDBServerUpgradeBlockedReason - a major version upgrade was requested + // but cannot start because the cluster is not fully available + MariaDBServerUpgradeBlockedReason condition.Reason = "UpgradeBlocked" + // MariaDBServerUpgradeVersionMismatchReason - the cluster came back up after + // an upgrade but the running MariaDB server version does not match + // spec.targetVersion (e.g. containerImage was not bumped, or targetVersion is + // wrong) + MariaDBServerUpgradeVersionMismatchReason condition.Reason = "UpgradeVersionMismatch" ) // MariaDB Messages used by API objects. @@ -114,6 +133,24 @@ const ( MariaDBAccountFinalizersRemainMessage = "Waiting for finalizers %s to be removed before dropping username" MariaDBAccountReadyForDeleteMessage = "MariaDBAccount ready for delete" + + // MariaDBServerUpgradeReadyInitMessage - upgrade tracking not yet started + MariaDBServerUpgradeReadyInitMessage = "MariaDB server version upgrade state not yet determined" + + // MariaDBServerUpgradeReadyMessage - no version upgrade pending + MariaDBServerUpgradeReadyMessage = "MariaDB server version up to date" + + // MariaDBServerUpgradeInProgressMessage - upgrade underway (expects target version) + MariaDBServerUpgradeInProgressMessage = "MariaDB server version upgrade to %s in progress" + + // MariaDBServerUpgradeBlockedMessage - upgrade requested but cluster not + // fully available (expects target version, ready node count, total node count) + MariaDBServerUpgradeBlockedMessage = "MariaDB server version upgrade to %s blocked: cluster not fully available (%d/%d nodes ready); resolve the offline node before upgrading" + + // MariaDBServerUpgradeVersionMismatchMessage - the cluster is running a + // different MariaDB version than spec.targetVersion (expects detected + // running version, then requested target version) + MariaDBServerUpgradeVersionMismatchMessage = "MariaDB server version mismatch: cluster is running %s but spec.targetVersion is %s" ) // GaleraBackup Condition Types used by API objects. diff --git a/api/v1beta1/galera_types.go b/api/v1beta1/galera_types.go index d9ce5c0a..09ec5513 100644 --- a/api/v1beta1/galera_types.go +++ b/api/v1beta1/galera_types.go @@ -110,6 +110,14 @@ type GaleraSpecCore struct { // +kubebuilder:validation:Optional // Override, provides the ability to override the generated manifest of several child resources. Override GaleraOverrideSpec `json:"override,omitempty"` + + // +kubebuilder:validation:Optional + // TargetVersion is the MariaDB major.minor version that the cluster should + // run after an upgrade (e.g. "10.11"). When set and different from the + // version recorded in the existing data files, the operator performs a full + // cluster stop and runs mariadb-upgrade on each node before restarting. + // Leave empty when no version upgrade is in progress. + TargetVersion string `json:"targetVersion,omitempty"` } // GaleraOverrideSpec to override the generated manifest of several child resources diff --git a/api/v1beta1/mariadbdatabase_funcs.go b/api/v1beta1/mariadbdatabase_funcs.go index 1585d11d..6f9836e8 100644 --- a/api/v1beta1/mariadbdatabase_funcs.go +++ b/api/v1beta1/mariadbdatabase_funcs.go @@ -941,20 +941,22 @@ func ensureMariaDBAccount(ctx context.Context, } } - _, err = createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, labels) + op, err := createOrPatchAccountAndSecret(ctx, helper, account, dbSecret, labels) if err != nil { return nil, nil, err } - util.LogForObject( - helper, - fmt.Sprintf( - "Successfully ensured MariaDBAccount %s exists; database username is %s", - accountName, - account.Spec.UserName, - ), - account, - ) + if op != controllerutil.OperationResultNone { + util.LogForObject( + helper, + fmt.Sprintf( + "Successfully ensured MariaDBAccount %s exists; database username is %s", + accountName, + account.Spec.UserName, + ), + account, + ) + } return account, dbSecret, nil diff --git a/config/crd/bases/mariadb.openstack.org_galeras.yaml b/config/crd/bases/mariadb.openstack.org_galeras.yaml index c58c04fe..7f3c9377 100644 --- a/config/crd/bases/mariadb.openstack.org_galeras.yaml +++ b/config/crd/bases/mariadb.openstack.org_galeras.yaml @@ -308,6 +308,14 @@ spec: storageRequest: description: Storage size allocated for the mariadb databases type: string + targetVersion: + description: |- + TargetVersion is the MariaDB major.minor version that the cluster should + run after an upgrade (e.g. "10.11"). When set and different from the + version recorded in the existing data files, the operator performs a full + cluster stop and runs mariadb-upgrade on each node before restarting. + Leave empty when no version upgrade is in progress. + type: string tls: description: TLS settings for MySQL service and internal Galera replication properties: diff --git a/hack/Containerfile.mariadb-10.11 b/hack/Containerfile.mariadb-10.11 new file mode 100644 index 00000000..59eb4252 --- /dev/null +++ b/hack/Containerfile.mariadb-10.11 @@ -0,0 +1,53 @@ +# Test image: MariaDB 10.11 on CentOS Stream 9 kolla base +# +# Derives from the standard 10.5 kolla image to preserve all kolla scripts, +# uid/gid setup, and filesystem layout. Swaps the MariaDB packages to 10.11 +# from mariadb.org community repos. +# +# This is a temporary test image for developing/validating the 10.5->10.11 +# upgrade path (OSPRH-31231). The real target image will be RHEL 10 based +# and built via the production pipeline. +# +# Build: +# podman build -t quay.io//openstack-mariadb:10.11-test \ +# -f hack/Containerfile.mariadb-10.11 . +# +# Verify: +# podman run --rm quay.io//openstack-mariadb:10.11-test mysqld --version +# podman run --rm quay.io//openstack-mariadb:10.11-test which mariadb-upgrade + +FROM quay.io/podified-antelope-centos9/openstack-mariadb:current-podified + +USER root + +# Remove the CentOS 9 MariaDB 10.5 RPMs. The community MariaDB-server package +# conflicts on /usr/sbin/mysqld so these must be removed first. +RUN rpm -e --nodeps \ + mariadb-server-galera \ + mariadb-server \ + mariadb-server-utils \ + mariadb-backup \ + mariadb \ + mariadb-errmsg \ + mariadb-common \ + galera + +# Add the MariaDB.org 10.11 repo and install the community packages. +# MariaDB-server, MariaDB-client, MariaDB-backup use uppercase names so there +# is no RPM naming conflict with the packages removed above. +RUN curl -LsS https://downloads.mariadb.com/MariaDB/mariadb_repo_setup | \ + bash -s -- --mariadb-server-version=mariadb-10.11 --skip-check-installed && \ + dnf install -y \ + MariaDB-server \ + MariaDB-client \ + MariaDB-backup \ + galera-4 && \ + dnf clean all && \ + # RHEL packages differ from MariaDB.org community packages in two paths + # that the operator scripts and config reference. Add symlinks so this test + # image works without touching operator code. The production RHEL 10 image + # won't need these. + mkdir -p /usr/libexec && ln -s /usr/sbin/mysqld /usr/libexec/mysqld && \ + mkdir -p /usr/lib64/galera && ln -s /usr/lib64/galera-4/libgalera_smm.so /usr/lib64/galera/libgalera_smm.so + +USER mysql diff --git a/internal/controller/galera_controller.go b/internal/controller/galera_controller.go index 76a5d3dd..fbba1952 100644 --- a/internal/controller/galera_controller.go +++ b/internal/controller/galera_controller.go @@ -23,6 +23,7 @@ import ( "encoding/json" "errors" "fmt" + "regexp" "slices" "sort" "strconv" @@ -87,6 +88,10 @@ var ( ErrOpenStackSecretNotFound = errors.New("OpenStack secret not found") // ErrOpenStackSecretMissingField indicates that the OpenStack secret is missing a required field ErrOpenStackSecretMissingField = errors.New("OpenStack secret missing required field") + // ErrNoReadyGaleraPods indicates no ready pods are available to probe + ErrNoReadyGaleraPods = errors.New("no ready galera pods available to probe server version") + // ErrServerVersionUnparseable indicates the server version output could not be parsed + ErrServerVersionUnparseable = errors.New("could not parse server version") ) var allWatchFields = []string{ @@ -445,6 +450,53 @@ func retrieveSequenceNumber(ctx context.Context, helper *helper.Helper, config * return } +// serverVersionRe captures the leading major.minor.patch of a MariaDB version +// string such as "mysqld Ver 10.11.6-MariaDB for Linux" or a bare "10.11". +var serverVersionRe = regexp.MustCompile(`(\d+)\.(\d+)(?:\.\d+)?`) + +// normalizeMajorMinor extracts the leading major.minor (e.g. "10.11") from a +// MariaDB version string. Only major.minor is returned because spec.targetVersion +// is documented as a major.minor value; the patch level is intentionally ignored +// so that a 10.11.6 binary satisfies a "10.11" target. Returns "" if no version +// number can be parsed, which lets the caller treat an unparseable spec (e.g. +// "bananas") as a mismatch. +func normalizeMajorMinor(version string) string { + m := serverVersionRe.FindStringSubmatch(version) + if m == nil { + return "" + } + return m[1] + "." + m[2] +} + +// probeGaleraServerVersion execs `mysqld --version` in one ready galera pod and +// returns the running server's major.minor version (e.g. "10.11"). It returns an +// error when no ready pod is available or the version cannot be parsed, so the +// caller requeues rather than advancing upgrade status on partial data. Probing +// the running server (rather than trusting spec.targetVersion) is what lets the +// operator detect a containerImage/targetVersion mismatch as an error instead of +// recording a silent false success. +func probeGaleraServerVersion(ctx context.Context, h *helper.Helper, config *rest.Config, instance *mariadbv1.Galera, pods []corev1.Pod) (string, error) { + ready := getReadyPods(pods) + if len(ready) == 0 { + return "", ErrNoReadyGaleraPods + } + pod := ready[0] + var actual string + err := mariadb.ExecInPod(ctx, h, config, instance.Namespace, pod.Name, "galera", + []string{"/bin/bash", "-c", "mysqld --version"}, + func(stdout *bytes.Buffer, _ *bytes.Buffer) error { + actual = normalizeMajorMinor(stdout.String()) + if actual == "" { + return fmt.Errorf("%w: %q", ErrServerVersionUnparseable, strings.TrimSpace(stdout.String())) + } + return nil + }) + if err != nil { + return "", err + } + return actual, nil +} + // clearPodAttributes clears information known by the operator about a pod func clearPodAttributes(ctx context.Context, instance *mariadbv1.Galera, podName string) { delete(instance.Status.Attributes, podName) @@ -590,6 +642,8 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res condition.UnknownCondition(condition.ServiceConfigReadyCondition, condition.InitReason, condition.ServiceConfigReadyInitMessage), // cluster bootstrap condition.UnknownCondition(condition.DeploymentReadyCondition, condition.InitReason, condition.DeploymentReadyInitMessage), + // major version upgrade (Spec.TargetVersion) + condition.UnknownCondition(mariadbv1.MariaDBServerUpgradeReadyCondition, condition.InitReason, mariadbv1.MariaDBServerUpgradeReadyInitMessage), // service account, role, rolebinding condition.UnknownCondition(condition.ServiceAccountReadyCondition, condition.InitReason, condition.ServiceAccountReadyInitMessage), condition.UnknownCondition(condition.RoleReadyCondition, condition.InitReason, condition.RoleReadyInitMessage), @@ -947,10 +1001,59 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // build state of the restart hash. this is used to decide whether the // statefulset must stop all its pods before applying a config update clusterPropertiesEnv["GCommTLS"] = env.SetValue(strconv.FormatBool(instance.Spec.TLS.Enabled() && instance.Spec.TLS.CaBundleSecretName != "")) + if instance.Spec.TargetVersion != "" { + clusterPropertiesEnv["TargetVersion"] = env.SetValue(instance.Spec.TargetVersion) + } clusterPropertiesHash, err := util.HashOfInputHashes(clusterPropertiesEnv) if err != nil { return ctrl.Result{}, err } + + // compare requested TargetVersion with what was last observed as the + // actual version of MariaDB running on a pod. upgrade pending state + // is established when these are out of sync + upgradePending := instance.Spec.TargetVersion != "" && + instance.Spec.TargetVersion != instance.Status.ClusterProperties["TargetVersion"] + + // Load the existing StatefulSet, if any; Name is empty when it does not + // yet exist. Loading this up front where it first plays a role in some + // decisionmaking regarding upgrades, then later where we either update it + // or create a new one. + statefulset := appsv1.StatefulSet{} + if err := r.Get(ctx, client.ObjectKey{Name: mariadb.StatefulSetName(instance.Name), Namespace: instance.Namespace}, &statefulset); err != nil { + if !k8s_errors.IsNotFound(err) { + return ctrl.Result{}, err + } + } + statefulSetExists := statefulset.Name != "" + + // indicates upgrade is pending but was blocked due to some nodes not being + // available. + upgradeBlocked := false + + // if upgrade pending, and ClusterProperties hash indicates we have yet to + // initiate for this, first check that all nodes are running + // before allowing the upgrade to proceed. + if upgradePending && instance.Status.Hash["ClusterProperties"] != clusterPropertiesHash { + clusterFullyAvailable := statefulSetExists && instance.Spec.Replicas != nil && + statefulset.Status.AvailableReplicas == *instance.Spec.Replicas + + // if cluster is already-running, and some nodes not available, then + // block an upgrade. if cluster is entirely stopped, or entirely + // running, then we can enter the upgrade + upgradeBlocked = statefulSetExists && !clusterFullyAvailable + if upgradeBlocked { + util.LogForObject(helper, fmt.Sprintf( + "Version upgrade to %s requested but cluster is not fully available (%d/%d ready); deferring upgrade until all nodes are online", + instance.Spec.TargetVersion, statefulset.Status.AvailableReplicas, *instance.Spec.Replicas), instance) + delete(clusterPropertiesEnv, "TargetVersion") + clusterPropertiesHash, err = util.HashOfInputHashes(clusterPropertiesEnv) + if err != nil { + return ctrl.Result{}, err + } + } + } + inputHashEnv["ClusterProperties"] = env.SetValue(clusterPropertiesHash) // @@ -964,12 +1067,19 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // Update ClusterProperties here, as from this point we are sure we can update // both `ClusterProperties` and `StopRequired` in this reconcile loop. + // + // TargetVersion in ClusterProperties tracks the running version of mariadb + // in the current pods. Save it before wiping the map and + // restore it after; it is only advanced to Spec.TargetVersion once the + // cluster is running after the upgrade. + observedRunningVersion := instance.Status.ClusterProperties["TargetVersion"] instance.Status.ClusterProperties = make(map[string]string) for k, s := range clusterPropertiesEnv { var envVar corev1.EnvVar s(&envVar) instance.Status.ClusterProperties[k] = envVar.Value } + instance.Status.ClusterProperties["TargetVersion"] = observedRunningVersion // check whether we need to stop the cluster after a cluster-wide change if oldPropertiesHash, exists := instance.Status.Hash["ClusterProperties"]; exists { @@ -981,6 +1091,26 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res } } + // detect if Spec.ContainerImage was changed while we were already mid-upgrade. + // this could happen for example if user first updated TargetVersion, saved + // the CR, then came back and changed ContainerImage afterwards. Re-set + // StopRequired if this happens because this otherwise can lead into a + // rolling upgrade scenario. + if upgradePending && !upgradeBlocked && !instance.Status.StopRequired { + if statefulSetExists { + for i := range statefulset.Spec.Template.Spec.Containers { + if statefulset.Spec.Template.Spec.Containers[i].Name == "galera" && + statefulset.Spec.Template.Spec.Containers[i].Image != instance.Spec.ContainerImage { + util.LogForObject(helper, fmt.Sprintf( + "Container image changed (%s -> %s) while upgrade to %s is pending; forcing full cluster stop to avoid mixed MariaDB versions", + statefulset.Spec.Template.Spec.Containers[i].Image, instance.Spec.ContainerImage, instance.Spec.TargetVersion), instance) + instance.Status.StopRequired = true + break + } + } + } + } + if hashMap, changed := util.SetHash(instance.Status.Hash, common.InputHashName, hashOfHashes); changed { // Hash changed and instance status should be updated (which will be done by main defer func), // so update all the input hashes and return to reconcile again @@ -1029,22 +1159,24 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res instance.Status.LastAppliedTopology = nil } - stsSpec, err := mariadb.StatefulSet(instance, hashOfHashes, topology) - // an error is detected while creating the StatefulSet spec - if err != nil { - return ctrl.Result{}, err - } - commonstatefulset := commonstatefulset.NewStatefulSet(stsSpec, 5) - sfres, sferr := commonstatefulset.CreateOrPatch(ctx, helper) - if sferr != nil { - if k8s_errors.IsNotFound(sferr) { - return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + // if upgrade is not blocked, create or patch the statefulset with the new, + // desired spec. + if !upgradeBlocked { + stsSpec, err := mariadb.StatefulSet(instance, hashOfHashes, topology) + if err != nil { + return ctrl.Result{}, err + } + commonsts := commonstatefulset.NewStatefulSet(stsSpec, 5) + sfres, sferr := commonsts.CreateOrPatch(ctx, helper) + if sferr != nil { + if k8s_errors.IsNotFound(sferr) { + return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + } + return sfres, sferr } - return sfres, sferr + statefulset = commonsts.GetStatefulSet() } - statefulset := commonstatefulset.GetStatefulSet() - // If a full cluster restart was requested, // check whether it is still in progress if instance.Status.StopRequired && statefulset.Status.Replicas == 0 { @@ -1087,6 +1219,80 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res // . Cluster is bootstrapped as soon as one pod is available instance.Status.Bootstrapped = statefulset.Status.AvailableReplicas > 0 + // Probe the running MariaDB version and record it in + // ClusterProperties["TargetVersion"] whenever the cluster is healthy. + // This serves two purposes: + // - Auto-detect: populate the version on clusters that never set + // Spec.TargetVersion, so the status always reflects reality. + // - Upgrade verification: after an upgrade, confirm the running + // version matches what was requested. + allReplicasReady := instance.Spec.Replicas != nil && + statefulset.Status.AvailableReplicas == *instance.Spec.Replicas + + var detectedVersion string + versionMismatch := false + if allReplicasReady && !instance.Status.StopRequired { + needsProbe := false + if instance.Status.ClusterProperties["TargetVersion"] == "" { + needsProbe = true + } else if instance.Spec.TargetVersion != "" && + instance.Status.ClusterProperties["TargetVersion"] != instance.Spec.TargetVersion { + needsProbe = true + } + if needsProbe { + actual, probeErr := probeGaleraServerVersion(ctx, helper, r.config, instance, podList.Items) + if probeErr != nil { + if instance.Spec.TargetVersion != "" { + util.LogForObject(helper, fmt.Sprintf("Version upgrade to %s not yet verifiable (%v); will retry", instance.Spec.TargetVersion, probeErr), instance) + } else { + util.LogForObject(helper, fmt.Sprintf("Server version not yet available (%v); will retry", probeErr), instance) + } + return ctrl.Result{RequeueAfter: time.Duration(3) * time.Second}, nil + } + + instance.Status.ClusterProperties["TargetVersion"] = actual + detectedVersion = actual + if instance.Spec.TargetVersion != "" && + normalizeMajorMinor(instance.Spec.TargetVersion) != actual { + versionMismatch = true + } else if instance.Spec.TargetVersion != "" { + util.LogForObject(helper, fmt.Sprintf("Version upgrade to %s verified on all %d replicas", actual, *instance.Spec.Replicas), instance) + } else { + util.LogForObject(helper, fmt.Sprintf("Detected running MariaDB version %s", actual), instance) + } + } + } + + // set appropriate conditions for the fully observed upgrade state + switch { + case upgradeBlocked: + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBServerUpgradeReadyCondition, + mariadbv1.MariaDBServerUpgradeBlockedReason, + condition.SeverityError, + mariadbv1.MariaDBServerUpgradeBlockedMessage, + instance.Spec.TargetVersion, statefulset.Status.AvailableReplicas, *instance.Spec.Replicas)) + case versionMismatch: + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBServerUpgradeReadyCondition, + mariadbv1.MariaDBServerUpgradeVersionMismatchReason, + condition.SeverityError, + mariadbv1.MariaDBServerUpgradeVersionMismatchMessage, + detectedVersion, instance.Spec.TargetVersion)) + case instance.Spec.TargetVersion != "" && + instance.Status.ClusterProperties["TargetVersion"] != instance.Spec.TargetVersion: + instance.Status.Conditions.Set(condition.FalseCondition( + mariadbv1.MariaDBServerUpgradeReadyCondition, + mariadbv1.MariaDBServerUpgradeInProgressReason, + condition.SeverityInfo, + mariadbv1.MariaDBServerUpgradeInProgressMessage, + instance.Spec.TargetVersion)) + default: + instance.Status.Conditions.MarkTrue( + mariadbv1.MariaDBServerUpgradeReadyCondition, + mariadbv1.MariaDBServerUpgradeReadyMessage) + } + // Clear transient in-memory bootstrap tracker now that a pod is available if instance.Status.Bootstrapped { r.clearBootstrapState(instance) diff --git a/internal/mariadb/statefulset.go b/internal/mariadb/statefulset.go index 0cd61bb8..490ad0cd 100644 --- a/internal/mariadb/statefulset.go +++ b/internal/mariadb/statefulset.go @@ -99,7 +99,19 @@ func StatefulSet(g *mariadbv1.Galera, configHash string, topology *topologyv1.To } func getGaleraInitContainers(g *mariadbv1.Galera) []corev1.Container { - return []corev1.Container{{ + var initContainers []corev1.Container + + if g.Spec.TargetVersion != "" && g.Spec.TargetVersion != g.Status.ClusterProperties["TargetVersion"] { + initContainers = append(initContainers, corev1.Container{ + Image: g.Spec.ContainerImage, + Name: "mysql-upgrade", + Command: []string{"bash", "/var/lib/operator-scripts/mysql_version_upgrade.sh"}, + Resources: g.Spec.Resources, + VolumeMounts: getGaleraInitVolumeMounts(g), + }) + } + + initContainers = append(initContainers, corev1.Container{ Image: g.Spec.ContainerImage, Name: "mysql-bootstrap", Command: []string{"bash", "/var/lib/operator-scripts/mysql_bootstrap.sh"}, @@ -112,7 +124,9 @@ func getGaleraInitContainers(g *mariadbv1.Galera) []corev1.Container { }}, Resources: g.Spec.Resources, VolumeMounts: getGaleraInitVolumeMounts(g), - }} + }) + + return initContainers } func getGaleraContainers(g *mariadbv1.Galera, configHash string) ([]corev1.Container, error) { diff --git a/internal/mariadb/volumes.go b/internal/mariadb/volumes.go index 6dd08275..461048f1 100644 --- a/internal/mariadb/volumes.go +++ b/internal/mariadb/volumes.go @@ -112,6 +112,10 @@ func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { Key: "mysql_root_auth.sh", Path: "mysql_root_auth.sh", }, + { + Key: "mysql_version_upgrade.sh", + Path: "mysql_version_upgrade.sh", + }, }, }, }, diff --git a/templates/galera/bin/mysql_root_auth.sh b/templates/galera/bin/mysql_root_auth.sh index 44076da8..61d2d938 100755 --- a/templates/galera/bin/mysql_root_auth.sh +++ b/templates/galera/bin/mysql_root_auth.sh @@ -1,6 +1,10 @@ #!/bin/bash set +eu +if [ "$(id -u)" != "$(id -u mysql)" ]; then + echo "WARNING: mysql_root_auth.sh running as $(id -un) but expects to run as mysql" >&2 +fi + POD_NAME=$(hostname) # API server config @@ -197,12 +201,20 @@ else echo "Wrote new credentials to ${PW_CACHE_FILE}" >&2 fi -# Set restrictive permissions on .my.cnf (only if file was successfully written) +# Set ownership and permissions on cache dir and file so that scripts +# called by mysqld (e.g. wsrep_notify) can read the cached credentials, +# even if this script was invoked as root (which is not expected) +if [ -d "${PW_CACHE_DIR}" ]; then + if ! chown mysql:mysql "${PW_CACHE_DIR}" 2>/dev/null; then + echo "Did not yet set ownership on ${PW_CACHE_DIR}; will try again later" >&2 + fi +fi if [ -f "${PW_CACHE_FILE}" ]; then + if ! chown mysql:mysql "${PW_CACHE_FILE}" 2>/dev/null; then + echo "Did not yet set ownership on ${PW_CACHE_FILE}; will try again later" >&2 + fi if ! chmod 600 "${PW_CACHE_FILE}" 2>/dev/null; then echo "Did not yet set permissions on ${PW_CACHE_FILE}; will try again later" >&2 - else - echo "Set chmod 600 on ${PW_CACHE_FILE}" >&2 fi fi diff --git a/templates/galera/bin/mysql_version_upgrade.sh b/templates/galera/bin/mysql_version_upgrade.sh new file mode 100755 index 00000000..7adafa38 --- /dev/null +++ b/templates/galera/bin/mysql_version_upgrade.sh @@ -0,0 +1,91 @@ +#!/bin/bash +# pipefail is required so a failed `mariadb-upgrade` (which is piped to tee +# below) aborts this script with a non-zero exit. Kubernetes then keeps the +# pod out of the Available count, which is how the controller knows the +# upgrade has NOT yet succeeded on every replica. +set -euo pipefail + +# Run mariadb-upgrade if the data files are from a different MariaDB version +# than the current binary. This init container script runs before +# mysql_bootstrap.sh so the system tables are upgraded before the cluster forms. +# +# The check is intentionally based on mysql_upgrade_info rather than comparing +# version strings from spec, so the script is idempotent: it exits immediately +# on a node that has already been upgraded or was freshly initialized. + +# Extract the leading X.Y.Z numeric version from a MariaDB version string, +# tolerating any prefix (e.g. "mysqld Ver 10.5.22-MariaDB for Linux ...") +# and any suffix (e.g. "-MariaDB", "-log", or distro packaging strings). +# mysql_upgrade_info stores the raw server version whose exact suffix varies +# by build, so comparing raw strings could report "versions differ" even when +# they are the same X.Y.Z and re-run the upgrade on every pod start. Comparing +# the normalized numeric version avoids that. +version_num() { + printf '%s\n' "$1" | grep -oP '\d+\.\d+\.\d+' | head -1 || true +} + +DATA_VERSION=$(version_num "$(cat /var/lib/mysql/mysql_upgrade_info 2>/dev/null || true)") +BINARY_VERSION=$(version_num "$(mysqld --version 2>/dev/null || true)") + +if [[ -z "$DATA_VERSION" ]]; then + echo "No existing data version (mysql_upgrade_info absent or unparseable), skipping upgrade" + exit 0 +fi + +if [[ -z "$BINARY_VERSION" ]]; then + echo "Could not determine mysqld binary version; aborting upgrade check" >&2 + exit 1 +fi + +if [[ "$DATA_VERSION" = "$BINARY_VERSION" ]]; then + echo "Data already at $BINARY_VERSION, no upgrade needed" + exit 0 +fi + +echo "Upgrading data files: $DATA_VERSION -> $BINARY_VERSION" + +UPGRADE_PIDFILE=/var/tmp/upgrade.pid +MYSQLD_LOGFILE=/var/tmp/upgrade.log +UPGRADE_LOGFILE=/var/lib/mysql/mariadb_upgrade_${DATA_VERSION}_${BINARY_VERSION}.log + +rm -f "${UPGRADE_PIDFILE}" "${MYSQLD_LOGFILE}" + +mysqld_safe --wsrep-on=OFF --skip-grant-tables \ + --pid-file="${UPGRADE_PIDFILE}" \ + --log-error="${MYSQLD_LOGFILE}" & + +# Wait for mysqld to be ready +TIMEOUT=${DB_MAX_TIMEOUT:-60} +while [[ ! -S /var/lib/mysql/mysql.sock ]] || \ + [[ ! -f "${UPGRADE_PIDFILE}" ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 + else + echo "Timed out waiting for mysqld to start for upgrade" + cat "${MYSQLD_LOGFILE}" + exit 1 + fi +done + +mariadb-upgrade 2>&1 | tee "${UPGRADE_LOGFILE}" + +# do a graceful shutdown of mysqld using mysqladmin. note that if all of the +# below fails, the container exits anyway. +echo "Upgrade complete. Will shut down mysql now" + +# mariadb-upgrade runs FLUSH PRIVILEGES which re-enables the grant tables, +# disabling --skip-grant-tables. Fetch the root password from the K8s API +# so mysqladmin can authenticate for shutdown. + +# only use the k8s API to get the current root password; root password +# cache will not be set up yet +MYSQL_ROOT_AUTH_BYPASS_CHECKS=true + +# get the root password +source /var/lib/operator-scripts/mysql_root_auth.sh + +# do the shutdown +mysqladmin -uroot -p"${DB_ROOT_PASSWORD}" shutdown + +echo "shutdown complete" diff --git a/templates/galera/config/galera.cnf.in b/templates/galera/config/galera.cnf.in index efef5e95..bde6ad0c 100644 --- a/templates/galera/config/galera.cnf.in +++ b/templates/galera/config/galera.cnf.in @@ -12,11 +12,9 @@ bind-address = { PODNAME } binlog_format = ROW datadir = /var/lib/mysql default-storage-engine = innodb -expire_logs_days = 10 innodb_autoinc_lock_mode = 2 innodb_file_per_table = ON innodb_flush_log_at_trx_commit = 1 -innodb_locks_unsafe_for_binlog = 1 innodb_strict_mode = OFF key_buffer_size = 16M {{if .logToDisk}} @@ -28,8 +26,6 @@ max_connections = 4096 open_files_limit = 65536 pid-file = /var/lib/mysql/mariadb.pid port = 3306 -query_cache_limit = 1M -query_cache_size = 16M skip-external-locking skip-name-resolve = 1 socket = /var/lib/mysql/mysql.sock @@ -39,13 +35,10 @@ tmpdir = /tmp user = mysql wsrep_notify_cmd = /usr/local/bin/mysql_wsrep_notify.sh wsrep_auto_increment_control = 1 -wsrep_causal_reads = 0 -wsrep_certify_nonPK = 1 # wsrep_cluster_address = gcomm://database-0.internalapi.redhat.local,database-1.internalapi.redhat.local,database-2.internalapi.redhat.local wsrep_cluster_name = galera_cluster wsrep_convert_LOCK_to_trx = 0 wsrep_debug = 0 -wsrep_drupal_282555_workaround = 0 wsrep_on = ON wsrep_provider = /usr/lib64/galera/libgalera_smm.so wsrep_provider_options = pc.wait_prim_timeout=PT5S;gcache.recover=no;gmcast.listen_addr=tcp://{ PODIP }:4567 @@ -53,6 +46,15 @@ wsrep_retry_autocommit = 1 wsrep_slave_threads = 1 wsrep_sst_method = rsync +# fields removed in 10.11 — tolerated as no-ops in 10.11, remove when 10.5 support is dropped +expire_logs_days = 10 +innodb_locks_unsafe_for_binlog = 1 +query_cache_limit = 1M +query_cache_size = 16M +wsrep_causal_reads = 0 +wsrep_certify_nonPK = 1 +wsrep_drupal_282555_workaround = 0 + [mysqld_safe] {{if .logToDisk}} log-error = /var/log/mariadb/mariadb.log diff --git a/test/chainsaw/common/galera-assert.yaml b/test/chainsaw/common/galera-assert.yaml index a664062d..5990022a 100644 --- a/test/chainsaw/common/galera-assert.yaml +++ b/test/chainsaw/common/galera-assert.yaml @@ -31,6 +31,10 @@ status: reason: Ready status: "True" type: MariaDBAccountReady + - message: MariaDB server version up to date + reason: Ready + status: "True" + type: MariaDBServerUpgradeReady - message: PodDisruptionBudget completed reason: Ready status: "True" diff --git a/test/chainsaw/common/galera-no-secret-assert.yaml b/test/chainsaw/common/galera-no-secret-assert.yaml index 7fd6d024..d2d5e0c0 100644 --- a/test/chainsaw/common/galera-no-secret-assert.yaml +++ b/test/chainsaw/common/galera-no-secret-assert.yaml @@ -30,6 +30,10 @@ status: reason: Ready status: "True" type: MariaDBAccountReady + - message: MariaDB server version up to date + reason: Ready + status: "True" + type: MariaDBServerUpgradeReady - message: PodDisruptionBudget completed reason: Ready status: "True" diff --git a/test/chainsaw/tests/galera-name-with-galera/galera-assert.yaml b/test/chainsaw/tests/galera-name-with-galera/galera-assert.yaml index a408a7ec..bbe151d5 100644 --- a/test/chainsaw/tests/galera-name-with-galera/galera-assert.yaml +++ b/test/chainsaw/tests/galera-name-with-galera/galera-assert.yaml @@ -29,6 +29,10 @@ status: reason: Ready status: "True" type: MariaDBAccountReady + - message: MariaDB server version up to date + reason: Ready + status: "True" + type: MariaDBServerUpgradeReady - message: PodDisruptionBudget completed reason: Ready status: "True" diff --git a/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml b/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml index 8015b394..3cc90978 100644 --- a/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml +++ b/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml @@ -33,6 +33,10 @@ status: reason: Ready status: "True" type: MariaDBAccountReady + - message: MariaDB server version up to date + reason: Ready + status: "True" + type: MariaDBServerUpgradeReady - message: PodDisruptionBudget completed reason: Ready status: "True" diff --git a/test/chainsaw/tests/missing-dbrootpassword/galera-assert.yaml b/test/chainsaw/tests/missing-dbrootpassword/galera-assert.yaml index 96818532..cb664d92 100644 --- a/test/chainsaw/tests/missing-dbrootpassword/galera-assert.yaml +++ b/test/chainsaw/tests/missing-dbrootpassword/galera-assert.yaml @@ -26,6 +26,10 @@ status: severity: Warning status: "False" type: InputReady + - message: MariaDB server version upgrade state not yet determined + reason: Init + status: Unknown + type: MariaDBServerUpgradeReady - message: PodDisruptionBudget completed reason: Ready status: "True" diff --git a/test/chainsaw/tests/upgrade-blocked/chainsaw-test.yaml b/test/chainsaw/tests/upgrade-blocked/chainsaw-test.yaml new file mode 100644 index 00000000..4cfe2f3b --- /dev/null +++ b/test/chainsaw/tests/upgrade-blocked/chainsaw-test.yaml @@ -0,0 +1,63 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: upgrade-blocked +spec: + description: | + Test that a mariadb upgrade is blocked if all cluster nodes are not + available. + steps: + - name: Deploy a healthy 3-node cluster + description: Establish a known-good baseline before taking a node offline. + bindings: + - name: replicas + value: 3 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: Force one node's readiness probe to fail + description: | + Set wsrep_desync=ON on openstack-galera-2. This changes the node's + wsrep_local_state_comment from Synced to Donor/Desynced, which fails the + readiness probe while the liveness probe (wsrep_cluster_status=Primary) + continues to pass. The pod stays Running but becomes NotReady, dropping + availableReplicas from 3 to 2. + try: + - script: + content: | + set -e + oc exec -n "${NAMESPACE}" -c galera openstack-galera-2 -- /bin/sh -c 'source /var/lib/operator-scripts/mysql_root_auth.sh; mysql -uroot -e "SET GLOBAL wsrep_desync=ON"' + # Wait until the kubelet sees the failed readiness probe and removes + # the pod from the available count. + oc wait -n "${NAMESPACE}" --for=jsonpath='{.status.availableReplicas}'=2 --timeout=120s statefulset openstack-galera + + - name: Request an upgrade while a node is unready + description: Bump spec.targetVersion to 19.999 with the cluster at 2/3 availability. + try: + - script: + content: | + oc patch -n "${NAMESPACE}" galera openstack --type=merge -p='{"spec":{"targetVersion":"19.999"}}' + + - name: Assert the upgrade is blocked + description: | + The operator must report MariaDBServerUpgradeReady=False with reason + UpgradeBlocked. The node is still unready (desynced, not deleted), so the + blocked state is stable. + try: + - assert: + file: upgrade-blocked-assert.yaml + + - name: Restore the node and clear the upgrade request + description: | + Disable wsrep_desync and clear targetVersion so the cluster can recover + to 3/3 before cleanup. + try: + - script: + content: | + set -e + oc patch -n "${NAMESPACE}" galera openstack --type=merge -p='{"spec":{"targetVersion":null}}' + oc exec -n "${NAMESPACE}" -c galera openstack-galera-2 -- /bin/sh -c 'source /var/lib/operator-scripts/mysql_root_auth.sh; mysql -uroot -e "SET GLOBAL wsrep_desync=OFF"' + oc wait -n "${NAMESPACE}" --for=jsonpath='{.status.availableReplicas}'=3 --timeout=300s statefulset openstack-galera || true diff --git a/test/chainsaw/tests/upgrade-blocked/upgrade-blocked-assert.yaml b/test/chainsaw/tests/upgrade-blocked/upgrade-blocked-assert.yaml new file mode 100644 index 00000000..1893fb14 --- /dev/null +++ b/test/chainsaw/tests/upgrade-blocked/upgrade-blocked-assert.yaml @@ -0,0 +1,43 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +status: + conditions: + - reason: UpgradeBlocked + status: "False" + type: Ready + - reason: Ready + status: "True" + type: CreateServiceReady + - reason: Ready + status: "True" + type: DeploymentReady + - reason: Ready + status: "True" + type: InputReady + - reason: Ready + status: "True" + type: MariaDBAccountReady + - reason: UpgradeBlocked + status: "False" + type: MariaDBServerUpgradeReady + - message: PodDisruptionBudget completed + reason: Ready + status: "True" + type: PDBReady + - reason: Ready + status: "True" + type: RoleBindingReady + - reason: Ready + status: "True" + type: RoleReady + - reason: Ready + status: "True" + type: ServiceAccountReady + - reason: Ready + status: "True" + type: ServiceConfigReady + - reason: Ready + status: "True" + type: TLSInputReady diff --git a/test/chainsaw/tests/upgrade-version-mismatch/chainsaw-test.yaml b/test/chainsaw/tests/upgrade-version-mismatch/chainsaw-test.yaml new file mode 100644 index 00000000..b7ce1626 --- /dev/null +++ b/test/chainsaw/tests/upgrade-version-mismatch/chainsaw-test.yaml @@ -0,0 +1,52 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: upgrade-version-mismatch +spec: + description: | + Verify that requesting an upgrade to a MariaDB version that does not + match containerImage will be detected as an UpgradeVersionMismatch error. + steps: + - name: Deploy a healthy 1-node cluster + description: Establish a known-good baseline before requesting the upgrade. + bindings: + - name: replicas + value: 1 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: Request an upgrade to a non-existent version + description: Bump spec.targetVersion to 19.999, a version no image will provide. + try: + - script: + content: | + oc patch -n "${NAMESPACE}" galera openstack --type=merge -p='{"spec":{"targetVersion":"19.999"}}' + + - name: Assert the operator reports a version mismatch + description: | + The operator must probe the running pods and report + MariaDBServerUpgradeReady=False with reason UpgradeVersionMismatch rather + than silently claiming the upgrade succeeded. + try: + - assert: + file: version-mismatch-assert.yaml + + - name: Assert status records the real running version + description: | + status.clusterProperties.TargetVersion must hold the version scanned from + the pods, not the one that was entered in the Spec + try: + - script: + content: | + set -eu + CR_VERSION=$(oc get -n "${NAMESPACE}" galera openstack -o jsonpath='{.status.clusterProperties.TargetVersion}') + RAW=$(oc exec -n "${NAMESPACE}" -c galera openstack-galera-0 -- /bin/sh -c 'mysqld --version') + POD_VERSION=$(printf '%s\n' "${RAW}" | grep -oE '[0-9]+\.[0-9]+' | head -n1) + echo "pod=${POD_VERSION} cr=${CR_VERSION} targetVersion=19.999" + [ -n "${CR_VERSION}" ] || { echo "FAIL: status.clusterProperties.TargetVersion is empty"; exit 1; } + [ "${CR_VERSION}" != "19.999" ] || { echo "FAIL: status recorded the bogus target 19.999"; exit 1; } + [ "${POD_VERSION}" = "${CR_VERSION}" ] || { echo "FAIL: pod ${POD_VERSION} != cr ${CR_VERSION}"; exit 1; } + echo "PASS: upgrade to 19.999 rejected as version mismatch; status records real running version ${CR_VERSION}" diff --git a/test/chainsaw/tests/upgrade-version-mismatch/version-mismatch-assert.yaml b/test/chainsaw/tests/upgrade-version-mismatch/version-mismatch-assert.yaml new file mode 100644 index 00000000..3f1b9c62 --- /dev/null +++ b/test/chainsaw/tests/upgrade-version-mismatch/version-mismatch-assert.yaml @@ -0,0 +1,43 @@ +apiVersion: mariadb.openstack.org/v1beta1 +kind: Galera +metadata: + name: openstack +status: + conditions: + - reason: UpgradeVersionMismatch + status: "False" + type: Ready + - reason: Ready + status: "True" + type: CreateServiceReady + - reason: Ready + status: "True" + type: DeploymentReady + - reason: Ready + status: "True" + type: InputReady + - reason: Ready + status: "True" + type: MariaDBAccountReady + - reason: UpgradeVersionMismatch + status: "False" + type: MariaDBServerUpgradeReady + - message: PodDisruptionBudget completed + reason: Ready + status: "True" + type: PDBReady + - reason: Ready + status: "True" + type: RoleBindingReady + - reason: Ready + status: "True" + type: RoleReady + - reason: Ready + status: "True" + type: ServiceAccountReady + - reason: Ready + status: "True" + type: ServiceConfigReady + - reason: Ready + status: "True" + type: TLSInputReady diff --git a/test/chainsaw/tests/upgrade-version-probe/chainsaw-test.yaml b/test/chainsaw/tests/upgrade-version-probe/chainsaw-test.yaml new file mode 100644 index 00000000..cdcc7f2b --- /dev/null +++ b/test/chainsaw/tests/upgrade-version-probe/chainsaw-test.yaml @@ -0,0 +1,40 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: upgrade-version-probe +spec: + description: | + test that status.clusterProperties.TargetVersion is populated with the + actual version of mariadb running in the pods, by default. + steps: + - name: Deploy a healthy 1-node cluster + bindings: + - name: replicas + value: 1 + try: + - apply: + file: ../../common/galera.yaml + - assert: + file: ../../common/galera-assert.yaml + + - name: Verify status TargetVersion matches the running pod version + description: | + Probe mysqld --version on a pod, normalize to major.minor, and compare to + Galera status.clusterProperties.TargetVersion. + try: + - script: + content: | + set -eu + CR_VERSION="" + for i in $(seq 1 60); do + CR_VERSION=$(oc get -n "${NAMESPACE}" galera openstack -o jsonpath='{.status.clusterProperties.TargetVersion}' 2>/dev/null || true) + [ -n "${CR_VERSION}" ] && break + sleep 2 + done + [ -n "${CR_VERSION}" ] || { echo "FAIL: status.clusterProperties.TargetVersion never populated"; exit 1; } + RAW=$(oc exec -n "${NAMESPACE}" -c galera openstack-galera-0 -- /bin/sh -c 'mysqld --version') + POD_VERSION=$(printf '%s\n' "${RAW}" | grep -oE '[0-9]+\.[0-9]+' | head -n1) + echo "pod=${POD_VERSION} cr=${CR_VERSION}" + [ -n "${POD_VERSION}" ] || { echo "FAIL: could not parse pod version from: ${RAW}"; exit 1; } + [ "${POD_VERSION}" = "${CR_VERSION}" ] || { echo "FAIL: pod version ${POD_VERSION} != cr version ${CR_VERSION}"; exit 1; } + echo "PASS: status.clusterProperties.TargetVersion (${CR_VERSION}) matches running pod version (${POD_VERSION})"