diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0933208d..c513f63f 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -150,7 +150,7 @@ rules: - apiGroups: - security.openshift.io resourceNames: - - anyuid + - nonroot-v2 resources: - securitycontextconstraints verbs: diff --git a/internal/controller/galera_controller.go b/internal/controller/galera_controller.go index 76a5d3dd..c9abd82b 100644 --- a/internal/controller/galera_controller.go +++ b/internal/controller/galera_controller.go @@ -499,7 +499,7 @@ func clearOldPodsAttributesOnScaleDown(ctx context.Context, instance *mariadbv1. // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // RBAC required to grant the service account role these capabilities -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch // +kubebuilder:rbac:groups=core,resources=persistentvolumeclaims,verbs=get;list;watch;create;update;delete;patch @@ -622,7 +622,7 @@ func (r *GaleraReconciler) Reconcile(ctx context.Context, req ctrl.Request) (res rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, diff --git a/internal/controller/galerabackup_controller.go b/internal/controller/galerabackup_controller.go index b4dc8578..ee14a4ce 100644 --- a/internal/controller/galerabackup_controller.go +++ b/internal/controller/galerabackup_controller.go @@ -69,7 +69,7 @@ type GaleraBackupReconciler struct { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // RBAC required to grant the service account role these capabilities -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch // RBAC for PVC @@ -172,7 +172,7 @@ func (r *GaleraBackupReconciler) Reconcile(ctx context.Context, req ctrl.Request rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, diff --git a/internal/controller/galerarestore_controller.go b/internal/controller/galerarestore_controller.go index 06a5acf6..37beccc2 100644 --- a/internal/controller/galerarestore_controller.go +++ b/internal/controller/galerarestore_controller.go @@ -74,7 +74,7 @@ type GaleraRestoreReconciler struct { // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch // RBAC required to grant the service account role these capabilities -// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use +// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use // +kubebuilder:rbac:groups="",resources=pods,verbs=create;delete;get;list;patch;update;watch // Reconcile handles the reconciliation logic for GaleraRestore resources @@ -162,7 +162,7 @@ func (r *GaleraRestoreReconciler) Reconcile(ctx context.Context, req ctrl.Reques rbacRules := []rbacv1.PolicyRule{ { APIGroups: []string{"security.openshift.io"}, - ResourceNames: []string{"anyuid"}, + ResourceNames: []string{"nonroot-v2"}, Resources: []string{"securitycontextconstraints"}, Verbs: []string{"use"}, }, diff --git a/internal/mariadb/account.go b/internal/mariadb/account.go index 744662ce..0a24e7f7 100644 --- a/internal/mariadb/account.go +++ b/internal/mariadb/account.go @@ -74,7 +74,8 @@ func CreateOrUpdateDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.Mar }, }, }, - VolumeMounts: getGaleraRootOnlyVolumeMounts(), + VolumeMounts: getGaleraRootOnlyVolumeMounts(), + SecurityContext: GaleraSecurityContext(), }, }, Volumes: getGaleraRootOnlyVolumes(galera), @@ -115,10 +116,11 @@ func DeleteDbAccountJob(galera *mariadbv1.Galera, account *mariadbv1.MariaDBAcco ServiceAccountName: serviceAccountName, Containers: []corev1.Container{ { - Name: "mariadb-account-delete", - Image: containerImage, - Command: []string{"/bin/sh", "-c", delCmd}, - VolumeMounts: getGaleraRootOnlyVolumeMounts(), + Name: "mariadb-account-delete", + Image: containerImage, + Command: []string{"/bin/sh", "-c", delCmd}, + VolumeMounts: getGaleraRootOnlyVolumeMounts(), + SecurityContext: GaleraSecurityContext(), }, }, Volumes: getGaleraRootOnlyVolumes(galera), diff --git a/internal/mariadb/backup/cronjob.go b/internal/mariadb/backup/cronjob.go index 8f097cab..63ac76e1 100644 --- a/internal/mariadb/backup/cronjob.go +++ b/internal/mariadb/backup/cronjob.go @@ -9,7 +9,6 @@ import ( batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" ) // BackupCronJob returns a CronJob object for the galera backup @@ -54,9 +53,6 @@ func getBackupPodTemplate(b *mariadbv1.GaleraBackup, g *mariadbv1.Galera, config }, { Name: "DB", Value: g.Name, - }, { - Name: "KOLLA_CONFIG_STRATEGY", - Value: "COPY_ALWAYS", }, { Name: "RETENTION", Value: retentionTime, @@ -79,15 +75,13 @@ func getBackupPodTemplate(b *mariadbv1.GaleraBackup, g *mariadbv1.Galera, config Subdomain: svcName, RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: b.RbacResourceName(), - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To[int64](42434), - }, - InitContainers: []corev1.Container{}, + SecurityContext: mariadb.PodSecurityContext(), Containers: []corev1.Container{{ - Image: g.Spec.ContainerImage, - Name: "backup", - Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"}, - Env: environ, + Image: g.Spec.ContainerImage, + Name: "backup", + Command: []string{"/usr/bin/dumb-init", "--", "/var/lib/backup-scripts/backup_galera"}, + Env: environ, + SecurityContext: mariadb.GaleraSecurityContext(), Ports: []corev1.ContainerPort{{ ContainerPort: 4567, Name: "galera", diff --git a/internal/mariadb/backup/restorepod.go b/internal/mariadb/backup/restorepod.go index 965fa3be..f2733f65 100644 --- a/internal/mariadb/backup/restorepod.go +++ b/internal/mariadb/backup/restorepod.go @@ -3,10 +3,10 @@ package mariadbbackup import ( mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" + mariadb "github.com/openstack-k8s-operators/mariadb-operator/internal/mariadb" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/utils/ptr" ) // RestorePod returns a Pod object for a galera restore CR @@ -22,9 +22,6 @@ func RestorePod(restoreCR *mariadbv1.GaleraRestore, backupCR *mariadbv1.GaleraBa environ := []corev1.EnvVar{{ Name: "DB", Value: backupCR.Spec.DatabaseInstance, - }, { - Name: "KOLLA_CONFIG_STRATEGY", - Value: "COPY_ALWAYS", }} // The restore pod uses the same container image as the configured backup CR, @@ -43,19 +40,14 @@ func RestorePod(restoreCR *mariadbv1.GaleraRestore, backupCR *mariadbv1.GaleraBa Spec: corev1.PodSpec{ RestartPolicy: corev1.RestartPolicyOnFailure, ServiceAccountName: restoreCR.RbacResourceName(), - SecurityContext: &corev1.PodSecurityContext{ - FSGroup: ptr.To[int64](42434), - }, - InitContainers: []corev1.Container{}, + SecurityContext: mariadb.PodSecurityContext(), Containers: []corev1.Container{{ - Image: backupPodSpec.Containers[0].Image, - Name: "restore", - Command: []string{"/usr/bin/dumb-init", "--", "/bin/bash", "-c", - "sudo -E /usr/local/bin/kolla_set_configs;" + - "sudo -E /usr/local/bin/kolla_copy_cacerts;" + - "sleep infinity"}, - Env: environ, - VolumeMounts: RestoreVolumeMounts(backupCR, galeraCR), + Image: backupPodSpec.Containers[0].Image, + Name: "restore", + Command: []string{"/usr/bin/dumb-init", "--", "sleep", "infinity"}, + Env: environ, + SecurityContext: mariadb.GaleraSecurityContext(), + VolumeMounts: RestoreVolumeMounts(backupCR, galeraCR), }}, Volumes: RestoreVolumes(backupCR, galeraCR), }, diff --git a/internal/mariadb/backup/volumes.go b/internal/mariadb/backup/volumes.go index fa792a61..e8ded420 100644 --- a/internal/mariadb/backup/volumes.go +++ b/internal/mariadb/backup/volumes.go @@ -14,22 +14,12 @@ const ( ) // baseVolumes returns the volumes shared by both backup and restore pods: -// kolla-config, operator-scripts, backup-scripts, and backup-data PVC. +// var-local, operator-scripts, backup-scripts, and backup-data PVC. func baseVolumes(b *mariadbv1.GaleraBackup, g *mariadbv1.Galera) []corev1.Volume { return []corev1.Volume{{ - Name: "kolla-config", + Name: "var-local", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: b.Name + "-backup-config", - }, - Items: []corev1.KeyToPath{ - { - Key: "backup-config.json", - Path: "config.json", - }, - }, - }, + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, { Name: "operator-scripts", @@ -82,8 +72,8 @@ func tlsVolumes(b *mariadbv1.GaleraBackup, g *mariadbv1.Galera) []corev1.Volume if g.Spec.TLS.Enabled() { svc := tls.Service{ SecretName: *g.Spec.TLS.SecretName, - CertMount: nil, - KeyMount: nil, + CertMount: ptr.To("/etc/pki/tls/certs/galera.crt"), + KeyMount: ptr.To("/etc/pki/tls/private/galera.key"), CaMount: nil, } volumes = append(volumes, svc.CreateVolume(GaleraCertPrefix)) @@ -112,13 +102,9 @@ func tlsVolumes(b *mariadbv1.GaleraBackup, g *mariadbv1.Galera) []corev1.Volume } // baseVolumeMounts returns the volume mounts shared by both backup and restore pods: -// kolla-config, operator-scripts, and backup-scripts. +// operator-scripts, backup-scripts, and var-local. func baseVolumeMounts() []corev1.VolumeMount { return []corev1.VolumeMount{{ - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", - }, { MountPath: "/var/lib/operator-scripts", ReadOnly: true, Name: "operator-scripts", @@ -126,6 +112,9 @@ func baseVolumeMounts() []corev1.VolumeMount { MountPath: "/var/lib/backup-scripts", ReadOnly: true, Name: "backup-scripts", + }, { + MountPath: "/var/local", + Name: "var-local", }} } @@ -135,8 +124,8 @@ func tlsVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { if g.Spec.TLS.Enabled() { svc := tls.Service{ SecretName: *g.Spec.TLS.SecretName, - CertMount: nil, - KeyMount: nil, + CertMount: ptr.To("/etc/pki/tls/certs/galera.crt"), + KeyMount: ptr.To("/etc/pki/tls/private/galera.key"), CaMount: nil, } mounts = append(mounts, svc.CreateVolumeMounts(GaleraCertPrefix)...) diff --git a/internal/mariadb/const.go b/internal/mariadb/const.go index 5afe93c6..90048428 100644 --- a/internal/mariadb/const.go +++ b/internal/mariadb/const.go @@ -9,4 +9,7 @@ const ( // StartupProbeTimeout is the time allowed during the startup probe (in seconds) StartupProbeTimeout = 240 + + // MysqlUID is the UID/GID of the mysql user in the container image (from TCIB) + MysqlUID int64 = 42434 ) diff --git a/internal/mariadb/database.go b/internal/mariadb/database.go index 9e38c2d9..f2bdad76 100644 --- a/internal/mariadb/database.go +++ b/internal/mariadb/database.go @@ -81,11 +81,12 @@ func DbDatabaseJob(galera *mariadbv1.Galera, database *mariadbv1.MariaDBDatabase ServiceAccountName: serviceAccountName, Containers: []corev1.Container{ { - Name: "mariadb-database-create", - Image: containerImage, - Command: []string{"/bin/sh", "-c", dbCmd}, - Env: scriptEnv, - VolumeMounts: getGaleraRootOnlyVolumeMounts(), + Name: "mariadb-database-create", + Image: containerImage, + Command: []string{"/bin/sh", "-c", dbCmd}, + Env: scriptEnv, + VolumeMounts: getGaleraRootOnlyVolumeMounts(), + SecurityContext: GaleraSecurityContext(), }, }, Volumes: getGaleraRootOnlyVolumes(galera), @@ -155,11 +156,12 @@ func DeleteDbDatabaseJob(galera *mariadbv1.Galera, database *mariadbv1.MariaDBDa ServiceAccountName: serviceAccountName, Containers: []corev1.Container{ { - Name: "mariadb-database-create", - Image: containerImage, - Command: []string{"/bin/sh", "-c", delCmd}, - Env: scriptEnv, - VolumeMounts: getGaleraRootOnlyVolumeMounts(), + Name: "mariadb-database-create", + Image: containerImage, + Command: []string{"/bin/sh", "-c", delCmd}, + Env: scriptEnv, + VolumeMounts: getGaleraRootOnlyVolumeMounts(), + SecurityContext: GaleraSecurityContext(), }, }, Volumes: getGaleraRootOnlyVolumes(galera), diff --git a/internal/mariadb/security.go b/internal/mariadb/security.go new file mode 100644 index 00000000..abd099e7 --- /dev/null +++ b/internal/mariadb/security.go @@ -0,0 +1,34 @@ +package mariadb + +import ( + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" +) + +// PodSecurityContext returns a PodSecurityContext with FSGroup set to the +// mysql UID. This ensures that EmptyDir and ConfigMap volumes are group-owned +// by the mysql user, which is required for pods that write to those volumes +// (e.g. galera pods writing to config-data-generated and var-local, or +// backup/restore pods writing to their working directories). +func PodSecurityContext() *corev1.PodSecurityContext { + return &corev1.PodSecurityContext{ + FSGroup: ptr.To(MysqlUID), + } +} + +// GaleraSecurityContext returns a SecurityContext for galera and related containers +func GaleraSecurityContext() *corev1.SecurityContext { + return &corev1.SecurityContext{ + RunAsUser: ptr.To(MysqlUID), + RunAsGroup: ptr.To(MysqlUID), + RunAsNonRoot: ptr.To(true), + AllowPrivilegeEscalation: ptr.To(false), + ReadOnlyRootFilesystem: ptr.To(false), + Capabilities: &corev1.Capabilities{ + Drop: []corev1.Capability{"ALL"}, + }, + SeccompProfile: &corev1.SeccompProfile{ + Type: corev1.SeccompProfileTypeRuntimeDefault, + }, + } +} diff --git a/internal/mariadb/statefulset.go b/internal/mariadb/statefulset.go index 0cd61bb8..1e0cbce3 100644 --- a/internal/mariadb/statefulset.go +++ b/internal/mariadb/statefulset.go @@ -52,6 +52,7 @@ func StatefulSet(g *mariadbv1.Galera, configHash string, topology *topologyv1.To }, Spec: corev1.PodSpec{ ServiceAccountName: g.RbacResourceName(), + SecurityContext: PodSecurityContext(), InitContainers: getGaleraInitContainers(g), Containers: containers, Volumes: getGaleraVolumes(g), @@ -100,18 +101,12 @@ func StatefulSet(g *mariadbv1.Galera, configHash string, topology *topologyv1.To func getGaleraInitContainers(g *mariadbv1.Galera) []corev1.Container { return []corev1.Container{{ - Image: g.Spec.ContainerImage, - Name: "mysql-bootstrap", - Command: []string{"bash", "/var/lib/operator-scripts/mysql_bootstrap.sh"}, - Env: []corev1.EnvVar{{ - Name: "KOLLA_BOOTSTRAP", - Value: "True", - }, { - Name: "KOLLA_CONFIG_STRATEGY", - Value: "COPY_ALWAYS", - }}, - Resources: g.Spec.Resources, - VolumeMounts: getGaleraInitVolumeMounts(g), + Image: g.Spec.ContainerImage, + Name: "mysql-bootstrap", + Command: []string{"bash", "/var/lib/operator-scripts/mysql_bootstrap.sh"}, + Resources: g.Spec.Resources, + VolumeMounts: getGaleraInitVolumeMounts(g), + SecurityContext: GaleraSecurityContext(), }} } @@ -141,11 +136,11 @@ func getGaleraContainers(g *mariadbv1.Galera, configHash string) ([]corev1.Conta StartupProbes: &startupConf, LivenessProbes: &probes.ProbeConf{ Type: probes.ProbeHandlerExec, - Command: []string{"/bin/bash", "/var/lib/operator-scripts/mysql_probe.sh", "liveness"}, + Command: []string{"/var/lib/operator-scripts/mysql_probe.sh", "liveness"}, }, ReadinessProbes: &probes.ProbeConf{ Type: probes.ProbeHandlerExec, - Command: []string{"/bin/bash", "/var/lib/operator-scripts/mysql_probe.sh", "readiness"}, + Command: []string{"/var/lib/operator-scripts/mysql_probe.sh", "readiness"}, }, }, ) @@ -156,13 +151,10 @@ func getGaleraContainers(g *mariadbv1.Galera, configHash string) ([]corev1.Conta containers := []corev1.Container{{ Image: g.Spec.ContainerImage, Name: "galera", - Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"}, + Command: []string{"/usr/bin/dumb-init", "--", "/var/lib/operator-scripts/detect_gcomm_and_start.sh"}, Env: []corev1.EnvVar{{ Name: "CR_CONFIG_HASH", Value: configHash, - }, { - Name: "KOLLA_CONFIG_STRATEGY", - Value: "COPY_ALWAYS", }}, Ports: []corev1.ContainerPort{{ ContainerPort: 3306, @@ -171,24 +163,26 @@ func getGaleraContainers(g *mariadbv1.Galera, configHash string) ([]corev1.Conta ContainerPort: 4567, Name: "galera", }}, - Resources: g.Spec.Resources, - VolumeMounts: getGaleraVolumeMounts(g), - StartupProbe: probeSet.Startup, - LivenessProbe: probeSet.Liveness, - ReadinessProbe: probeSet.Readiness, + Resources: g.Spec.Resources, + VolumeMounts: getGaleraVolumeMounts(g), + StartupProbe: probeSet.Startup, + LivenessProbe: probeSet.Liveness, + ReadinessProbe: probeSet.Readiness, + SecurityContext: GaleraSecurityContext(), Lifecycle: &corev1.Lifecycle{ PreStop: &corev1.LifecycleHandler{ Exec: &corev1.ExecAction{ - Command: []string{"/bin/bash", "/var/lib/operator-scripts/mysql_shutdown.sh"}, + Command: []string{"/var/lib/operator-scripts/mysql_shutdown.sh"}, }, }, }, }} logSideCar := corev1.Container{ - Image: g.Spec.ContainerImage, - Name: "log", - Command: []string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", "tail -n+1 -F /var/log/mariadb/mariadb.log"}, - VolumeMounts: getGaleraVolumeMounts(g), + Image: g.Spec.ContainerImage, + Name: "log", + Command: []string{"/usr/bin/dumb-init", "--", "/bin/sh", "-c", "tail -n+1 -F /var/log/mariadb/mariadb.log"}, + VolumeMounts: getGaleraVolumeMounts(g), + SecurityContext: GaleraSecurityContext(), } if g.Spec.LogToDisk { diff --git a/internal/mariadb/volumes.go b/internal/mariadb/volumes.go index 6dd08275..5548d28b 100644 --- a/internal/mariadb/volumes.go +++ b/internal/mariadb/volumes.go @@ -4,6 +4,7 @@ import ( tls "github.com/openstack-k8s-operators/lib-common/modules/common/tls" mariadbv1 "github.com/openstack-k8s-operators/mariadb-operator/api/v1beta1" corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" ) const ( @@ -40,23 +41,13 @@ func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { volumes := []corev1.Volume{ { - Name: "kolla-config", + Name: "config-data-generated", VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - LocalObjectReference: corev1.LocalObjectReference{ - Name: g.Name + "-config-data", - }, - Items: []corev1.KeyToPath{ - { - Key: "config.json", - Path: "config.json", - }, - }, - }, + EmptyDir: &corev1.EmptyDirVolumeSource{}, }, }, { - Name: "config-data-generated", + Name: "var-local", VolumeSource: corev1.VolumeSource{ EmptyDir: &corev1.EmptyDirVolumeSource{}, }, @@ -79,6 +70,7 @@ func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { LocalObjectReference: corev1.LocalObjectReference{ Name: g.Name + "-scripts", }, + DefaultMode: ptr.To[int32](0755), Items: []corev1.KeyToPath{ { Key: "mysql_bootstrap.sh", @@ -121,8 +113,8 @@ func getGaleraVolumes(g *mariadbv1.Galera) []corev1.Volume { if g.Spec.TLS.Enabled() { svc := tls.Service{ SecretName: *g.Spec.TLS.SecretName, - CertMount: nil, - KeyMount: nil, + CertMount: ptr.To("/etc/pki/tls/certs/galera.crt"), + KeyMount: ptr.To("/etc/pki/tls/private/galera.key"), CaMount: nil, } serviceVolume := svc.CreateVolume(GaleraCertPrefix) @@ -145,6 +137,7 @@ func getGaleraRootOnlyVolumes(g *mariadbv1.Galera) []corev1.Volume { LocalObjectReference: corev1.LocalObjectReference{ Name: g.Name + "-scripts", }, + DefaultMode: ptr.To[int32](0755), Items: []corev1.KeyToPath{ { Key: "mysql_root_auth.sh", @@ -154,6 +147,12 @@ func getGaleraRootOnlyVolumes(g *mariadbv1.Galera) []corev1.Volume { }, }, }, + { + Name: "var-local", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, } return volumes @@ -172,14 +171,16 @@ func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { }, { MountPath: "/var/lib/config-data/generated", Name: "config-data-generated", + }, { + MountPath: "/etc/my.cnf.d", + Name: "config-data-generated", }, { MountPath: "/var/lib/operator-scripts", ReadOnly: true, Name: "operator-scripts", }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", + MountPath: "/var/local", + Name: "var-local", }, } @@ -190,8 +191,8 @@ func getGaleraVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { if g.Spec.TLS.Enabled() { svc := tls.Service{ SecretName: *g.Spec.TLS.SecretName, - CertMount: nil, - KeyMount: nil, + CertMount: ptr.To("/etc/pki/tls/certs/galera.crt"), + KeyMount: ptr.To("/etc/pki/tls/private/galera.key"), CaMount: nil, } serviceVolumeMounts := svc.CreateVolumeMounts(GaleraCertPrefix) @@ -212,6 +213,10 @@ func getGaleraRootOnlyVolumeMounts() []corev1.VolumeMount { ReadOnly: true, Name: "operator-scripts", }, + { + MountPath: "/var/local", + Name: "var-local", + }, } return volumeMounts @@ -230,14 +235,16 @@ func getGaleraInitVolumeMounts(g *mariadbv1.Galera) []corev1.VolumeMount { }, { MountPath: "/var/lib/config-data/generated", Name: "config-data-generated", + }, { + MountPath: "/etc/my.cnf.d", + Name: "config-data-generated", }, { MountPath: "/var/lib/operator-scripts", ReadOnly: true, Name: "operator-scripts", }, { - MountPath: "/var/lib/kolla/config_files", - ReadOnly: true, - Name: "kolla-config", + MountPath: "/var/local", + Name: "var-local", }, } diff --git a/templates/galera/bin/detect_last_commit.sh b/templates/galera/bin/detect_last_commit.sh index 079d35b9..d8606468 100755 --- a/templates/galera/bin/detect_last_commit.sh +++ b/templates/galera/bin/detect_last_commit.sh @@ -82,7 +82,6 @@ fi if [ -z "$seqno" ] || [ "$seqno" = "-1" ]; then tmp=$(mktemp) - chown mysql:mysql $tmp # if we pass here because grastate.dat doesn't exist, report it if [ ! -f /var/lib/mysql/grastate.dat ]; then diff --git a/templates/galera/bin/mysql_bootstrap.sh b/templates/galera/bin/mysql_bootstrap.sh index 996315a7..cb47641e 100755 --- a/templates/galera/bin/mysql_bootstrap.sh +++ b/templates/galera/bin/mysql_bootstrap.sh @@ -2,10 +2,31 @@ set +eux +function wait_for_mysqld_readiness { + # Wait for the mariadb server to be "Ready" before running commands + # Querying the cluster status has to be executed after the existence of mysql.sock and mariadb.pid. + local mysql_pid_file=$1 + local mysql_log_file=$2 + + ORIG_TIMEOUT=${DB_MAX_TIMEOUT:-60} + TIMEOUT=${ORIG_TIMEOUT} + while [[ ! -S /var/lib/mysql/mysql.sock ]] || \ + [[ ! -f "${mysql_pid_file}" ]]; do + + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo -e "Surpassed timeout of ${ORIG_TIMEOUT} without seeing a pidfile" + echo -e "Dump of ${mysql_log_file}" + cat "${mysql_log_file}" + exit 1 + fi + done +} -function kolla_update_db_root_pw { +function update_db_root_pw { # update the root password given a set of mariadb datafiles - # ported from kolla_extend_start with major changes # because galera controller generates a new root password if one was # not sent via pre-existing secret, the root pw has to be updated if @@ -63,24 +84,7 @@ function kolla_update_db_root_pw { echo -e "Running with --skip-grant-tables to reset root password" rm -fv ${CHANGE_PW_PIDFILE} ${CHANGE_PW_LOGFILE} mysqld_safe --skip-grant-tables --wsrep-on=OFF --log-error=${CHANGE_PW_LOGFILE} --pid-file=${CHANGE_PW_PIDFILE} & - - # Wait for the mariadb server to be "Ready" before running root update commands - # Querying the cluster status has to be executed after the existence of mysql.sock and mariadb.pid. - ORIG_TIMEOUT=${DB_MAX_TIMEOUT:-60} - TIMEOUT=${ORIG_TIMEOUT} - while [[ ! -S /var/lib/mysql/mysql.sock ]] || \ - [[ ! -f "${CHANGE_PW_PIDFILE}" ]]; do - - if [[ ${TIMEOUT} -gt 0 ]]; then - let TIMEOUT-=1 - sleep 1 - else - echo -e "Surpassed timeout of ${ORIG_TIMEOUT} without seeing a pidfile" - echo -e "Dump of ${CHANGE_PW_LOGFILE}" - cat ${CHANGE_PW_LOGFILE} - exit 1 - fi - done + wait_for_mysqld_readiness "$CHANGE_PW_PIDFILE" "$CHANGE_PW_LOGFILE" echo -e "Refreshing root passwords" mysql -u root </var/lib/config-data/generated/galera.cnf [client] !includedir /var/local/my.cnf/ @@ -171,13 +190,13 @@ bind_address=localhost wsrep_provider=none EOF - kolla_set_all_configs - - kolla_extend_start + setup_configs_and_credentials + setup_new_database fi -# Generate the mariadb configs from the templates, these will get -# copied by `kolla_start` when the pod's main container will start +# Generate the mariadb configs from the templates; they are written +# to the config-data-generated EmptyDir which is also mounted at +# /etc/my.cnf.d/ so the main container picks them up directly. if [ "$(sysctl -n crypto.fips_enabled)" == "1" ]; then echo FIPS enabled SSL_CIPHER='ECDHE-RSA-AES256-GCM-SHA384' diff --git a/templates/galera/bin/mysql_probe.sh b/templates/galera/bin/mysql_probe.sh index 79733fd7..df968a85 100755 --- a/templates/galera/bin/mysql_probe.sh +++ b/templates/galera/bin/mysql_probe.sh @@ -111,7 +111,10 @@ function check_mysql_startup { # . only at this point, InnoDB is initialized, mysql pidfile and # mysql socket are created on disk - if pgrep -f detect_gcomm_and_start.sh >/dev/null ; then + # detect_gcomm_and_start is running only until a gcomm URI is + # created on disk, but its name shows in dumb_init (pid 1), so + # remove this match in the pgrep below + if pgrep -f detect_gcomm_and_start.sh | grep -v 1 >/dev/null ; then log_state "waiting for gcomm URI" return 1 fi diff --git a/templates/galera/bin/mysql_root_auth.sh b/templates/galera/bin/mysql_root_auth.sh index 44076da8..01c0c8c9 100755 --- a/templates/galera/bin/mysql_root_auth.sh +++ b/templates/galera/bin/mysql_root_auth.sh @@ -189,9 +189,8 @@ user=root password="${PASSWORD}" EOF then - # we are called for the first time from detect_gcomm_and_start.sh which is - # called **before** kolla can set directory permissions; so when writing - # the file, proceed even if we can't write the file yet + # /var/local is an EmptyDir which may not exist yet on the very first call; + # proceed even if we can't write the file yet echo "Did not yet write to ${PW_CACHE_FILE} due to permissions; will try again later" >&2 else echo "Wrote new credentials to ${PW_CACHE_FILE}" >&2 diff --git a/templates/galera/config/config.json b/templates/galera/config/config.json deleted file mode 100644 index 181c302f..00000000 --- a/templates/galera/config/config.json +++ /dev/null @@ -1,70 +0,0 @@ -{ - "command": "/usr/local/bin/detect_gcomm_and_start.sh", - "config_files": [ - { - "source": "/var/lib/config-data/generated/galera.cnf", - "dest": "/etc/my.cnf.d/galera.cnf", - "owner": "root", - "perm": "0644" - }, - { - "source": "/var/lib/config-data/generated/galera_tls.cnf", - "dest": "/etc/my.cnf.d/galera_tls.cnf", - "owner": "root", - "perm": "0644", - "optional": true - }, - { - "source": "/var/lib/config-data/generated/galera_external_tls.cnf", - "dest": "/etc/my.cnf.d/galera_external_tls.cnf", - "owner": "root", - "perm": "0644", - "optional": true - }, - { - "source": "/var/lib/config-data/generated/zzz_galera_custom.cnf", - "dest": "/etc/my.cnf.d/zzz_galera_custom.cnf", - "owner": "root", - "perm": "0644", - "optional": true - }, - { - "source": "/var/lib/operator-scripts", - "dest": "/usr/local/bin", - "owner": "root", - "perm": "0755", - "merge": "true" - }, - { - "source": "/var/lib/config-data/tls/private/galera.key", - "dest": "/etc/pki/tls/private/galera.key", - "owner": "mysql", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/config-data/tls/certs/galera.crt", - "dest": "/etc/pki/tls/certs/galera.crt", - "owner": "mysql", - "perm": "0755", - "optional": true - } - ], - "permissions": [ - { - "path": "/var/lib/mysql", - "owner": "mysql:mysql", - "recurse": "true" - }, - { - "path": "/var/log/mariadb", - "owner": "mysql:mysql", - "recurse": "true" - }, - { - "path": "/var/local", - "owner": "mysql:mysql", - "recurse": "true" - } - ] -} diff --git a/templates/galera/config/galera.cnf.in b/templates/galera/config/galera.cnf.in index efef5e95..fffa0bee 100644 --- a/templates/galera/config/galera.cnf.in +++ b/templates/galera/config/galera.cnf.in @@ -37,7 +37,7 @@ thread_cache_size = 8 thread_stack = 256K tmpdir = /tmp user = mysql -wsrep_notify_cmd = /usr/local/bin/mysql_wsrep_notify.sh +wsrep_notify_cmd = /var/lib/operator-scripts/mysql_wsrep_notify.sh wsrep_auto_increment_control = 1 wsrep_causal_reads = 0 wsrep_certify_nonPK = 1 diff --git a/templates/galerabackup/bin/backup_galera b/templates/galerabackup/bin/backup_galera index a25226e7..42748476 100755 --- a/templates/galerabackup/bin/backup_galera +++ b/templates/galerabackup/bin/backup_galera @@ -135,15 +135,15 @@ trap finally EXIT # - prepare local storage prior to backup # if [ "${TLS:-}" = "true" ]; then - MYSQL_OPTS="--defaults-extra-file=/etc/mysql_backup_tls.cnf" - WSREP_SST_CFG=/etc/mysql_backup_tls.cnf + MYSQL_OPTS="--defaults-extra-file=/var/lib/config-data/default/mysql_backup_tls.cnf" + WSREP_SST_CFG=/var/lib/config-data/default/mysql_backup_tls.cnf if [ "$(sysctl -n crypto.fips_enabled)" == "1" ]; then echo FIPS enabled GARBD_CIPHER='ECDHE-RSA-AES256-GCM-SHA384' else GARBD_CIPHER='AES128-SHA256' fi - . /etc/garbd_backup_tls.cnf + . /var/lib/config-data/default/garbd_backup_tls.cnf GARBD_OPTS=";${GARBD_TLS_OPTS}" else MYSQL_OPTS= diff --git a/templates/galerabackup/bin/restore_galera b/templates/galerabackup/bin/restore_galera index e8516eca..a452ff08 100755 --- a/templates/galerabackup/bin/restore_galera +++ b/templates/galerabackup/bin/restore_galera @@ -114,8 +114,8 @@ if [ "${confirm}" != "y" ]; then exit 0 fi -if [ -f /etc/mysql_backup_tls.cnf ]; then -MYSQL_OPTS="--defaults-extra-file=/etc/mysql_backup_tls.cnf" +if [ -f /var/lib/config-data/default/mysql_backup_tls.cnf ]; then +MYSQL_OPTS="--defaults-extra-file=/var/lib/config-data/default/mysql_backup_tls.cnf" else MYSQL_OPTS= fi diff --git a/templates/galerabackup/config/backup-config.json b/templates/galerabackup/config/backup-config.json deleted file mode 100644 index e148b1bf..00000000 --- a/templates/galerabackup/config/backup-config.json +++ /dev/null @@ -1,40 +0,0 @@ -{ - "command": "/var/lib/backup-scripts/backup_galera", - "config_files": [ - { - "source": "/var/lib/config-data/default/mysql_backup_tls.cnf", - "dest": "/etc/mysql_backup_tls.cnf", - "owner": "root", - "perm": "0644", - "optional": true - }, - { - "source": "/var/lib/config-data/default/garbd_backup_tls.cnf", - "dest": "/etc/garbd_backup_tls.cnf", - "owner": "root", - "perm": "0644", - "optional": true - }, - { - "source": "/var/lib/config-data/tls/private/galera.key", - "dest": "/etc/pki/tls/private/galera.key", - "owner": "mysql", - "perm": "0600", - "optional": true - }, - { - "source": "/var/lib/config-data/tls/certs/galera.crt", - "dest": "/etc/pki/tls/certs/galera.crt", - "owner": "mysql", - "perm": "0755", - "optional": true - } - ], - "permissions": [ - { - "path": "/var/local", - "owner": "mysql:mysql", - "recurse": "true" - } - ] -} diff --git a/test/chainsaw/common/galera-assert.yaml b/test/chainsaw/common/galera-assert.yaml index a664062d..3ca6c4d7 100644 --- a/test/chainsaw/common/galera-assert.yaml +++ b/test/chainsaw/common/galera-assert.yaml @@ -79,7 +79,7 @@ spec: - command: - /usr/bin/dumb-init - -- - - /usr/local/bin/kolla_start + - /var/lib/operator-scripts/detect_gcomm_and_start.sh name: galera ports: - containerPort: 3306 diff --git a/test/chainsaw/common/galera-no-secret-assert.yaml b/test/chainsaw/common/galera-no-secret-assert.yaml index 7fd6d024..8ce0267b 100644 --- a/test/chainsaw/common/galera-no-secret-assert.yaml +++ b/test/chainsaw/common/galera-no-secret-assert.yaml @@ -78,7 +78,7 @@ spec: - command: - /usr/bin/dumb-init - -- - - /usr/local/bin/kolla_start + - /var/lib/operator-scripts/detect_gcomm_and_start.sh name: galera ports: - containerPort: 3306 diff --git a/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml b/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml index 8015b394..455a2a8f 100644 --- a/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml +++ b/test/chainsaw/tests/galera-topology/galera-topology-assert.yaml @@ -93,7 +93,7 @@ spec: - command: - /usr/bin/dumb-init - -- - - /usr/local/bin/kolla_start + - /var/lib/operator-scripts/detect_gcomm_and_start.sh name: galera ports: - containerPort: 3306 diff --git a/test/chainsaw/tests/probe-config/chainsaw-test.yaml b/test/chainsaw/tests/probe-config/chainsaw-test.yaml index 0b45ea27..26e68ce6 100644 --- a/test/chainsaw/tests/probe-config/chainsaw-test.yaml +++ b/test/chainsaw/tests/probe-config/chainsaw-test.yaml @@ -45,11 +45,11 @@ spec: timeoutSeconds: 20 periodSeconds: 15 failureThreshold: 5 - initialDelaySeconds: 45 + initialDelaySeconds: 5 startupProbes: timeoutSeconds: 300 failureThreshold: 10 - initialDelaySeconds: 60 + initialDelaySeconds: 10 readinessProbes: timeoutSeconds: 10 periodSeconds: 5 @@ -67,11 +67,11 @@ spec: timeoutSeconds: 20 periodSeconds: 15 failureThreshold: 5 - initialDelaySeconds: 45 + initialDelaySeconds: 5 startupProbe: timeoutSeconds: 300 failureThreshold: 10 - initialDelaySeconds: 60 + initialDelaySeconds: 10 readinessProbe: timeoutSeconds: 10 periodSeconds: 5