Harden Redis, InstanceHA, Memcached, DNSMasq, and RabbitMQ workload security

config/rbac/role.yaml

@@ -348,7 +348,7 @@ rules:
 - apiGroups:
   - security.openshift.io
   resourceNames:
-  - anyuid
+  - nonroot-v2
   resources:
   - securitycontextconstraints
   verbs:

internal/controller/instanceha/instanceha_controller.go

@@ -95,7 +95,7 @@ func (r *Reconciler) GetLogger(ctx context.Context) logr.Logger {
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch
 // service account permissions that are needed to grant permission to the above
-// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use
+// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use
 // +kubebuilder:rbac:groups="",resources=pods,verbs=list
 // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
 // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update
@@ -205,7 +205,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 	rbacRules := []rbacv1.PolicyRule{
 		{
 			APIGroups:     []string{"security.openshift.io"},
-			ResourceNames: []string{"anyuid"},
+			ResourceNames: []string{"nonroot-v2"},
 			Resources:     []string{"securitycontextconstraints"},
 			Verbs:         []string{"use"},
 		},

internal/controller/memcached/memcached_controller.go

@@ -102,7 +102,7 @@ func (r *Reconciler) GetLogger(ctx context.Context) logr.Logger {
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch
 // service account permissions that are needed to grant permission to the above
-// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use
+// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use
 // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update
 
 // Reconcile - Memcached
@@ -223,7 +223,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 	rbacRules := []rbacv1.PolicyRule{
 		{
 			APIGroups:     []string{"security.openshift.io"},
-			ResourceNames: []string{"anyuid"},
+			ResourceNames: []string{"nonroot-v2"},
 			Resources:     []string{"securitycontextconstraints"},
 			Verbs:         []string{"use"},
 		},
@@ -489,8 +489,8 @@ func (r *Reconciler) generateConfigMaps(
 	if instance.Spec.TLS.Enabled() {
 		memcachedTLSListen = "| sed 's/\\(.*\\)/\\1\\nnotls:\\1:11211/'"
 		memcachedTLSOptions = "-Z " +
-			"-o ssl_chain_cert=/etc/pki/tls/certs/memcached.crt " +
-			"-o ssl_key=/etc/pki/tls/private/memcached.key " +
+			"-o ssl_chain_cert=/var/lib/config-data/tls/certs/memcached.crt " +
+			"-o ssl_key=/var/lib/config-data/tls/private/memcached.key " +
 			"-o ssl_ca_cert=/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem"
 
 		switch instance.Spec.TLS.MTLS.SslVerifyMode {

internal/controller/network/dnsmasq_controller.go

@@ -97,7 +97,7 @@ func (r *DNSMasqReconciler) GetLogger(ctx context.Context) logr.Logger {
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update;patch
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update;patch
 // service account permissions that are needed to grant permission to the above
-// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use
+// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use
 // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update
 
 // Reconcile is part of the main kubernetes reconciliation loop which aims to
@@ -204,7 +204,7 @@ func (r *DNSMasqReconciler) Reconcile(ctx context.Context, req ctrl.Request) (re
 	rbacRules := []rbacv1.PolicyRule{
 		{
 			APIGroups:     []string{"security.openshift.io"},
-			ResourceNames: []string{"anyuid"},
+			ResourceNames: []string{"nonroot-v2"},
 			Resources:     []string{"securitycontextconstraints"},
 			Verbs:         []string{"use"},
 		},

internal/controller/rabbitmq/proxy.go

@@ -8,6 +8,7 @@ import (
 	rabbitmqv1beta1 "github.com/openstack-k8s-operators/infra-operator/apis/rabbitmq/v1beta1"
 	"github.com/openstack-k8s-operators/infra-operator/internal/rabbitmq"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/helper"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -153,12 +154,11 @@ func BuildProxySidecarContainer(instance *rabbitmqv1beta1.RabbitMq, IPv6Enabled
 			TimeoutSeconds:      3,
 			FailureThreshold:    3,
 		},
-		SecurityContext: &corev1.SecurityContext{
-			AllowPrivilegeEscalation: ptr.To(false),
-			Capabilities: &corev1.Capabilities{
-				Drop: []corev1.Capability{"ALL"},
-			},
-		},
+		SecurityContext: func() *corev1.SecurityContext {
+			sc := pod.RestrictiveSecurityContext(999)
+			sc.ReadOnlyRootFilesystem = ptr.To(false)
+			return sc
+		}(),
 	}
 
 	// Mount TLS certificates if TLS is enabled

internal/controller/rabbitmq/rabbitmq_controller.go

@@ -130,11 +130,15 @@ type Reconciler struct {
 // +kubebuilder:rbac:groups=config.openshift.io,resources=networks,verbs=get;list;watch;
 // +kubebuilder:rbac:groups="",resources=nodes,verbs=get;list
 
+// Required to label and delete pods during CR deletion
 // +kubebuilder:rbac:groups=core,resources=pods,verbs=list;watch;update;delete
 
 // Required to manage PodDisruptionBudgets for multi-replica deployments
 // +kubebuilder:rbac:groups=policy,resources=poddisruptionbudgets,verbs=get;list;watch;create;update;patch;delete
 
+// Required to grant nonroot-v2 SCC to RabbitMQ workload SA
+// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use
+
 // Required to create per-pod LoadBalancer services
 // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
 
@@ -575,7 +579,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 	rbacRules := []rbacv1.PolicyRule{
 		{
 			APIGroups:     []string{"security.openshift.io"},
-			ResourceNames: []string{"anyuid"},
+			ResourceNames: []string{"nonroot-v2"},
 			Resources:     []string{"securitycontextconstraints"},
 			Verbs:         []string{"use"},
 		},

internal/controller/redis/redis_controller.go

@@ -105,7 +105,7 @@ type Reconciler struct {
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=roles,verbs=get;list;watch;create;update
 // +kubebuilder:rbac:groups="rbac.authorization.k8s.io",resources=rolebindings,verbs=get;list;watch;create;update
 // service account permissions that are needed to grant permission to the above
-// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=anyuid,resources=securitycontextconstraints,verbs=use
+// +kubebuilder:rbac:groups="security.openshift.io",resourceNames=nonroot-v2,resources=securitycontextconstraints,verbs=use
 // +kubebuilder:rbac:groups="",resources=pods,verbs=patch
 // +kubebuilder:rbac:groups=topology.openstack.org,resources=topologies,verbs=get;list;watch;update
 
@@ -211,7 +211,7 @@ func (r *Reconciler) Reconcile(ctx context.Context, req ctrl.Request) (result ct
 	rbacRules := []rbacv1.PolicyRule{
 		{
 			APIGroups:     []string{"security.openshift.io"},
-			ResourceNames: []string{"anyuid"},
+			ResourceNames: []string{"nonroot-v2"},
 			Resources:     []string{"securitycontextconstraints"},
 			Verbs:         []string{"use"},
 		},

internal/dnsmasq/const.go

@@ -24,4 +24,7 @@ const (
 	DNSPort int32 = 53
 	// DNSTargetPort - port used the service is listening on in the pod
 	DNSTargetPort int32 = 5353
+
+	// DnsmasqUID - https://github.com/openstack/kolla/blob/master/kolla/common/users.py
+	DnsmasqUID int64 = 42435
 )

internal/dnsmasq/deployment.go

@@ -26,6 +26,7 @@ import (
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/env"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
@@ -121,21 +122,20 @@ func Deployment(
 					Labels:      labels,
 				},
 				Spec: corev1.PodSpec{
-					ServiceAccountName: instance.RbacResourceName(),
-					Volumes:            getVolumes(instance.Name, cms),
+					ServiceAccountName:           instance.RbacResourceName(),
+					AutomountServiceAccountToken: ptr.To(false),
+					Volumes:                      getVolumes(instance.Name, cms),
 					InitContainers: []corev1.Container{
 						{
 							Name:    "init",
 							Command: command,
 							Args:    initArgs,
 							Image:   instance.Spec.ContainerImage,
-							SecurityContext: &corev1.SecurityContext{
-								RunAsNonRoot:             ptr.To(true),
-								AllowPrivilegeEscalation: ptr.To(false),
-								SeccompProfile: &corev1.SeccompProfile{
-									Type: corev1.SeccompProfileTypeRuntimeDefault,
-								},
-							},
+							SecurityContext: func() *corev1.SecurityContext {
+								sc := pod.RestrictiveSecurityContext(DnsmasqUID)
+								sc.ReadOnlyRootFilesystem = ptr.To(false)
+								return sc
+							}(),
 							Env:          env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts: getVolumeMounts(instance.Name, cms),
 						},
@@ -146,13 +146,11 @@ func Deployment(
 							Command: command,
 							Args:    args,
 							Image:   instance.Spec.ContainerImage,
-							SecurityContext: &corev1.SecurityContext{
-								RunAsNonRoot:             ptr.To(true),
-								AllowPrivilegeEscalation: ptr.To(false),
-								SeccompProfile: &corev1.SeccompProfile{
-									Type: corev1.SeccompProfileTypeRuntimeDefault,
-								},
-							},
+							SecurityContext: func() *corev1.SecurityContext {
+								sc := pod.RestrictiveSecurityContext(DnsmasqUID)
+								sc.ReadOnlyRootFilesystem = ptr.To(false)
+								return sc
+							}(),
 							Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
 							VolumeMounts:   getVolumeMounts(instance.Name, cms),
 							ReadinessProbe: readinessProbe,

internal/instanceha/funcs.go

@@ -14,16 +14,17 @@ limitations under the License.
 package instanceha
 
 import (
+	"fmt"
+
 	instancehav1 "github.com/openstack-k8s-operators/infra-operator/apis/instanceha/v1beta1"
 	topologyv1 "github.com/openstack-k8s-operators/infra-operator/apis/topology/v1beta1"
 	env "github.com/openstack-k8s-operators/lib-common/modules/common/env"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/serviceaccount"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/tls"
-
-	"fmt"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/utils/ptr"
 )
@@ -172,33 +173,23 @@ func Deployment(
 					Annotations: annotations,
 				},
 				Spec: corev1.PodSpec{
-					ServiceAccountName: instance.RbacResourceName(),
+					ServiceAccountName:           instance.RbacResourceName(),
+					AutomountServiceAccountToken: ptr.To(false),
 					SecurityContext: &corev1.PodSecurityContext{
 						FSGroup: ptr.To(instanceHaUID),
 					},
-					Volumes:                       volumes,
+					Volumes:                       append(volumes, serviceaccount.KubeAPIAccessVolume()),
 					TerminationGracePeriodSeconds: ptr.To[int64](45),
 					Containers: []corev1.Container{{
-						Name:    "instanceha",
-						Image:   containerImage,
-						Command: []string{"/usr/bin/python3", "-u", "/var/lib/instanceha/instanceha.py"},
-						SecurityContext: &corev1.SecurityContext{
-							RunAsUser:                ptr.To(instanceHaUID),
-							RunAsGroup:               ptr.To(instanceHaUID),
-							RunAsNonRoot:             ptr.To(true),
-							ReadOnlyRootFilesystem:   ptr.To(true),
-							AllowPrivilegeEscalation: ptr.To(false),
-							Capabilities: &corev1.Capabilities{
-								Drop: []corev1.Capability{
-									"ALL",
-								},
-							},
-						},
-						Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
-						Ports:          instancehaPorts(instance),
-						VolumeMounts:   volumeMounts,
-						LivenessProbe:  livenessProbe,
-						ReadinessProbe: readinessProbe,
+						Name:            "instanceha",
+						Image:           containerImage,
+						Command:         []string{"/usr/bin/python3", "-u", "/var/lib/instanceha/instanceha.py"},
+						SecurityContext: pod.RestrictiveSecurityContext(instanceHaUID),
+						Env:             env.MergeEnvs([]corev1.EnvVar{}, envVars),
+						Ports:           instancehaPorts(instance),
+						VolumeMounts:    append(volumeMounts, serviceaccount.KubeAPIAccessVolumeMount()),
+						LivenessProbe:   livenessProbe,
+						ReadinessProbe:  readinessProbe,
 					}},
 				},
 			},

internal/instanceha/funcs_test.go

@@ -6,6 +6,7 @@ import (
 	. "github.com/onsi/gomega" //revive:disable:dot-imports
 
 	instancehav1 "github.com/openstack-k8s-operators/infra-operator/apis/instanceha/v1beta1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -101,6 +102,51 @@ func TestDeploymentSecurityContext(t *testing.T) {
 	g.Expect(container.SecurityContext.ReadOnlyRootFilesystem).NotTo(BeNil())
 	g.Expect(*container.SecurityContext.ReadOnlyRootFilesystem).To(BeTrue())
 
+	g.Expect(container.SecurityContext.RunAsNonRoot).NotTo(BeNil())
+	g.Expect(*container.SecurityContext.RunAsNonRoot).To(BeTrue())
+
+	g.Expect(container.SecurityContext.RunAsUser).NotTo(BeNil())
+	g.Expect(*container.SecurityContext.RunAsUser).To(Equal(int64(42401)))
+
+	g.Expect(container.SecurityContext.RunAsGroup).NotTo(BeNil())
+	g.Expect(*container.SecurityContext.RunAsGroup).To(Equal(int64(42401)))
+
+	g.Expect(container.SecurityContext.AllowPrivilegeEscalation).NotTo(BeNil())
+	g.Expect(*container.SecurityContext.AllowPrivilegeEscalation).To(BeFalse())
+
+	g.Expect(container.SecurityContext.Capabilities).NotTo(BeNil())
+	g.Expect(container.SecurityContext.Capabilities.Drop).To(ContainElement(corev1.Capability("ALL")))
+
+	g.Expect(container.SecurityContext.SeccompProfile).NotTo(BeNil())
+	g.Expect(container.SecurityContext.SeccompProfile.Type).To(Equal(corev1.SeccompProfileTypeRuntimeDefault))
+
+	// Verify AutomountServiceAccountToken is disabled
+	g.Expect(dep.Spec.Template.Spec.AutomountServiceAccountToken).NotTo(BeNil())
+	g.Expect(*dep.Spec.Template.Spec.AutomountServiceAccountToken).To(BeFalse())
+
+	// Verify kube-api-access projected volume exists
+	var kubeAPIVolumeFound bool
+	for _, vol := range dep.Spec.Template.Spec.Volumes {
+		if vol.Name == "kube-api-access" {
+			kubeAPIVolumeFound = true
+			g.Expect(vol.VolumeSource.Projected).NotTo(BeNil())
+			g.Expect(vol.VolumeSource.Projected.Sources).To(HaveLen(3))
+			break
+		}
+	}
+	g.Expect(kubeAPIVolumeFound).To(BeTrue(), "kube-api-access projected volume should exist")
+
+	// Verify kube-api-access mount exists
+	var kubeAPIMountFound bool
+	for _, mount := range container.VolumeMounts {
+		if mount.Name == "kube-api-access" && mount.MountPath == "/var/run/secrets/kubernetes.io/serviceaccount" {
+			kubeAPIMountFound = true
+			g.Expect(mount.ReadOnly).To(BeTrue())
+			break
+		}
+	}
+	g.Expect(kubeAPIMountFound).To(BeTrue(), "kube-api-access volume mount should exist")
+
 	// Verify /tmp emptyDir volume exists
 	var tmpVolumeFound bool
 	for _, vol := range dep.Spec.Template.Spec.Volumes {

internal/memcached/statefulset.go

@@ -6,6 +6,7 @@ import (
 	common "github.com/openstack-k8s-operators/lib-common/modules/common"
 	"github.com/openstack-k8s-operators/lib-common/modules/common/affinity"
 	labels "github.com/openstack-k8s-operators/lib-common/modules/common/labels"
+	"github.com/openstack-k8s-operators/lib-common/modules/common/pod"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -63,19 +64,21 @@ func StatefulSet(
 					Labels: ls,
 				},
 				Spec: corev1.PodSpec{
-					ServiceAccountName: m.RbacResourceName(),
+					ServiceAccountName:           m.RbacResourceName(),
+					AutomountServiceAccountToken: ptr.To(false),
+					SecurityContext: &corev1.PodSecurityContext{
+						FSGroup: ptr.To(MemcachedUID),
+					},
 					Containers: []corev1.Container{{
 						Image:   m.Spec.ContainerImage,
 						Name:    "memcached",
-						Command: []string{"/usr/bin/dumb-init", "--", "/usr/local/bin/kolla_start"},
-						SecurityContext: &corev1.SecurityContext{
-							RunAsUser:  ptr.To(MemcachedUID),
-							RunAsGroup: ptr.To(MemcachedUID),
-						},
+						Command: []string{"/usr/bin/dumb-init", "--", "bash", "-c", "source /etc/sysconfig/memcached && exec /usr/bin/memcached -p ${PORT} -u ${USER} -m ${CACHESIZE} -c ${MAXCONN} $OPTIONS"},
+						SecurityContext: func() *corev1.SecurityContext {
+							sc := pod.RestrictiveSecurityContext(MemcachedUID)
+							sc.ReadOnlyRootFilesystem = ptr.To(false)
+							return sc
+						}(),
 						Env: []corev1.EnvVar{{
-							Name:  "KOLLA_CONFIG_STRATEGY",
-							Value: "COPY_ALWAYS",
-						}, {
 							Name: "POD_IPS",
 							ValueFrom: &corev1.EnvVarSource{
 								FieldRef: &corev1.ObjectFieldSelector{