Add securityContext to ARO worker and master operator deployments as…

… required by k8s v1.27.x

pkg/cluster/cluster.go

@@ -20,6 +20,7 @@ import (
 	extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/cluster/graph"
@@ -89,6 +90,7 @@ type manager struct {
 	subnet  subnet.Manager
 	graph   graph.Manager
 
+	client           client.Client
 	kubernetescli    kubernetes.Interface
 	dynamiccli       dynamic.Interface
 	extensionscli    extensionsclient.Interface

pkg/cluster/install.go

@@ -19,6 +19,8 @@ import (
 	extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	"k8s.io/client-go/dynamic"
 	"k8s.io/client-go/kubernetes"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 
 	"github.com/Azure/ARO-RP/pkg/api"
 	"github.com/Azure/ARO-RP/pkg/containerinstall"
@@ -499,13 +501,25 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error {
 	}
 
 	m.imageregistrycli, err = imageregistryclient.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
+	mapper, err := apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery)
+	if err != nil {
+		return err
+	}
+
+	m.client, err = client.New(restConfig, client.Options{
+		Mapper: mapper,
+	})
 	return err
 }
 
 // initializeKubernetesClients initializes clients which are used
 // once the cluster is up later on in the install process.
 func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) {
-	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli)
+	m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli)
 	return
 }
 

pkg/operator/controllers/genevalogging/genevalogging.go

@@ -22,6 +22,12 @@ import (
 	"github.com/Azure/ARO-RP/pkg/util/version"
 )
 
+var privilegedNamespaceLabels = map[string]string{
+	"pod-security.kubernetes.io/enforce": "privileged",
+	"pod-security.kubernetes.io/audit":   "privileged",
+	"pod-security.kubernetes.io/warn":    "privileged",
+}
+
 func (r *Reconciler) securityContextConstraints(ctx context.Context, name, serviceAccountName string) (*securityv1.SecurityContextConstraints, error) {
 	scc := &securityv1.SecurityContextConstraints{}
 	err := r.Client.Get(ctx, types.NamespacedName{Name: "privileged"}, scc)
@@ -284,6 +290,7 @@ func (r *Reconciler) resources(ctx context.Context, cluster *arov1alpha1.Cluster
 			ObjectMeta: metav1.ObjectMeta{
 				Name:        kubeNamespace,
 				Annotations: map[string]string{projectv1.ProjectNodeSelector: ""},
+				Labels:      privilegedNamespaceLabels,
 			},
 		},
 		&corev1.Secret{

pkg/operator/deploy/deploy.go

@@ -28,6 +28,7 @@ import (
 	appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1"
 	corev1client "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/util/retry"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	"github.com/Azure/ARO-RP/pkg/api"
@@ -63,12 +64,13 @@ type operator struct {
 	oc  *api.OpenShiftCluster
 
 	arocli        aroclient.Interface
+	client        client.Client
 	extensionscli extensionsclient.Interface
 	kubernetescli kubernetes.Interface
 	dh            dynamichelper.Interface
 }
 
-func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
+func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) {
 	restConfig, err := restconfig.RestConfig(env, oc)
 	if err != nil {
 		return nil, err
@@ -84,6 +86,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli
 		oc:  oc,
 
 		arocli:        arocli,
+		client:        client,
 		extensionscli: extensionscli,
 		kubernetescli: kubernetescli,
 		dh:            dh,
@@ -125,7 +128,7 @@ func templateManifests(data deploymentData) ([][]byte, error) {
 	return templatedFiles, nil
 }
 
-func (o *operator) createDeploymentData() deploymentData {
+func (o *operator) createDeploymentData() (deploymentData, error) {
 	image := o.env.AROOperatorImage()
 
 	// HACK: Override for ARO_IMAGE env variable setup in local-dev mode
@@ -145,11 +148,15 @@ func (o *operator) createDeploymentData() deploymentData {
 		IsLocalDevelopment: o.env.IsLocalDevelopmentMode(),
 		Image:              image,
 		Version:            version,
-	}
+	}, nil
 }
 
 func (o *operator) createObjects() ([]kruntime.Object, error) {
-	deploymentData := o.createDeploymentData()
+	deploymentData, err := o.createDeploymentData()
+	if err != nil {
+		return nil, err
+	}
+
 	templated, err := templateManifests(deploymentData)
 	if err != nil {
 		return nil, err

pkg/operator/deploy/deploy_test.go

@@ -137,6 +137,7 @@ func TestCreateDeploymentData(t *testing.T) {
 		mock                    func(*mock_env.MockInterface, *api.OpenShiftCluster)
 		operatorVersionOverride string
 		expected                deploymentData
+		wantErr                 string
 	}{
 		{
 			name: "no image override, use default",
@@ -147,7 +148,8 @@ func TestCreateDeploymentData(t *testing.T) {
 			},
 			expected: deploymentData{
 				Image:   operatorImageWithTag,
-				Version: operatorImageTag},
+				Version: operatorImageTag,
+			},
 		},
 		{
 			name: "no image tag, use latest version",
@@ -158,7 +160,8 @@ func TestCreateDeploymentData(t *testing.T) {
 			},
 			expected: deploymentData{
 				Image:   operatorImageUntagged,
-				Version: "latest"},
+				Version: "latest",
+			},
 		},
 		{
 			name: "OperatorVersion override set",
@@ -174,7 +177,20 @@ func TestCreateDeploymentData(t *testing.T) {
 			},
 			expected: deploymentData{
 				Image:   "docker.io/aro:override",
-				Version: "override"},
+				Version: "override",
+			},
+		},
+		{
+			name: "version supports pod security admission",
+			mock: func(env *mock_env.MockInterface, oc *api.OpenShiftCluster) {
+				env.EXPECT().
+					AROOperatorImage().
+					Return(operatorImageWithTag)
+			},
+			expected: deploymentData{
+				Image:   operatorImageWithTag,
+				Version: operatorImageTag,
+			},
 		},
 	} {
 		t.Run(tt.name, func(t *testing.T) {
@@ -192,7 +208,8 @@ func TestCreateDeploymentData(t *testing.T) {
 				env: env,
 			}
 
-			deploymentData := o.createDeploymentData()
+			deploymentData, err := o.createDeploymentData()
+			utilerror.AssertErrorMessage(t, err, tt.wantErr)
 			if !reflect.DeepEqual(deploymentData, tt.expected) {
 				t.Errorf("actual deployment: %v, expected %v", deploymentData, tt.expected)
 			}

pkg/operator/deploy/staticresources/master/deployment.yaml.tmpl

@@ -55,8 +55,22 @@ spec:
           httpGet:
             path: /healthz/ready
             port: 8080
+        {{ if .SupportsPodSecurityAdmission }} 
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          runAsNonRoot: true  
+        {{ end }}            
       nodeSelector:
         node-role.kubernetes.io/master: ""
+      {{ if .SupportsPodSecurityAdmission }}
+      securityContext:
+        runAsNonRoot: true        
+        seccompProfile: 
+          type: RuntimeDefault
+      {{ end }}         
       serviceAccountName: aro-operator-master
       serviceAccount: aro-operator-master
       priorityClassName: system-cluster-critical

pkg/operator/deploy/staticresources/worker/deployment.yaml.tmpl

@@ -37,8 +37,22 @@ spec:
           httpGet:
             path: /healthz/ready
             port: 8080
+        {{ if .SupportsPodSecurityAdmission }}
+        securityContext:
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop:
+            - ALL
+          runAsNonRoot: true  
+        {{ end }}            
       nodeSelector:
         node-role.kubernetes.io/worker: ""
+      {{ if .SupportsPodSecurityAdmission }}
+      securityContext:
+        runAsNonRoot: true        
+        seccompProfile: 
+          type: RuntimeDefault
+      {{ end }}         
       serviceAccountName: aro-operator-worker
       serviceAccount: aro-operator-worker
       priorityClassName: system-cluster-critical

pkg/operator/helpers_test.go

@@ -0,0 +1,44 @@
+package operator
+
+// Copyright (c) Microsoft Corporation.
+// Licensed under the Apache License 2.0.
+
+import (
+	"testing"
+
+	arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1"
+	_ "github.com/Azure/ARO-RP/pkg/util/scheme"
+)
+
+func aroCluster(domains []string) *arov1alpha1.Cluster {
+	return &arov1alpha1.Cluster{
+		Spec: arov1alpha1.ClusterSpec{
+			GatewayDomains: domains,
+		},
+	}
+}
+
+func TestGatewayEnabled(t *testing.T) {
+	tests := []struct {
+		name        string
+		cluster     *arov1alpha1.Cluster
+		wantEnabled bool
+	}{
+		{
+			name:    "gateway disabled",
+			cluster: aroCluster([]string{}),
+		},
+		{
+			name:        "gateway enabled",
+			cluster:     aroCluster([]string{"domain1", "domain2"}),
+			wantEnabled: true,
+		},
+	}
+
+	for _, tt := range tests {
+		gotEnabled := GatewayEnabled(tt.cluster)
+		if gotEnabled != tt.wantEnabled {
+			t.Errorf("got: %v\nwant: %v\n", gotEnabled, tt.wantEnabled)
+		}
+	}
+}