From 2459d5e3ec852bd2a64545e2686632ed74a96759 Mon Sep 17 00:00:00 2001 From: bennerv <10840174+bennerv@users.noreply.github.com> Date: Fri, 12 Jan 2024 17:16:02 -0500 Subject: [PATCH] Add securityContext to ARO worker and master operator deployments as required by k8s v1.27.x --- pkg/cluster/cluster.go | 2 + pkg/cluster/install.go | 16 ++++++- .../genevalogging/genevalogging.go | 7 +++ pkg/operator/deploy/deploy.go | 15 +++++-- pkg/operator/deploy/deploy_test.go | 25 +++++++++-- .../master/deployment.yaml.tmpl | 14 ++++++ .../worker/deployment.yaml.tmpl | 14 ++++++ pkg/operator/helpers_test.go | 44 +++++++++++++++++++ 8 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 pkg/operator/helpers_test.go diff --git a/pkg/cluster/cluster.go b/pkg/cluster/cluster.go index 606aca59bf1..32196f7f170 100644 --- a/pkg/cluster/cluster.go +++ b/pkg/cluster/cluster.go @@ -20,6 +20,7 @@ import ( extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/cluster/graph" @@ -89,6 +90,7 @@ type manager struct { subnet subnet.Manager graph graph.Manager + client client.Client kubernetescli kubernetes.Interface dynamiccli dynamic.Interface extensionscli extensionsclient.Interface diff --git a/pkg/cluster/install.go b/pkg/cluster/install.go index 6e43d971b82..b56b2b0aba7 100644 --- a/pkg/cluster/install.go +++ b/pkg/cluster/install.go @@ -19,6 +19,8 @@ import ( extensionsclient "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" "k8s.io/client-go/dynamic" "k8s.io/client-go/kubernetes" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" "github.com/Azure/ARO-RP/pkg/api" "github.com/Azure/ARO-RP/pkg/containerinstall" @@ -499,13 +501,25 @@ func (m *manager) initializeKubernetesClients(ctx context.Context) error { } m.imageregistrycli, err = imageregistryclient.NewForConfig(restConfig) + if err != nil { + return err + } + + mapper, err := apiutil.NewDynamicRESTMapper(restConfig, apiutil.WithLazyDiscovery) + if err != nil { + return err + } + + m.client, err = client.New(restConfig, client.Options{ + Mapper: mapper, + }) return err } // initializeKubernetesClients initializes clients which are used // once the cluster is up later on in the install process. func (m *manager) initializeOperatorDeployer(ctx context.Context) (err error) { - m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.extensionscli, m.kubernetescli) + m.aroOperatorDeployer, err = deploy.New(m.log, m.env, m.doc.OpenShiftCluster, m.arocli, m.client, m.extensionscli, m.kubernetescli) return } diff --git a/pkg/operator/controllers/genevalogging/genevalogging.go b/pkg/operator/controllers/genevalogging/genevalogging.go index e5a4aef6db8..a654e4f1363 100644 --- a/pkg/operator/controllers/genevalogging/genevalogging.go +++ b/pkg/operator/controllers/genevalogging/genevalogging.go @@ -22,6 +22,12 @@ import ( "github.com/Azure/ARO-RP/pkg/util/version" ) +var privilegedNamespaceLabels = map[string]string{ + "pod-security.kubernetes.io/enforce": "privileged", + "pod-security.kubernetes.io/audit": "privileged", + "pod-security.kubernetes.io/warn": "privileged", +} + func (r *Reconciler) securityContextConstraints(ctx context.Context, name, serviceAccountName string) (*securityv1.SecurityContextConstraints, error) { scc := &securityv1.SecurityContextConstraints{} err := r.Client.Get(ctx, types.NamespacedName{Name: "privileged"}, scc) @@ -284,6 +290,7 @@ func (r *Reconciler) resources(ctx context.Context, cluster *arov1alpha1.Cluster ObjectMeta: metav1.ObjectMeta{ Name: kubeNamespace, Annotations: map[string]string{projectv1.ProjectNodeSelector: ""}, + Labels: privilegedNamespaceLabels, }, }, &corev1.Secret{ diff --git a/pkg/operator/deploy/deploy.go b/pkg/operator/deploy/deploy.go index f41294e2780..0995b458ff2 100644 --- a/pkg/operator/deploy/deploy.go +++ b/pkg/operator/deploy/deploy.go @@ -28,6 +28,7 @@ import ( appsv1client "k8s.io/client-go/kubernetes/typed/apps/v1" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" "github.com/Azure/ARO-RP/pkg/api" @@ -63,12 +64,13 @@ type operator struct { oc *api.OpenShiftCluster arocli aroclient.Interface + client client.Client extensionscli extensionsclient.Interface kubernetescli kubernetes.Interface dh dynamichelper.Interface } -func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) { +func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli aroclient.Interface, client client.Client, extensionscli extensionsclient.Interface, kubernetescli kubernetes.Interface) (Operator, error) { restConfig, err := restconfig.RestConfig(env, oc) if err != nil { return nil, err @@ -84,6 +86,7 @@ func New(log *logrus.Entry, env env.Interface, oc *api.OpenShiftCluster, arocli oc: oc, arocli: arocli, + client: client, extensionscli: extensionscli, kubernetescli: kubernetescli, dh: dh, @@ -125,7 +128,7 @@ func templateManifests(data deploymentData) ([][]byte, error) { return templatedFiles, nil } -func (o *operator) createDeploymentData() deploymentData { +func (o *operator) createDeploymentData() (deploymentData, error) { image := o.env.AROOperatorImage() // HACK: Override for ARO_IMAGE env variable setup in local-dev mode @@ -145,11 +148,15 @@ func (o *operator) createDeploymentData() deploymentData { IsLocalDevelopment: o.env.IsLocalDevelopmentMode(), Image: image, Version: version, - } + }, nil } func (o *operator) createObjects() ([]kruntime.Object, error) { - deploymentData := o.createDeploymentData() + deploymentData, err := o.createDeploymentData() + if err != nil { + return nil, err + } + templated, err := templateManifests(deploymentData) if err != nil { return nil, err diff --git a/pkg/operator/deploy/deploy_test.go b/pkg/operator/deploy/deploy_test.go index 2f2187f7b2b..561d20702ec 100644 --- a/pkg/operator/deploy/deploy_test.go +++ b/pkg/operator/deploy/deploy_test.go @@ -137,6 +137,7 @@ func TestCreateDeploymentData(t *testing.T) { mock func(*mock_env.MockInterface, *api.OpenShiftCluster) operatorVersionOverride string expected deploymentData + wantErr string }{ { name: "no image override, use default", @@ -147,7 +148,8 @@ func TestCreateDeploymentData(t *testing.T) { }, expected: deploymentData{ Image: operatorImageWithTag, - Version: operatorImageTag}, + Version: operatorImageTag, + }, }, { name: "no image tag, use latest version", @@ -158,7 +160,8 @@ func TestCreateDeploymentData(t *testing.T) { }, expected: deploymentData{ Image: operatorImageUntagged, - Version: "latest"}, + Version: "latest", + }, }, { name: "OperatorVersion override set", @@ -174,7 +177,20 @@ func TestCreateDeploymentData(t *testing.T) { }, expected: deploymentData{ Image: "docker.io/aro:override", - Version: "override"}, + Version: "override", + }, + }, + { + name: "version supports pod security admission", + mock: func(env *mock_env.MockInterface, oc *api.OpenShiftCluster) { + env.EXPECT(). + AROOperatorImage(). + Return(operatorImageWithTag) + }, + expected: deploymentData{ + Image: operatorImageWithTag, + Version: operatorImageTag, + }, }, } { t.Run(tt.name, func(t *testing.T) { @@ -192,7 +208,8 @@ func TestCreateDeploymentData(t *testing.T) { env: env, } - deploymentData := o.createDeploymentData() + deploymentData, err := o.createDeploymentData() + utilerror.AssertErrorMessage(t, err, tt.wantErr) if !reflect.DeepEqual(deploymentData, tt.expected) { t.Errorf("actual deployment: %v, expected %v", deploymentData, tt.expected) } diff --git a/pkg/operator/deploy/staticresources/master/deployment.yaml.tmpl b/pkg/operator/deploy/staticresources/master/deployment.yaml.tmpl index 6746ab94889..a425c35f422 100644 --- a/pkg/operator/deploy/staticresources/master/deployment.yaml.tmpl +++ b/pkg/operator/deploy/staticresources/master/deployment.yaml.tmpl @@ -55,8 +55,22 @@ spec: httpGet: path: /healthz/ready port: 8080 + {{ if .SupportsPodSecurityAdmission }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + {{ end }} nodeSelector: node-role.kubernetes.io/master: "" + {{ if .SupportsPodSecurityAdmission }} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + {{ end }} serviceAccountName: aro-operator-master serviceAccount: aro-operator-master priorityClassName: system-cluster-critical diff --git a/pkg/operator/deploy/staticresources/worker/deployment.yaml.tmpl b/pkg/operator/deploy/staticresources/worker/deployment.yaml.tmpl index d218d995d20..697cb34b17b 100644 --- a/pkg/operator/deploy/staticresources/worker/deployment.yaml.tmpl +++ b/pkg/operator/deploy/staticresources/worker/deployment.yaml.tmpl @@ -37,8 +37,22 @@ spec: httpGet: path: /healthz/ready port: 8080 + {{ if .SupportsPodSecurityAdmission }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + runAsNonRoot: true + {{ end }} nodeSelector: node-role.kubernetes.io/worker: "" + {{ if .SupportsPodSecurityAdmission }} + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + {{ end }} serviceAccountName: aro-operator-worker serviceAccount: aro-operator-worker priorityClassName: system-cluster-critical diff --git a/pkg/operator/helpers_test.go b/pkg/operator/helpers_test.go new file mode 100644 index 00000000000..f26e695ed05 --- /dev/null +++ b/pkg/operator/helpers_test.go @@ -0,0 +1,44 @@ +package operator + +// Copyright (c) Microsoft Corporation. +// Licensed under the Apache License 2.0. + +import ( + "testing" + + arov1alpha1 "github.com/Azure/ARO-RP/pkg/operator/apis/aro.openshift.io/v1alpha1" + _ "github.com/Azure/ARO-RP/pkg/util/scheme" +) + +func aroCluster(domains []string) *arov1alpha1.Cluster { + return &arov1alpha1.Cluster{ + Spec: arov1alpha1.ClusterSpec{ + GatewayDomains: domains, + }, + } +} + +func TestGatewayEnabled(t *testing.T) { + tests := []struct { + name string + cluster *arov1alpha1.Cluster + wantEnabled bool + }{ + { + name: "gateway disabled", + cluster: aroCluster([]string{}), + }, + { + name: "gateway enabled", + cluster: aroCluster([]string{"domain1", "domain2"}), + wantEnabled: true, + }, + } + + for _, tt := range tests { + gotEnabled := GatewayEnabled(tt.cluster) + if gotEnabled != tt.wantEnabled { + t.Errorf("got: %v\nwant: %v\n", gotEnabled, tt.wantEnabled) + } + } +}