Add runPrivileged/runAsUser options, for running on more restricted/s…

…ecured K8s clusters

These options make it possible to run the operator and es-clusters on a Kubernetes cluster,
that has Pod Security Policies in place, that:
- disallows running containers as root
- (and/or) disallows running containers in privileged mode

Note: the default elasticsearch image (upmcenterprises/docker-elasticsearch-kubernetes:6.1.3_0 as of writing)
will not work if you don't run it as root (uid 0), as its wrapper script tries a 'ulimit -l unlimited',
and eventually su-execs to elasticsearch user with uid 1000; both actions will fail.
Setting ulimit should not be neccessary with IPC_LOCK/SYS_RESOURCE capabilities,
however they get wiped when running a container as non-root.
So running this image will require some modifications,
e.g. chowning folders, setcap cap_ipc_lock=+ep on java binary + dependencies.

Most (recent) information on this topic that allowed me to solve the puzzle:
https://medium.com/@thejasongerard/resource-limits-mlock-and-containers-oh-my-cca1e5d1f259

Change-Id: I600e9dd4a49cab15a289fc50cc2a605c83ac3aa9

cmd/operator/main.go

@@ -54,6 +54,8 @@ var (
 	enableInitDaemonset    bool
 	initDaemonsetNamespace string
 	busyboxImage           string
+	runPrivileged          bool
+	runAsUser              int64
 )
 
 func init() {
@@ -64,6 +66,8 @@ func init() {
 	flag.BoolVar(&enableInitDaemonset, "enableInitDaemonset", true, "Set to false to disable the sysctl init daemonset")
 	flag.StringVar(&initDaemonsetNamespace, "initDaemonsetNamespace", "default", "Namespace to deploy the sysctl init daemonset into")
 	flag.StringVar(&busyboxImage, "busybox-image", "busybox:1.26.2", "Image to use for sysctl init daemonset")
+	flag.BoolVar(&runPrivileged, "runPrivileged", true, "Run pods as privileged. Set to false if your Kubernetes cluster doesn't allow running containers in privileged mode. Setting does not affect InitDaemonset.")
+	flag.Int64Var(&runAsUser, "runAsUser", 0, "Run the first process in the container as this uid. Change this if your Kubernetes cluster doesn't allow running containers as root. Setting does not affect InitDaemonset.")
 	flag.Parse()
 }
 
@@ -78,11 +82,17 @@ func Main() int {
 
 	// Print params configured
 	logrus.Info("Using Variables:")
+	logrus.Infof("   kubecfg-file: %s", kubeCfgFile)
+	logrus.Infof("   masterhost: %s", masterHost)
 	logrus.Infof("   enableInitDaemonset: %t", enableInitDaemonset)
+	logrus.Infof("   initDaemonsetNamespace: %s", initDaemonsetNamespace)
 	logrus.Infof("   baseImage: %s", baseImage)
+	logrus.Infof("   busybox-image: %s", busyboxImage)
+	logrus.Infof("   runPrivileged: %t", runPrivileged)
+	logrus.Infof("   runAsUser: %d", runAsUser)
 
 	// Init
-	k8sclient, err := k8sutil.New(kubeCfgFile, masterHost, enableInitDaemonset, initDaemonsetNamespace, busyboxImage)
+	k8sclient, err := k8sutil.New(kubeCfgFile, masterHost, enableInitDaemonset, initDaemonsetNamespace, busyboxImage, runPrivileged, runAsUser)
 	if err != nil {
 		logrus.Error("Could not init k8sclient! ", err)
 		return 1

pkg/k8sutil/deployments.go

@@ -160,14 +160,19 @@ func (k *K8sutil) CreateClientDeployment(baseImage string, replicas *int32, java
 						},
 					},
 					Spec: v1.PodSpec{
+						SecurityContext: &v1.PodSecurityContext{
+							RunAsUser: &k.RunAsUser,
+							FSGroup: &k.RunAsUser,
+						},
 						Containers: []v1.Container{
 							v1.Container{
 								Name: deploymentName,
 								SecurityContext: &v1.SecurityContext{
-									Privileged: &[]bool{true}[0],
+									Privileged: &k.RunPrivileged,
 									Capabilities: &v1.Capabilities{
 										Add: []v1.Capability{
 											"IPC_LOCK",
+											"SYS_RESOURCE",
 										},
 									},
 								},

pkg/k8sutil/k8sutil.go

@@ -89,10 +89,12 @@ type K8sutil struct {
 	EnableInitDaemonset    bool
 	InitDaemonsetNamespace string
 	BusyboxImage           string
+	RunPrivileged          bool
+	RunAsUser              int64
 }
 
 // New creates a new instance of k8sutil
-func New(kubeCfgFile, masterHost string, enableInitDaemonset bool, initDaemonsetNamespace, busyboxImage string) (*K8sutil, error) {
+func New(kubeCfgFile, masterHost string, enableInitDaemonset bool, initDaemonsetNamespace, busyboxImage string, runPrivileged bool, runAsUser int64) (*K8sutil, error) {
 
 	crdClient, kubeClient, kubeExt, k8sVersion, err := newKubeClient(kubeCfgFile)
 
@@ -109,6 +111,8 @@ func New(kubeCfgFile, masterHost string, enableInitDaemonset bool, initDaemonset
 		EnableInitDaemonset:    enableInitDaemonset,
 		InitDaemonsetNamespace: initDaemonsetNamespace,
 		BusyboxImage:           busyboxImage,
+		RunPrivileged:          runPrivileged,
+		RunAsUser:              runAsUser,
 	}
 
 	return k, nil
@@ -396,7 +400,8 @@ func processDeploymentType(deploymentType string, clusterName string) (string, s
 }
 
 func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, storageClass, dataDiskSize, javaOptions, serviceAccountName,
-	statsdEndpoint, networkHost string, replicas *int32, useSSL *bool, resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy string) *apps.StatefulSet {
+	statsdEndpoint, networkHost string, replicas *int32, useSSL *bool, resources myspec.Resources, imagePullSecrets []myspec.ImagePullSecrets,
+	imagePullPolicy string, runPrivileged *bool, runAsUser *int64) *apps.StatefulSet {
 
 	_, role, isNodeMaster, isNodeData := processDeploymentType(deploymentType, clusterName)
 
@@ -473,6 +478,10 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s
 					},
 				},
 				Spec: v1.PodSpec{
+					SecurityContext: &v1.PodSecurityContext{
+						RunAsUser: runAsUser,
+						FSGroup: runAsUser,
+					},
 					Affinity: &v1.Affinity{
 						PodAntiAffinity: &v1.PodAntiAffinity{
 							PreferredDuringSchedulingIgnoredDuringExecution: []v1.WeightedPodAffinityTerm{
@@ -497,10 +506,11 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s
 						v1.Container{
 							Name: statefulSetName,
 							SecurityContext: &v1.SecurityContext{
-								Privileged: &[]bool{true}[0],
+								Privileged: runPrivileged,
 								Capabilities: &v1.Capabilities{
 									Add: []v1.Capability{
 										"IPC_LOCK",
+										"SYS_RESOURCE",
 									},
 								},
 							},
@@ -653,7 +663,8 @@ func buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, s
 
 // CreateDataNodeDeployment creates the data node deployment
 func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int32, baseImage, storageClass string, dataDiskSize string, resources myspec.Resources,
-	imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string, useSSL *bool, esUrl string) error {
+	imagePullSecrets []myspec.ImagePullSecrets, imagePullPolicy, serviceAccountName, clusterName, statsdEndpoint, networkHost, namespace, javaOptions string,
+	useSSL *bool, esUrl string) error {
 
 	deploymentName, _, _, _ := processDeploymentType(deploymentType, clusterName)
 
@@ -667,7 +678,7 @@ func (k *K8sutil) CreateDataNodeDeployment(deploymentType string, replicas *int3
 		logrus.Infof("StatefulSet %s not found, creating...", statefulSetName)
 
 		statefulSet := buildStatefulSet(statefulSetName, clusterName, deploymentType, baseImage, storageClass, dataDiskSize, javaOptions, serviceAccountName,
-			statsdEndpoint, networkHost, replicas, useSSL, resources, imagePullSecrets, imagePullPolicy)
+			statsdEndpoint, networkHost, replicas, useSSL, resources, imagePullSecrets, imagePullPolicy, &k.RunPrivileged, &k.RunAsUser)
 
 		if _, err := k.Kclient.AppsV1beta2().StatefulSets(namespace).Create(statefulSet); err != nil {
 			logrus.Error("Could not create stateful set: ", err)

pkg/k8sutil/k8sutil_test.go

@@ -39,8 +39,10 @@ func TestSSLCertConfig(t *testing.T) {
 	}
 	clusterName := "test"
 	useSSL := false
+	runPrivileged := true
+	var runAsUser int64 = 0
 	statefulSet := buildStatefulSet("test", clusterName, "master", "foo/image", "test", "1G", "",
-		"", "", "", nil, &useSSL, resources, nil, "")
+		"", "", "", nil, &useSSL, resources, nil, "", &runPrivileged, &runAsUser)
 
 	for _, volume := range statefulSet.Spec.Template.Spec.Volumes {
 		if volume.Name == fmt.Sprintf("%s-%s", secretName, clusterName) {
@@ -50,7 +52,7 @@ func TestSSLCertConfig(t *testing.T) {
 
 	useSSL = true
 	statefulSet = buildStatefulSet("test", clusterName, "master", "foo/image", "test", "1G", "",
-		"", "", "", nil, &useSSL, resources, nil, "")
+		"", "", "", nil, &useSSL, resources, nil, "", &runPrivileged, &runAsUser)
 
 	found := false
 	for _, volume := range statefulSet.Spec.Template.Spec.Volumes {