Merge pull request #115 from FabianKramm/master

leaderelection & cleanup

Dockerfile

@@ -2,8 +2,8 @@
 FROM golang:1.15 as builder
 
 WORKDIR /workspace
-ARG TARGETOS
-ARG TARGETARCH
+ARG TARGETOS=linux
+ARG TARGETARCH=amd64
 
 # Install Helm 3
 RUN bash -c "curl -s https://get.helm.sh/helm-v3.5.2-linux-${TARGETARCH}.tar.gz > helm3.tar.gz" && tar -zxvf helm3.tar.gz linux-${TARGETARCH}/helm && chmod +x linux-${TARGETARCH}/helm && mv linux-${TARGETARCH}/helm /workspace/helm && rm helm3.tar.gz && rm -R linux-${TARGETARCH}

cmd/kiosk/main.go

@@ -23,25 +23,29 @@ import (
 	tenancyv1alpha1 "github.com/loft-sh/kiosk/pkg/apis/tenancy/v1alpha1"
 	"github.com/loft-sh/kiosk/pkg/apiserver"
 	_ "github.com/loft-sh/kiosk/pkg/apiserver/registry"
+	"github.com/loft-sh/kiosk/pkg/leaderelection"
 	"github.com/loft-sh/kiosk/pkg/manager/blockingcacheclient"
+	"github.com/loft-sh/kiosk/pkg/manager/quota/controller"
 	"github.com/loft-sh/kiosk/pkg/openapi"
 	"github.com/loft-sh/kiosk/pkg/store/apiservice"
 	"github.com/loft-sh/kiosk/pkg/store/crd"
 	"github.com/loft-sh/kiosk/pkg/store/validatingwebhookconfiguration"
 	"github.com/loft-sh/kiosk/pkg/util/certhelper"
 	"github.com/loft-sh/kiosk/pkg/util/log"
+	"github.com/loft-sh/kiosk/pkg/util/secret"
+	"github.com/pkg/errors"
 	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
-	"k8s.io/client-go/rest"
 	"k8s.io/klog"
 	apiregistrationv1 "k8s.io/kube-aggregator/pkg/apis/apiregistration/v1"
+	"math/rand"
 	"os"
 	client2 "sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
+	"time"
 
 	configv1alpha1 "github.com/loft-sh/kiosk/pkg/apis/config/v1alpha1"
 	"github.com/loft-sh/kiosk/pkg/manager/controllers"
-	"github.com/loft-sh/kiosk/pkg/manager/quota"
 	"github.com/loft-sh/kiosk/pkg/manager/webhooks"
 	"k8s.io/apimachinery/pkg/runtime"
 	genericfeatures "k8s.io/apiserver/pkg/features"
@@ -77,20 +81,15 @@ func init() {
 }
 
 func main() {
+	rand.Seed(time.Now().UnixNano())
+
 	// set global logger
 	if os.Getenv("DEBUG") == "true" {
 		ctrl.SetLogger(log.NewLog(0))
 	} else {
 		ctrl.SetLogger(log.NewLog(2))
 	}
 
-	// Make sure the certificates are there
-	err := certhelper.WriteCertificates()
-	if err != nil {
-		setupLog.Error(err, "unable to generate certificates")
-		os.Exit(1)
-	}
-
 	// retrieve in cluster config
 	config := ctrl.GetConfigOrDie()
 
@@ -99,8 +98,22 @@ func main() {
 	config.Burst = 100
 	config.Timeout = 0
 
+	// create a new temporary client
+	uncachedClient, err := client2.New(config, client2.Options{Scheme: scheme})
+	if err != nil {
+		setupLog.Error(err, "unable to create client")
+		os.Exit(1)
+	}
+
+	// Make sure the certificates are there
+	err = secret.EnsureCertSecrets(context.Background(), uncachedClient)
+	if err != nil {
+		setupLog.Error(err, "unable to generate certificates")
+		os.Exit(1)
+	}
+
 	// Make sure the needed crds are installed in the cluster
-	err = initialize(config)
+	err = installCRDS(uncachedClient)
 	if err != nil {
 		klog.Fatal(err)
 	}
@@ -119,12 +132,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	// create an uncached client for api routes
-	uncachedClient, err := client2.New(mgr.GetConfig(), client2.Options{
-		Scheme: mgr.GetScheme(),
-		Mapper: mgr.GetRESTMapper(),
-	})
-
 	// Inject the cached, uncached client and scheme
 	injectClient(mgr.GetClient(), uncachedClient, scheme)
 
@@ -139,20 +146,6 @@ func main() {
 	ctx := signals.SetupSignalHandler()
 	ctrlCtx := controllers.NewControllerContext(mgr, stopChan)
 
-	// Register generic controllers
-	err = controllers.Register(mgr)
-	if err != nil {
-		setupLog.Error(err, "unable to register controller")
-		os.Exit(1)
-	}
-
-	// Register quota controller
-	err = quota.Register(ctrlCtx)
-	if err != nil {
-		setupLog.Error(err, "unable to register quota controller")
-		os.Exit(1)
-	}
-
 	// Register webhooks
 	err = webhooks.Register(ctrlCtx)
 	if err != nil {
@@ -195,42 +188,56 @@ func main() {
 			panic(err)
 		}
 	}()
+
+	// start leader election for controllers
+	go func() {
+		err = leaderelection.StartLeaderElection(ctx, scheme, config, func() error {
+			// setup ValidatingWebhookConfiguration
+			if os.Getenv("UPDATE_WEBHOOK") != "false" {
+				err = validatingwebhookconfiguration.EnsureValidatingWebhookConfiguration(context.Background(), mgr.GetClient())
+				if err != nil {
+					setupLog.Error(err, "unable to set up validating webhook configuration")
+					os.Exit(1)
+				}
+			}
 
-	// setup validatingwebhookconfiguration
-	if os.Getenv("UPDATE_WEBHOOK") != "false" {
-		err = validatingwebhookconfiguration.EnsureValidatingWebhookConfiguration(context.Background(), mgr.GetClient())
-		if err != nil {
-			setupLog.Error(err, "unable to set up validating webhook configuration")
-			os.Exit(1)
-		}
-	}
+			// setup APIService
+			if os.Getenv("UPDATE_APISERVICE") != "false" {
+				err = apiservice.EnsureAPIService(context.Background(), mgr.GetClient())
+				if err != nil {
+					setupLog.Error(err, "unable to set up apiservice")
+					os.Exit(1)
+				}
+			}
+
+			// Register generic controllers
+			err = controllers.Register(mgr)
+			if err != nil {
+				return errors.Wrap(err, "unable to register controller")
+			}
 
-	// setup apiservice
-	if os.Getenv("UPDATE_APISERVICE") != "false" {
-		err = apiservice.EnsureAPIService(context.Background(), mgr.GetClient())
+			// Register quota controller
+			err = controller.Register(ctrlCtx)
+			if err != nil {
+				return errors.Wrap(err, "unable to register quota controller")
+			}
+
+			return nil
+		})
 		if err != nil {
-			setupLog.Error(err, "unable to set up apiservice")
-			os.Exit(1)
+			klog.Fatalf("Error starting leader election: %v", err)
 		}
-	}
+	}()
 
 	// Wait till stopChan is closed
 	<-stopChan
 }
 
-func initialize(config *rest.Config) error {
-	klog.Info("Initialize...")
-	defer klog.Info("Done initializing...")
-
-	client, err := client2.New(config, client2.Options{Scheme: scheme})
-	if err != nil {
-		return err
-	}
-
+func installCRDS(uncachedClient client2.Client) error {
 	klog.Info("Installing crds...")
 
-	builder := crd.NewBuilder(client)
-	_, err = builder.CreateCRDs(context.Background(), apis.TypeDefinitions...)
+	builder := crd.NewBuilder(uncachedClient)
+	_, err := builder.CreateCRDs(context.Background(), apis.TypeDefinitions...)
 	return err
 }
 

devspace.yaml

@@ -77,3 +77,4 @@ profiles:
               name: ./chart
             values:
               image: kiosksh/kiosktest
+              replicaCount: 3

examples/template-helm.yaml

@@ -12,4 +12,6 @@ resources:
     values: |
       redisPort: 6379
       # Use a predefined parameter here
-      myOtherValue: ${NAMESPACE}
+      # You can also use annotations on the namespace 
+      # with ${namespace.metadata.annotations.myAnnotation}
+      myOtherValue: ${namespace}

examples/template-manifests.yaml

@@ -21,9 +21,10 @@ resources:
       kind: LimitRange
       metadata:
         name: space-limit-range
-        annotations:
+        annotations: 
           # Parameters can also be used inside a string
-          example-parameter: "Hello from ${NAMESPACE} - ${WILL_NOT_BE_REPLACED}"
+          # with ${namespace.metadata.annotations.myAnnotation}
+          example-parameter: "Hello from ${namespace} - ${WILL_NOT_BE_REPLACED}"
       spec:
         limits:
           - default:

go.mod

@@ -35,7 +35,7 @@ require (
 	k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd
 	k8s.io/kubectl v0.20.2
 	sigs.k8s.io/apiserver-builder-alpha v1.18.0
-	sigs.k8s.io/controller-runtime v0.8.2
+	sigs.k8s.io/controller-runtime v0.8.3
 )
 
 go 1.13

go.sum

@@ -1547,6 +1547,8 @@ sigs.k8s.io/controller-runtime v0.7.0 h1:bU20IBBEPccWz5+zXpLnpVsgBYxqclaHu1pVDl/
 sigs.k8s.io/controller-runtime v0.7.0/go.mod h1:pJ3YBrJiAqMAZKi6UVGuE98ZrroV1p+pIhoHsMm9wdU=
 sigs.k8s.io/controller-runtime v0.8.2 h1:SBWmI0b3uzMIUD/BIXWNegrCeZmPJ503pOtwxY0LPHM=
 sigs.k8s.io/controller-runtime v0.8.2/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
+sigs.k8s.io/controller-runtime v0.8.3 h1:GMHvzjTmaWHQB8HadW+dIvBoJuLvZObYJ5YoZruPRao=
+sigs.k8s.io/controller-runtime v0.8.3/go.mod h1:U/l+DUopBc1ecfRZ5aviA9JDmGFQKvLf5YkZNx2e0sU=
 sigs.k8s.io/controller-tools v0.1.11-0.20190405182121-56d42b19e55a/go.mod h1:5B/NyJEvqwZNl6sJgTvQJGlxRK4TOJMdELRFDsUwSeY=
 sigs.k8s.io/controller-tools v0.1.12/go.mod h1:6g08p9m9G/So3sBc1AOQifHfhxH/mb6Sc4z0LMI8XMw=
 sigs.k8s.io/kubebuilder v1.0.8/go.mod h1:tjcjlckQhPDkbbOw9sUkf1hB1EMiKvqVJhxMXh13npA=

pkg/apis/crds.go

@@ -3,7 +3,7 @@ package apis
 import (
 	configv1alpha1 "github.com/loft-sh/kiosk/pkg/apis/config/v1alpha1"
 	"github.com/loft-sh/kiosk/pkg/store/crd"
-	apiextensionsv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1"
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 )
 
 var (
@@ -13,25 +13,25 @@ var (
 			GVK:      configv1alpha1.SchemeGroupVersion.WithKind("Account"),
 			Singular: "account",
 			Plural:   "accounts",
-			Scope:    apiextensionsv1beta1.ClusterScoped,
+			Scope:    apiextensionsv1.ClusterScoped,
 		},
 		&crd.TypeDefinition{
 			GVK:      configv1alpha1.SchemeGroupVersion.WithKind("AccountQuota"),
 			Singular: "accountquota",
 			Plural:   "accountquotas",
-			Scope:    apiextensionsv1beta1.ClusterScoped,
+			Scope:    apiextensionsv1.ClusterScoped,
 		},
 		&crd.TypeDefinition{
 			GVK:      configv1alpha1.SchemeGroupVersion.WithKind("Template"),
 			Singular: "template",
 			Plural:   "templates",
-			Scope:    apiextensionsv1beta1.ClusterScoped,
+			Scope:    apiextensionsv1.ClusterScoped,
 		},
 		&crd.TypeDefinition{
 			GVK:      configv1alpha1.SchemeGroupVersion.WithKind("TemplateInstance"),
 			Singular: "templateinstance",
 			Plural:   "templateinstances",
-			Scope:    apiextensionsv1beta1.NamespaceScoped,
+			Scope:    apiextensionsv1.NamespaceScoped,
 		},
 	}
 )