Merge pull request #135 from SovereignCloudStack/hook-server-one-cont…

…ainer

✨ Add hook server into same container as operator

Tiltfile

@@ -18,7 +18,6 @@ settings = {
         "kind-cso",
     ],
     "local_mode": False,
-    "runtime_sdk": True,
     "deploy_cert_manager": True,
     "preload_images_for_kind": True,
     "kind_cluster_name": "cso",
@@ -110,12 +109,6 @@ def fixup_yaml_empty_arrays(yaml_str):
     yaml_str = yaml_str.replace("conditions: null", "conditions: []")
     return yaml_str.replace("storedVersions: null", "storedVersions: []")
 
-tilt_dockerfile_header_runtime = """
-FROM docker.io/library/alpine:3.18.0
-WORKDIR /
-COPY extension/.tiltbuild/manager .
-"""
-
 ## This should have the same versions as the Dockerfile
 tilt_dockerfile_header_cso = """
 FROM docker.io/alpine/helm:3.12.2 as helm
@@ -191,53 +184,11 @@ def deploy_cso():
         labels = ["CSO"],
     )
 
-def deploy_runtime_extension():
-    yaml = str(kustomizesub("./extension/config/default"))
-
-    local_resource(
-        name = "runtime-components",
-        cmd = ["sh", "-ec", sed_cmd, yaml, "|", envsubst_cmd],
-        labels = ["runtime"],
-    )
-
-    local_resource(
-        "runtime-manager",
-        cmd = 'cd extension; mkdir -p .tiltbuild;CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags \'-extldflags "-static"\' -o .tiltbuild/manager main.go',
-        deps = ["extension/handlers", "extension/main.go"],
-        labels = ["runtime"],
-    )
-
-    entrypoint = ["/manager"]
-    extra_args = settings.get("extra_args")
-    if extra_args:
-        entrypoint.extend(extra_args)
-
-    docker_build_with_restart(
-    ref = "ghcr.io/sovereigncloudstack/runtime-sdk-staging",
-    context = ".",
-    dockerfile_contents = tilt_dockerfile_header_runtime,
-    entrypoint = entrypoint,
-    live_update = [
-        sync("extension/.tiltbuild/manager", "/manager"),
-        # sync(".release", "/tmp/cluster-stacks"),
-    ],
-    ignore = ["templates"],
-    )
+def deploy_capd():
+    yaml = './capd.yaml'
+    cmd = "kubectl apply -f capd.yaml"
+    local(cmd, quiet = True)
 
-    k8s_yaml(blob(yaml))
-    k8s_resource(workload = "test-runtime-sdk", labels = ["runtime"])
-    k8s_resource(
-        objects = [
-            "runtimesdk:namespace",
-            # "test-runtime-sdk:deployment",
-            "test-runtime-sdk-sa:serviceaccount",
-            "test-runtime-sdk-role:clusterrole",
-            "test-runtime-sdk-role-rolebinding:clusterrolebinding",
-            "runtime-sdk-selfsigned-issuer:issuer",
-        ],
-        new_name = "runtime-misc",
-        labels = ["runtime"],
-    )
 
 def clusterstack():
     k8s_resource(objects = ["clusterstack:clusterstack"], new_name = "clusterstack", labels = ["CLUSTERSTACK"])
@@ -306,9 +257,6 @@ deploy_cso()
 
 clusterstack()
 
-if settings.get("runtime_sdk"):
-    deploy_runtime_extension()
-
 waitforsystem()
 
 prepare_environment()

cmd/main.go

@@ -26,6 +26,7 @@ import (
 
 	//+kubebuilder:scaffold:imports
 	csov1alpha1 "github.com/SovereignCloudStack/cluster-stack-operator/api/v1alpha1"
+	"github.com/SovereignCloudStack/cluster-stack-operator/extension/handlers"
 	"github.com/SovereignCloudStack/cluster-stack-operator/internal/controller"
 	"github.com/SovereignCloudStack/cluster-stack-operator/pkg/csoversion"
 	githubclient "github.com/SovereignCloudStack/cluster-stack-operator/pkg/github/client"
@@ -37,6 +38,10 @@ import (
 	clientgoscheme "k8s.io/client-go/kubernetes/scheme"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
+	"sigs.k8s.io/cluster-api/controllers/remote"
+	runtimecatalog "sigs.k8s.io/cluster-api/exp/runtime/catalog"
+	runtimehooksv1 "sigs.k8s.io/cluster-api/exp/runtime/hooks/api/v1alpha1"
+	"sigs.k8s.io/cluster-api/exp/runtime/server"
 	dockerv1beta1 "sigs.k8s.io/cluster-api/test/infrastructure/docker/api/v1beta1"
 	"sigs.k8s.io/cluster-api/util/record"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -47,9 +52,15 @@ import (
 var (
 	scheme   = runtime.NewScheme()
 	setupLog = ctrl.Log.WithName("setup")
+
+	// catalog contains all information about RuntimeHooks.
+	catalog = runtimecatalog.New()
 )
 
 func init() {
+	// Adds to the catalog all the RuntimeHooks defined in cluster API.
+	_ = runtimehooksv1.AddToCatalog(catalog)
+
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(csov1alpha1.AddToScheme(scheme))
 	utilruntime.Must(dockerv1beta1.AddToScheme(scheme))
@@ -71,6 +82,8 @@ var (
 	releaseDir                     string
 	qps                            float64
 	burst                          int
+	hookPort                       int
+	hookCertDir                    string
 )
 
 func main() {
@@ -87,7 +100,8 @@ func main() {
 	flag.StringVar(&releaseDir, "release-dir", "/tmp/downloads/", "Specify release directory for cluster-stack releases")
 	flag.Float64Var(&qps, "qps", 50, "Enable custom query per second for kubernetes API server")
 	flag.IntVar(&burst, "burst", 100, "Enable custom burst defines how many queries the API server will accept before enforcing the limit established by qps")
-
+	flag.IntVar(&hookPort, "hook-port", 9442, "hook server port")
+	flag.StringVar(&hookCertDir, "hook-cert-dir", "/tmp/k8s-hook-server/serving-certs/", "hook cert dir, only used when hook-port is specified.")
 	flag.Parse()
 
 	ctrl.SetLogger(utillog.GetDefaultLogger(logLevel))
@@ -130,7 +144,6 @@ func main() {
 	}
 
 	var wg sync.WaitGroup
-	wg.Add(1)
 
 	if err = (&controller.ClusterStackReconciler{
 		Client:              mgr.GetClient(),
@@ -185,13 +198,87 @@ func main() {
 		os.Exit(1)
 	}
 
-	setupLog.Info("starting manager", "version", csoversion.Get().String())
-	if err := mgr.Start(ctx); err != nil {
-		setupLog.Error(err, "problem running manager")
+	// Create a http server for serving runtime extensions
+	hookServer, err := server.New(server.Options{
+		Catalog: catalog,
+		Port:    hookPort,
+		CertDir: hookCertDir,
+	})
+	if err != nil {
+		setupLog.Error(err, "error creating webhook server")
 		os.Exit(1)
 	}
 
-	wg.Done()
-	// Wait for all target cluster managers to gracefully shut down.
+	// Lifecycle Hooks
+	// Gets a client to access the Kubernetes cluster where this RuntimeExtension will be deployed to;
+	// this is a requirement specific of the lifecycle hooks implementation for Cluster APIs E2E tests.
+	restConfig.UserAgent = remote.DefaultClusterAPIUserAgent("cluster-stack-operator-extension-manager")
+
+	// Create the ExtensionHandlers for the lifecycle hooks
+	lifecycleExtensionHandlers := handlers.NewExtensionHandlers(mgr.GetClient(), scheme)
+
+	setupLog.Info("Add extension handlers")
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.BeforeClusterUpgrade,
+		Name:        "before-cluster-upgrade",
+		HandlerFunc: lifecycleExtensionHandlers.DoBeforeClusterUpgrade,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
+		os.Exit(1)
+	}
+
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.AfterClusterUpgrade,
+		Name:        "after-cluster-upgrade",
+		HandlerFunc: lifecycleExtensionHandlers.DoAfterClusterUpgrade,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
+		os.Exit(1)
+	}
+
+	if err := hookServer.AddExtensionHandler(server.ExtensionHandler{
+		Hook:        runtimehooksv1.AfterControlPlaneInitialized,
+		Name:        "after-control-plane-initialized",
+		HandlerFunc: lifecycleExtensionHandlers.DoAfterControlPlaneInitialized,
+	}); err != nil {
+		setupLog.Error(err, "error adding handler")
+		os.Exit(1)
+	}
+
+	errChan := make(chan error, 1)
+
+	wg.Add(1)
+
+	go func() {
+		setupLog.Info("starting manager", "version", csoversion.Get().String())
+		if err := mgr.Start(ctx); err != nil {
+			setupLog.Error(err, "problem running manager")
+			errChan <- err
+		}
+		wg.Done()
+	}()
+
+	wg.Add(1)
+
+	go func() {
+		setupLog.Info("starting hook server")
+		if err := hookServer.Start(ctx); err != nil {
+			setupLog.Error(err, "problem running hook server")
+			errChan <- err
+		}
+		wg.Done()
+	}()
+
+	go func() {
+		select {
+		case err := <-errChan:
+			setupLog.Error(err, "Received error")
+			os.Exit(1)
+		case <-ctx.Done():
+			setupLog.Info("shutting down")
+		}
+	}()
+
+	// wait for all processes to shut down
 	wg.Wait()
 }

config/certmanager/certificate.yaml

@@ -25,4 +25,21 @@ spec:
   secretName: cso-webhook-server-cert  # this secret will not be prefixed, since it's not managed by kustomize
   subject:
     organizations:
-    - k8s-sig-cluster-lifecycle
+    - k8s-sig-cluster-lifecycle
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: hook-server-serving-cert  # this name should match the one appeared in kustomizeconfig.yaml
+  namespace: system
+spec:
+  dnsNames:
+  - $(HOOK_SERVER_SERVICE_NAME).$(HOOK_SERVER_SERVICE_NAMESPACE).svc
+  - $(HOOK_SERVER_SERVICE_NAME).$(HOOK_SERVER_SERVICE_NAMESPACE).svc.cluster.local
+  issuerRef:
+    kind: Issuer
+    name: selfsigned-issuer
+  secretName: cso-hook-server-server-cert  # this secret will not be prefixed, since it's not managed by kustomize
+  subject:
+    organizations:
+    - k8s-sig-cluster-lifecycle

config/default/kustomization.yaml

@@ -6,38 +6,70 @@ commonLabels:
   cluster.x-k8s.io/provider: "infrastructure-cluster-stack-operator"
 
 resources:
-  - ../crd
-  - ../rbac
-  - ../manager
-  - ../certmanager
+- ../crd
+- ../rbac
+- ../manager
+- ../webhook
+- ../hookserver
+- ../certmanager
 
 patchesStrategicMerge:
-  - manager_config_patch.yaml
-  - manager_pull_policy.yaml
-# vars:
-# - name: CERTIFICATE_NAMESPACE  # namespace of the certificate CR
-#   objref:
-#     kind: Certificate
-#     group: cert-manager.io
-#     version: v1
-#     name: serving-cert  # this name should match the one in certificate.yaml
-#   fieldref:
-#     fieldpath: metadata.namespace
-# - name: CERTIFICATE_NAME
-#   objref:
-#     kind: Certificate
-#     group: cert-manager.io
-#     version: v1
-#     name: serving-cert  # this name should match the one in certificate.yaml
-# - name: SERVICE_NAMESPACE  # namespace of the service
-#   objref:
-#     kind: Service
-#     version: v1
-#     name: webhook-service
-#   fieldref:
-#     fieldpath: metadata.namespace
-# - name: SERVICE_NAME
-#   objref:
-#     kind: Service
-#     version: v1
-#     name: webhook-service
+- manager_config_patch.yaml
+- manager_webhook_patch.yaml
+- manager_hookserver_patch.yaml
+- webhookcainjection_patch.yaml
+- manager_pull_policy.yaml
+
+vars:
+- name: CERTIFICATE_NAMESPACE  # namespace of the certificate CR
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert  # this name should match the one in certificate.yaml
+  fieldref:
+    fieldpath: metadata.namespace
+- name: HOOK_SERVER_CERTIFICATE_NAMESPACE  # namespace of the certificate CR
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: hook-server-serving-cert  # this name should match the one in certificate.yaml
+  fieldref:
+    fieldpath: metadata.namespace
+- name: CERTIFICATE_NAME
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: serving-cert  # this name should match the one in certificate.yaml
+- name: HOOK_SERVER_CERTIFICATE_NAME
+  objref:
+    kind: Certificate
+    group: cert-manager.io
+    version: v1
+    name: hook-server-serving-cert  # this name should match the one in certificate.yaml
+- name: SERVICE_NAMESPACE  # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
+  fieldref:
+    fieldpath: metadata.namespace
+- name: HOOK_SERVER_SERVICE_NAMESPACE  # namespace of the service
+  objref:
+    kind: Service
+    version: v1
+    name: hook-server-svc
+  fieldref:
+    fieldpath: metadata.namespace
+- name: SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: webhook-service
+- name: HOOK_SERVER_SERVICE_NAME
+  objref:
+    kind: Service
+    version: v1
+    name: hook-server-svc

config/default/manager_hookserver_patch.yaml

@@ -0,0 +1,23 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: controller-manager
+  namespace: system
+spec:
+  template:
+    spec:
+      containers:
+      - name: manager
+        ports:
+        - containerPort: 9442
+          name: hook-server-svc
+          protocol: TCP
+        volumeMounts:
+        - mountPath: /tmp/k8s-hook-server/serving-certs
+          name: hook-server-cert
+          readOnly: true
+      volumes:
+      - name: hook-server-cert
+        secret:
+          defaultMode: 420
+          secretName: cso-hook-server-server-cert