diff --git a/.gitignore b/.gitignore index 90717c04..d19960ea 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ certs .DS_Store .claude +.config \ No newline at end of file diff --git a/Makefile b/Makefile index 7821dd39..9dd0abfe 100644 --- a/Makefile +++ b/Makefile @@ -158,20 +158,35 @@ manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and Cust generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations. GOWORK=off GO111MODULE=on $(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths=./api/... paths=./internal/... paths=./cmd/... -# source secret.env && make start-sample-workflow TEMPORAL_CLOUD_API_KEY=$TEMPORAL_CLOUD_API_KEY .PHONY: start-sample-workflow +.SILENT: start-sample-workflow start-sample-workflow: ## Start a sample workflow. - @$(TEMPORAL) workflow start --type "HelloWorld" --task-queue "default/helloworld" \ - --tls-cert-path certs/client.pem \ - --tls-key-path certs/client.key \ - --address "worker-controller-test.a2dd6.tmprl.cloud:7233" \ - -n "worker-controller-test.a2dd6" -# --address replay-2025.ktasd.tmprl.cloud:7233 \ -# --api-key $(TEMPORAL_CLOUD_API_KEY) + @set -e; \ + # Load env vars from skaffold.env if present so address/namespace aren't hardcoded + if [ -f skaffold.env ]; then set -a; . skaffold.env; set +a; fi; \ + API_KEY_VAL=""; \ + if [ -n "$$TEMPORAL_API_KEY" ]; then API_KEY_VAL="$$TEMPORAL_API_KEY"; \ + elif [ -f certs/api-key.txt ]; then API_KEY_VAL="$$(tr -d '\r\n' < certs/api-key.txt)"; fi; \ + if [ -n "$$API_KEY_VAL" ]; then \ + $(TEMPORAL) workflow start --type "HelloWorld" --task-queue "default/helloworld" \ + --address "$$TEMPORAL_ADDRESS" \ + --namespace "$$TEMPORAL_NAMESPACE" \ + --api-key "$$API_KEY_VAL"; \ + else \ + $(TEMPORAL) workflow start --type "HelloWorld" --task-queue "default/helloworld" \ + --tls-cert-path certs/client.pem \ + --tls-key-path certs/client.key \ + --address "$$TEMPORAL_ADDRESS" \ + --namespace "$$TEMPORAL_NAMESPACE"; \ + fi .PHONY: apply-load-sample-workflow +.SILENT: apply-load-sample-workflow apply-load-sample-workflow: ## Start a sample workflow every 15 seconds - watch --interval 0.1 -- $(TEMPORAL) workflow start --type "HelloWorld" --task-queue "default/helloworld" + @while true; do \ + $(MAKE) -s start-sample-workflow; \ + sleep 15; \ + done .PHONY: list-workflow-build-ids list-workflow-build-ids: ## List workflow executions and their build IDs. @@ -300,6 +315,11 @@ create-cloud-mtls-secret: --cert=certs/client.pem \ --key=certs/client.key +.PHONY: create-api-key-secret +create-api-key-secret: + kubectl create secret generic temporal-api-key --namespace default \ + --from-file=api-key=certs/api-key.txt + ##### Checks ##### goimports: fmt-imports $(GOIMPORTS) @printf $(COLOR) "Run goimports for all files..." diff --git a/api/v1alpha1/temporalconnection_types.go b/api/v1alpha1/temporalconnection_types.go index 2d424fe5..b27f7bc8 100644 --- a/api/v1alpha1/temporalconnection_types.go +++ b/api/v1alpha1/temporalconnection_types.go @@ -5,6 +5,7 @@ package v1alpha1 import ( + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -19,6 +20,7 @@ type SecretReference struct { } // TemporalConnectionSpec defines the desired state of TemporalConnection +// +kubebuilder:validation:XValidation:rule="!(has(self.mutualTLSSecretRef) && has(self.apiKeySecretRef))",message="Only one of mutualTLSSecretRef or apiKeySecretRef may be set" type TemporalConnectionSpec struct { // The host and port of the Temporal server. // +kubebuilder:validation:Pattern=`^[a-zA-Z0-9.-]+:[0-9]+$` @@ -32,6 +34,11 @@ type TemporalConnectionSpec struct { // https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets // +optional MutualTLSSecretRef *SecretReference `json:"mutualTLSSecretRef,omitempty"` + + // APIKeyRef is the name of the Secret that contains the API key. The secret must be `type: kubernetes.io/opaque` and exist + // in the same Kubernetes namespace as the TemporalConnection resource. + // +optional + APIKeySecretRef *corev1.SecretKeySelector `json:"apiKeySecretRef,omitempty"` } // TemporalConnectionStatus defines the observed state of TemporalConnection diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index b9b11ac7..1decf431 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -332,6 +332,11 @@ func (in *TemporalConnectionSpec) DeepCopyInto(out *TemporalConnectionSpec) { *out = new(SecretReference) **out = **in } + if in.APIKeySecretRef != nil { + in, out := &in.APIKeySecretRef, &out.APIKeySecretRef + *out = new(v1.SecretKeySelector) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemporalConnectionSpec. diff --git a/helm/temporal-worker-controller/crds/temporal.io_temporalconnections.yaml b/helm/temporal-worker-controller/crds/temporal.io_temporalconnections.yaml index 8a211d28..e2a21715 100644 --- a/helm/temporal-worker-controller/crds/temporal.io_temporalconnections.yaml +++ b/helm/temporal-worker-controller/crds/temporal.io_temporalconnections.yaml @@ -37,6 +37,19 @@ spec: type: object spec: properties: + apiKeySecretRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic hostPort: pattern: ^[a-zA-Z0-9.-]+:[0-9]+$ type: string @@ -51,6 +64,9 @@ spec: required: - hostPort type: object + x-kubernetes-validations: + - message: Only one of mutualTLSSecretRef or apiKeySecretRef may be set + rule: '!(has(self.mutualTLSSecretRef) && has(self.apiKeySecretRef))' status: type: object type: object diff --git a/internal/controller/clientpool/clientpool.go b/internal/controller/clientpool/clientpool.go index d332df8c..03cf2ec2 100644 --- a/internal/controller/clientpool/clientpool.go +++ b/internal/controller/clientpool/clientpool.go @@ -21,16 +21,35 @@ import ( runtimeclient "sigs.k8s.io/controller-runtime/pkg/client" ) +type AuthMode string + +const ( + AuthModeTLS AuthMode = "TLS" + AuthModeAPIKey AuthMode = "API_KEY" + AuthModeNoCredentials AuthMode = "NO_CREDENTIALS" // no TLS/API keys + // Add more auth modes here as they are supported +) + type ClientPoolKey struct { - HostPort string - Namespace string - MutualTLSSecret string // Include secret name in key to invalidate cache when the secret name changes + HostPort string + Namespace string + SecretName string // Include secret name in key to invalidate cache when the secret name changes + AuthMode AuthMode // Include auth mode in key to invalidate cache when the auth mode changes for the secret +} + +type MTLSAuth struct { + tlsConfig *tls.Config + expiryTime time.Time // Time we consider the cert expired (NotAfter minus safety buffer) +} + +type ClientAuth struct { + mode AuthMode + mTLS *MTLSAuth // non-nil when mode == AuthMTLS, nil when mode == AuthAPIKey } type ClientInfo struct { - client sdkclient.Client - tls *tls.Config // Storing the TLS config associated with the client to check certificate expiration. If the certificate is expired, a new client will be created. - expiryTime time.Time // Effective expiration time (cert.NotAfter - buffer) for efficient expiration checking + client sdkclient.Client + auth ClientAuth } type ClientPool struct { @@ -48,7 +67,7 @@ func New(l log.Logger, c runtimeclient.Client) *ClientPool { } } -func (cp *ClientPool) GetSDKClient(key ClientPoolKey, withMTLS bool) (sdkclient.Client, bool) { +func (cp *ClientPool) GetSDKClient(key ClientPoolKey) (sdkclient.Client, bool) { cp.mux.RLock() defer cp.mux.RUnlock() @@ -57,9 +76,9 @@ func (cp *ClientPool) GetSDKClient(key ClientPoolKey, withMTLS bool) (sdkclient. return nil, false } - if withMTLS { + if key.AuthMode == AuthModeTLS { // Check if any certificate is expired - expired, err := isCertificateExpired(info.expiryTime) + expired, err := isCertificateExpired(info.auth.mTLS.expiryTime) if err != nil { cp.logger.Error("Error checking certificate expiration", "error", err) return nil, false @@ -79,56 +98,41 @@ type NewClientOptions struct { Spec v1alpha1.TemporalConnectionSpec } -func (cp *ClientPool) UpsertClient(ctx context.Context, opts NewClientOptions) (sdkclient.Client, error) { +func (cp *ClientPool) fetchClientUsingMTLSSecret(secret corev1.Secret, opts NewClientOptions) (sdkclient.Client, error) { + clientOpts := sdkclient.Options{ Logger: cp.logger, HostPort: opts.Spec.HostPort, Namespace: opts.TemporalNamespace, - // TODO(jlegrone): Make API Keys work } var pemCert []byte var expiryTime time.Time - // Get the connection secret if it exists - if opts.Spec.MutualTLSSecretRef != nil { - var secret corev1.Secret - if err := cp.k8sClient.Get(ctx, types.NamespacedName{ - Name: opts.Spec.MutualTLSSecretRef.Name, - Namespace: opts.K8sNamespace, - }, &secret); err != nil { - return nil, err - } - if secret.Type != corev1.SecretTypeTLS { - err := fmt.Errorf("secret %s must be of type kubernetes.io/tls", secret.Name) - return nil, err - } - - // Extract the certificate to calculate the effective expiration time - pemCert = secret.Data["tls.crt"] + // Extract the certificate to calculate the effective expiration time + pemCert = secret.Data["tls.crt"] - // Check if certificate is expired before creating the client - exp, err := calculateCertificateExpirationTime(pemCert, 5*time.Minute) - if err != nil { - return nil, fmt.Errorf("failed to check certificate expiration: %v", err) - } - expired, err := isCertificateExpired(exp) - if err != nil { - return nil, fmt.Errorf("failed to check certificate expiration: %v", err) - } - if expired { - return nil, fmt.Errorf("certificate is expired or is going to expire soon") - } + // Check if certificate is expired before creating the client + exp, err := calculateCertificateExpirationTime(pemCert, 5*time.Minute) + if err != nil { + return nil, fmt.Errorf("failed to check certificate expiration: %v", err) + } + expired, err := isCertificateExpired(exp) + if err != nil { + return nil, fmt.Errorf("failed to check certificate expiration: %v", err) + } + if expired { + return nil, fmt.Errorf("certificate is expired or is going to expire soon") + } - cert, err := tls.X509KeyPair(secret.Data["tls.crt"], secret.Data["tls.key"]) - if err != nil { - return nil, err - } - clientOpts.ConnectionOptions.TLS = &tls.Config{ - Certificates: []tls.Certificate{cert}, - } - expiryTime = exp + cert, err := tls.X509KeyPair(secret.Data["tls.crt"], secret.Data["tls.key"]) + if err != nil { + return nil, err + } + clientOpts.ConnectionOptions.TLS = &tls.Config{ + Certificates: []tls.Certificate{cert}, } + expiryTime = exp c, err := sdkclient.Dial(clientOpts) if err != nil { @@ -142,24 +146,129 @@ func (cp *ClientPool) UpsertClient(ctx context.Context, opts NewClientOptions) ( cp.mux.Lock() defer cp.mux.Unlock() - var mutualTLSSecret string - if opts.Spec.MutualTLSSecretRef != nil { - mutualTLSSecret = opts.Spec.MutualTLSSecretRef.Name + key := ClientPoolKey{ + HostPort: opts.Spec.HostPort, + Namespace: opts.TemporalNamespace, + SecretName: opts.Spec.MutualTLSSecretRef.Name, + AuthMode: AuthModeTLS, + } + cp.clients[key] = ClientInfo{ + client: c, + auth: ClientAuth{ + mode: AuthModeTLS, + mTLS: &MTLSAuth{tlsConfig: clientOpts.ConnectionOptions.TLS, expiryTime: expiryTime}, + }, + } + + return c, nil +} + +func (cp *ClientPool) fetchClientUsingAPIKeySecret(secret corev1.Secret, opts NewClientOptions) (sdkclient.Client, error) { + clientOpts := sdkclient.Options{ + Logger: cp.logger, + HostPort: opts.Spec.HostPort, + Namespace: opts.TemporalNamespace, + ConnectionOptions: sdkclient.ConnectionOptions{ + TLS: &tls.Config{}, + }, + } + + clientOpts.Credentials = sdkclient.NewAPIKeyDynamicCredentials(func(ctx context.Context) (string, error) { + return string(secret.Data[opts.Spec.APIKeySecretRef.Key]), nil + }) + + c, err := sdkclient.Dial(clientOpts) + if err != nil { + return nil, err + } + + cp.mux.Lock() + defer cp.mux.Unlock() + + key := ClientPoolKey{ + HostPort: opts.Spec.HostPort, + Namespace: opts.TemporalNamespace, + SecretName: opts.Spec.APIKeySecretRef.Name, + AuthMode: AuthModeAPIKey, + } + cp.clients[key] = ClientInfo{ + client: c, + auth: ClientAuth{ + mode: AuthModeAPIKey, + mTLS: nil, + }, } + + return c, nil +} + +func (cp *ClientPool) fetchClientUsingNoCredentials(opts NewClientOptions) (sdkclient.Client, error) { + clientOpts := sdkclient.Options{ + Logger: cp.logger, + HostPort: opts.Spec.HostPort, + Namespace: opts.TemporalNamespace, + } + + c, err := sdkclient.Dial(clientOpts) + if err != nil { + return nil, err + } + key := ClientPoolKey{ - HostPort: opts.Spec.HostPort, - Namespace: opts.TemporalNamespace, - MutualTLSSecret: mutualTLSSecret, + HostPort: opts.Spec.HostPort, + Namespace: opts.TemporalNamespace, + SecretName: "", + AuthMode: AuthModeNoCredentials, } cp.clients[key] = ClientInfo{ - client: c, - tls: clientOpts.ConnectionOptions.TLS, - expiryTime: expiryTime, + client: c, + auth: ClientAuth{ + mode: AuthModeNoCredentials, + mTLS: nil, + }, } return c, nil } +func (cp *ClientPool) UpsertClient(ctx context.Context, secretName string, authMode AuthMode, opts NewClientOptions) (sdkclient.Client, error) { + + // Fetch the secret from k8s cluster, if it exists. Otherwise, create a connection with the server without using any credentials. + var secret corev1.Secret + if secretName != "" { + if err := cp.k8sClient.Get(ctx, types.NamespacedName{ + Name: secretName, + Namespace: opts.K8sNamespace, + }, &secret); err != nil { + return nil, err + } + } + + // Check the secret type + switch authMode { + case AuthModeTLS: + if secret.Type != corev1.SecretTypeTLS { + err := fmt.Errorf("secret %s must be of type kubernetes.io/tls", secret.Name) + return nil, err + } + return cp.fetchClientUsingMTLSSecret(secret, opts) + + case AuthModeAPIKey: + if secret.Type != corev1.SecretTypeOpaque { + err := fmt.Errorf("secret %s must be of type kubernetes.io/opaque", secret.Name) + return nil, err + } + return cp.fetchClientUsingAPIKeySecret(secret, opts) + + case AuthModeNoCredentials: + return cp.fetchClientUsingNoCredentials(opts) + + default: + return nil, fmt.Errorf("invalid auth mode: %s", authMode) + } + +} + func (cp *ClientPool) Close() { cp.mux.Lock() defer cp.mux.Unlock() diff --git a/internal/controller/worker_controller.go b/internal/controller/worker_controller.go index ba463890..fec3f080 100644 --- a/internal/controller/worker_controller.go +++ b/internal/controller/worker_controller.go @@ -14,6 +14,7 @@ import ( "github.com/temporalio/temporal-worker-controller/internal/k8s" "github.com/temporalio/temporal-worker-controller/internal/temporal" appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" @@ -36,12 +37,44 @@ const ( buildIDLabel = "temporal.io/build-id" ) -// getMutualTLSSecretName extracts the mutual TLS secret name from a secret reference -func getMutualTLSSecretName(secretRef *temporaliov1alpha1.SecretReference) (string, bool) { +// getAPIKeySecretName extracts the secret name from a SecretKeySelector +func getAPIKeySecretName(secretRef *corev1.SecretKeySelector) (string, error) { if secretRef != nil { - return secretRef.Name, true + return secretRef.Name, nil } - return "", false + + return "", fmt.Errorf("API key secret name is not set") +} + +func getTLSSecretName(secretRef *temporaliov1alpha1.SecretReference) (string, error) { + if secretRef != nil { + return secretRef.Name, nil + } + + return "", fmt.Errorf("TLS secret name is not set") +} + +func resolveAuthSecretName(tc *temporaliov1alpha1.TemporalConnection) (clientpool.AuthMode, string, error) { + auth := getAuthMode(tc) + switch auth { + case clientpool.AuthModeTLS: + name, err := getTLSSecretName(tc.Spec.MutualTLSSecretRef) + return auth, name, err + case clientpool.AuthModeAPIKey: + name, err := getAPIKeySecretName(tc.Spec.APIKeySecretRef) + return auth, name, err + default: + return auth, "", nil + } +} + +func getAuthMode(temporalConnection *temporaliov1alpha1.TemporalConnection) clientpool.AuthMode { + if temporalConnection.Spec.MutualTLSSecretRef != nil { + return clientpool.AuthModeTLS + } else if temporalConnection.Spec.APIKeySecretRef != nil { + return clientpool.AuthModeAPIKey + } + return clientpool.AuthModeNoCredentials } // TemporalWorkerDeploymentReconciler reconciles a TemporalWorkerDeployment object @@ -127,15 +160,22 @@ func (r *TemporalWorkerDeploymentReconciler) Reconcile(ctx context.Context, req return ctrl.Result{}, err } + // Get the Auth Mode and Secret Name + authMode, secretName, err := resolveAuthSecretName(&temporalConnection) + if err != nil { + l.Error(err, "unable to resolve auth secret name") + return ctrl.Result{}, err + } + // Get or update temporal client for connection - mutualTLSSecretName, hasMutualTLS := getMutualTLSSecretName(temporalConnection.Spec.MutualTLSSecretRef) temporalClient, ok := r.TemporalClientPool.GetSDKClient(clientpool.ClientPoolKey{ - HostPort: temporalConnection.Spec.HostPort, - Namespace: workerDeploy.Spec.WorkerOptions.TemporalNamespace, - MutualTLSSecret: mutualTLSSecretName, - }, hasMutualTLS) + HostPort: temporalConnection.Spec.HostPort, + Namespace: workerDeploy.Spec.WorkerOptions.TemporalNamespace, + SecretName: secretName, + AuthMode: authMode, + }) if !ok { - c, err := r.TemporalClientPool.UpsertClient(ctx, clientpool.NewClientOptions{ + c, err := r.TemporalClientPool.UpsertClient(ctx, secretName, authMode, clientpool.NewClientOptions{ K8sNamespace: workerDeploy.Namespace, TemporalNamespace: workerDeploy.Spec.WorkerOptions.TemporalNamespace, Spec: temporalConnection.Spec, diff --git a/internal/demo/README.md b/internal/demo/README.md index 8a726f98..44a5adf8 100644 --- a/internal/demo/README.md +++ b/internal/demo/README.md @@ -8,7 +8,7 @@ This guide will help you set up and run the Temporal Worker Controller locally u - [Helm](https://helm.sh/docs/intro/install/) - [Skaffold](https://skaffold.dev/docs/install/) - [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) -- Temporal Cloud account with mTLS certificates +- Temporal Cloud account with API key or mTLS certificates - Understanding of [Worker Versioning concepts](https://docs.temporal.io/production-deployment/worker-deployments/worker-versioning) (Pinned and Auto-Upgrade versioning behaviors) > **Note**: This demo specifically showcases **Pinned** workflow behavior. All workflows in the demo will remain on the worker version where they started, demonstrating how the controller safely manages multiple worker versions simultaneously during deployments. @@ -20,7 +20,15 @@ This guide will help you set up and run the Temporal Worker Controller locally u minikube start ``` -2. Set up Temporal Cloud mTLS certificates and update `skaffold.env`: +2. Create the `skaffold.env` file: + - Run: + ```bash + cp skaffold.example.env skaffold.env + ``` + + - Update the value of `TEMPORAL_NAMESPACE`, `TEMPORAL_ADDRESS` in `skaffold.env` to match your configuration. + +2. Set up Temporal Cloud Authentication: - Create a `certs` directory in the project root - Save your Temporal Cloud mTLS client certificates as: - `certs/client.pem` @@ -29,45 +37,65 @@ This guide will help you set up and run the Temporal Worker Controller locally u ```bash make create-cloud-mtls-secret ``` - - Create a `skaffold.env` file: + - In `skaffold.env`, set: + ```env + TEMPORAL_API_KEY_SECRET_NAME="" + TEMPORAL_MTLS_SECRET_NAME=temporal-cloud-mtls-secret + ``` + + NOTE: Alternatively, if you are using API keys, follow the steps below instead of mTLS: + + #### Using API Keys (alternative to mTLS) + - Create a `certs` directory in the project root if not already present + - Save your Temporal Cloud API key in a file (single line, no newline): ```bash - cp skaffold.example.env skaffold.env + echo -n "" > certs/api-key.txt + ``` + - Create the Kubernetes Secret: + ```bash + make create-api-key-secret + ``` + - In `skaffold.env`, set: + ```env + TEMPORAL_API_KEY_SECRET_NAME=temporal-api-key + TEMPORAL_MTLS_SECRET_NAME="" ``` - - Update the value of `TEMPORAL_NAMESPACE` in `skaffold.env` to match your Temporal cloud namespace. + - Note: Do not set both mTLS and API key for the same connection. If both present, the TemporalConnection Custom Resource + Instance will not get installed in the k8s environment. -3. Build and deploy the Controller image to the local k8s cluster: +4. Build and deploy the Controller image to the local k8s cluster: ```bash skaffold run --profile worker-controller ``` ### Testing Progressive Deployments -4. **Deploy the v1 worker**: +5. **Deploy the v1 worker**: ```bash skaffold run --profile helloworld-worker ``` This deploys a TemporalWorkerDeployment and TemporalConnection Custom Resource using the **Progressive strategy**. Note that when there is no current version (as in an initial versioned worker deployment), the progressive steps are skipped and v1 becomes the current version immediately. All new workflow executions will now start on v1. -5. Watch the deployment status: +6. Watch the deployment status: ```bash watch kubectl get twd ``` -6. **Apply load** to the v1 worker to simulate production traffic: +7. **Apply load** to the v1 worker to simulate production traffic: ```bash make apply-load-sample-workflow ``` #### **Progressive Rollout of v2** (Non-Replay-Safe Change) -7. **Deploy a non-replay-safe workflow change**: +8. **Deploy a non-replay-safe workflow change**: ```bash git apply internal/demo/helloworld/changes/no-version-gate.patch skaffold run --profile helloworld-worker ``` This applies a **non-replay-safe change** (switching an activity response type from string to a struct). -8. **Observe the progressive rollout managing incompatible versions**: +9. **Observe the progressive rollout managing incompatible versions**: - New workflow executions gradually shift from v1 to v2 following the configured rollout steps (1% → 5% → 10% → 50% → 100%) - **Both worker versions run simultaneously** - this is critical since the code changes are incompatible - v1 workers continue serving existing workflows (which would fail to replay on v2) diff --git a/internal/demo/helloworld/helm/helloworld/templates/connection.yaml b/internal/demo/helloworld/helm/helloworld/templates/connection.yaml index 37a51433..c6277834 100644 --- a/internal/demo/helloworld/helm/helloworld/templates/connection.yaml +++ b/internal/demo/helloworld/helm/helloworld/templates/connection.yaml @@ -8,6 +8,15 @@ metadata: {{- include "helloworld.labels" . | nindent 4 }} spec: hostPort: "{{ .Values.temporal.address }}" + {{- $apiSecretName := .Values.temporal.apiKey.name | trim }} + {{- $apiSecretKey := .Values.temporal.apiKey.key | trim }} + {{- if $apiSecretName }} + apiKeySecretRef: + name: {{ $apiSecretName | quote }} + key: {{ $apiSecretKey | quote }} + {{- end }} + {{- if .Values.temporal.mtlsSecretName }} mutualTLSSecretRef: - name: {{ .Values.temporal.mtlsSecretName }} -{{- end }} + name: {{ .Values.temporal.mtlsSecretName | quote }} + {{- end }} +{{- end }} \ No newline at end of file diff --git a/internal/demo/helloworld/helm/helloworld/values.schema.json b/internal/demo/helloworld/helm/helloworld/values.schema.json index dfffaa9c..790edd62 100644 --- a/internal/demo/helloworld/helm/helloworld/values.schema.json +++ b/internal/demo/helloworld/helm/helloworld/values.schema.json @@ -35,6 +35,20 @@ "mtlsSecretName": { "type": "string", "description": "Name of the secret containing mTLS certificates (required if connectionName is empty)" + }, + "apiKey": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": { + "type": "string", + "description": "Name of the secret containing API key (required if connectionName is empty)" + }, + "key": { + "type": "string", + "description": "Data key inside the API key secret" + } + } } }, "required": ["namespace"], @@ -51,6 +65,15 @@ }, { "required": ["address", "mtlsSecretName"] + }, + { + "properties": { + "apiKey": { + "type": "object", + "required": ["name", "key"] + } + }, + "required": ["address", "apiKey"] } ] }, diff --git a/internal/demo/helloworld/helm/helloworld/values.yaml b/internal/demo/helloworld/helm/helloworld/values.yaml index 7e10f445..7143e741 100644 --- a/internal/demo/helloworld/helm/helloworld/values.yaml +++ b/internal/demo/helloworld/helm/helloworld/values.yaml @@ -3,12 +3,15 @@ image: tag: latest temporal: - namespace: "" # e.g. default + namespace: "default" # e.g. default # Use existing connection (leave empty to create new one) connectionName: "" # e.g. dev-server # Connection details (required if connectionName is empty) address: "" # e.g. .tmprl.cloud:7233 mtlsSecretName: "" # e.g. temporal-cloud-mtls - + apiKey: + name: "" # e.g. temporal-api-key + key: "" # data key inside the secret + rollout: strategy: Progressive diff --git a/internal/demo/util/client.go b/internal/demo/util/client.go index 348928eb..6a1b7f38 100644 --- a/internal/demo/util/client.go +++ b/internal/demo/util/client.go @@ -38,10 +38,6 @@ func newClient(buildID string, metricsPort int) (c client.Client, stopFunc func( panic(err) } - if _, err := c.CheckHealth(context.Background(), &client.CheckHealthRequest{}); err != nil { - panic(err) - } - if _, err := c.ListWorkflow(context.Background(), &workflowservice.ListWorkflowExecutionsRequest{ Namespace: opts.Namespace, }); err != nil { diff --git a/internal/k8s/deployments.go b/internal/k8s/deployments.go index ac5da9c3..9e5d069f 100644 --- a/internal/k8s/deployments.go +++ b/internal/k8s/deployments.go @@ -261,6 +261,18 @@ func NewDeploymentWithOwnerRef( }, }, }) + } else if connection.APIKeySecretRef != nil { + for i, container := range podSpec.Containers { + container.Env = append(container.Env, + corev1.EnvVar{ + Name: "TEMPORAL_API_KEY", + ValueFrom: &corev1.EnvVarSource{ + SecretKeyRef: connection.APIKeySecretRef, + }, + }, + ) + podSpec.Containers[i] = container + } } // Build pod annotations @@ -306,6 +318,7 @@ func NewDeploymentWithOwnerRef( } } +// TODO (Shivam): Change hash when secret name is updated as well. func ComputeConnectionSpecHash(connection temporaliov1alpha1.TemporalConnectionSpec) string { // HostPort is required, but MutualTLSSecret can be empty for non-mTLS connections if connection.HostPort == "" { @@ -318,6 +331,8 @@ func ComputeConnectionSpecHash(connection temporaliov1alpha1.TemporalConnectionS _, _ = hasher.Write([]byte(connection.HostPort)) if connection.MutualTLSSecretRef != nil { _, _ = hasher.Write([]byte(connection.MutualTLSSecretRef.Name)) + } else if connection.APIKeySecretRef != nil { + _, _ = hasher.Write([]byte(connection.APIKeySecretRef.Name)) } return hex.EncodeToString(hasher.Sum(nil)) diff --git a/internal/k8s/deployments_test.go b/internal/k8s/deployments_test.go index f11581dd..ae1bafd9 100644 --- a/internal/k8s/deployments_test.go +++ b/internal/k8s/deployments_test.go @@ -8,6 +8,7 @@ import ( "context" "crypto/rand" "crypto/rsa" + "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/pem" @@ -562,6 +563,64 @@ func TestComputeConnectionSpecHash(t *testing.T) { assert.NotEqual(t, hash1, hash2, "Empty vs non-empty mTLS secret should produce different hashes") assert.NotEmpty(t, hash1, "Hash should still be generated even with empty mTLS secret") }) + + t.Run("different API key secrets produce different hashes", func(t *testing.T) { + spec1 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret1"}, + Key: "api-key1"}, + } + spec2 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret2"}, + Key: "api-key2"}, + } + hash1 := k8s.ComputeConnectionSpecHash(spec1) + hash2 := k8s.ComputeConnectionSpecHash(spec2) + + assert.NotEqual(t, hash1, hash2, "Different API key secrets should produce different hashes") + }) + + t.Run("empty API key secret vs non-empty produce different hashes", func(t *testing.T) { + spec1 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: nil, + } + spec2 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret"}, + Key: "api-key"}, + } + hash1 := k8s.ComputeConnectionSpecHash(spec1) + hash2 := k8s.ComputeConnectionSpecHash(spec2) + + assert.NotEqual(t, hash1, hash2, "Empty vs non-empty API key secret should produce different hashes") + assert.NotEmpty(t, hash1, "Hash should still be generated even with empty API key secret") + }) + + t.Run("same API key secret name produce the same hash", func(t *testing.T) { + spec1 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret"}, + Key: "api-key"}, + } + spec2 := temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "localhost:7233", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "secret"}, + Key: "api-key"}, + } + + hash1 := k8s.ComputeConnectionSpecHash(spec1) + hash2 := k8s.ComputeConnectionSpecHash(spec2) + + assert.Equal(t, hash1, hash2, "Same API key secret name should produce the same hash") + }) + } func TestNewDeploymentWithOwnerRef_EnvironmentVariablesAndVolumes(t *testing.T) { @@ -726,6 +785,15 @@ func TestNewDeploymentWithOwnerRef_EnvConfigSDKCompatibility(t *testing.T) { }, namespace: "test-namespace-with-tls", }, + "with API key": { + connection: temporaliov1alpha1.TemporalConnectionSpec{ + HostPort: "test.temporal.example:9999", + APIKeySecretRef: &corev1.SecretKeySelector{ + LocalObjectReference: corev1.LocalObjectReference{Name: "test-api-key-secret"}, + Key: "api-key"}, + }, + namespace: "test-namespace-with-api-key", + }, } for name, tt := range tests { @@ -759,7 +827,7 @@ func TestNewDeploymentWithOwnerRef_EnvConfigSDKCompatibility(t *testing.T) { container := deployment.Spec.Template.Spec.Containers[0] // Infer whether TLS is expected from connection spec - expectTLS := tt.connection.MutualTLSSecretRef != nil + expectTLS := tt.connection.MutualTLSSecretRef != nil || tt.connection.APIKeySecretRef != nil if expectTLS { // Create temporary test certificate files @@ -777,10 +845,23 @@ func TestNewDeploymentWithOwnerRef_EnvConfigSDKCompatibility(t *testing.T) { container.Env[i].Value = keyPath } } + } else if tt.connection.APIKeySecretRef != nil { + for i := range container.Env { + if container.Env[i].Name == "TEMPORAL_API_KEY" { + assert.Equal(t, tt.connection.APIKeySecretRef.Name, container.Env[i].ValueFrom.SecretKeyRef.Name, "API key secret name should match") + assert.Equal(t, "api-key", container.Env[i].ValueFrom.SecretKeyRef.Key, "API key secret key should be 'api-key'") + } + } } // Set environment variables using t.Setenv() to simulate the runtime environment for _, env := range container.Env { + if env.Name == "TEMPORAL_API_KEY" { + // setting a dummy value here since API values are read from an actual secret object in runtime + // moreover, env.Value is nil for TEMPORAL_API_KEY since it used the ValueFrom field (check deployments.go) + t.Setenv(env.Name, "test-api-key-value") + continue + } t.Setenv(env.Name, env.Value) } @@ -791,6 +872,9 @@ func TestNewDeploymentWithOwnerRef_EnvConfigSDKCompatibility(t *testing.T) { // Verify that the parsed client options match our expectations assert.Equal(t, tt.connection.HostPort, clientOptions.HostPort, "HostPort should be parsed from TEMPORAL_ADDRESS") assert.Equal(t, tt.namespace, clientOptions.Namespace, "Namespace should be parsed from TEMPORAL_NAMESPACE") + if tt.connection.APIKeySecretRef != nil { + assert.Equal(t, "test-api-key-value", os.Getenv("TEMPORAL_API_KEY"), "API key should be parsed from TEMPORAL_API_KEY") + } // Verify other client option fields that should have default/empty values assert.Empty(t, clientOptions.Identity, "Identity should be empty when not set via env vars") @@ -800,8 +884,11 @@ func TestNewDeploymentWithOwnerRef_EnvConfigSDKCompatibility(t *testing.T) { if expectTLS { assert.NotNil(t, clientOptions.ConnectionOptions.TLS, "TLS should be configured for mTLS connection") + } else if tt.connection.APIKeySecretRef != nil { + // An empty TLS config is configured when an API key is used without mTLS secrets + assert.Equal(t, clientOptions.ConnectionOptions.TLS, &tls.Config{}, "Empty TLS config should be configured for API key connection") } else { - assert.Nil(t, clientOptions.ConnectionOptions.TLS, "TLS should not be configured for non-mTLS connection") + assert.Nil(t, clientOptions.ConnectionOptions.TLS, "TLS config should not be configured for non-mTLS connection") } // Note: TEMPORAL_DEPLOYMENT_NAME and TEMPORAL_WORKER_BUILD_ID are not part of client options diff --git a/internal/tests/internal/deployment_controller.go b/internal/tests/internal/deployment_controller.go index c44b4fe1..b009d12a 100644 --- a/internal/tests/internal/deployment_controller.go +++ b/internal/tests/internal/deployment_controller.go @@ -91,8 +91,8 @@ func applyDeployment(t *testing.T, ctx context.Context, k8sClient client.Client, go testhelpers.RunHelloWorldWorker(ctx, deployment.Spec.Template, workerCallback(i)) } - // wait 10s for all expected workers to be healthy - timedOut := waitTimeout(&wg, 10*time.Second) + // wait 30s for all expected workers to be healthy (under suite load startup can be slower) + timedOut := waitTimeout(&wg, 30*time.Second) if timedOut { t.Fatalf("could not start workers, errors were: %+v", workerErrors) diff --git a/internal/tests/internal/validation_helpers.go b/internal/tests/internal/validation_helpers.go index 76719af8..b76e762f 100644 --- a/internal/tests/internal/validation_helpers.go +++ b/internal/tests/internal/validation_helpers.go @@ -49,7 +49,7 @@ func waitForVersionRegistrationInDeployment( deploymentHandler := ts.GetDefaultClient().WorkerDeploymentClient().GetHandle(version.DeploymentName) - eventually(t, 30*time.Second, time.Second, func() error { + eventually(t, 60*time.Second, time.Second, func() error { resp, err := deploymentHandler.Describe(ctx, sdkclient.WorkerDeploymentDescribeOptions{}) if err != nil { return fmt.Errorf("unable to describe worker deployment %s: %w", version.DeploymentName, err) @@ -77,7 +77,7 @@ func setCurrentVersion( }) } deploymentHandler := ts.GetDefaultClient().WorkerDeploymentClient().GetHandle(workerDeploymentName) - eventually(t, 30*time.Second, time.Second, func() error { + eventually(t, 60*time.Second, time.Second, func() error { _, err := deploymentHandler.SetCurrentVersion(ctx, sdkclient.WorkerDeploymentSetCurrentVersionOptions{ BuildID: buildID, Identity: defaults.ControllerIdentity, diff --git a/skaffold.example.env b/skaffold.example.env index dcb492b4..a29be8ed 100644 --- a/skaffold.example.env +++ b/skaffold.example.env @@ -13,8 +13,16 @@ TEMPORAL_NAMESPACE=your-namespace-here # Temporal server address (optionally templated from namespace, if using Temporal Cloud) TEMPORAL_ADDRESS=${TEMPORAL_NAMESPACE}.tmprl.cloud:7233 +# Set one of mTLS or API key names + # Temporal mTLS secret name -TEMPORAL_MTLS_SECRET_NAME=temporal-cloud-mtls +TEMPORAL_MTLS_SECRET_NAME="" + +# K8s secret storing the Temporal API key +TEMPORAL_API_KEY_SECRET_NAME="" + +# Secret key whose corresponding value is the API key +TEMPORAL_API_KEY_SECRET_KEY="" # Kubernetes context to use for deployment SKAFFOLD_KUBE_CONTEXT=minikube diff --git a/skaffold.yaml b/skaffold.yaml index d9660546..64ba7096 100644 --- a/skaffold.yaml +++ b/skaffold.yaml @@ -58,6 +58,8 @@ profiles: temporal.namespace: "{{ .TEMPORAL_NAMESPACE }}" temporal.address: "{{ .TEMPORAL_ADDRESS }}" temporal.mtlsSecretName: "{{ .TEMPORAL_MTLS_SECRET_NAME }}" + temporal.apiKey.name: "{{ .TEMPORAL_API_KEY_SECRET_NAME }}" + temporal.apiKey.key: "{{ .TEMPORAL_API_KEY_SECRET_KEY }}" valuesFiles: - internal/demo/helloworld/helm/helloworld/values.yaml deploy: