Create vizier and cloud cert-manager compatible secrets (#2391)

Summary: Create vizier and cloud cert-manager compatible secrets

Pixie's certificate management predates cert-manager becoming the
definitive method for managing k8s certs. As a result, Pixie's
certificates are created in an incompatible way to how cert-manager
creates its TLS secrets -- Pixie's are of type generic and bundle client
and server certs while cert-manager uses the [tls secret
type](https://kubernetes.io/docs/concepts/configuration/secret/#tls-secrets)
and only can store a single CA, key and cert file.

This PR updates Pixie's certificate management to create the existing
generic secret alongside two tls type secrets.

Future PRs will move the consumers of these secrets to use the newer
cert-manager compatible equivalents

Relevant Issues: N/A

Type of change: /kind cleanup

Test Plan: Used this as part of a larger change to deploy a cloud with
cert-manager service tls certs

Changelog Message: Update Pixie's vizier and cloud certificate
management to create cert-manager compatible kubernetes secrets

---------

Signed-off-by: Dom Del Nano <ddelnano@gmail.com>
GitOrigin-RevId: 7622689973b118d71a5563e891c44cbc25101aeb

Author: ddelnano

scripts/create_cloud_secrets.sh

@@ -91,6 +91,20 @@ kubectl create secret generic -n "${namespace}" \
   --from-file=server.crt=./server.crt \
   --from-file=server.key=./server.key
 
+kubectl create secret generic -n "${namespace}" \
+  service-tls-server-certs \
+  --type=kubernetes.io/tls \
+  --from-file=tls.crt=./server.crt \
+  --from-file=tls.key=./server.key \
+  --from-file=ca.crt=./ca.crt
+
+kubectl create secret generic -n "${namespace}" \
+  service-tls-client-certs \
+  --type=kubernetes.io/tls \
+  --from-file=tls.crt=./client.crt \
+  --from-file=tls.key=./client.key \
+  --from-file=ca.crt=./ca.crt
+
 popd || exit 1
 
 PROXY_TLS_CERTS="$(mktemp -d)"

src/operator/controllers/monitor.go

@@ -250,12 +250,30 @@ func (m *VizierMonitor) watchCerts() {
 	}()
 }
 
+// getServerTLSCerts returns the server certificate and CA certificate PEM bytes for
+// Vizier's TLS material. It prefers the kubernetes.io/tls typed service-tls-server-certs
+// secret (which is also the shape produced by cert-manager), falling back to the legacy
+// service-tls-certs secret so existing clusters that have not regenerated certs keep working.
+func (m *VizierMonitor) getServerTLSCerts() ([]byte, []byte, error) {
+	tlsSecret, err := m.clientset.CoreV1().Secrets(m.namespace).Get(context.Background(), "service-tls-server-certs", metav1.GetOptions{})
+	if err == nil {
+		return tlsSecret.Data["tls.crt"], tlsSecret.Data["ca.crt"], nil
+	}
+
+	legacySecret, legacyErr := m.clientset.CoreV1().Secrets(m.namespace).Get(context.Background(), "service-tls-certs", metav1.GetOptions{})
+	if legacyErr != nil {
+		// Return the original error referencing the preferred secret.
+		return nil, nil, err
+	}
+	return legacySecret.Data["server.crt"], legacySecret.Data["ca.crt"], nil
+}
+
 func (m *VizierMonitor) checkCerts() error {
-	tlsSecret, err := m.clientset.CoreV1().Secrets(m.namespace).Get(context.Background(), "service-tls-certs", metav1.GetOptions{})
+	serverCert, _, err := m.getServerTLSCerts()
 	if err != nil {
 		return err
 	}
-	cert, _ := pem.Decode(tlsSecret.Data["server.crt"])
+	cert, _ := pem.Decode(serverCert)
 	x509cert, err := x509.ParseCertificate(cert.Bytes)
 	if err != nil {
 		log.WithError(err).Error("failed to parse cert")
@@ -273,14 +291,14 @@ func (m *VizierMonitor) getTLSConfig() *tls.Config {
 	// This is used as a fallback incase we somehow fail to get the CA for the vizier.
 	fallbackInsecureConfig := &tls.Config{InsecureSkipVerify: true} // lgtm [go/disabled-certificate-check]
 
-	tlsSecret, err := m.clientset.CoreV1().Secrets(m.namespace).Get(context.Background(), "service-tls-certs", metav1.GetOptions{})
+	_, caCert, err := m.getServerTLSCerts()
 	if err != nil {
 		log.WithError(err).Warn("failed to get certs secret, monitor will use insecure tls to check /statusz")
 		return fallbackInsecureConfig
 	}
 
 	certPool := x509.NewCertPool()
-	if ok := certPool.AppendCertsFromPEM(tlsSecret.Data["ca.crt"]); !ok {
+	if ok := certPool.AppendCertsFromPEM(caCert); !ok {
 		log.WithError(err).Warn("failed add CA to pool, monitor will use insecure tls to check /statusz")
 		return fallbackInsecureConfig
 	}

src/utils/shared/certs/BUILD.bazel

@@ -21,5 +21,8 @@ go_library(
     srcs = ["certs.go"],
     importpath = "px.dev/pixie/src/utils/shared/certs",
     visibility = ["//src:__subpackages__"],
-    deps = ["//src/utils/shared/k8s"],
+    deps = [
+        "//src/utils/shared/k8s",
+        "@io_k8s_api//core/v1:core",
+    ],
 )

src/utils/shared/certs/certs.go

@@ -29,6 +29,8 @@ import (
 	"strings"
 	"time"
 
+	v1 "k8s.io/api/core/v1"
+
 	"px.dev/pixie/src/utils/shared/k8s"
 )
 
@@ -172,12 +174,46 @@ func GenerateCloudCertYAMLs(namespace string) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	yaml, err := k8s.ConvertResourceToYAML(tlsCert)
+
+	var yamls []string
+
+	tcYaml, err := k8s.ConvertResourceToYAML(tlsCert)
+	if err != nil {
+		return "", err
+	}
+	yamls = append(yamls, tcYaml)
+
+	serverTLSCert, err := k8s.CreateGenericSecretFromLiterals(namespace, "service-tls-server-certs", map[string]string{
+		"tls.crt": string(serverCert),
+		"tls.key": string(serverKey),
+		"ca.crt":  string(caCert),
+	})
+	if err != nil {
+		return "", err
+	}
+	serverTLSCert.Type = v1.SecretTypeTLS
+	stYaml, err := k8s.ConvertResourceToYAML(serverTLSCert)
 	if err != nil {
 		return "", err
 	}
+	yamls = append(yamls, stYaml)
 
-	return fmt.Sprintf("---\n%s\n", yaml), nil
+	clientTLSCert, err := k8s.CreateGenericSecretFromLiterals(namespace, "service-tls-client-certs", map[string]string{
+		"tls.crt": string(clientCert),
+		"tls.key": string(clientKey),
+		"ca.crt":  string(caCert),
+	})
+	if err != nil {
+		return "", err
+	}
+	clientTLSCert.Type = v1.SecretTypeTLS
+	ctYaml, err := k8s.ConvertResourceToYAML(clientTLSCert)
+	if err != nil {
+		return "", err
+	}
+	yamls = append(yamls, ctYaml)
+
+	return "---\n" + strings.Join(yamls, "\n---\n"), nil
 }
 
 // GenerateVizierCertYAMLs generates the yamls for vizier certs.
@@ -231,6 +267,41 @@ func GenerateVizierCertYAMLs(namespace string) (string, error) {
 	}
 	yamls = append(yamls, tcYaml)
 
+	// service-tls-server-certs and service-tls-client-certs mirror the data in
+	// service-tls-certs, but are kubernetes.io/tls typed secrets. This matches the
+	// shape produced by cert-manager Certificate resources, allowing workloads to
+	// consume TLS material via a projected volume regardless of whether the certs
+	// are generated by Pixie or provisioned externally by cert-manager.
+	serverTLSCert, err := k8s.CreateGenericSecretFromLiterals(namespace, "service-tls-server-certs", map[string]string{
+		"tls.crt": string(serverCert),
+		"tls.key": string(serverKey),
+		"ca.crt":  string(caCert),
+	})
+	if err != nil {
+		return "", err
+	}
+	serverTLSCert.Type = v1.SecretTypeTLS
+	stYaml, err := k8s.ConvertResourceToYAML(serverTLSCert)
+	if err != nil {
+		return "", err
+	}
+	yamls = append(yamls, stYaml)
+
+	clientTLSCert, err := k8s.CreateGenericSecretFromLiterals(namespace, "service-tls-client-certs", map[string]string{
+		"tls.crt": string(clientCert),
+		"tls.key": string(clientKey),
+		"ca.crt":  string(caCert),
+	})
+	if err != nil {
+		return "", err
+	}
+	clientTLSCert.Type = v1.SecretTypeTLS
+	ctYaml, err := k8s.ConvertResourceToYAML(clientTLSCert)
+	if err != nil {
+		return "", err
+	}
+	yamls = append(yamls, ctYaml)
+
 	etcdPeerCert, err := k8s.CreateGenericSecretFromLiterals(namespace, "etcd-peer-tls-certs", map[string]string{
 		"peer.key":    string(serverKey),
 		"peer.crt":    string(serverCert),