Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Compliance multi-tenant changes #3111

Merged
merged 6 commits into from
Jan 22, 2024
Merged

Compliance multi-tenant changes #3111

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4b4f463
Compliance multi-tenant changes
Josh-Tigera Jan 9, 2024
6f683ef
Merge branch 'master' into josh.compliance-mt
Josh-Tigera Jan 10, 2024
638cf2a
Fix typo
Josh-Tigera Jan 11, 2024
2f2fe10
Address PR feedback
Josh-Tigera Jan 15, 2024
8e8ba20
More PR feedback
Josh-Tigera Jan 17, 2024
3777c0a
Small fixes
Josh-Tigera Jan 17, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pkg/controller/authentication/authentication_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved.
// Copyright (c) 2020-2024 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
2 changes: 1 addition & 1 deletion pkg/controller/clusterconnection/clusterconnection_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved.
// Copyright (c) 2020-2024 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand Down
147 changes: 90 additions & 57 deletions pkg/controller/compliance/compliance_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
// Copyright (c) 2020-2023 Tigera, Inc. All rights reserved.
// Copyright (c) 2020-2024 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
Expand All @@ -18,6 +18,8 @@ import (
"context"
"fmt"

"github.com/tigera/operator/pkg/controller/tenancy"

"k8s.io/apimachinery/pkg/types"

v3 "github.com/tigera/api/pkg/apis/projectcalico/v3"
Expand Down Expand Up @@ -64,69 +66,58 @@ func Add(mgr manager.Manager, opts options.AddOptions) error {
reconciler := newReconciler(mgr, opts, licenseAPIReady, tierWatchReady)

// Create a new controller
controller, err := controller.New("compliance-controller", mgr, controller.Options{Reconciler: reconcile.Reconciler(reconciler)})
complianceController, err := controller.New("compliance-controller", mgr, controller.Options{Reconciler: reconciler})
if err != nil {
return err
}

// Determine how to handle watch events for cluster-scoped resources. For multi-tenant clusters,
// we should update all tenants whenever one changes. For single-tenant clusters, we can just queue the object.
var eventHandler handler.EventHandler = &handler.EnqueueRequestForObject{}
if opts.MultiTenant {
eventHandler = utils.EnqueueAllTenants(mgr.GetClient())
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.Tenant{}}, &handler.EnqueueRequestForObject{}); err != nil {
return fmt.Errorf("compliance-controller failed to watch Tenant resource: %w", err)
}
}

k8sClient, err := kubernetes.NewForConfig(mgr.GetConfig())
if err != nil {
log.Error(err, "Failed to establish a connection to k8s")
return err
}

go utils.WaitToAddLicenseKeyWatch(controller, k8sClient, log, licenseAPIReady)
installNS, _, watchNamespaces := tenancy.GetWatchNamespaces(opts.MultiTenant, render.PolicyRecommendationNamespace)

go utils.WaitToAddTierWatch(networkpolicy.TigeraComponentTierName, controller, k8sClient, log, tierWatchReady)
go utils.WaitToAddNetworkPolicyWatches(controller, k8sClient, log, []types.NamespacedName{
{Name: render.ComplianceAccessPolicyName, Namespace: render.ComplianceNamespace},
{Name: render.ComplianceServerPolicyName, Namespace: render.ComplianceNamespace},
{Name: networkpolicy.TigeraComponentDefaultDenyPolicyName, Namespace: render.ComplianceNamespace},
})

return add(mgr, controller)
}
go utils.WaitToAddLicenseKeyWatch(complianceController, k8sClient, log, licenseAPIReady)

// newReconciler returns a new *reconcile.Reconciler
func newReconciler(mgr manager.Manager, opts options.AddOptions, licenseAPIReady *utils.ReadyFlag, tierWatchReady *utils.ReadyFlag) reconcile.Reconciler {
r := &ReconcileCompliance{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
provider: opts.DetectedProvider,
status: status.New(mgr.GetClient(), "compliance", opts.KubernetesVersion),
clusterDomain: opts.ClusterDomain,
licenseAPIReady: licenseAPIReady,
tierWatchReady: tierWatchReady,
usePSP: opts.UsePSP,
}
r.status.Run(opts.ShutdownContext)
return r
}

// add adds watches for resources that are available at startup
func add(mgr manager.Manager, c controller.Controller) error {
var err error
go utils.WaitToAddTierWatch(networkpolicy.TigeraComponentTierName, complianceController, k8sClient, log, tierWatchReady)
go utils.WaitToAddNetworkPolicyWatches(complianceController, k8sClient, log, []types.NamespacedName{
{Name: render.ComplianceAccessPolicyName, Namespace: installNS},
{Name: render.ComplianceServerPolicyName, Namespace: installNS},
{Name: networkpolicy.TigeraComponentDefaultDenyPolicyName, Namespace: installNS},
})

// Watch for changes to primary resource Compliance
err = c.Watch(&source.Kind{Type: &operatorv1.Compliance{}}, &handler.EnqueueRequestForObject{})
err = complianceController.Watch(&source.Kind{Type: &operatorv1.Compliance{}}, &handler.EnqueueRequestForObject{})
if err != nil {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this needs to be eventHandler

return err
}

if err = utils.AddInstallationWatch(c); err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.Installation{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch Installation resource: %w", err)
}

if err = imageset.AddImageSetWatch(c); err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.ImageSet{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch ImageSet: %w", err)
}

if err = utils.AddAPIServerWatch(c); err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.APIServer{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch APIServer resource: %w", err)
}

// Watch the given secrets in each both the compliance and operator namespaces
for _, namespace := range []string{common.OperatorNamespace(), render.ComplianceNamespace} {
for _, namespace := range watchNamespaces {
for _, secretName := range []string{
render.TigeraElasticsearchGatewaySecret, render.ElasticsearchComplianceBenchmarkerUserSecret,
render.ElasticsearchComplianceControllerUserSecret, render.ElasticsearchComplianceReporterUserSecret,
Expand All @@ -135,37 +126,51 @@ func add(mgr manager.Manager, c controller.Controller) error {
render.TigeraLinseedSecret, render.VoltronLinseedTLS,
render.VoltronLinseedPublicCert,
} {
if err = utils.AddSecretsWatch(c, secretName, namespace); err != nil {
if err = utils.AddSecretsWatch(complianceController, secretName, namespace); err != nil {
return fmt.Errorf("compliance-controller failed to watch the secret '%s' in '%s' namespace: %w", secretName, namespace, err)
}
}
}

// Watch for changes to primary resource ManagementCluster
err = c.Watch(&source.Kind{Type: &operatorv1.ManagementCluster{}}, &handler.EnqueueRequestForObject{})
if err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.ManagementCluster{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch primary resource: %w", err)
}

// Watch for changes to primary resource ManagementClusterConnection
err = c.Watch(&source.Kind{Type: &operatorv1.ManagementClusterConnection{}}, &handler.EnqueueRequestForObject{})
if err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.ManagementClusterConnection{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch primary resource: %w", err)
}

err = c.Watch(&source.Kind{Type: &operatorv1.Authentication{}}, &handler.EnqueueRequestForObject{})
if err != nil {
if err = complianceController.Watch(&source.Kind{Type: &operatorv1.Authentication{}}, eventHandler); err != nil {
return fmt.Errorf("compliance-controller failed to watch resource: %w", err)
}

// Watch for changes to TigeraStatus.
if err = utils.AddTigeraStatusWatch(c, ResourceName); err != nil {
if err = utils.AddTigeraStatusWatch(complianceController, ResourceName); err != nil {
return fmt.Errorf("compliance-controller failed to watch compliance Tigerastatus: %w", err)
}

return nil
}

// newReconciler returns a new *reconcile.Reconciler
func newReconciler(mgr manager.Manager, opts options.AddOptions, licenseAPIReady *utils.ReadyFlag, tierWatchReady *utils.ReadyFlag) reconcile.Reconciler {
r := &ReconcileCompliance{
client: mgr.GetClient(),
scheme: mgr.GetScheme(),
provider: opts.DetectedProvider,
status: status.New(mgr.GetClient(), "compliance", opts.KubernetesVersion),
clusterDomain: opts.ClusterDomain,
licenseAPIReady: licenseAPIReady,
tierWatchReady: tierWatchReady,
usePSP: opts.UsePSP,
multiTenant: opts.MultiTenant,
}
r.status.Run(opts.ShutdownContext)
return r
}

// blank assignment to verify that ReconcileCompliance implements reconcile.Reconciler
var _ reconcile.Reconciler = &ReconcileCompliance{}

Expand All @@ -181,11 +186,16 @@ type ReconcileCompliance struct {
licenseAPIReady *utils.ReadyFlag
tierWatchReady *utils.ReadyFlag
usePSP bool
multiTenant bool
}

func GetCompliance(ctx context.Context, cli client.Client) (*operatorv1.Compliance, error) {
func GetCompliance(ctx context.Context, cli client.Client, mt bool, ns string) (*operatorv1.Compliance, error) {
key := client.ObjectKey{Name: "tigera-secure"}
if mt {
key.Namespace = ns
}
instance := &operatorv1.Compliance{}
err := cli.Get(ctx, utils.DefaultTSEEInstanceKey, instance)
err := cli.Get(ctx, key, instance)
if err != nil {
return nil, err
}
Expand All @@ -198,11 +208,27 @@ func GetCompliance(ctx context.Context, cli client.Client) (*operatorv1.Complian
// The Controller will requeue the Request to be processed again if the returned error is non-nil or
// Result.Requeue is true, otherwise upon completion it will remove the work from the queue.
func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) {
reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name)
helper := utils.NewNamespaceHelper(r.multiTenant, render.ComplianceNamespace, request.Namespace)
reqLogger := log.WithValues("Request.Namespace", request.Namespace, "Request.Name", request.Name, "installNS", helper.InstallNamespace(), "truthNS", helper.TruthNamespace())
reqLogger.Info("Reconciling Compliance")

// We skip requests without a namespace specified in multi-tenant setups.
if r.multiTenant && request.Namespace == "" {
return reconcile.Result{}, nil
}

// Check if this is a tenant-scoped request.
tenant, _, err := utils.GetTenant(ctx, r.multiTenant, r.client, request.Namespace)
if errors.IsNotFound(err) {
reqLogger.Info("No Tenant in this Namespace, skip")
return reconcile.Result{}, nil
} else if err != nil {
r.status.SetDegraded(operatorv1.ResourceReadError, "An error occurred while querying Tenant", err, reqLogger)
return reconcile.Result{}, err
}

// Fetch the Compliance instance
instance, err := GetCompliance(ctx, r.client)
instance, err := GetCompliance(ctx, r.client, r.multiTenant, request.Namespace)
if err != nil {
if errors.IsNotFound(err) {
// Request object not found, could have been deleted after reconcile request.
Expand Down Expand Up @@ -328,14 +354,18 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
return reconcile.Result{}, err
}

certificateManager, err := certificatemanager.Create(r.client, network, r.clusterDomain, common.OperatorNamespace())
var opts []certificatemanager.Option

opts = append(opts, certificatemanager.WithTenant(tenant), certificatemanager.WithLogger(reqLogger))

certificateManager, err := certificatemanager.Create(r.client, network, r.clusterDomain, helper.TruthNamespace(), opts...)
if err != nil {
r.status.SetDegraded(operatorv1.ResourceCreateError, "Unable to create the Tigera CA", err, reqLogger)
return reconcile.Result{}, err
}
var managerInternalTLSSecret certificatemanagement.CertificateInterface
if managementCluster != nil {
managerInternalTLSSecret, err = certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, common.OperatorNamespace())
managerInternalTLSSecret, err = certificateManager.GetCertificate(r.client, render.ManagerInternalTLSSecretName, helper.TruthNamespace())
if err != nil {
r.status.SetDegraded(operatorv1.ResourceValidationError, fmt.Sprintf("failed to retrieve / validate %s", render.ManagerInternalTLSSecretName), err, reqLogger)
return reconcile.Result{}, err
Expand All @@ -350,7 +380,7 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
// management cluster into the managed cluster by kube-controllers.
linseedCertLocation = render.VoltronLinseedPublicCert
}
linseedCertificate, err := certificateManager.GetCertificate(r.client, linseedCertLocation, common.OperatorNamespace())
linseedCertificate, err := certificateManager.GetCertificate(r.client, linseedCertLocation, helper.TruthNamespace())
if err != nil {
r.status.SetDegraded(operatorv1.ResourceValidationError, fmt.Sprintf("Failed to retrieve / validate %s", render.TigeraLinseedSecret), err, reqLogger)
return reconcile.Result{}, err
Expand All @@ -374,7 +404,7 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
// These key pairs are only used as client credentials for mTLS with Linseed, and so do not need DNS names listed
// as they do not act as server certs.
dnsNames := []string{"localhost"}
kp.Interface, err = certificateManager.GetOrCreateKeyPair(r.client, kp.SecretName, common.OperatorNamespace(), dnsNames)
kp.Interface, err = certificateManager.GetOrCreateKeyPair(r.client, kp.SecretName, helper.TruthNamespace(), dnsNames)
if err != nil {
r.status.SetDegraded(operatorv1.ResourceValidationError, fmt.Sprintf("failed to retrieve / validate %s", kp.SecretName), err, reqLogger)
return reconcile.Result{}, err
Expand All @@ -386,14 +416,14 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
complianceServerKeyPair, err = certificateManager.GetOrCreateKeyPair(
r.client,
render.ComplianceServerCertSecret,
common.OperatorNamespace(),
dns.GetServiceDNSNames(render.ComplianceServiceName, render.ComplianceNamespace, r.clusterDomain))
helper.TruthNamespace(),
dns.GetServiceDNSNames(render.ComplianceServiceName, helper.InstallNamespace(), r.clusterDomain))
if err != nil {
r.status.SetDegraded(operatorv1.ResourceValidationError, fmt.Sprintf("failed to retrieve / validate %s", render.ComplianceServerCertSecret), err, reqLogger)
return reconcile.Result{}, err
}
}
certificateManager.AddToStatusManager(r.status, render.ComplianceNamespace)
certificateManager.AddToStatusManager(r.status, helper.InstallNamespace())

// Fetch the Authentication spec. If present, we use to configure user authentication.
authenticationCR, err := utils.GetAuthentication(ctx, r.client)
Expand All @@ -417,7 +447,7 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R

reqLogger.V(3).Info("rendering components")

namespaceComp := render.NewPassthrough(render.CreateNamespace(render.ComplianceNamespace, network.KubernetesProvider, render.PSSPrivileged))
namespaceComp := render.NewPassthrough(render.CreateNamespace(helper.InstallNamespace(), network.KubernetesProvider, render.PSSPrivileged))

hasNoLicense := !utils.IsFeatureActive(license, common.ComplianceFeature)
openshift := r.provider == operatorv1.ProviderOpenShift
Expand All @@ -438,6 +468,8 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
ClusterDomain: r.clusterDomain,
HasNoLicense: hasNoLicense,
UsePSP: r.usePSP,
Namespace: helper.InstallNamespace(),
Tenant: tenant,
}

// Render the desired objects from the CRD and create or update them.
Expand All @@ -452,7 +484,8 @@ func (r *ReconcileCompliance) Reconcile(ctx context.Context, request reconcile.R
return reconcile.Result{}, err
}
certificateComponent := rcertificatemanagement.CertificateManagement(&rcertificatemanagement.Config{
Namespace: render.ComplianceNamespace,
Namespace: helper.InstallNamespace(),
TruthNamespace: helper.TruthNamespace(),
ServiceAccounts: []string{render.ComplianceServerServiceAccount, render.ComplianceBenchmarkerServiceAccount, render.ComplianceSnapshotterServiceAccount, render.ComplianceControllerServiceAccount, render.ComplianceReporterServiceAccount},
KeyPairOptions: []rcertificatemanagement.KeyPairOption{
rcertificatemanagement.NewKeyPairOption(complianceServerKeyPair, true, true),
Expand Down
Loading
Loading