[wip] service + deployment mode for applicationlayer

api/v1/applicationlayer_types.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2021-2024 Tigera, Inc. All rights reserved.
+// Copyright (c) 2021-2025 Tigera, Inc. All rights reserved.
 /*
 
 Licensed under the Apache License, Version 2.0 (the "License");
@@ -44,6 +44,18 @@ type ApplicationLayerSpec struct {
 	// The per-host proxy will continue to be used for pods without this label.
 	// +optional
 	SidecarInjection *SidecarStatusType `json:"sidecarInjection,omitempty"`
+
+	// DeploymentMode controls the deployment mode of the L7LogCollector.
+	// When set to ServiceDeployment, the L7LogCollector will be deployed as a service and deployment.
+	// When set to DaemonSetSocket, the L7LogCollector will be deployed as a daemonset listening to a socket with predetermined hostpath path.
+	// When not set, the default value is DaemonSetSocket.
+	// +optional
+	DeploymentMode *ApplicationLayerDeploymentModeType `json:"deploymentMode,omitempty"`
+
+	// ServiceDeploymentReplicas controls the number of replicas for the L7LogCollector service deployment.
+	// Only works for DeploymentMode=ServiceDeployment.
+	// +optional
+	ServiceDeploymentReplicas *int32 `json:"serviceDeploymentReplicas,omitempty"`
 }
 
 // +kubebuilder:validation:Enum=Enabled;Disabled
@@ -58,20 +70,25 @@ type ApplicationLayerPolicyStatusType string
 // +kubebuilder:validation:Enum=Enabled;Disabled
 type SidecarStatusType string
 
+// +kubebuilder:validation:Enum=ServiceDeployment;DaemonSetSocket
+type ApplicationLayerDeploymentModeType string
+
 // +kubebuilder:validation:Enum=Enabled;Disabled
 type SidecarWebhookStateType string
 
 const (
-	WAFDisabled                    WAFStatusType                    = "Disabled"
-	WAFEnabled                     WAFStatusType                    = "Enabled"
-	L7LogCollectionDisabled        LogCollectionStatusType          = "Disabled"
-	L7LogCollectionEnabled         LogCollectionStatusType          = "Enabled"
-	ApplicationLayerPolicyEnabled  ApplicationLayerPolicyStatusType = "Enabled"
-	ApplicationLayerPolicyDisabled ApplicationLayerPolicyStatusType = "Disabled"
-	SidecarEnabled                 SidecarStatusType                = "Enabled"
-	SidecarDisabled                SidecarStatusType                = "Disabled"
-	SidecarWebhookStateEnabled     SidecarWebhookStateType          = "Enabled"
-	SidecarWebhookStateDisabled    SidecarWebhookStateType          = "Disabled"
+	WAFDisabled                                     WAFStatusType                      = "Disabled"
+	WAFEnabled                                      WAFStatusType                      = "Enabled"
+	L7LogCollectionDisabled                         LogCollectionStatusType            = "Disabled"
+	L7LogCollectionEnabled                          LogCollectionStatusType            = "Enabled"
+	ApplicationLayerPolicyEnabled                   ApplicationLayerPolicyStatusType   = "Enabled"
+	ApplicationLayerPolicyDisabled                  ApplicationLayerPolicyStatusType   = "Disabled"
+	SidecarEnabled                                  SidecarStatusType                  = "Enabled"
+	SidecarDisabled                                 SidecarStatusType                  = "Disabled"
+	SidecarWebhookStateEnabled                      SidecarWebhookStateType            = "Enabled"
+	SidecarWebhookStateDisabled                     SidecarWebhookStateType            = "Disabled"
+	ApplicationLayerDeploymentModeServiceDeployment ApplicationLayerDeploymentModeType = "ServiceDeployment"
+	ApplicationLayerDeploymentModeDaemonSetSocket   ApplicationLayerDeploymentModeType = "DaemonSetSocket"
 )
 
 type EnvoySettings struct {

pkg/controller/applicationlayer/applicationlayer_controller.go

@@ -281,6 +281,8 @@ func (r *ReconcileApplicationLayer) Reconcile(ctx context.Context, request recon
 		PerHostLogsEnabled:          r.isLogsCollectionEnabled(&instance.Spec),
 		PerHostALPEnabled:           r.isALPEnabled(&instance.Spec),
 		SidecarInjectionEnabled:     r.isSidecarInjectionEnabled(&instance.Spec),
+		DeploymentMode:              instance.Spec.DeploymentMode,
+		DeploymentReplicas:          instance.Spec.ServiceDeploymentReplicas,
 		LogRequestsPerInterval:      lcSpec.LogRequestsPerInterval,
 		LogIntervalSeconds:          lcSpec.LogIntervalSeconds,
 		WAFRulesetConfigMap:         wafRulesetConfig,

pkg/crds/enterprise/crd.projectcalico.org_felixconfigurations.yaml

@@ -578,8 +578,8 @@ spec:
 
                   This setting is ignored by eBPF and BPFDNSPolicyMode is used instead.
 
-                  Inline policy mode is not supported in NFTables mode. Default mode in DelayDeniedPacket in case of NFTables.
-                  [Default: DelayDeniedPacket]
+                  This field has no effect in NFTables mode. Please use NFTablesDNSPolicyMode instead.
+                  [Default: Inline]
                 enum:
                 - NoDelay
                 - DelayDeniedPacket
@@ -1351,6 +1351,24 @@ spec:
                   are used to report flow verdicts from the kernel.  Warning: currently increasing the value may cause errors
                   due to a bug in the netlink library.
                 type: string
+              nftablesDNSPolicyMode:
+                description: |-
+                  NFTablesDNSPolicyMode specifies how DNS policy programming will be handled for NFTables.
+                  DelayDeniedPacket - Felix delays any denied packet that traversed a policy that included egress domain matches,
+                  but did not match. The packet is released after a fixed time, or after the destination IP address was programmed.
+                  DelayDNSResponse - Felix delays any DNS response until related IPSets are programmed. This introduces some
+                  latency to all DNS packets (even when no IPSet programming is required), but it ensures policy hit statistics
+                  are accurate. This is the recommended setting when you are making use of staged policies or policy rule hit
+                  statistics.
+                  NoDelay - Felix does not introduce any delay to the packets. DNS rules may not have been programmed by the time
+                  the first packet traverses the policy rules. Client applications need to handle reconnection attempts if initial
+                  connection attempts fail. This may be problematic for some applications or for very low DNS TTLs.
+                  [Default: DelayDeniedPacket]
+                enum:
+                - NoDelay
+                - DelayDeniedPacket
+                - DelayDNSResponse
+                type: string
               nftablesFilterAllowAction:
                 description: |-
                   NftablesFilterAllowAction controls the nftables action that Felix uses to represent the "allow" policy verdict

pkg/crds/operator/operator.tigera.io_applicationlayers.yaml

@@ -46,6 +46,16 @@ spec:
                 - Enabled
                 - Disabled
                 type: string
+              deploymentMode:
+                description: |-
+                  DeploymentMode controls the deployment mode of the L7LogCollector.
+                  When set to ServiceDeployment, the L7LogCollector will be deployed as a service and deployment.
+                  When set to DaemonSetSocket, the L7LogCollector will be deployed as a daemonset listening to a socket with predetermined hostpath path.
+                  When not set, the default value is DaemonSetSocket.
+                enum:
+                - ServiceDeployment
+                - DaemonSetSocket
+                type: string
               envoy:
                 description: User-configurable settings for the Envoy proxy.
                 properties:
@@ -261,6 +271,12 @@ spec:
                     format: int64
                     type: integer
                 type: object
+              serviceDeploymentReplicas:
+                description: |-
+                  ServiceDeploymentReplicas controls the number of replicas for the L7LogCollector service deployment.
+                  Only works for DeploymentMode=ServiceDeployment.
+                format: int32
+                type: integer
               sidecarInjection:
                 description: |-
                   SidecarInjection controls whether or not sidecar injection is enabled for the cluster.

pkg/render/applicationlayer/applicationlayer.go

@@ -62,6 +62,13 @@ const (
 	WAFConfigHashAnnotation       = "hash.operator.tigera.io/tigera-waf-config"
 	CalicoLogsVolumeName          = "var-log-calico"
 	CalicologsVolumePath          = "/var/log/calico"
+
+	L7CollectorTCPListenPort = 8080
+	DikastesTCPListenPort    = 8081
+)
+
+var (
+	DefaultDeploymentReplicas int32 = 1
 )
 
 func ApplicationLayer(
@@ -98,6 +105,8 @@ type Config struct {
 
 	// Optional config for SidecarInjection
 	SidecarInjectionEnabled bool
+	DeploymentMode          *operatorv1.ApplicationLayerDeploymentModeType
+	DeploymentReplicas      *int32
 
 	// Calculated internal fields.
 	proxyImage            string
@@ -115,6 +124,30 @@ type Config struct {
 	ApplicationLayer *operatorv1.ApplicationLayer
 }
 
+func (cfg *Config) isDeploymentAndService() bool {
+	if cfg.DeploymentMode == nil {
+		return false
+	}
+	return *cfg.DeploymentMode == operatorv1.ApplicationLayerDeploymentModeServiceDeployment
+}
+
+func (cfg *Config) serviceDeploymentReplicas() *int32 {
+	if cfg.DeploymentReplicas != nil {
+		return cfg.DeploymentReplicas
+	}
+	return &DefaultDeploymentReplicas
+}
+
+func (cfg *Config) dikastesListenFlagValues() []string {
+	if cfg.isDeploymentAndService() {
+		return []string{
+			"--listen-network", "tcp",
+			"--listen", fmt.Sprintf("0.0.0.0:%d", DikastesTCPListenPort),
+		}
+	}
+	return []string{"--listen", "/var/run/dikastes/dikastes.sock"}
+}
+
 func (c *component) ResolveImages(is *operatorv1.ImageSet) error {
 	reg := c.config.Installation.Registry
 	path := c.config.Installation.ImagePath
@@ -182,13 +215,17 @@ func (c *component) Objects() ([]client.Object, []client.Object) {
 		objs = append(objs, c.config.envoyConfigMap)
 	}
 
-	// Envoy, Dikastes & L7 log collector Daemonset
+	// Envoy, Dikastes & L7 log collector Daemonset or Deployment+Service
 	// It always installed, even with sidecars enabled, the sidecars contain
 	// only envoy proxy inside them, and if only sidcar is enabled the
 	// daemonset will contain only Dikastes and L7 log collector to be
 	// prepared to do some action depending on the envoy inside application
 	// pod configuration.
-	objs = append(objs, c.daemonset())
+	if c.config.isDeploymentAndService() {
+		objs = append(objs, c.deploymentAndService()...)
+	} else {
+		objs = append(objs, c.daemonset())
+	}
 
 	if c.config.Installation.KubernetesProvider.IsDockerEE() {
 		objs = append(objs, c.clusterAdminClusterRoleBinding())
@@ -205,6 +242,85 @@ func (c *component) Ready() bool {
 	return true
 }
 
+func (c *component) deploymentAndService() []client.Object {
+	maxUnavailable := intstr.FromInt(1)
+
+	annots := map[string]string{}
+
+	if c.config.envoyConfigMap != nil {
+		annots[EnvoyConfigMapName] = rmeta.AnnotationHash(c.config.envoyConfigMap)
+	}
+
+	if c.config.WAFRulesetConfigMap != nil {
+		annots[WAFConfigHashAnnotation] = rmeta.AnnotationHash(c.config.WAFRulesetConfigMap.Data)
+	}
+
+	podTemplate := corev1.PodTemplateSpec{
+		ObjectMeta: metav1.ObjectMeta{
+			Annotations: annots,
+		},
+		Spec: corev1.PodSpec{
+			HostIPC:            true,
+			HostNetwork:        true,
+			ServiceAccountName: APLName,
+			DNSPolicy:          corev1.DNSClusterFirstWithHostNet,
+			Tolerations:        rmeta.TolerateAll,
+			ImagePullSecrets:   secret.GetReferenceList(c.config.PullSecrets),
+			Containers:         c.containers(),
+			Volumes:            c.volumes(),
+		},
+	}
+
+	service := &corev1.Service{
+		TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ApplicationLayerDaemonsetName,
+			Namespace: common.CalicoNamespace,
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"app": ApplicationLayerDaemonsetName,
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Name:       "http-dikastes",
+					Port:       DikastesTCPListenPort,
+					TargetPort: intstr.FromInt(8080),
+				},
+				{
+					Name:       "http-l7-collector",
+					Port:       L7CollectorTCPListenPort,
+					TargetPort: intstr.FromInt(8080),
+				},
+			},
+		},
+	}
+
+	deployment := &appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "apps/v1"},
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      ApplicationLayerDaemonsetName,
+			Namespace: common.CalicoNamespace,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: c.config.serviceDeploymentReplicas(),
+			Template: podTemplate,
+			Strategy: appsv1.DeploymentStrategy{
+				RollingUpdate: &appsv1.RollingUpdateDeployment{
+					MaxUnavailable: &maxUnavailable,
+				},
+			},
+		},
+	}
+
+	if c.config.ApplicationLayer != nil {
+		if overrides := c.config.ApplicationLayer.Spec.L7LogCollectorDaemonSet; overrides != nil {
+			rcomponents.ApplyDeploymentOverrides(deployment, overrides)
+		}
+	}
+	return []client.Object{deployment, service}
+}
+
 // daemonset creates a daemonset for the L7 log collector component.
 func (c *component) daemonset() *appsv1.DaemonSet {
 	maxUnavailable := intstr.FromInt(1)
@@ -305,9 +421,10 @@ func (c *component) containers() []corev1.Container {
 			"/dikastes",
 			"server",
 			"--dial", "/var/run/felix/nodeagent/socket",
-			"--listen", "/var/run/dikastes/dikastes.sock",
 		}
 
+		commandArgs = append(commandArgs, c.config.dikastesListenFlagValues()...)
+
 		volMounts := []corev1.VolumeMount{
 			{Name: FelixSync, MountPath: "/var/run/felix"},
 			{Name: DikastesSyncVolumeName, MountPath: "/var/run/dikastes"},
@@ -395,6 +512,19 @@ func (c *component) collectorEnv() []corev1.EnvVar {
 		})
 	}
 
+	if c.config.isDeploymentAndService() {
+		envs = append(envs,
+			corev1.EnvVar{
+				Name:  "LISTEN_NETWORK",
+				Value: "tcp",
+			},
+			corev1.EnvVar{
+				Name:  "LISTEN_ADDRESS",
+				Value: fmt.Sprintf("0.0.0.0:%d", L7CollectorTCPListenPort),
+			},
+		)
+	}
+
 	return envs
 }