From 14b83fd3c718acfe8f6415141c08a25e6503bf88 Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Thu, 2 May 2024 15:42:02 +0200 Subject: [PATCH 1/3] Add fetch from k8s and proxy modes --- kubernetes-ingress/templates/clusterrole.yaml | 8 + .../templates/controller-deployment.yaml | 13 +- .../controller-proxy-deployment.yaml | 271 ++++++++++++++++++ .../templates/controller-proxy-service.yaml | 61 ++++ kubernetes-ingress/values.yaml | 13 + 5 files changed, 365 insertions(+), 1 deletion(-) create mode 100644 kubernetes-ingress/templates/controller-proxy-deployment.yaml create mode 100644 kubernetes-ingress/templates/controller-proxy-service.yaml diff --git a/kubernetes-ingress/templates/clusterrole.yaml b/kubernetes-ingress/templates/clusterrole.yaml index 999c4c8..9d4a4eb 100644 --- a/kubernetes-ingress/templates/clusterrole.yaml +++ b/kubernetes-ingress/templates/clusterrole.yaml @@ -140,4 +140,12 @@ rules: - get - list - watch +{{- if and (eq .Values.controller.sync.mode "fetch") (eq .Values.controller.sync.fetchParams.source "proxy") }} +- apiGroups: + - "coordination.k8s.io" + resources: + - leases + verbs: + - "*" +{{- end }} {{- end -}} diff --git a/kubernetes-ingress/templates/controller-deployment.yaml b/kubernetes-ingress/templates/controller-deployment.yaml index abd8b12..68db999 100644 --- a/kubernetes-ingress/templates/controller-deployment.yaml +++ b/kubernetes-ingress/templates/controller-deployment.yaml @@ -21,7 +21,7 @@ metadata: name: {{ include "kubernetes-ingress.fullname" . }} namespace: {{ include "kubernetes-ingress.namespace" . }} labels: - app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }} + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy helm.sh/chart: {{ include "kubernetes-ingress.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} @@ -120,6 +120,17 @@ spec: {{- if .Values.controller.service.enablePorts.prometheus }} - --prometheus {{- end }} +{{- if eq .Values.controller.sync.mode "fetch" }} + {{- if .Values.controller.sync.fetchParams.period }} + - --proxy-k8s-fetch-period={{ .Values.controller.sync.fetchParams.period }} + {{- end }} + {{- if eq .Values.controller.sync.fetchParams.source "k8s" }} + - --k8s-api-sync-type=fetch_from_k8s_api + {{- else if eq .Values.controller.sync.fetchParams.source "proxy" }} + - --k8s-api-sync-type=fetch_from_proxy + - --proxy-svc-label-selector={{ .Values.controller.sync.proxyParams.proxySvcLabelSelector }} + {{- end }} +{{- end }} {{- range .Values.controller.extraArgs }} - {{ . }} {{- end }} diff --git a/kubernetes-ingress/templates/controller-proxy-deployment.yaml b/kubernetes-ingress/templates/controller-proxy-deployment.yaml new file mode 100644 index 0000000..9d95500 --- /dev/null +++ b/kubernetes-ingress/templates/controller-proxy-deployment.yaml @@ -0,0 +1,271 @@ +{{/* +Copyright 2019 HAProxy Technologies LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (eq .Values.controller.sync.mode "fetch") (eq .Values.controller.sync.fetchParams.source "proxy") }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "kubernetes-ingress.fullname" . }}-proxy + namespace: {{ include "kubernetes-ingress.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + helm.sh/chart: {{ include "kubernetes-ingress.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + {{- if .Values.controller.extraLabels }} +{{ toYaml .Values.controller.extraLabels | indent 4 }} + {{- end }} +spec: + {{- if and (not .Values.controller.autoscaling.enabled) (not .Values.controller.keda.enabled) }} + replicas: {{ .Values.controller.sync.proxyParams.replicaCount }} + {{- end }} + minReadySeconds: {{ .Values.controller.minReadySeconds }} + selector: + matchLabels: + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + app.kubernetes.io/instance: {{ .Release.Name }} + {{- with .Values.controller.strategy }} + strategy: + {{- toYaml . | nindent 4 }} + {{- end }} + template: + metadata: + labels: + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.controller.podLabels }} +{{ toYaml .Values.controller.podLabels | indent 8 }} + {{- end }} + {{- if .Values.controller.podAnnotations }} + annotations: +{{ toYaml .Values.controller.podAnnotations | indent 8 }} + {{- end }} + spec: + enableServiceLinks: {{ .Values.controller.enableServiceLinks }} + serviceAccountName: {{ include "kubernetes-ingress.serviceAccountName" . }} + terminationGracePeriodSeconds: {{ .Values.controller.terminationGracePeriodSeconds }} +{{- with .Values.controller.topologySpreadConstraints }} + topologySpreadConstraints: + {{- toYaml . | nindent 8 }} +{{- end }} +{{- if .Values.controller.dnsConfig }} + dnsConfig: +{{ toYaml .Values.controller.dnsConfig | indent 8 }} +{{- end }} + dnsPolicy: {{ .Values.controller.dnsPolicy }} +{{- if .Values.controller.imageCredentials.registry }} + imagePullSecrets: + - name: {{ include "kubernetes-ingress.fullname" . }} +{{- else if .Values.controller.existingImagePullSecret }} + imagePullSecrets: + - name: {{ .Values.controller.existingImagePullSecret }} +{{- end }} +{{- if .Values.controller.priorityClassName }} + priorityClassName: {{ .Values.controller.priorityClassName }} +{{- end }} +{{- if .Values.controller.runtimeClassName }} + runtimeClassName: {{ .Values.controller.runtimeClassName }} +{{- end }} +{{- if .Values.controller.unprivileged }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 +{{- end }} + containers: + - name: {{ include "kubernetes-ingress.name" . }}-{{ .Values.controller.name }} + image: "{{ .Values.controller.image.repository }}:{{ tpl .Values.controller.image.tag . }}" + imagePullPolicy: {{ .Values.controller.image.pullPolicy }} + args: +{{- if .Values.controller.defaultTLSSecret.enabled -}} +{{- if and .Values.controller.defaultTLSSecret.secret .Values.controller.defaultTLSSecret.secretNamespace }} + - --default-ssl-certificate={{ tpl .Values.controller.defaultTLSSecret.secretNamespace . }}/{{ .Values.controller.defaultTLSSecret.secret }} +{{- else }} + - --default-ssl-certificate={{ include "kubernetes-ingress.namespace" . }}/{{ include "kubernetes-ingress.defaultTLSSecret.fullname" . }} +{{- end }} +{{- end }} + - --configmap={{ include "kubernetes-ingress.namespace" . }}/{{ include "kubernetes-ingress.fullname" . }} + - --http-bind-port={{ .Values.controller.containerPort.http }} + - --https-bind-port={{ .Values.controller.containerPort.https }} +{{- if and (semverCompare ">=1.24.0-0" .Capabilities.KubeVersion.Version) .Values.controller.service.enablePorts.quic }} + - --quic-bind-port={{ .Values.controller.containerPort.https }} + - --quic-announce-port={{ .Values.controller.service.ports.https }} +{{- end }} +{{- if .Values.controller.ingressClass }} + - --ingress.class={{ .Values.controller.ingressClass }} +{{- end }} +{{- if and .Values.controller.kubernetesGateway.enabled .Values.controller.kubernetesGateway.gatewayControllerName }} + - --gateway-controller-name={{ .Values.controller.kubernetesGateway.gatewayControllerName }} +{{- end }} +{{- if .Values.controller.publishService.enabled }} + - --publish-service={{ include "kubernetes-ingress.publishServicePath" . }} +{{- end }} +{{- if .Values.controller.logging.level }} + - --log={{ .Values.controller.logging.level }} +{{- end }} +{{- if .Values.controller.service.enablePorts.prometheus }} + - --prometheus +{{- end }} + - --proxy-server-mode + - --k8s-api-sync-type=fetch_from_k8s_api + - --proxy-svc-label-selector={{ .Values.controller.sync.proxyParams.proxySvcLabelSelector }} +{{- if .Values.controller.sync.fetchParams.period }} + - --proxy-k8s-fetch-period={{ .Values.controller.sync.fetchParams.period }} +{{- end }} +{{- range .Values.controller.extraArgs }} + - {{ . }} +{{- end }} + {{- if .Values.controller.unprivileged }} + securityContext: + runAsNonRoot: true + runAsUser: 1000 + runAsGroup: 1000 + allowPrivilegeEscalation: {{ .Values.controller.allowPrivilegeEscalation }} + capabilities: + drop: + - ALL + add: + - NET_BIND_SERVICE + {{- if .Values.controller.enableRuntimeDefaultSeccompProfile }} + seccompProfile: + type: RuntimeDefault + {{- end }} + {{- end }} + ports: + {{- range $key, $value := .Values.controller.containerPort }} + - name: {{ $key }} + containerPort: {{ $value }} + protocol: TCP + {{- end }} + {{- if and (semverCompare ">=1.24.0-0" .Capabilities.KubeVersion.Version) .Values.controller.service.enablePorts.quic }} + - name: quic + containerPort: {{ .Values.controller.containerPort.https }} + protocol: UDP + {{- end }} + {{- range .Values.controller.service.tcpPorts }} + - name: {{ .name }}-tcp + containerPort: {{ .targetPort }} + protocol: TCP + {{- end }} + {{- with .Values.controller.livenessProbe }} + livenessProbe: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.controller.readinessProbe }} + readinessProbe: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + {{- with .Values.controller.startupProbe }} + startupProbe: + {{- toYaml . | trim | nindent 12 }} + {{- end }} + env: + {{- if .Values.aws.licenseConfigSecretName }} + - name: AWS_WEB_IDENTITY_REFRESH_TOKEN_FILE + value: "/var/run/secrets/product-license/license_token" + - name: AWS_ROLE_ARN + valueFrom: + secretKeyRef: + name: {{ .Values.aws.licenseConfigSecretName }} + key: iam_role + {{- end }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + {{- if .Values.controller.extraEnvs -}} + {{- toYaml .Values.controller.extraEnvs | nindent 10 }} + {{- end }} + resources: + {{- toYaml .Values.controller.resources | nindent 12 }} + {{- if .Values.controller.lifecycle }} + lifecycle: + {{- if eq "string" (printf "%T" .Values.controller.lifecycle) }} +{{ tpl .Values.controller.lifecycle . | indent 12 }} + {{- else }} +{{ toYaml .Values.controller.lifecycle | indent 12 }} + {{- end }} + {{- end }} + volumeMounts: + - name: tmp + mountPath: /tmp + subPath: tmp + - name: tmp + mountPath: /run + subPath: run + {{- if .Values.aws.licenseConfigSecretName }} + - name: aws-product-license + readOnly: true + mountPath: /var/run/secrets/product-license + {{- end }} + {{- if eq "string" (printf "%T" .Values.controller.extraVolumeMounts) }} +{{ tpl .Values.controller.extraVolumeMounts . | indent 12 }} + {{- else if gt (len .Values.controller.extraVolumeMounts) 0 }} +{{ toYaml .Values.controller.extraVolumeMounts | indent 12 }} + {{- end }} + {{- if .Values.controller.extraContainers }} + {{- if eq "string" (printf "%T" .Values.controller.extraContainers) }} +{{ tpl .Values.controller.extraContainers . | indent 8 }} + {{- else }} +{{ toYaml .Values.controller.extraContainers | indent 8 }} + {{- end }} + {{- end }} + volumes: + - name: tmp + {{- if semverCompare ">=1.21.0-0" .Capabilities.KubeVersion.Version }} + emptyDir: + medium: Memory + sizeLimit: 64Mi + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.aws.licenseConfigSecretName }} + - name: aws-product-license + secret: + secretName: {{ .Values.aws.licenseConfigSecretName }} + optional: true + {{- end }} + {{- if eq "string" (printf "%T" .Values.controller.extraVolumes) }} +{{ tpl .Values.controller.extraVolumes . | indent 8 }} + {{- else if gt (len .Values.controller.extraVolumes) 0 }} +{{ toYaml .Values.controller.extraVolumes | indent 8 }} + {{- end }} + {{- with.Values.controller.initContainers }} + initContainers: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.controller.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} +{{- end }} diff --git a/kubernetes-ingress/templates/controller-proxy-service.yaml b/kubernetes-ingress/templates/controller-proxy-service.yaml new file mode 100644 index 0000000..f4075fe --- /dev/null +++ b/kubernetes-ingress/templates/controller-proxy-service.yaml @@ -0,0 +1,61 @@ +{{/* +Copyright 2019 HAProxy Technologies LLC + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (eq .Values.controller.sync.mode "fetch") (eq .Values.controller.sync.fetchParams.source "proxy") }} +apiVersion: v1 +kind: Service +metadata: + name: {{ include "kubernetes-ingress.fullname" . }}-proxy + namespace: {{ include "kubernetes-ingress.namespace" . }} + labels: + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + helm.sh/chart: {{ include "kubernetes-ingress.chart" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/version: {{ .Chart.AppVersion }} + {{ (split ":" .Values.controller.sync.proxyParams.proxySvcLabelSelector)._0 }}: {{ (split ":" .Values.controller.sync.proxyParams.proxySvcLabelSelector)._1 }} +{{- if .Values.controller.service.labels }} +{{ toYaml .Values.controller.service.labels | indent 4 }} +{{- end }} + annotations: +{{- range $key, $value := .Values.controller.service.annotations }} + {{ $key }}: {{ $value | quote }} +{{- end }} +spec: + type: ClusterIP + {{- if .Values.controller.service.healthCheckNodePort }} + healthCheckNodePort: {{ .Values.controller.service.healthCheckNodePort }} + {{- end }} + ports: + {{- if .Values.controller.service.enablePorts.http }} + - name: http + port: {{ .Values.controller.service.ports.http }} + protocol: TCP + {{- if semverCompare ">=1.20.0-0" .Capabilities.KubeVersion.Version }} + appProtocol: http + {{- end }} + targetPort: {{ .Values.controller.service.targetPorts.http }} + {{- if .Values.controller.service.nodePorts.http }} + nodePort: {{ .Values.controller.service.nodePorts.http }} + {{- end }} + {{- end }} + selector: + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + app.kubernetes.io/instance: {{ .Release.Name }} + {{- if .Values.controller.service.sessionAffinity }} + sessionAffinity: {{ .Values.controller.service.sessionAffinity }} + {{- end }} +{{- end }} diff --git a/kubernetes-ingress/values.yaml b/kubernetes-ingress/values.yaml index b7ae90a..edb4363 100644 --- a/kubernetes-ingress/values.yaml +++ b/kubernetes-ingress/values.yaml @@ -560,3 +560,16 @@ controller: path: /metrics scheme: http interval: 30s + + ## Specified how the controller will sync data from Kubernetes + ## Possbile values: 'default' or 'fetch' + ## - 'default': the sync is done based on K8s informers (event based) + ## - 'fetch': the controller pulls data periodically (from k8s or from proxy) + sync: + mode: default # can be 'default' or 'fetch' + fetchParams: # Mandatory if mode is 'fetch' + # period: 3s # optional, default is 5s + source: k8s # possible values are: 'proxy', 'k8s' + proxyParams: # Mandatory if source is 'proxy' + replicaCount: 3 # number of replicas of the proxy, mandatory if source is 'proxy' + proxySvcLabelSelector: run:haproxy-ingress-proxy # label selector of the proxy service, mandatory if source is 'proxy' From e37601ea413056318cd9307c5a70f32085b86a63 Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Fri, 3 May 2024 14:47:38 +0200 Subject: [PATCH 2/3] Fix wrong -proxy suffix in controller-deployment --- kubernetes-ingress/templates/controller-deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes-ingress/templates/controller-deployment.yaml b/kubernetes-ingress/templates/controller-deployment.yaml index 68db999..ddc3c61 100644 --- a/kubernetes-ingress/templates/controller-deployment.yaml +++ b/kubernetes-ingress/templates/controller-deployment.yaml @@ -21,7 +21,7 @@ metadata: name: {{ include "kubernetes-ingress.fullname" . }} namespace: {{ include "kubernetes-ingress.namespace" . }} labels: - app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }}-proxy + app.kubernetes.io/name: {{ include "kubernetes-ingress.name" . }} helm.sh/chart: {{ include "kubernetes-ingress.chart" . }} app.kubernetes.io/managed-by: {{ .Release.Service }} app.kubernetes.io/instance: {{ .Release.Name }} From 3bbf3ce07bfd405dfba3f32e93cbb32bfb639ae1 Mon Sep 17 00:00:00 2001 From: Helene Durand Date: Mon, 6 May 2024 11:02:31 +0200 Subject: [PATCH 3/3] Update flag k8s-api-sync-type with new values (normal, k8s, proxy) Old possible values were: normal, fetch_from_k8s_api, fetch_from_proxy New possible values are: normal, k8s, proxy --- kubernetes-ingress/templates/controller-deployment.yaml | 4 ++-- kubernetes-ingress/templates/controller-proxy-deployment.yaml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/kubernetes-ingress/templates/controller-deployment.yaml b/kubernetes-ingress/templates/controller-deployment.yaml index ddc3c61..a0c4a4d 100644 --- a/kubernetes-ingress/templates/controller-deployment.yaml +++ b/kubernetes-ingress/templates/controller-deployment.yaml @@ -125,9 +125,9 @@ spec: - --proxy-k8s-fetch-period={{ .Values.controller.sync.fetchParams.period }} {{- end }} {{- if eq .Values.controller.sync.fetchParams.source "k8s" }} - - --k8s-api-sync-type=fetch_from_k8s_api + - --k8s-api-sync-type=k8s {{- else if eq .Values.controller.sync.fetchParams.source "proxy" }} - - --k8s-api-sync-type=fetch_from_proxy + - --k8s-api-sync-type=proxy - --proxy-svc-label-selector={{ .Values.controller.sync.proxyParams.proxySvcLabelSelector }} {{- end }} {{- end }} diff --git a/kubernetes-ingress/templates/controller-proxy-deployment.yaml b/kubernetes-ingress/templates/controller-proxy-deployment.yaml index 9d95500..0c0f4b1 100644 --- a/kubernetes-ingress/templates/controller-proxy-deployment.yaml +++ b/kubernetes-ingress/templates/controller-proxy-deployment.yaml @@ -121,7 +121,7 @@ spec: - --prometheus {{- end }} - --proxy-server-mode - - --k8s-api-sync-type=fetch_from_k8s_api + - --k8s-api-sync-type=k8s - --proxy-svc-label-selector={{ .Values.controller.sync.proxyParams.proxySvcLabelSelector }} {{- if .Values.controller.sync.fetchParams.period }} - --proxy-k8s-fetch-period={{ .Values.controller.sync.fetchParams.period }}