[EKS] Change Kubernetes control plane to EKS (the eks branch)

cluster/config-defaults.yaml

@@ -677,9 +677,9 @@ teapot_admission_controller_configmap_deletion_protection_factories_enabled: "tr
 teapot_admission_controller_enable_rolebinding_webhook: "true"
 
 # enable the generic deny-all admission webhook which rejects all requests it receives
-teapot_admission_controller_enable_write_protection_webhook: "false"
+teapot_admission_controller_enable_write_protection_webhook: "true"
 # configure the behaviour of the deny-all admission webhook, `true` blocks everything, `false` allows everything
-teapot_admission_controller_prevent_write_operations: "false"
+teapot_admission_controller_prevent_write_operations: "true"
 
 # Enable and configure Pod Security Policy rules implemented in admission-controller.
 teapot_admission_controller_pod_security_policy_enabled: "true"
@@ -1194,3 +1194,20 @@ role_sync_controller_enabled: "true"
 {{ else }}
 role_sync_controller_enabled: "false"
 {{ end }}
+
+# EKS specific configuration
+eks_control_plane_logging: "false"
+eks_ip_family: "ipv4"
+eks_zalando_iam_aws_proxy_cpu: "100m"
+eks_zalando_iam_aws_proxy_memory: "512Mi"
+eks_zalando_iam_aws_proxy_hpa_max_replicas: "10"
+eks_zalando_iam_aws_proxy_hpa_cpu_target: "80"
+eks_zalando_iam_aws_proxy_hpa_memory_target: "80"
+eks_okta_identity_provider: "true"
+
+# prefix delegation can only be configured for ipv4. For ipv6 it can only be true.
+aws_vpc_cni_prefix_delegation: "true"
+# enable network policy enforcement in the cluster.
+aws_vpc_cni_enable_network_policy: "false"
+# specify the network policy enforcement mode.
+aws_vpc_cni_network_policy_enforcing_mode: "standard"

cluster/manifests/01-aws-node/config.yaml

@@ -0,0 +1,22 @@
+{{- if eq .Cluster.Provider "zalando-eks" }}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: amazon-vpc-cni
+  namespace: kube-system
+  labels:
+    app.kubernetes.io/instance: aws-vpc-cni
+    app.kubernetes.io/name: aws-node
+    app.kubernetes.io/version: v1.19.0
+    k8s-app: aws-node
+    application: kubernetes
+    component: aws-node
+data:
+  branch-eni-cooldown: "60"
+  enable-network-policy-controller: "{{.Cluster.ConfigItems.aws_vpc_cni_enable_network_policy}}"
+  enable-windows-ipam: "false"
+  enable-windows-prefix-delegation: "false"
+  minimum-ip-target: "3"
+  warm-ip-target: "1"
+  warm-prefix-target: "0"
+{{- end }}

cluster/manifests/01-aws-node/daemonset.yaml

@@ -0,0 +1,261 @@
+{{- if eq .Cluster.Provider "zalando-eks" }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app.kubernetes.io/instance: aws-vpc-cni
+    app.kubernetes.io/name: aws-node
+    app.kubernetes.io/version: v1.19.0
+    k8s-app: aws-node
+    application: kubernetes
+    component: aws-node
+  name: aws-node
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      k8s-app: aws-node
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: aws-vpc-cni
+        app.kubernetes.io/name: aws-node
+        k8s-app: aws-node
+        application: kubernetes
+        component: aws-node
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: kubernetes.io/os
+                operator: In
+                values:
+                - linux
+              - key: kubernetes.io/arch
+                operator: In
+                values:
+                - amd64
+                - arm64
+              - key: eks.amazonaws.com/compute-type
+                operator: NotIn
+                values:
+                - fargate
+                - hybrid
+                - auto
+      containers:
+      - env:
+        - name: ADDITIONAL_ENI_TAGS
+          value: '{}'
+        - name: ANNOTATE_POD_IP
+          value: "false"
+        - name: AWS_VPC_CNI_NODE_PORT_SUPPORT
+          value: "true"
+        - name: AWS_VPC_ENI_MTU
+          value: "9001"
+        - name: AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG
+          value: "false"
+        - name: AWS_VPC_K8S_CNI_EXTERNALSNAT
+          value: "false"
+        - name: AWS_VPC_K8S_CNI_LOGLEVEL
+          value: DEBUG
+        - name: AWS_VPC_K8S_CNI_LOG_FILE
+          value: /host/var/log/aws-routed-eni/ipamd.log
+        - name: AWS_VPC_K8S_CNI_RANDOMIZESNAT
+          value: prng
+        - name: AWS_VPC_K8S_CNI_VETHPREFIX
+          value: eni
+        - name: AWS_VPC_K8S_PLUGIN_LOG_FILE
+          value: /var/log/aws-routed-eni/plugin.log
+        - name: AWS_VPC_K8S_PLUGIN_LOG_LEVEL
+          value: DEBUG
+        - name: CLUSTER_NAME
+          value: "{{ .Cluster.Name }}"
+        - name: DISABLE_INTROSPECTION
+          value: "false"
+        - name: DISABLE_METRICS
+          value: "false"
+        - name: DISABLE_NETWORK_RESOURCE_PROVISIONING
+          value: "false"
+        - name: ENABLE_IPv4
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}true{{else}}false{{end}}"
+        - name: ENABLE_IPv6
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
+        - name: ENABLE_POD_ENI
+          value: "false"
+        - name: ENABLE_PREFIX_DELEGATION
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}{{.Cluster.ConfigItems.aws_vpc_cni_prefix_delegation}}{{else}}true{{end}}"
+        - name: ENABLE_SUBNET_DISCOVERY
+          value: "true"
+        - name: NETWORK_POLICY_ENFORCING_MODE
+          value: "{{.Cluster.ConfigItems.aws_vpc_cni_network_policy_enforcing_mode}}"
+        - name: VPC_CNI_VERSION
+          value: v1.19.0
+        - name: VPC_ID
+          value: "{{ .Cluster.ConfigItems.vpc_id }}"
+        - name: WARM_ENI_TARGET
+          value: "1"
+        - name: WARM_PREFIX_TARGET
+          value: "1"
+        - name: MY_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        - name: MY_POD_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni:v1.19.0-eksbuild.1
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          exec:
+            command:
+            - /app/grpc-health-probe
+            - -addr=:50051
+            - -connect-timeout=5s
+            - -rpc-timeout=5s
+          failureThreshold: 3
+          initialDelaySeconds: 60
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 10
+        name: aws-node
+        ports:
+        - containerPort: 61678
+          hostPort: 61678
+          name: metrics
+          protocol: TCP
+        readinessProbe:
+          exec:
+            command:
+            - /app/grpc-health-probe
+            - -addr=:50051
+            - -connect-timeout=5s
+            - -rpc-timeout=5s
+          failureThreshold: 3
+          initialDelaySeconds: 1
+          periodSeconds: 10
+          successThreshold: 1
+          timeoutSeconds: 10
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /host/etc/cni/net.d
+          name: cni-net-dir
+        - mountPath: /host/var/log/aws-routed-eni
+          name: log-dir
+        - mountPath: /var/run/aws-node
+          name: run-dir
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+      - args:
+        - --enable-ipv6={{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}
+        - --enable-network-policy={{.Cluster.ConfigItems.aws_vpc_cni_enable_network_policy}}
+        - --enable-cloudwatch-logs=false
+        - --enable-policy-event-logs=false
+        - --log-file=/var/log/aws-routed-eni/network-policy-agent.log
+        - --metrics-bind-addr=:8162
+        - --health-probe-bind-addr=:8163
+        - --conntrack-cache-cleanup-period=300
+        env:
+        - name: MY_NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon/aws-network-policy-agent:v1.1.5-eksbuild.1
+        imagePullPolicy: IfNotPresent
+        name: aws-eks-nodeagent
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+        - mountPath: /sys/fs/bpf
+          name: bpf-pin-path
+        - mountPath: /var/log/aws-routed-eni
+          name: log-dir
+        - mountPath: /var/run/aws-node
+          name: run-dir
+      dnsPolicy: ClusterFirst
+      hostNetwork: true
+      initContainers:
+      - env:
+        - name: DISABLE_TCP_EARLY_DEMUX
+          value: "false"
+        - name: ENABLE_IPv6
+          value: "{{ if eq .Cluster.ConfigItems.eks_ip_family "ipv4" }}false{{else}}true{{end}}"
+        image: 602401143452.dkr.ecr.eu-central-1.amazonaws.com/amazon-k8s-cni-init:v1.19.0-eksbuild.1
+        imagePullPolicy: IfNotPresent
+        name: aws-vpc-cni-init
+        resources:
+          requests:
+            cpu: 25m
+        securityContext:
+          privileged: true
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /host/opt/cni/bin
+          name: cni-bin-dir
+      priorityClassName: system-node-critical
+      restartPolicy: Always
+      schedulerName: default-scheduler
+      securityContext: {}
+      serviceAccount: aws-node
+      serviceAccountName: aws-node
+      terminationGracePeriodSeconds: 10
+      tolerations:
+      - operator: Exists
+      volumes:
+      - hostPath:
+          path: /sys/fs/bpf
+          type: ""
+        name: bpf-pin-path
+      - hostPath:
+          path: /opt/cni/bin
+          type: ""
+        name: cni-bin-dir
+      - hostPath:
+          path: /etc/cni/net.d
+          type: ""
+        name: cni-net-dir
+      - hostPath:
+          path: /var/log/aws-routed-eni
+          type: DirectoryOrCreate
+        name: log-dir
+      - hostPath:
+          path: /var/run/aws-node
+          type: DirectoryOrCreate
+        name: run-dir
+      - hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+        name: xtables-lock
+  updateStrategy:
+    rollingUpdate:
+      maxSurge: 0
+      maxUnavailable: 10%
+    type: RollingUpdate
+{{- end }}

cluster/manifests/01-aws-node/sa.yaml

@@ -0,0 +1,11 @@
+{{- if eq .Cluster.Provider "zalando-eks"}}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aws-node
+  namespace: kube-system
+  labels:
+    application: kubernetes
+    component: aws-node
+{{- end}}

cluster/manifests/coredns-local/configmap-local.yaml → cluster/manifests/01-coredns-local/configmap-local.yaml

@@ -71,7 +71,11 @@ data:
 {{ end }}
         template IN A {
             match "^.*[.]ingress[.]cluster[.]local"
+            {{- if eq .Cluster.Provider "zalando-eks" }}
+            answer "{{"{{"}} .Name {{"}}"}} 60 IN AAAA {{ nthAddressFromCIDR .Cluster.ConfigItems.service_cidr 50 }}"
+            {{- else}}
             answer "{{"{{"}} .Name {{"}}"}} 60 IN A 10.5.99.99"
+            {{- end}}
             fallthrough
         }
         template IN AAAA {
@@ -83,7 +87,7 @@ data:
 
     # Defines that this server is authority for reverse
     # lookups for these ranges.
-    cluster.local:9254 10.2.0.0/15:9254 10.5.0.0/16:9254 {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}{{ range $src := split .Cluster.ConfigItems.tracing_coredns_global_traces_endpoint  "," }}{{ $src }}:9254 {{ end }} {{ end }} {
+    cluster.local:9254 {{if eq .Cluster.Provider "zalando-eks"}}in-addr.arpa:9254 ip6.arpa:9254{{else}}10.2.0.0/15:9254 10.5.0.0/16:9254{{end}} {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}{{ range $src := split .Cluster.ConfigItems.tracing_coredns_global_traces_endpoint  "," }}{{ $src }}:9254 {{ end }} {{ end }} {
         errors
         {{ if eq .Cluster.ConfigItems.tracing_coredns_route_traces_to_local_zone "true"}}
           {{- with $cluster := .Cluster }}
@@ -94,6 +98,9 @@ data:
         {{ end }}
         kubernetes {
             pods insecure
+            {{- if eq .Cluster.Provider "zalando-eks"}}
+            fallthrough in-addr.arpa ip6.arpa
+            {{- end}}
         }
         cache 30
 {{ if eq .Cluster.ConfigItems.coredns_log_svc_names "true"}}
@@ -118,7 +125,7 @@ data:
 {{ else }}
         forward . /etc/resolv.conf
 {{ end }}
-        pprof 127.0.0.1:9156
+        pprof :9156
         cache 30
         reload
     }