feat: 优化coredns

ansible.hosts.ha.publicnetwork.tpl

@@ -44,6 +44,8 @@ flannel_image_tag="v0.20.1"
 # subnet
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
+# cluster dns, docker0 ip
+local_dns_address="172.17.0.1"
 
 # helm
 helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560

ansible.hosts.ha.tpl

@@ -44,6 +44,8 @@ flannel_image_tag="v0.20.1"
 # subnet
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
+# cluster dns, docker0 ip
+local_dns_address="172.17.0.1"
 
 # helm
 helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560

ansible.hosts.ha.vip.tpl

@@ -38,6 +38,9 @@ flannel_image_tag="v0.20.1"
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
 
+# node local dns
+local_dns_address="169.254.20.10"
+
 # api server 
 master_vip="172.16.92.250"
 master_vip_advertise_address="172.16.92.250"

ansible.hosts.tpl

@@ -45,6 +45,8 @@ flannel_image_tag="v0.20.1"
 # subnet
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
+# cluster dns, default docker0 ip
+local_dns_address="172.17.0.1"
 
 # helm
 helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560

roles/host-init/tasks/installKubeadm.yml

@@ -26,7 +26,7 @@
 - name: config kubelet
   template: src=kubelet.j2 dest=/etc/sysconfig/kubelet  owner=root group=root mode=644 backup=yes
   notify: restart kubelet
-  tags: kubeadm
+  tags: kubeadm,kubelet-config
   when: OS_ID == "centos"
 
 - name: Enable service kubelet and start

roles/host-init/templates/kubelet.j2

@@ -1 +1 @@
-KUBELET_EXTRA_ARGS=""
+KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}"

roles/k8s-masters/files/check-nodelocaldns.sh

@@ -0,0 +1,40 @@
+#!/bin/bash
+set -e
+
+BASE_DIR=$(cd `dirname $0` && pwd)
+cd $BASE_DIR
+
+check_ds(){
+  desiredNumberScheduled=1
+  numberReady=0
+
+  name=$1
+
+  if [ -z "$name" ]; then
+  	echo "$0 <name>"
+  	exit 1
+  fi
+
+  get_cmd="kubectl get ds $name -n kube-system"
+
+  get_status(){
+  	desiredNumberScheduled=`$get_cmd -o jsonpath='{.status.desiredNumberScheduled}'`
+  	numberReady=`$get_cmd -o jsonpath='{.status.numberReady}'`
+  }
+
+  i=1
+  while [[ "$desiredNumberScheduled" -ne "$numberReady" ]] || [[ "$desiredNumberScheduled" -eq '' ]]
+  do
+  	get_status
+  	if [ "$i" -gt 60 ];then
+  		echo "check $name status timeout !!!"
+  		exit 1
+  	fi
+  	let i=i+1
+  	sleep 1
+  done
+
+  echo "$name ds is runing!"
+}
+
+check_ds node-local-dns

roles/k8s-masters/tasks/kubelet.yml

@@ -1,4 +1,4 @@
 ---
 - name: config /etc/sysconfig/kubelet
   template: src=kubelet.j2 dest=/etc/sysconfig/kubelet mode=0644
-  tags: join-node
+  tags: join-node,kubelet-config

roles/k8s-masters/tasks/main.yml

@@ -11,3 +11,4 @@
   when: hostvars[ groups['masters'][0] ].inventory_hostname == inventory_hostname and flannel_enable == True
 - include: kubedns.yml
   when: public_network_node == False
+#- include: nodeLocalDns.yml

roles/k8s-masters/tasks/nodeLocalDns.yml

@@ -0,0 +1,20 @@
+---
+- name: check node local dns  is installed
+  command: kubectl get ds node-local-dns -n kube-system
+  register: check_nodelocaldns_ret
+  ignore_errors: True
+  tags: nodelocaldns
+
+- name: create nodelocaldns.yml
+  template: src=nodelocaldns.yml.j2 dest=$HOME/k8s_config/nodelocaldns.yml  owner=root group=root mode=644
+  when: check_nodelocaldns_ret.rc == 1
+  tags: nodelocaldns
+
+- name: install nodelocaldns.yml
+  command: kubectl apply -f $HOME/k8s_config/nodelocaldns.yml
+  when: check_nodelocaldns_ret.rc == 1
+  tags: nodelocaldns
+
+- name: check node local dns status
+  script: check-nodelocaldns.sh
+  tags: nodelocaldns

roles/k8s-masters/templates/coredns-config.yml.j2

@@ -8,10 +8,11 @@ data:
         }
         ready
         kubernetes cluster.local in-addr.arpa ip6.arpa {
-           pods insecure
+           pods verified
            fallthrough in-addr.arpa ip6.arpa
            ttl 30
         }
+        autopath @kubernetes
         prometheus :9153
         forward . {{upstream_dns_ips}} {
            max_concurrent 1000

roles/k8s-masters/templates/kube-dns.conf.j2

@@ -7,6 +7,6 @@ dns-forward-max=10000
 cache-size=10000
 bind-dynamic
 min-port=1024
-interface={{LOCAL_ENNAME}}
-#except-interface=lo
+interface={{LOCAL_ENNAME}},docker0
+except-interface=lo,nodelocaldns,kube-ipvs0,flannel.1
 # End of config

roles/k8s-masters/templates/kubelet.j2

@@ -1,5 +1,5 @@
 {% if advertise_address is defined %}
-KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}"
+KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }} --cluster-dns={{ local_dns_address }}"
 {% else %}
-KUBELET_EXTRA_ARGS=""
+KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}"
 {% endif %}

roles/k8s-masters/templates/nodelocaldns.yml.j2

@@ -0,0 +1,209 @@
+# Copyright 2018 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: kube-dns-upstream
+  namespace: kube-system
+  labels:
+    k8s-app: kube-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+    kubernetes.io/name: "KubeDNSUpstream"
+spec:
+  ports:
+  - name: dns
+    port: 53
+    protocol: UDP
+    targetPort: 53
+  - name: dns-tcp
+    port: 53
+    protocol: TCP
+    targetPort: 53
+  selector:
+    k8s-app: kube-dns
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    addonmanager.kubernetes.io/mode: Reconcile
+data:
+  Corefile: |
+    cluster.local:53 {
+        errors
+        cache {
+                success 9984 30
+                denial 9984 5
+        }
+        reload
+        loop
+        bind {{ local_dns_address }} __PILLAR__DNS__SERVER__
+        forward . 10.96.0.10 {
+                force_tcp
+        }
+        prometheus :9253
+        health {{ local_dns_address }}:8080
+        }
+    in-addr.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ local_dns_address }} __PILLAR__DNS__SERVER__
+        forward . 10.96.0.10 {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    ip6.arpa:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ local_dns_address }} __PILLAR__DNS__SERVER__
+        forward . 10.96.0.10 {
+                force_tcp
+        }
+        prometheus :9253
+        }
+    .:53 {
+        errors
+        cache 30
+        reload
+        loop
+        bind {{ local_dns_address }} __PILLAR__DNS__SERVER__
+        forward . __PILLAR__UPSTREAM__SERVERS__
+        prometheus :9253
+        }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: node-local-dns
+  namespace: kube-system
+  labels:
+    k8s-app: node-local-dns
+    kubernetes.io/cluster-service: "true"
+    addonmanager.kubernetes.io/mode: Reconcile
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 10%
+  selector:
+    matchLabels:
+      k8s-app: node-local-dns
+  template:
+    metadata:
+      labels:
+        k8s-app: node-local-dns
+      annotations:
+        prometheus.io/port: "9253"
+        prometheus.io/scrape: "true"
+    spec:
+      priorityClassName: system-node-critical
+      serviceAccountName: node-local-dns
+      hostNetwork: true
+      dnsPolicy: Default  # Don't use cluster DNS.
+      tolerations:
+      - key: "CriticalAddonsOnly"
+        operator: "Exists"
+      - effect: "NoExecute"
+        operator: "Exists"
+      - effect: "NoSchedule"
+        operator: "Exists"
+      containers:
+      - name: node-cache
+        image: {{ registry_repo }}/dns/k8s-dns-node-cache:1.17.0
+        resources:
+          requests:
+            cpu: 25m
+            memory: 5Mi
+        args: [ "-localip", "{{ local_dns_address }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ]
+        securityContext:
+          privileged: true
+        ports:
+        - containerPort: 53
+          name: dns
+          protocol: UDP
+        - containerPort: 53
+          name: dns-tcp
+          protocol: TCP
+        - containerPort: 9253
+          name: metrics
+          protocol: TCP
+        livenessProbe:
+          httpGet:
+            host: {{ local_dns_address }}
+            path: /health
+            port: 8080
+          initialDelaySeconds: 60
+          timeoutSeconds: 5
+        volumeMounts:
+        - mountPath: /run/xtables.lock
+          name: xtables-lock
+          readOnly: false
+        - name: config-volume
+          mountPath: /etc/coredns
+        - name: kube-dns-config
+          mountPath: /etc/kube-dns
+      volumes:
+      - name: xtables-lock
+        hostPath:
+          path: /run/xtables.lock
+          type: FileOrCreate
+      - name: kube-dns-config
+        configMap:
+          name: kube-dns
+          optional: true
+      - name: config-volume
+        configMap:
+          name: node-local-dns
+          items:
+            - key: Corefile
+              path: Corefile.base
+---
+# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods.
+# We use this to expose metrics to Prometheus.
+apiVersion: v1
+kind: Service
+metadata:
+  annotations:
+    prometheus.io/port: "9253"
+    prometheus.io/scrape: "true"
+  labels:
+    k8s-app: node-local-dns
+  name: node-local-dns
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+    - name: metrics
+      port: 9253
+      targetPort: 9253
+  selector:
+    k8s-app: node-local-dns

roles/k8s-nodes/tasks/joinNode.yml

@@ -2,11 +2,11 @@
   stat:
     path: /etc/kubernetes/kubelet.conf
   register: check_kubelet_conf
-  tags: join-node
+  tags: join-node,kubelet-config
 
 - name: config /etc/sysconfig/kubelet
   template: src=kubelet.j2 dest=/etc/sysconfig/kubelet mode=0644
-  tags: join-node
+  tags: join-node,kubelet-config
 
 - name: join k8s nodes
   command: "{{hostvars[ groups['masters'][0] ].join_command.stdout}}"

roles/k8s-nodes/templates/kube-dns.conf.j2

@@ -7,6 +7,6 @@ dns-forward-max=10000
 cache-size=10000
 bind-dynamic
 min-port=1024
-interface={{LOCAL_ENNAME}}
-#except-interface=lo
+interface={{LOCAL_ENNAME}},docker0
+except-interface=lo,nodelocaldns,kube-ipvs0,flannel.1
 # End of config

roles/k8s-nodes/templates/kubelet.j2

@@ -1,5 +1,5 @@
 {% if advertise_address is defined %}
-KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}"
+KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }} --cluster-dns={{ local_dns_address }}"
 {% else %}
-KUBELET_EXTRA_ARGS=""
+KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}"
 {% endif %}