diff --git a/ansible.hosts.ha.publicnetwork.tpl b/ansible.hosts.ha.publicnetwork.tpl index 7970bae..96fef25 100644 --- a/ansible.hosts.ha.publicnetwork.tpl +++ b/ansible.hosts.ha.publicnetwork.tpl @@ -44,6 +44,8 @@ flannel_image_tag="v0.20.1" # subnet service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 +# cluster dns, docker0 ip +local_dns_address="172.17.0.1" # helm helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl index 1701996..fb5a05f 100644 --- a/ansible.hosts.ha.tpl +++ b/ansible.hosts.ha.tpl @@ -44,6 +44,8 @@ flannel_image_tag="v0.20.1" # subnet service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 +# cluster dns, docker0 ip +local_dns_address="172.17.0.1" # helm helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 diff --git a/ansible.hosts.ha.vip.tpl b/ansible.hosts.ha.vip.tpl index b5e94c2..d09ae7d 100644 --- a/ansible.hosts.ha.vip.tpl +++ b/ansible.hosts.ha.vip.tpl @@ -38,6 +38,9 @@ flannel_image_tag="v0.20.1" service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 +# node local dns +local_dns_address="169.254.20.10" + # api server master_vip="172.16.92.250" master_vip_advertise_address="172.16.92.250" diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl index 9b18879..e9345e4 100644 --- a/ansible.hosts.tpl +++ b/ansible.hosts.tpl @@ -45,6 +45,8 @@ flannel_image_tag="v0.20.1" # subnet service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 +# cluster dns, default docker0 ip +local_dns_address="172.17.0.1" # helm helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 diff --git a/roles/host-init/tasks/installKubeadm.yml b/roles/host-init/tasks/installKubeadm.yml index b1ae37e..b49117a 100644 --- a/roles/host-init/tasks/installKubeadm.yml +++ b/roles/host-init/tasks/installKubeadm.yml @@ -26,7 +26,7 @@ - name: config kubelet template: src=kubelet.j2 dest=/etc/sysconfig/kubelet owner=root group=root mode=644 backup=yes notify: restart kubelet - tags: kubeadm + tags: kubeadm,kubelet-config when: OS_ID == "centos" - name: Enable service kubelet and start diff --git a/roles/host-init/templates/kubelet.j2 b/roles/host-init/templates/kubelet.j2 index adb7d63..c9945b9 100644 --- a/roles/host-init/templates/kubelet.j2 +++ b/roles/host-init/templates/kubelet.j2 @@ -1 +1 @@ -KUBELET_EXTRA_ARGS="" +KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}" diff --git a/roles/k8s-masters/files/check-nodelocaldns.sh b/roles/k8s-masters/files/check-nodelocaldns.sh new file mode 100755 index 0000000..195c68b --- /dev/null +++ b/roles/k8s-masters/files/check-nodelocaldns.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +BASE_DIR=$(cd `dirname $0` && pwd) +cd $BASE_DIR + +check_ds(){ + desiredNumberScheduled=1 + numberReady=0 + + name=$1 + + if [ -z "$name" ]; then + echo "$0 " + exit 1 + fi + + get_cmd="kubectl get ds $name -n kube-system" + + get_status(){ + desiredNumberScheduled=`$get_cmd -o jsonpath='{.status.desiredNumberScheduled}'` + numberReady=`$get_cmd -o jsonpath='{.status.numberReady}'` + } + + i=1 + while [[ "$desiredNumberScheduled" -ne "$numberReady" ]] || [[ "$desiredNumberScheduled" -eq '' ]] + do + get_status + if [ "$i" -gt 60 ];then + echo "check $name status timeout !!!" + exit 1 + fi + let i=i+1 + sleep 1 + done + + echo "$name ds is runing!" +} + +check_ds node-local-dns diff --git a/roles/k8s-masters/tasks/kubelet.yml b/roles/k8s-masters/tasks/kubelet.yml index ba1e2de..62fff2d 100644 --- a/roles/k8s-masters/tasks/kubelet.yml +++ b/roles/k8s-masters/tasks/kubelet.yml @@ -1,4 +1,4 @@ --- - name: config /etc/sysconfig/kubelet template: src=kubelet.j2 dest=/etc/sysconfig/kubelet mode=0644 - tags: join-node + tags: join-node,kubelet-config diff --git a/roles/k8s-masters/tasks/main.yml b/roles/k8s-masters/tasks/main.yml index b83b8bc..a64b823 100644 --- a/roles/k8s-masters/tasks/main.yml +++ b/roles/k8s-masters/tasks/main.yml @@ -11,3 +11,4 @@ when: hostvars[ groups['masters'][0] ].inventory_hostname == inventory_hostname and flannel_enable == True - include: kubedns.yml when: public_network_node == False +#- include: nodeLocalDns.yml diff --git a/roles/k8s-masters/tasks/nodeLocalDns.yml b/roles/k8s-masters/tasks/nodeLocalDns.yml new file mode 100644 index 0000000..ab54daf --- /dev/null +++ b/roles/k8s-masters/tasks/nodeLocalDns.yml @@ -0,0 +1,20 @@ +--- +- name: check node local dns is installed + command: kubectl get ds node-local-dns -n kube-system + register: check_nodelocaldns_ret + ignore_errors: True + tags: nodelocaldns + +- name: create nodelocaldns.yml + template: src=nodelocaldns.yml.j2 dest=$HOME/k8s_config/nodelocaldns.yml owner=root group=root mode=644 + when: check_nodelocaldns_ret.rc == 1 + tags: nodelocaldns + +- name: install nodelocaldns.yml + command: kubectl apply -f $HOME/k8s_config/nodelocaldns.yml + when: check_nodelocaldns_ret.rc == 1 + tags: nodelocaldns + +- name: check node local dns status + script: check-nodelocaldns.sh + tags: nodelocaldns diff --git a/roles/k8s-masters/templates/coredns-config.yml.j2 b/roles/k8s-masters/templates/coredns-config.yml.j2 index b630a73..78e8d93 100644 --- a/roles/k8s-masters/templates/coredns-config.yml.j2 +++ b/roles/k8s-masters/templates/coredns-config.yml.j2 @@ -8,10 +8,11 @@ data: } ready kubernetes cluster.local in-addr.arpa ip6.arpa { - pods insecure + pods verified fallthrough in-addr.arpa ip6.arpa ttl 30 } + autopath @kubernetes prometheus :9153 forward . {{upstream_dns_ips}} { max_concurrent 1000 diff --git a/roles/k8s-masters/templates/kube-dns.conf.j2 b/roles/k8s-masters/templates/kube-dns.conf.j2 index 3efd0ca..b08e091 100644 --- a/roles/k8s-masters/templates/kube-dns.conf.j2 +++ b/roles/k8s-masters/templates/kube-dns.conf.j2 @@ -7,6 +7,6 @@ dns-forward-max=10000 cache-size=10000 bind-dynamic min-port=1024 -interface={{LOCAL_ENNAME}} -#except-interface=lo +interface={{LOCAL_ENNAME}},docker0 +except-interface=lo,nodelocaldns,kube-ipvs0,flannel.1 # End of config diff --git a/roles/k8s-masters/templates/kubelet.j2 b/roles/k8s-masters/templates/kubelet.j2 index 8002dff..f7620e7 100644 --- a/roles/k8s-masters/templates/kubelet.j2 +++ b/roles/k8s-masters/templates/kubelet.j2 @@ -1,5 +1,5 @@ {% if advertise_address is defined %} -KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}" +KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }} --cluster-dns={{ local_dns_address }}" {% else %} -KUBELET_EXTRA_ARGS="" +KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}" {% endif %} diff --git a/roles/k8s-masters/templates/nodelocaldns.yml.j2 b/roles/k8s-masters/templates/nodelocaldns.yml.j2 new file mode 100644 index 0000000..f4295d6 --- /dev/null +++ b/roles/k8s-masters/templates/nodelocaldns.yml.j2 @@ -0,0 +1,209 @@ +# Copyright 2018 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-local-dns + namespace: kube-system + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns-upstream + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile + kubernetes.io/name: "KubeDNSUpstream" +spec: + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-local-dns + namespace: kube-system + labels: + addonmanager.kubernetes.io/mode: Reconcile +data: + Corefile: | + cluster.local:53 { + errors + cache { + success 9984 30 + denial 9984 5 + } + reload + loop + bind {{ local_dns_address }} __PILLAR__DNS__SERVER__ + forward . 10.96.0.10 { + force_tcp + } + prometheus :9253 + health {{ local_dns_address }}:8080 + } + in-addr.arpa:53 { + errors + cache 30 + reload + loop + bind {{ local_dns_address }} __PILLAR__DNS__SERVER__ + forward . 10.96.0.10 { + force_tcp + } + prometheus :9253 + } + ip6.arpa:53 { + errors + cache 30 + reload + loop + bind {{ local_dns_address }} __PILLAR__DNS__SERVER__ + forward . 10.96.0.10 { + force_tcp + } + prometheus :9253 + } + .:53 { + errors + cache 30 + reload + loop + bind {{ local_dns_address }} __PILLAR__DNS__SERVER__ + forward . __PILLAR__UPSTREAM__SERVERS__ + prometheus :9253 + } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-local-dns + namespace: kube-system + labels: + k8s-app: node-local-dns + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 10% + selector: + matchLabels: + k8s-app: node-local-dns + template: + metadata: + labels: + k8s-app: node-local-dns + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + spec: + priorityClassName: system-node-critical + serviceAccountName: node-local-dns + hostNetwork: true + dnsPolicy: Default # Don't use cluster DNS. + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - effect: "NoExecute" + operator: "Exists" + - effect: "NoSchedule" + operator: "Exists" + containers: + - name: node-cache + image: {{ registry_repo }}/dns/k8s-dns-node-cache:1.17.0 + resources: + requests: + cpu: 25m + memory: 5Mi + args: [ "-localip", "{{ local_dns_address }}", "-conf", "/etc/Corefile", "-upstreamsvc", "kube-dns-upstream" ] + securityContext: + privileged: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9253 + name: metrics + protocol: TCP + livenessProbe: + httpGet: + host: {{ local_dns_address }} + path: /health + port: 8080 + initialDelaySeconds: 60 + timeoutSeconds: 5 + volumeMounts: + - mountPath: /run/xtables.lock + name: xtables-lock + readOnly: false + - name: config-volume + mountPath: /etc/coredns + - name: kube-dns-config + mountPath: /etc/kube-dns + volumes: + - name: xtables-lock + hostPath: + path: /run/xtables.lock + type: FileOrCreate + - name: kube-dns-config + configMap: + name: kube-dns + optional: true + - name: config-volume + configMap: + name: node-local-dns + items: + - key: Corefile + path: Corefile.base +--- +# A headless service is a service with a service IP but instead of load-balancing it will return the IPs of our associated Pods. +# We use this to expose metrics to Prometheus. +apiVersion: v1 +kind: Service +metadata: + annotations: + prometheus.io/port: "9253" + prometheus.io/scrape: "true" + labels: + k8s-app: node-local-dns + name: node-local-dns + namespace: kube-system +spec: + clusterIP: None + ports: + - name: metrics + port: 9253 + targetPort: 9253 + selector: + k8s-app: node-local-dns diff --git a/roles/k8s-nodes/tasks/joinNode.yml b/roles/k8s-nodes/tasks/joinNode.yml index 77c9db0..29fae86 100644 --- a/roles/k8s-nodes/tasks/joinNode.yml +++ b/roles/k8s-nodes/tasks/joinNode.yml @@ -2,11 +2,11 @@ stat: path: /etc/kubernetes/kubelet.conf register: check_kubelet_conf - tags: join-node + tags: join-node,kubelet-config - name: config /etc/sysconfig/kubelet template: src=kubelet.j2 dest=/etc/sysconfig/kubelet mode=0644 - tags: join-node + tags: join-node,kubelet-config - name: join k8s nodes command: "{{hostvars[ groups['masters'][0] ].join_command.stdout}}" diff --git a/roles/k8s-nodes/templates/kube-dns.conf.j2 b/roles/k8s-nodes/templates/kube-dns.conf.j2 index 3efd0ca..b08e091 100644 --- a/roles/k8s-nodes/templates/kube-dns.conf.j2 +++ b/roles/k8s-nodes/templates/kube-dns.conf.j2 @@ -7,6 +7,6 @@ dns-forward-max=10000 cache-size=10000 bind-dynamic min-port=1024 -interface={{LOCAL_ENNAME}} -#except-interface=lo +interface={{LOCAL_ENNAME}},docker0 +except-interface=lo,nodelocaldns,kube-ipvs0,flannel.1 # End of config diff --git a/roles/k8s-nodes/templates/kubelet.j2 b/roles/k8s-nodes/templates/kubelet.j2 index 8002dff..f7620e7 100644 --- a/roles/k8s-nodes/templates/kubelet.j2 +++ b/roles/k8s-nodes/templates/kubelet.j2 @@ -1,5 +1,5 @@ {% if advertise_address is defined %} -KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}" +KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }} --cluster-dns={{ local_dns_address }}" {% else %} -KUBELET_EXTRA_ARGS="" +KUBELET_EXTRA_ARGS="--cluster-dns={{ local_dns_address }}" {% endif %}