From 374cb5c3541b0db8f26fc3cf75bf3ae6d2844835 Mon Sep 17 00:00:00 2001 From: ss75710541 <75710541@qq.com> Date: Fri, 25 Nov 2022 16:54:47 +0800 Subject: [PATCH] fix: install k8s 1.22.16 bugs, update helm --- ansible.cfg | 2 +- ansible.hosts.ha.publicnetwork.tpl | 5 +-- ansible.hosts.ha.tpl | 5 +-- ansible.hosts.ha.vip.tpl | 6 ++-- ansible.hosts.tpl | 20 ++++++------ ...\250\347\275\262\345\215\225master-k8s.md" | 31 +++++++++++-------- ...3\230\345\217\257\347\224\250k8s1.22.2.md" | 5 +-- roles/k8s-masters/files/check_flannel.sh | 4 +-- roles/k8s-masters/tasks/kube-flannel.yml | 3 +- roles/k8s-masters/tasks/kubeadmInit.yml | 15 +++++++-- roles/k8s-nodes/templates/kubelet.j2 | 4 +++ roles/k8s-services/tasks/installHelm.yml | 13 ++++---- 12 files changed, 68 insertions(+), 45 deletions(-) diff --git a/ansible.cfg b/ansible.cfg index 31b33c8..6c77d76 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -21,7 +21,7 @@ gathering = smart fact_caching = jsonfile fact_caching_connection = $HOME/ansible/facts fact_caching_timeout = 600 -callback_whitelist = profile_tasks +callbacks_enabled = profile_tasks inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt, .ini # work around privilege escalation timeouts in ansible: timeout = 30 diff --git a/ansible.hosts.ha.publicnetwork.tpl b/ansible.hosts.ha.publicnetwork.tpl index 5e164b2..7970bae 100644 --- a/ansible.hosts.ha.publicnetwork.tpl +++ b/ansible.hosts.ha.publicnetwork.tpl @@ -46,8 +46,9 @@ service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 # helm -helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz # os id, centos|ubuntu OS_ID="centos" diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl index 6ccedba..1701996 100644 --- a/ansible.hosts.ha.tpl +++ b/ansible.hosts.ha.tpl @@ -46,8 +46,9 @@ service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 # helm -helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz # os id, centos|ubuntu OS_ID="centos" diff --git a/ansible.hosts.ha.vip.tpl b/ansible.hosts.ha.vip.tpl index 80f8975..b5e94c2 100644 --- a/ansible.hosts.ha.vip.tpl +++ b/ansible.hosts.ha.vip.tpl @@ -58,9 +58,11 @@ keepalived_auth_pass=solarfs{{keepalived_router_id}} ingress_nodeport_http=32080 ingress_nodeport_https=32443 + # helm -helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz # os id, centos|ubuntu OS_ID="centos" diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl index 181f75f..9b18879 100644 --- a/ansible.hosts.tpl +++ b/ansible.hosts.tpl @@ -8,7 +8,7 @@ new_nodes [k8sCluster:vars] # SSH user, this user should allow ssh based auth without requiring a password ansible_ssh_user=root -#ansible_ssh_pass=xxxxxx +#ansible_ssh_pass=xxxxxxxxxxxx ansible_port=22 # If ansible_ssh_user is not root, ansible_become must be set to true @@ -24,8 +24,8 @@ public_network_node = False flannel_enable=True # api server 域名, 单master 写master ip, 多master 写vip -master_vip="172.16.195.211" -master_vip_advertise_address="x.x.x.x" +master_vip="172.30.1.251" +master_vip_advertise_address="172.30.1.251" node_domain=solarfs.k8s install_domain=install.{{node_domain}} api_server_domain="api-server.{{node_domain}}" @@ -37,9 +37,9 @@ k8s_version=1.22.16 registry_domain=registry.hisun.netwarps.com registry_repo="{{registry_domain}}" kubeadm_registry_repo="{{registry_domain}}" -coredns_image_repo="docker.io/coredns" +coredns_image_repo="registry.hisun.netwarps.com/coredns" coredns_image_tag="1.8.4" -flannel_image_repo="quay.io" +flannel_image_repo="registry.hisun.netwarps.com" flannel_image_tag="v0.20.1" # subnet @@ -47,8 +47,9 @@ service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 # helm -helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz # os id, centos|ubuntu OS_ID="centos" @@ -57,11 +58,10 @@ OS_ID="centos" master1.solarfs.k8s [masters] -master1.solarfs.k8s ansible_host=172.16.195.211 +master1.solarfs.k8s ansible_host=172.30.1.251 [nodes] -infra1.solarfs.k8s ansible_host=172.16.3.85 -node1.solarfs.k8s ansible_host=172.16.128.250 +node1.solarfs.k8s ansible_host=172.30.1.252 [new_nodes] #node2.solarfs.k8s ansible_host=172.16.214.182 OS_ID="ubuntu" diff --git "a/docs/Rocky-Linux8.5\351\203\250\347\275\262\345\215\225master-k8s.md" "b/docs/Rocky-Linux8.5\351\203\250\347\275\262\345\215\225master-k8s.md" index be2ca2e..950413f 100644 --- "a/docs/Rocky-Linux8.5\351\203\250\347\275\262\345\215\225master-k8s.md" +++ "b/docs/Rocky-Linux8.5\351\203\250\347\275\262\345\215\225master-k8s.md" @@ -36,7 +36,7 @@ cp ansible.hosts.tpl ansible.hosts.tmp 修改 `ansible.hosts.tmp` 如下 -``` +```ini [k8sCluster:children] masters nodes @@ -63,42 +63,47 @@ public_network_node = False flannel_enable=True # api server 域名, 单master 写master ip, 多master 写vip -master_vip="172.30.1.198" -master_vip_advertise_address="x.x.x.x" +master_vip="172.30.1.251" +master_vip_advertise_address="172.30.1.251" node_domain=solarfs.k8s install_domain=install.{{node_domain}} api_server_domain="api-server.{{node_domain}}" api_server_port="6443" # k8s 版本 -k8s_version=1.22.2 +k8s_version=1.22.16 # 定义外部镜像仓库 registry_domain=registry.hisun.netwarps.com registry_repo="{{registry_domain}}" kubeadm_registry_repo="{{registry_domain}}" -coredns_image_repo="docker.io/coredns" -coredns_image_tag="1.8.0" -flannel_image_repo="quay.io" -flannel_image_tag="v0.15.1" +coredns_image_repo="registry.hisun.netwarps.com/coredns" +coredns_image_tag="1.8.4" +flannel_image_repo="registry.hisun.netwarps.com" +flannel_image_tag="v0.20.1" # subnet service_subnet=10.96.0.0/12 pod_subnet=10.128.0.0/16 # helm -helm_binary_md5=24b16800f8c7f44b5dd128e3355ecf1b -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.3-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz + +# os id, centos|ubuntu +OS_ID="centos" [install] -master1.kuggatest.k8s +master1.solarfs.k8s [masters] -master1.kuggatest.k8s ansible_host=172.30.1.198 +master1.solarfs.k8s ansible_host=172.30.1.251 [nodes] -infra1.kuggatest.k8s ansible_host=172.30.1.199 +node1.solarfs.k8s ansible_host=172.30.1.252 [new_nodes] +#node2.solarfs.k8s ansible_host=172.16.214.182 OS_ID="ubuntu" ``` ## 修改 config 配置 diff --git "a/docs/\351\203\250\347\275\262keepalived+haproxy\351\253\230\345\217\257\347\224\250k8s1.22.2.md" "b/docs/\351\203\250\347\275\262keepalived+haproxy\351\253\230\345\217\257\347\224\250k8s1.22.2.md" index 00c8eb4..62dfbe7 100644 --- "a/docs/\351\203\250\347\275\262keepalived+haproxy\351\253\230\345\217\257\347\224\250k8s1.22.2.md" +++ "b/docs/\351\203\250\347\275\262keepalived+haproxy\351\253\230\345\217\257\347\224\250k8s1.22.2.md" @@ -104,8 +104,9 @@ ingress_nodeport_http=32080 ingress_nodeport_https=32443 # helm -helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025 -helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz +helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560 +helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz +#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz [install] master1.solarfs.k8s diff --git a/roles/k8s-masters/files/check_flannel.sh b/roles/k8s-masters/files/check_flannel.sh index 65d310f..4e5507b 100755 --- a/roles/k8s-masters/files/check_flannel.sh +++ b/roles/k8s-masters/files/check_flannel.sh @@ -5,8 +5,8 @@ desiredNumberScheduled=1 numberReady=0 get_status(){ - desiredNumberScheduled=` kubectl get ds kube-flannel-ds -n kube-system -o jsonpath='{.status.desiredNumberScheduled}'` - numberReady=`kubectl get ds kube-flannel-ds -n kube-system -o jsonpath='{.status.numberReady}'` + desiredNumberScheduled=` kubectl get ds kube-flannel-ds -n kube-flannel -o jsonpath='{.status.desiredNumberScheduled}'` + numberReady=`kubectl get ds kube-flannel-ds -n kube-flannel -o jsonpath='{.status.numberReady}'` } i=1 diff --git a/roles/k8s-masters/tasks/kube-flannel.yml b/roles/k8s-masters/tasks/kube-flannel.yml index 7aa7fdf..080a18e 100644 --- a/roles/k8s-masters/tasks/kube-flannel.yml +++ b/roles/k8s-masters/tasks/kube-flannel.yml @@ -1,13 +1,12 @@ --- - name: check kube flannel is installed - command: kubectl get ds kube-flannel-ds -n kube-system + command: kubectl get ds kube-flannel-ds -n kube-flannel register: check_flannel_ret ignore_errors: True tags: kube-flannel - name: create kube-flannel.yml template: src=kube-flannel.yml.j2 dest=$HOME/k8s_config/kube-flannel.yml owner=root group=root mode=644 - tags: kubeadm_init when: check_flannel_ret.rc == 1 tags: kube-flannel diff --git a/roles/k8s-masters/tasks/kubeadmInit.yml b/roles/k8s-masters/tasks/kubeadmInit.yml index afea46a..691cdea 100644 --- a/roles/k8s-masters/tasks/kubeadmInit.yml +++ b/roles/k8s-masters/tasks/kubeadmInit.yml @@ -14,8 +14,17 @@ - name: kubeadm init command: kubeadm init --config $HOME/k8s_config/kubeadm-init.yaml --upload-certs + tags: kubeadm_init when: check_ret.rc == 1 -- name: copy kubeconfig - shell: mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config - when: check_ret.rc == 1 +- name: create .kube dir + ansible.builtin.file: + path: $HOME/.kube + state: directory + mode: '0755' + +- name: copy kube config + copy: + src: /etc/kubernetes/admin.conf + dest: $HOME/.kube/config + follow: no diff --git a/roles/k8s-nodes/templates/kubelet.j2 b/roles/k8s-nodes/templates/kubelet.j2 index 5f2055f..8002dff 100644 --- a/roles/k8s-nodes/templates/kubelet.j2 +++ b/roles/k8s-nodes/templates/kubelet.j2 @@ -1 +1,5 @@ +{% if advertise_address is defined %} KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}" +{% else %} +KUBELET_EXTRA_ARGS="" +{% endif %} diff --git a/roles/k8s-services/tasks/installHelm.yml b/roles/k8s-services/tasks/installHelm.yml index 4860973..6271b5d 100644 --- a/roles/k8s-services/tasks/installHelm.yml +++ b/roles/k8s-services/tasks/installHelm.yml @@ -6,28 +6,29 @@ tags: helm - name: create tmp helm directory - file: path=/tmp/helm.{{helm_binary_md5}} state=directory + file: path=/tmp/helm.{{helm_binary_checksum}} state=directory when: check_helm_ret.rc == 2 tags: helm - name: download helm get_url: url: "{{helm_binary_url}}" - dest: /tmp/helm.{{helm_binary_md5}}/helm.tar.gz - checksum: md5:{{helm_binary_md5}} + dest: /tmp/helm.{{helm_binary_checksum}}/helm.tar.gz + checksum: sha256:{{helm_binary_checksum}} + timeout: 60 when: check_helm_ret.rc == 2 tags: helm - name: Unarchive helm unarchive: - src: /tmp/helm.{{helm_binary_md5}}/helm.tar.gz - dest: /tmp/helm.{{helm_binary_md5}}/ + src: /tmp/helm.{{helm_binary_checksum}}/helm.tar.gz + dest: /tmp/helm.{{helm_binary_checksum}}/ remote_src: yes when: check_helm_ret.rc == 2 tags: helm - name: install helm - shell: mv /tmp/helm.{{helm_binary_md5}}/linux-amd64/helm /usr/local/bin/helm; rm -rf /tmp/helm.{{helm_binary_md5}} + shell: mv /tmp/helm.{{helm_binary_checksum}}/linux-amd64/helm /usr/local/bin/helm; rm -rf /tmp/helm.{{helm_binary_checksum}} when: check_helm_ret.rc == 2 tags: helm