fix: install k8s 1.22.16 bugs, update helm

paradeum-team · Nov 25, 2022 · 374cb5c · 374cb5c
1 parent cfdc11b
commit 374cb5c
Show file tree

Hide file tree

Showing 12 changed files with 68 additions and 45 deletions.
diff --git a/ansible.cfg b/ansible.cfg
@@ -21,7 +21,7 @@ gathering = smart
 fact_caching = jsonfile
 fact_caching_connection = $HOME/ansible/facts
 fact_caching_timeout = 600
-callback_whitelist = profile_tasks
+callbacks_enabled = profile_tasks
 inventory_ignore_extensions = secrets.py, .pyc, .cfg, .crt, .ini
 # work around privilege escalation timeouts in ansible:
 timeout = 30

diff --git a/ansible.hosts.ha.publicnetwork.tpl b/ansible.hosts.ha.publicnetwork.tpl
@@ -46,8 +46,9 @@ service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
 
 # helm
-helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
 
 # os id, centos|ubuntu
 OS_ID="centos"

diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl
@@ -46,8 +46,9 @@ service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
 
 # helm
-helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
 
 # os id, centos|ubuntu
 OS_ID="centos"

diff --git a/ansible.hosts.ha.vip.tpl b/ansible.hosts.ha.vip.tpl
@@ -58,9 +58,11 @@ keepalived_auth_pass=solarfs{{keepalived_router_id}}
 ingress_nodeport_http=32080
 ingress_nodeport_https=32443
 
+
 # helm
-helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
 
 # os id, centos|ubuntu
 OS_ID="centos"

diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl
@@ -8,7 +8,7 @@ new_nodes
 [k8sCluster:vars]
 # SSH user, this user should allow ssh based auth without requiring a password
 ansible_ssh_user=root
-#ansible_ssh_pass=xxxxxx
+#ansible_ssh_pass=xxxxxxxxxxxx
 ansible_port=22
 
 # If ansible_ssh_user is not root, ansible_become must be set to true
@@ -24,8 +24,8 @@ public_network_node = False
 flannel_enable=True
 
 # api server 域名, 单master 写master ip, 多master 写vip
-master_vip="172.16.195.211"
-master_vip_advertise_address="x.x.x.x"
+master_vip="172.30.1.251"
+master_vip_advertise_address="172.30.1.251"
 node_domain=solarfs.k8s
 install_domain=install.{{node_domain}}
 api_server_domain="api-server.{{node_domain}}"
@@ -37,18 +37,19 @@ k8s_version=1.22.16
 registry_domain=registry.hisun.netwarps.com
 registry_repo="{{registry_domain}}" 
 kubeadm_registry_repo="{{registry_domain}}"
-coredns_image_repo="docker.io/coredns"
+coredns_image_repo="registry.hisun.netwarps.com/coredns"
 coredns_image_tag="1.8.4"
-flannel_image_repo="quay.io"
+flannel_image_repo="registry.hisun.netwarps.com"
 flannel_image_tag="v0.20.1"
 
 # subnet
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
 
 # helm
-helm_binary_md5=77b16cb0ebc6266ac98fc9f2285e361f
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.7.1-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
 
 # os id, centos|ubuntu
 OS_ID="centos"
@@ -57,11 +58,10 @@ OS_ID="centos"
 master1.solarfs.k8s
 
 [masters]
-master1.solarfs.k8s ansible_host=172.16.195.211
+master1.solarfs.k8s ansible_host=172.30.1.251
 
 [nodes]
-infra1.solarfs.k8s ansible_host=172.16.3.85
-node1.solarfs.k8s ansible_host=172.16.128.250
+node1.solarfs.k8s ansible_host=172.30.1.252
 
 [new_nodes]
 #node2.solarfs.k8s ansible_host=172.16.214.182 OS_ID="ubuntu"
diff --git a/docs/Rocky-Linux8.5部署单master-k8s.md b/docs/Rocky-Linux8.5部署单master-k8s.md
@@ -36,7 +36,7 @@ cp ansible.hosts.tpl ansible.hosts.tmp
 
 修改 `ansible.hosts.tmp` 如下
 
-```
+```ini
 [k8sCluster:children]
 masters
 nodes
@@ -63,42 +63,47 @@ public_network_node = False
 flannel_enable=True
 
 # api server 域名, 单master 写master ip, 多master 写vip
-master_vip="172.30.1.198"
-master_vip_advertise_address="x.x.x.x"
+master_vip="172.30.1.251"
+master_vip_advertise_address="172.30.1.251"
 node_domain=solarfs.k8s
 install_domain=install.{{node_domain}}
 api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.22.2
+k8s_version=1.22.16
 # 定义外部镜像仓库
 registry_domain=registry.hisun.netwarps.com
 registry_repo="{{registry_domain}}"
 kubeadm_registry_repo="{{registry_domain}}"
-coredns_image_repo="docker.io/coredns"
-coredns_image_tag="1.8.0"
-flannel_image_repo="quay.io"
-flannel_image_tag="v0.15.1"
+coredns_image_repo="registry.hisun.netwarps.com/coredns"
+coredns_image_tag="1.8.4"
+flannel_image_repo="registry.hisun.netwarps.com"
+flannel_image_tag="v0.20.1"
 
 # subnet
 service_subnet=10.96.0.0/12
 pod_subnet=10.128.0.0/16
 
 # helm
-helm_binary_md5=24b16800f8c7f44b5dd128e3355ecf1b
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.3-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
+
+# os id, centos|ubuntu
+OS_ID="centos"
 
 [install]
-master1.kuggatest.k8s
+master1.solarfs.k8s
 
 [masters]
-master1.kuggatest.k8s ansible_host=172.30.1.198
+master1.solarfs.k8s ansible_host=172.30.1.251
 
 [nodes]
-infra1.kuggatest.k8s ansible_host=172.30.1.199
+node1.solarfs.k8s ansible_host=172.30.1.252
 
 [new_nodes]
+#node2.solarfs.k8s ansible_host=172.16.214.182 OS_ID="ubuntu"
 ```
 
 ## 修改 config 配置

diff --git a/docs/部署keepalived+haproxy高可用k8s1.22.2.md b/docs/部署keepalived+haproxy高可用k8s1.22.2.md
@@ -104,8 +104,9 @@ ingress_nodeport_http=32080
 ingress_nodeport_https=32443
 
 # helm
-helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025
-helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz
+helm_binary_checksum=31960ff2f76a7379d9bac526ddf889fb79241191f1dbe2a24f7864ddcb3f6560
+helm_binary_url=https://pnode.solarfs.io/dn/file/d5b5fd63f068c7a7e950afc840620baf/helm-v3.9.4-linux-amd64.tar.gz
+#helm_binary_url=https://get.helm.sh/helm-v3.9.4-linux-amd64.tar.gz
 
 [install]
 master1.solarfs.k8s

diff --git a/roles/k8s-masters/files/check_flannel.sh b/roles/k8s-masters/files/check_flannel.sh
@@ -5,8 +5,8 @@ desiredNumberScheduled=1
 numberReady=0
 
 get_status(){
-	desiredNumberScheduled=` kubectl get ds kube-flannel-ds -n kube-system -o jsonpath='{.status.desiredNumberScheduled}'`
-	numberReady=`kubectl get ds kube-flannel-ds -n kube-system -o jsonpath='{.status.numberReady}'`
+	desiredNumberScheduled=` kubectl get ds kube-flannel-ds -n kube-flannel -o jsonpath='{.status.desiredNumberScheduled}'`
+	numberReady=`kubectl get ds kube-flannel-ds -n kube-flannel -o jsonpath='{.status.numberReady}'`
 }
 
 i=1

diff --git a/roles/k8s-masters/tasks/kube-flannel.yml b/roles/k8s-masters/tasks/kube-flannel.yml
@@ -1,13 +1,12 @@
 ---
 - name: check kube flannel is installed
-  command: kubectl get ds kube-flannel-ds -n kube-system
+  command: kubectl get ds kube-flannel-ds -n kube-flannel
   register: check_flannel_ret
   ignore_errors: True
   tags: kube-flannel
 
 - name: create kube-flannel.yml
   template: src=kube-flannel.yml.j2 dest=$HOME/k8s_config/kube-flannel.yml  owner=root group=root mode=644
-  tags: kubeadm_init
   when: check_flannel_ret.rc == 1
   tags: kube-flannel
 

diff --git a/roles/k8s-masters/tasks/kubeadmInit.yml b/roles/k8s-masters/tasks/kubeadmInit.yml
@@ -14,8 +14,17 @@
 
 - name: kubeadm init
   command: kubeadm init --config $HOME/k8s_config/kubeadm-init.yaml --upload-certs
+  tags: kubeadm_init
   when: check_ret.rc == 1
 
-- name: copy kubeconfig
-  shell: mkdir -p $HOME/.kube && cp -i /etc/kubernetes/admin.conf $HOME/.kube/config && chown $(id -u):$(id -g) $HOME/.kube/config
-  when: check_ret.rc == 1
+- name: create .kube dir
+  ansible.builtin.file:
+    path: $HOME/.kube
+    state: directory
+    mode: '0755'
+
+- name: copy kube config
+  copy:
+    src: /etc/kubernetes/admin.conf
+    dest: $HOME/.kube/config
+    follow: no
diff --git a/roles/k8s-nodes/templates/kubelet.j2 b/roles/k8s-nodes/templates/kubelet.j2
@@ -1 +1,5 @@
+{% if advertise_address is defined %}
 KUBELET_EXTRA_ARGS="--node-ip {{ advertise_address }}"
+{% else %}
+KUBELET_EXTRA_ARGS=""
+{% endif %}
diff --git a/roles/k8s-services/tasks/installHelm.yml b/roles/k8s-services/tasks/installHelm.yml
@@ -6,28 +6,29 @@
   tags: helm
 
 - name: create tmp helm directory
-  file: path=/tmp/helm.{{helm_binary_md5}} state=directory
+  file: path=/tmp/helm.{{helm_binary_checksum}} state=directory
   when: check_helm_ret.rc == 2
   tags: helm
 
 - name: download helm 
   get_url:
     url: "{{helm_binary_url}}"
-    dest: /tmp/helm.{{helm_binary_md5}}/helm.tar.gz
-    checksum: md5:{{helm_binary_md5}}
+    dest: /tmp/helm.{{helm_binary_checksum}}/helm.tar.gz
+    checksum: sha256:{{helm_binary_checksum}}
+    timeout: 60
   when: check_helm_ret.rc == 2
   tags: helm
 
 - name: Unarchive helm
   unarchive:
-    src: /tmp/helm.{{helm_binary_md5}}/helm.tar.gz
-    dest: /tmp/helm.{{helm_binary_md5}}/
+    src: /tmp/helm.{{helm_binary_checksum}}/helm.tar.gz
+    dest: /tmp/helm.{{helm_binary_checksum}}/
     remote_src: yes
   when: check_helm_ret.rc == 2
   tags: helm
 
 - name: install helm
-  shell: mv /tmp/helm.{{helm_binary_md5}}/linux-amd64/helm /usr/local/bin/helm; rm -rf /tmp/helm.{{helm_binary_md5}}
+  shell: mv /tmp/helm.{{helm_binary_checksum}}/linux-amd64/helm /usr/local/bin/helm; rm -rf /tmp/helm.{{helm_binary_checksum}}
   when: check_helm_ret.rc == 2
   tags: helm