From 800a5470e71a9687129d30095213168b8f2fe983 Mon Sep 17 00:00:00 2001
From: ss75710541 <75710541@qq.com>
Date: Wed, 8 Sep 2021 17:05:43 +0800
Subject: [PATCH 1/4] ansible.cfg forks update to 50

---
 ansible.cfg | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ansible.cfg b/ansible.cfg
index 7f89590..31b33c8 100644
--- a/ansible.cfg
+++ b/ansible.cfg
@@ -10,7 +10,7 @@
 log_path = ~/k8s-ansible.log
 
 # Additional default options for OpenShift Ansible
-forks = 20
+forks = 50
 host_key_checking = False
 retry_files_enabled = False
 retry_files_save_path = ~/k8s-installer-retries

From 9010498b9ea50fd3f54b137bcde40f41c2b4576a Mon Sep 17 00:00:00 2001
From: ss75710541 <75710541@qq.com>
Date: Thu, 9 Sep 2021 17:43:27 +0800
Subject: [PATCH 2/4] =?UTF-8?q?add=20docs/=E4=BD=BF=E7=94=A8=E5=85=AC?=
 =?UTF-8?q?=E6=9C=89=E4=BA=91=E8=B7=A8=E6=9C=BA=E6=88=BF=E5=85=AC=E7=BD=91?=
 =?UTF-8?q?=E6=90=AD=E5=BB=BAk8s=201.21.6.md?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ...\221\346\220\255\345\273\272k8s 1.21.6.md" | 104 ++++++++++++++++++
 1 file changed, 104 insertions(+)
 create mode 100644 "docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md"

diff --git "a/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md" "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md"
new file mode 100644
index 0000000..c868282
--- /dev/null
+++ "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md"	
@@ -0,0 +1,104 @@
+# 使用公有云跨机房公网搭建k8s 1.21.6
+
+## 下载 ansible-k8s
+
+```
+yum install -y git ansible
+mkdir -p /data
+cd /data
+git clone https://github.com/paradeum-team/ansible-k8s.git
+```
+
+## 修改 ansible 配置
+
+```
+cd /data/ansible-k8s/
+cp ansible.hosts.ha.tpl ansible.hosts.tmp
+```
+
+修改 `ansible.hosts.tmp` 如下
+
+```
+[k8sCluster:children]
+masters
+nodes
+install
+new_nodes
+
+# Set variables common for all k8s-cluster hosts
+[k8sCluster:vars]
+# SSH user, this user should allow ssh based auth without requiring a password
+ansible_ssh_user=root
+ansible_port=22
+
+# If ansible_ssh_user is not root, ansible_become must be set to true
+ansible_become=false
+
+# 是否更新操作系统及内核
+is_system_update=True
+
+# 默认节点不是公网节点
+public_network_node = True
+
+# 是否开启 flannel
+flannel_enable=True
+
+# api server 域名
+master_vip="172.26.117.104"
+master_vip_advertise_address="8.142.71.35"
+node_domain=solarfs.k8s
+install_domain=install.{{node_domain}}
+api_server_domain="api-server.{{node_domain}}"
+api_server_port="6443"
+
+# k8s 版本
+k8s_version=1.21.6
+# 定义外部镜像仓库
+registry_domain=registry.hisun.netwarps.com
+registry_repo="{{registry_domain}}"
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="docker.io/coredns"
+coredns_image_tag="1.8.4"
+
+# subnet
+service_subnet=10.96.0.0/12
+pod_subnet=10.128.0.0/16
+
+# helm
+helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025
+helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz
+helm_repo_install=False
+#helm_repo_name=chartmuseum-hisun
+#helm_repo=https://charts.apps181227.hisun.k8s
+#helm_username=admin
+#helm_password=12345678
+
+[install]
+master1.solarfs.k8s
+
+[masters]
+master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35"
+
+[nodes]
+node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142"
+
+[new_nodes]
+```
+
+## 配置master1免密登录自己和其它主机
+
+略
+
+## 安装 k8s
+
+所有主机初始化
+
+```
+./base_init.sh
+```
+
+安装 k8s
+
+```
+./install_k8s.sh
+```

From 7c6ce76a5419e482f05c1c6a959f9401ed60b90c Mon Sep 17 00:00:00 2001
From: ss75710541 <liujinye@hisuntech.com>
Date: Thu, 9 Sep 2021 17:38:51 +0800
Subject: [PATCH 3/4] =?UTF-8?q?feat:=20=E6=9B=B4=E6=96=B0=E7=9B=B4?=
 =?UTF-8?q?=E6=8E=A5=E4=BB=8E=E5=A4=96=E9=83=A8=E6=8B=89=E5=8F=96=20k8s=20?=
 =?UTF-8?q?=E9=95=9C=E5=83=8F=E9=95=9C=E5=83=8F?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 ansible.hosts.ha.tpl                          | 17 +++---
 ansible.hosts.tpl                             | 61 +++++++++++++++++++
 config.cfg.example                            |  3 -
 roles/k8s-masters/tasks/main.yml              |  1 +
 .../k8s-masters/templates/kube-flannel.yml.j2 |  4 +-
 .../templates/kubeadm-init.yaml.j2            |  8 +--
 roles/k8s-services/tasks/installHelm.yml      |  7 ++-
 7 files changed, 80 insertions(+), 21 deletions(-)
 create mode 100644 ansible.hosts.tpl

diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl
index a9daa83..ba8935f 100644
--- a/ansible.hosts.ha.tpl
+++ b/ansible.hosts.ha.tpl
@@ -8,7 +8,6 @@ new_nodes
 [k8sCluster:vars]
 # SSH user, this user should allow ssh based auth without requiring a password
 ansible_ssh_user=root
-ansible_ssh_pass=12345678
 ansible_port=22
 
 # If ansible_ssh_user is not root, ansible_become must be set to true
@@ -32,11 +31,15 @@ api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.21.1
+k8s_version=1.21.6
 # 定义外部镜像仓库
-registry_domain=registry.hisun.netwarps.com
-registry_repo="{{registry_domain}}" 
-pod_infra_container_image="{{registry_repo}}/google_containers/pause:3.2"
+registry_domain=docker.io
+registry_repo="{{registry_domain}}"
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="{{registry_repo}}/coredns"
+coredns_image_tag="1.8.4"
+flannel_image_repo="quay.io"
+flannel_image_tag="v0.14.0"
 
 # subnet
 service_subnet=10.96.0.0/12
@@ -45,10 +48,6 @@ pod_subnet=10.128.0.0/16
 # helm
 helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025
 helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz
-helm_repo_name=chartmuseum-hisun
-helm_repo=https://charts.apps181227.hisun.k8s
-helm_username=admin
-helm_password=12345678
 
 [install]
 master1.solarfs.k8s
diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl
new file mode 100644
index 0000000..d63a071
--- /dev/null
+++ b/ansible.hosts.tpl
@@ -0,0 +1,61 @@
+[k8sCluster:children]
+masters
+nodes
+install
+new_nodes
+
+# Set variables common for all k8s-cluster hosts
+[k8sCluster:vars]
+# SSH user, this user should allow ssh based auth without requiring a password
+ansible_ssh_user=root
+ansible_port=22
+
+# If ansible_ssh_user is not root, ansible_become must be set to true
+ansible_become=false
+
+# 是否更新操作系统及内核
+is_system_update=True
+
+# 默认节点不是公网节点
+public_network_node = True
+
+# 是否开启 flannel
+flannel_enable=True
+
+# api server 域名
+master_vip="172.26.117.104"
+master_vip_advertise_address="8.142.71.35"
+node_domain=solarfs.k8s
+install_domain=install.{{node_domain}}
+api_server_domain="api-server.{{node_domain}}"
+api_server_port="6443"
+
+# k8s 版本
+k8s_version=1.21.6
+# 定义外部镜像仓库
+registry_domain=registry.hisun.netwarps.com
+registry_repo="{{registry_domain}}" 
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="docker.io/coredns"
+coredns_image_tag="1.8.4"
+flannel_image_repo="quay.io"
+flannel_image_tag="v0.14.0"
+
+# subnet
+service_subnet=10.96.0.0/12
+pod_subnet=10.128.0.0/16
+
+# helm
+helm_binary_md5=24b16800f8c7f44b5dd128e3355ecf1b
+helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.3-linux-amd64.tar.gz
+
+[install]
+master1.solarfs.k8s
+
+[masters]
+master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35"
+
+[nodes]
+node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142"
+
+[new_nodes]
diff --git a/config.cfg.example b/config.cfg.example
index 90456d1..5aafc35 100644
--- a/config.cfg.example
+++ b/config.cfg.example
@@ -10,9 +10,6 @@ LOCAL_ENNAME=eth0 ## Need to check
 # 是否线下安装
 is_offline=False
 
-# 负载均衡 IP
-master_vip="172.26.181.239"
-
 # 主机环境实际使用的上游dns server，酌情修改,公司内部实际使用的dns服务ip地址,poc建议为空（公司测试环境不建议配置，因为配置后所有节点就都可以访问外网，>有可能影响线>下包测试的准确性）
 upstream_dns_ips="172.26.164.91"
 
diff --git a/roles/k8s-masters/tasks/main.yml b/roles/k8s-masters/tasks/main.yml
index 8aa0ccb..e696b21 100644
--- a/roles/k8s-masters/tasks/main.yml
+++ b/roles/k8s-masters/tasks/main.yml
@@ -9,3 +9,4 @@
 - include: kube-flannel.yml
   when: hostvars[ groups['masters'][0] ].inventory_hostname == inventory_hostname and flannel_enable == True
 - include: kubedns.yml
+  when: public_network_node == False
diff --git a/roles/k8s-masters/templates/kube-flannel.yml.j2 b/roles/k8s-masters/templates/kube-flannel.yml.j2
index ecf0994..18d99db 100644
--- a/roles/k8s-masters/templates/kube-flannel.yml.j2
+++ b/roles/k8s-masters/templates/kube-flannel.yml.j2
@@ -166,7 +166,7 @@ spec:
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: {{registry_repo}}/coreos/flannel:v0.14.0
+        image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}}
         command:
         - cp
         args:
@@ -180,7 +180,7 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: {{registry_repo}}/coreos/flannel:v0.14.0
+        image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}}
         command:
         - /opt/bin/flanneld
         args:
diff --git a/roles/k8s-masters/templates/kubeadm-init.yaml.j2 b/roles/k8s-masters/templates/kubeadm-init.yaml.j2
index 2335e36..b42c892 100644
--- a/roles/k8s-masters/templates/kubeadm-init.yaml.j2
+++ b/roles/k8s-masters/templates/kubeadm-init.yaml.j2
@@ -25,8 +25,8 @@ controlPlaneEndpoint: {{api_server_domain}}:{{api_server_port}}
 controllerManager: {}
 dns:
   type: CoreDNS
-  #imageRepository: {{registry_repo}}
-  imageTag: 1.8.0
+  imageRepository: {{coredns_image_repo}}
+  imageTag: {{coredns_image_tag}}
 etcd:
   local:
     dataDir: /var/lib/etcd
@@ -34,9 +34,9 @@ etcd:
       listen-metrics-urls: http://0.0.0.0:2381
       listen-peer-urls: https://0.0.0.0:2380
       listen-client-urls: https://0.0.0.0:2379
-imageRepository: {{registry_repo}}/google_containers
+imageRepository: {{kubeadm_registry_repo}}/google_containers
 kind: ClusterConfiguration
-kubernetesVersion: 1.21.0
+kubernetesVersion: {{k8s_version}}
 networking:
   dnsDomain: cluster.local
   serviceSubnet: {{service_subnet}}
diff --git a/roles/k8s-services/tasks/installHelm.yml b/roles/k8s-services/tasks/installHelm.yml
index 5fced64..4860973 100644
--- a/roles/k8s-services/tasks/installHelm.yml
+++ b/roles/k8s-services/tasks/installHelm.yml
@@ -31,7 +31,8 @@
   when: check_helm_ret.rc == 2
   tags: helm
 
-- name: add helm private repo
-  command: helm repo add  {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password  {{helm_password}} --insecure-skip-tls-verify
-  tags: helm
+#- name: add helm private repo
+#  command: helm repo add  {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password  {{helm_password}} --insecure-skip-tls-verify
+#  when: helm_repo_install == False
+#  tags: helm
   

From 160624c7de22ad13aad886d9d16bd5cde1919718 Mon Sep 17 00:00:00 2001
From: ss75710541 <75710541@qq.com>
Date: Thu, 9 Sep 2021 19:04:29 +0800
Subject: [PATCH 4/4] fix: ansible.hosts.tpl

---
 ansible.hosts.ha.tpl                          |  2 +-
 ansible.hosts.tpl                             |  2 +-
 ...\221\346\220\255\345\273\272k8s 1.21.4.md" | 63 ++++++++++++++++++-
 3 files changed, 62 insertions(+), 5 deletions(-)
 rename "docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md" => "docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md" (55%)

diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl
index ba8935f..7a9e1be 100644
--- a/ansible.hosts.ha.tpl
+++ b/ansible.hosts.ha.tpl
@@ -31,7 +31,7 @@ api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.21.6
+k8s_version=1.21.4
 # 定义外部镜像仓库
 registry_domain=docker.io
 registry_repo="{{registry_domain}}"
diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl
index d63a071..2f2eda4 100644
--- a/ansible.hosts.tpl
+++ b/ansible.hosts.tpl
@@ -31,7 +31,7 @@ api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.21.6
+k8s_version=1.21.4
 # 定义外部镜像仓库
 registry_domain=registry.hisun.netwarps.com
 registry_repo="{{registry_domain}}" 
diff --git "a/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md" "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md"
similarity index 55%
rename from "docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md"
rename to "docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md"
index c868282..714dc74 100644
--- "a/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.6.md"	
+++ "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md"	
@@ -1,4 +1,4 @@
-# 使用公有云跨机房公网搭建k8s 1.21.6
+# 使用公有云跨机房公网搭建k8s 1.21.4
 
 ## 下载 ansible-k8s
 
@@ -13,7 +13,7 @@ git clone https://github.com/paradeum-team/ansible-k8s.git
 
 ```
 cd /data/ansible-k8s/
-cp ansible.hosts.ha.tpl ansible.hosts.tmp
+cp ansible.hosts.tpl ansible.hosts.tmp
 ```
 
 修改 `ansible.hosts.tmp` 如下
@@ -52,7 +52,7 @@ api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.21.6
+k8s_version=1.21.4
 # 定义外部镜像仓库
 registry_domain=registry.hisun.netwarps.com
 registry_repo="{{registry_domain}}"
@@ -102,3 +102,60 @@ node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142"
 ```
 ./install_k8s.sh
 ```
+
+## 安装 ingress-nginx
+
+```
+mkdir ~/ingress
+cd ~/ingress
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+helm repo update
+helm pull ingress-nginx/ingress-nginx --version 3.29.0
+```
+
+创建 values.yaml
+
+```
+controller:
+  hostNetwork: true
+  service:
+    externalIPs:
+      - 172.26.117.104
+  image:
+    repository: registry.hisun.netwarps.com/bitnami/nginx-ingress-controller
+    tag: 0.44.0
+    digest: sha256:278ad67a8f9f2008d213c86c43c3f37f69ccdecfded91bf57aaab3e4cd6ebc58
+  admissionWebhooks:
+    patch:
+      image:
+        repository:  registry.hisun.netwarps.com/jettech/kube-webhook-certgen
+  kind: DaemonSet
+  tolerations:
+    - effect: NoSchedule
+      operator: Exists
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: node-role.kubernetes.io/master
+            operator: Exists
+```
+
+安装 ingress
+
+```
+helm upgrade --install ingress-nginx ingress-nginx-3.29.0.tgz -f values.yaml -n ingress --create-namespace
+```
+
+检测安装的版本
+
+```
+# 查看pod状态
+kubectl get pod -n ingress
+# 获取pod name
+POD_NAME=$(kubectl get pods -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}' -n ingress)
+kubectl exec -it $POD_NAME -- /nginx-ingress-controller --version  -n ingress
+# 查看 版本
+kubectl -n ingress exec -it $POD_NAME -- /nginx-ingress-controller --version
+```
\ No newline at end of file