Merge pull request #1 from paradeum-team/jyliu

feat: 更新直接从外部拉取 k8s 镜像镜像, 优化部署参数，添加文档
paradeum-team · Sep 9, 2021 · 34756d2 · 34756d2
2 parents 8de2fa5 + 160624c
commit 34756d2
Show file tree

Hide file tree

Showing 9 changed files with 242 additions and 22 deletions.
diff --git a/ansible.cfg b/ansible.cfg
@@ -10,7 +10,7 @@
 log_path = ~/k8s-ansible.log
 
 # Additional default options for OpenShift Ansible
-forks = 20
+forks = 50
 host_key_checking = False
 retry_files_enabled = False
 retry_files_save_path = ~/k8s-installer-retries

diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl
@@ -8,7 +8,6 @@ new_nodes
 [k8sCluster:vars]
 # SSH user, this user should allow ssh based auth without requiring a password
 ansible_ssh_user=root
-ansible_ssh_pass=12345678
 ansible_port=22
 
 # If ansible_ssh_user is not root, ansible_become must be set to true
@@ -32,11 +31,15 @@ api_server_domain="api-server.{{node_domain}}"
 api_server_port="6443"
 
 # k8s 版本
-k8s_version=1.21.1
+k8s_version=1.21.4
 # 定义外部镜像仓库
-registry_domain=registry.hisun.netwarps.com
-registry_repo="{{registry_domain}}" 
-pod_infra_container_image="{{registry_repo}}/google_containers/pause:3.2"
+registry_domain=docker.io
+registry_repo="{{registry_domain}}"
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="{{registry_repo}}/coredns"
+coredns_image_tag="1.8.4"
+flannel_image_repo="quay.io"
+flannel_image_tag="v0.14.0"
 
 # subnet
 service_subnet=10.96.0.0/12
@@ -45,10 +48,6 @@ pod_subnet=10.128.0.0/16
 # helm
 helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025
 helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz
-helm_repo_name=chartmuseum-hisun
-helm_repo=https://charts.apps181227.hisun.k8s
-helm_username=admin
-helm_password=12345678
 
 [install]
 master1.solarfs.k8s

diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl
@@ -0,0 +1,61 @@
+[k8sCluster:children]
+masters
+nodes
+install
+new_nodes
+
+# Set variables common for all k8s-cluster hosts
+[k8sCluster:vars]
+# SSH user, this user should allow ssh based auth without requiring a password
+ansible_ssh_user=root
+ansible_port=22
+
+# If ansible_ssh_user is not root, ansible_become must be set to true
+ansible_become=false
+
+# 是否更新操作系统及内核
+is_system_update=True
+
+# 默认节点不是公网节点
+public_network_node = True
+
+# 是否开启 flannel
+flannel_enable=True
+
+# api server 域名
+master_vip="172.26.117.104"
+master_vip_advertise_address="8.142.71.35"
+node_domain=solarfs.k8s
+install_domain=install.{{node_domain}}
+api_server_domain="api-server.{{node_domain}}"
+api_server_port="6443"
+
+# k8s 版本
+k8s_version=1.21.4
+# 定义外部镜像仓库
+registry_domain=registry.hisun.netwarps.com
+registry_repo="{{registry_domain}}" 
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="docker.io/coredns"
+coredns_image_tag="1.8.4"
+flannel_image_repo="quay.io"
+flannel_image_tag="v0.14.0"
+
+# subnet
+service_subnet=10.96.0.0/12
+pod_subnet=10.128.0.0/16
+
+# helm
+helm_binary_md5=24b16800f8c7f44b5dd128e3355ecf1b
+helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.3-linux-amd64.tar.gz
+
+[install]
+master1.solarfs.k8s
+
+[masters]
+master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35"
+
+[nodes]
+node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142"
+
+[new_nodes]
diff --git a/config.cfg.example b/config.cfg.example
@@ -10,9 +10,6 @@ LOCAL_ENNAME=eth0 ## Need to check
 # 是否线下安装
 is_offline=False
 
-# 负载均衡 IP
-master_vip="172.26.181.239"
-
 # 主机环境实际使用的上游dns server，酌情修改,公司内部实际使用的dns服务ip地址,poc建议为空（公司测试环境不建议配置，因为配置后所有节点就都可以访问外网，>有可能影响线>下包测试的准确性）
 upstream_dns_ips="172.26.164.91"
 

diff --git a/docs/使用公有云跨机房公网搭建k8s 1.21.4.md b/docs/使用公有云跨机房公网搭建k8s 1.21.4.md
@@ -0,0 +1,161 @@
+# 使用公有云跨机房公网搭建k8s 1.21.4
+
+## 下载 ansible-k8s
+
+```
+yum install -y git ansible
+mkdir -p /data
+cd /data
+git clone https://github.com/paradeum-team/ansible-k8s.git
+```
+
+## 修改 ansible 配置
+
+```
+cd /data/ansible-k8s/
+cp ansible.hosts.tpl ansible.hosts.tmp
+```
+
+修改 `ansible.hosts.tmp` 如下
+
+```
+[k8sCluster:children]
+masters
+nodes
+install
+new_nodes
+
+# Set variables common for all k8s-cluster hosts
+[k8sCluster:vars]
+# SSH user, this user should allow ssh based auth without requiring a password
+ansible_ssh_user=root
+ansible_port=22
+
+# If ansible_ssh_user is not root, ansible_become must be set to true
+ansible_become=false
+
+# 是否更新操作系统及内核
+is_system_update=True
+
+# 默认节点不是公网节点
+public_network_node = True
+
+# 是否开启 flannel
+flannel_enable=True
+
+# api server 域名
+master_vip="172.26.117.104"
+master_vip_advertise_address="8.142.71.35"
+node_domain=solarfs.k8s
+install_domain=install.{{node_domain}}
+api_server_domain="api-server.{{node_domain}}"
+api_server_port="6443"
+
+# k8s 版本
+k8s_version=1.21.4
+# 定义外部镜像仓库
+registry_domain=registry.hisun.netwarps.com
+registry_repo="{{registry_domain}}"
+kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com"
+coredns_image_repo="docker.io/coredns"
+coredns_image_tag="1.8.4"
+
+# subnet
+service_subnet=10.96.0.0/12
+pod_subnet=10.128.0.0/16
+
+# helm
+helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025
+helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz
+helm_repo_install=False
+#helm_repo_name=chartmuseum-hisun
+#helm_repo=https://charts.apps181227.hisun.k8s
+#helm_username=admin
+#helm_password=12345678
+
+[install]
+master1.solarfs.k8s
+
+[masters]
+master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35"
+
+[nodes]
+node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142"
+
+[new_nodes]
+```
+
+## 配置master1免密登录自己和其它主机
+
+略
+
+## 安装 k8s
+
+所有主机初始化
+
+```
+./base_init.sh
+```
+
+安装 k8s
+
+```
+./install_k8s.sh
+```
+
+## 安装 ingress-nginx
+
+```
+mkdir ~/ingress
+cd ~/ingress
+helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
+helm repo update
+helm pull ingress-nginx/ingress-nginx --version 3.29.0
+```
+
+创建 values.yaml
+
+```
+controller:
+  hostNetwork: true
+  service:
+    externalIPs:
+      - 172.26.117.104
+  image:
+    repository: registry.hisun.netwarps.com/bitnami/nginx-ingress-controller
+    tag: 0.44.0
+    digest: sha256:278ad67a8f9f2008d213c86c43c3f37f69ccdecfded91bf57aaab3e4cd6ebc58
+  admissionWebhooks:
+    patch:
+      image:
+        repository:  registry.hisun.netwarps.com/jettech/kube-webhook-certgen
+  kind: DaemonSet
+  tolerations:
+    - effect: NoSchedule
+      operator: Exists
+  affinity:
+    nodeAffinity:
+      requiredDuringSchedulingIgnoredDuringExecution:
+        nodeSelectorTerms:
+        - matchExpressions:
+          - key: node-role.kubernetes.io/master
+            operator: Exists
+```
+
+安装 ingress
+
+```
+helm upgrade --install ingress-nginx ingress-nginx-3.29.0.tgz -f values.yaml -n ingress --create-namespace
+```
+
+检测安装的版本
+
+```
+# 查看pod状态
+kubectl get pod -n ingress
+# 获取pod name
+POD_NAME=$(kubectl get pods -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}' -n ingress)
+kubectl exec -it $POD_NAME -- /nginx-ingress-controller --version  -n ingress
+# 查看 版本
+kubectl -n ingress exec -it $POD_NAME -- /nginx-ingress-controller --version
+```
diff --git a/roles/k8s-masters/tasks/main.yml b/roles/k8s-masters/tasks/main.yml
@@ -9,3 +9,4 @@
 - include: kube-flannel.yml
   when: hostvars[ groups['masters'][0] ].inventory_hostname == inventory_hostname and flannel_enable == True
 - include: kubedns.yml
+  when: public_network_node == False
diff --git a/roles/k8s-masters/templates/kube-flannel.yml.j2 b/roles/k8s-masters/templates/kube-flannel.yml.j2
@@ -166,7 +166,7 @@ spec:
       serviceAccountName: flannel
       initContainers:
       - name: install-cni
-        image: {{registry_repo}}/coreos/flannel:v0.14.0
+        image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}}
         command:
         - cp
         args:
@@ -180,7 +180,7 @@ spec:
           mountPath: /etc/kube-flannel/
       containers:
       - name: kube-flannel
-        image: {{registry_repo}}/coreos/flannel:v0.14.0
+        image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}}
         command:
         - /opt/bin/flanneld
         args:

diff --git a/roles/k8s-masters/templates/kubeadm-init.yaml.j2 b/roles/k8s-masters/templates/kubeadm-init.yaml.j2
@@ -25,18 +25,18 @@ controlPlaneEndpoint: {{api_server_domain}}:{{api_server_port}}
 controllerManager: {}
 dns:
   type: CoreDNS
-  #imageRepository: {{registry_repo}}
-  imageTag: 1.8.0
+  imageRepository: {{coredns_image_repo}}
+  imageTag: {{coredns_image_tag}}
 etcd:
   local:
     dataDir: /var/lib/etcd
     extraArgs:
       listen-metrics-urls: http://0.0.0.0:2381
       listen-peer-urls: https://0.0.0.0:2380
       listen-client-urls: https://0.0.0.0:2379
-imageRepository: {{registry_repo}}/google_containers
+imageRepository: {{kubeadm_registry_repo}}/google_containers
 kind: ClusterConfiguration
-kubernetesVersion: 1.21.0
+kubernetesVersion: {{k8s_version}}
 networking:
   dnsDomain: cluster.local
   serviceSubnet: {{service_subnet}}

diff --git a/roles/k8s-services/tasks/installHelm.yml b/roles/k8s-services/tasks/installHelm.yml
@@ -31,7 +31,8 @@
   when: check_helm_ret.rc == 2
   tags: helm
 
-- name: add helm private repo
-  command: helm repo add  {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password  {{helm_password}} --insecure-skip-tls-verify
-  tags: helm
+#- name: add helm private repo
+#  command: helm repo add  {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password  {{helm_password}} --insecure-skip-tls-verify
+#  when: helm_repo_install == False
+#  tags: helm