diff --git a/ansible.cfg b/ansible.cfg index 7f89590..31b33c8 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -10,7 +10,7 @@ log_path = ~/k8s-ansible.log # Additional default options for OpenShift Ansible -forks = 20 +forks = 50 host_key_checking = False retry_files_enabled = False retry_files_save_path = ~/k8s-installer-retries diff --git a/ansible.hosts.ha.tpl b/ansible.hosts.ha.tpl index a9daa83..7a9e1be 100644 --- a/ansible.hosts.ha.tpl +++ b/ansible.hosts.ha.tpl @@ -8,7 +8,6 @@ new_nodes [k8sCluster:vars] # SSH user, this user should allow ssh based auth without requiring a password ansible_ssh_user=root -ansible_ssh_pass=12345678 ansible_port=22 # If ansible_ssh_user is not root, ansible_become must be set to true @@ -32,11 +31,15 @@ api_server_domain="api-server.{{node_domain}}" api_server_port="6443" # k8s 版本 -k8s_version=1.21.1 +k8s_version=1.21.4 # 定义外部镜像仓库 -registry_domain=registry.hisun.netwarps.com -registry_repo="{{registry_domain}}" -pod_infra_container_image="{{registry_repo}}/google_containers/pause:3.2" +registry_domain=docker.io +registry_repo="{{registry_domain}}" +kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com" +coredns_image_repo="{{registry_repo}}/coredns" +coredns_image_tag="1.8.4" +flannel_image_repo="quay.io" +flannel_image_tag="v0.14.0" # subnet service_subnet=10.96.0.0/12 @@ -45,10 +48,6 @@ pod_subnet=10.128.0.0/16 # helm helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025 helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz -helm_repo_name=chartmuseum-hisun -helm_repo=https://charts.apps181227.hisun.k8s -helm_username=admin -helm_password=12345678 [install] master1.solarfs.k8s diff --git a/ansible.hosts.tpl b/ansible.hosts.tpl new file mode 100644 index 0000000..2f2eda4 --- /dev/null +++ b/ansible.hosts.tpl @@ -0,0 +1,61 @@ +[k8sCluster:children] +masters +nodes +install +new_nodes + +# Set variables common for all k8s-cluster hosts +[k8sCluster:vars] +# SSH user, this user should allow ssh based auth without requiring a password +ansible_ssh_user=root +ansible_port=22 + +# If ansible_ssh_user is not root, ansible_become must be set to true +ansible_become=false + +# 是否更新操作系统及内核 +is_system_update=True + +# 默认节点不是公网节点 +public_network_node = True + +# 是否开启 flannel +flannel_enable=True + +# api server 域名 +master_vip="172.26.117.104" +master_vip_advertise_address="8.142.71.35" +node_domain=solarfs.k8s +install_domain=install.{{node_domain}} +api_server_domain="api-server.{{node_domain}}" +api_server_port="6443" + +# k8s 版本 +k8s_version=1.21.4 +# 定义外部镜像仓库 +registry_domain=registry.hisun.netwarps.com +registry_repo="{{registry_domain}}" +kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com" +coredns_image_repo="docker.io/coredns" +coredns_image_tag="1.8.4" +flannel_image_repo="quay.io" +flannel_image_tag="v0.14.0" + +# subnet +service_subnet=10.96.0.0/12 +pod_subnet=10.128.0.0/16 + +# helm +helm_binary_md5=24b16800f8c7f44b5dd128e3355ecf1b +helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.3-linux-amd64.tar.gz + +[install] +master1.solarfs.k8s + +[masters] +master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35" + +[nodes] +node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142" + +[new_nodes] diff --git a/config.cfg.example b/config.cfg.example index 90456d1..5aafc35 100644 --- a/config.cfg.example +++ b/config.cfg.example @@ -10,9 +10,6 @@ LOCAL_ENNAME=eth0 ## Need to check # 是否线下安装 is_offline=False -# 负载均衡 IP -master_vip="172.26.181.239" - # 主机环境实际使用的上游dns server,酌情修改,公司内部实际使用的dns服务ip地址,poc建议为空(公司测试环境不建议配置,因为配置后所有节点就都可以访问外网,>有可能影响线>下包测试的准确性) upstream_dns_ips="172.26.164.91" diff --git "a/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md" "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md" new file mode 100644 index 0000000..714dc74 --- /dev/null +++ "b/docs/\344\275\277\347\224\250\345\205\254\346\234\211\344\272\221\350\267\250\346\234\272\346\210\277\345\205\254\347\275\221\346\220\255\345\273\272k8s 1.21.4.md" @@ -0,0 +1,161 @@ +# 使用公有云跨机房公网搭建k8s 1.21.4 + +## 下载 ansible-k8s + +``` +yum install -y git ansible +mkdir -p /data +cd /data +git clone https://github.com/paradeum-team/ansible-k8s.git +``` + +## 修改 ansible 配置 + +``` +cd /data/ansible-k8s/ +cp ansible.hosts.tpl ansible.hosts.tmp +``` + +修改 `ansible.hosts.tmp` 如下 + +``` +[k8sCluster:children] +masters +nodes +install +new_nodes + +# Set variables common for all k8s-cluster hosts +[k8sCluster:vars] +# SSH user, this user should allow ssh based auth without requiring a password +ansible_ssh_user=root +ansible_port=22 + +# If ansible_ssh_user is not root, ansible_become must be set to true +ansible_become=false + +# 是否更新操作系统及内核 +is_system_update=True + +# 默认节点不是公网节点 +public_network_node = True + +# 是否开启 flannel +flannel_enable=True + +# api server 域名 +master_vip="172.26.117.104" +master_vip_advertise_address="8.142.71.35" +node_domain=solarfs.k8s +install_domain=install.{{node_domain}} +api_server_domain="api-server.{{node_domain}}" +api_server_port="6443" + +# k8s 版本 +k8s_version=1.21.4 +# 定义外部镜像仓库 +registry_domain=registry.hisun.netwarps.com +registry_repo="{{registry_domain}}" +kubeadm_registry_repo="registry.cn-hangzhou.aliyuncs.com" +coredns_image_repo="docker.io/coredns" +coredns_image_tag="1.8.4" + +# subnet +service_subnet=10.96.0.0/12 +pod_subnet=10.128.0.0/16 + +# helm +helm_binary_md5=e4500993ba21e5e6bdfbc084b4342025 +helm_binary_url=https://pnode.solarfs.io/dn/file/{{helm_binary_md5}}/helm-v3.6.0-linux-amd64.tar.gz +helm_repo_install=False +#helm_repo_name=chartmuseum-hisun +#helm_repo=https://charts.apps181227.hisun.k8s +#helm_username=admin +#helm_password=12345678 + +[install] +master1.solarfs.k8s + +[masters] +master1.solarfs.k8s ansible_host=172.26.117.104 advertise_address="8.142.71.35" + +[nodes] +node1.solarfs.k8s ansible_host=8.142.71.142 advertise_address="8.142.71.142" + +[new_nodes] +``` + +## 配置master1免密登录自己和其它主机 + +略 + +## 安装 k8s + +所有主机初始化 + +``` +./base_init.sh +``` + +安装 k8s + +``` +./install_k8s.sh +``` + +## 安装 ingress-nginx + +``` +mkdir ~/ingress +cd ~/ingress +helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx +helm repo update +helm pull ingress-nginx/ingress-nginx --version 3.29.0 +``` + +创建 values.yaml + +``` +controller: + hostNetwork: true + service: + externalIPs: + - 172.26.117.104 + image: + repository: registry.hisun.netwarps.com/bitnami/nginx-ingress-controller + tag: 0.44.0 + digest: sha256:278ad67a8f9f2008d213c86c43c3f37f69ccdecfded91bf57aaab3e4cd6ebc58 + admissionWebhooks: + patch: + image: + repository: registry.hisun.netwarps.com/jettech/kube-webhook-certgen + kind: DaemonSet + tolerations: + - effect: NoSchedule + operator: Exists + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: node-role.kubernetes.io/master + operator: Exists +``` + +安装 ingress + +``` +helm upgrade --install ingress-nginx ingress-nginx-3.29.0.tgz -f values.yaml -n ingress --create-namespace +``` + +检测安装的版本 + +``` +# 查看pod状态 +kubectl get pod -n ingress +# 获取pod name +POD_NAME=$(kubectl get pods -l app.kubernetes.io/name=ingress-nginx -o jsonpath='{.items[0].metadata.name}' -n ingress) +kubectl exec -it $POD_NAME -- /nginx-ingress-controller --version -n ingress +# 查看 版本 +kubectl -n ingress exec -it $POD_NAME -- /nginx-ingress-controller --version +``` \ No newline at end of file diff --git a/roles/k8s-masters/tasks/main.yml b/roles/k8s-masters/tasks/main.yml index 8aa0ccb..e696b21 100644 --- a/roles/k8s-masters/tasks/main.yml +++ b/roles/k8s-masters/tasks/main.yml @@ -9,3 +9,4 @@ - include: kube-flannel.yml when: hostvars[ groups['masters'][0] ].inventory_hostname == inventory_hostname and flannel_enable == True - include: kubedns.yml + when: public_network_node == False diff --git a/roles/k8s-masters/templates/kube-flannel.yml.j2 b/roles/k8s-masters/templates/kube-flannel.yml.j2 index ecf0994..18d99db 100644 --- a/roles/k8s-masters/templates/kube-flannel.yml.j2 +++ b/roles/k8s-masters/templates/kube-flannel.yml.j2 @@ -166,7 +166,7 @@ spec: serviceAccountName: flannel initContainers: - name: install-cni - image: {{registry_repo}}/coreos/flannel:v0.14.0 + image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}} command: - cp args: @@ -180,7 +180,7 @@ spec: mountPath: /etc/kube-flannel/ containers: - name: kube-flannel - image: {{registry_repo}}/coreos/flannel:v0.14.0 + image: {{flannel_image_repo}}/coreos/flannel:{{flannel_image_tag}} command: - /opt/bin/flanneld args: diff --git a/roles/k8s-masters/templates/kubeadm-init.yaml.j2 b/roles/k8s-masters/templates/kubeadm-init.yaml.j2 index 2335e36..b42c892 100644 --- a/roles/k8s-masters/templates/kubeadm-init.yaml.j2 +++ b/roles/k8s-masters/templates/kubeadm-init.yaml.j2 @@ -25,8 +25,8 @@ controlPlaneEndpoint: {{api_server_domain}}:{{api_server_port}} controllerManager: {} dns: type: CoreDNS - #imageRepository: {{registry_repo}} - imageTag: 1.8.0 + imageRepository: {{coredns_image_repo}} + imageTag: {{coredns_image_tag}} etcd: local: dataDir: /var/lib/etcd @@ -34,9 +34,9 @@ etcd: listen-metrics-urls: http://0.0.0.0:2381 listen-peer-urls: https://0.0.0.0:2380 listen-client-urls: https://0.0.0.0:2379 -imageRepository: {{registry_repo}}/google_containers +imageRepository: {{kubeadm_registry_repo}}/google_containers kind: ClusterConfiguration -kubernetesVersion: 1.21.0 +kubernetesVersion: {{k8s_version}} networking: dnsDomain: cluster.local serviceSubnet: {{service_subnet}} diff --git a/roles/k8s-services/tasks/installHelm.yml b/roles/k8s-services/tasks/installHelm.yml index 5fced64..4860973 100644 --- a/roles/k8s-services/tasks/installHelm.yml +++ b/roles/k8s-services/tasks/installHelm.yml @@ -31,7 +31,8 @@ when: check_helm_ret.rc == 2 tags: helm -- name: add helm private repo - command: helm repo add {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password {{helm_password}} --insecure-skip-tls-verify - tags: helm +#- name: add helm private repo +# command: helm repo add {{helm_repo_name}} {{helm_repo}} --username {{helm_username}} --password {{helm_password}} --insecure-skip-tls-verify +# when: helm_repo_install == False +# tags: helm