-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathfull-cluster.yaml
220 lines (210 loc) · 9.3 KB
/
full-cluster.yaml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
---
- hosts: k8s-global-group
become: true
become_user: root
tasks:
- name: install rsync, needed for ansible synchronize module
yum: name=rsync,unzip state=present
- name: install docker
yum: name=docker state=present
- name: enable docker service
service: name=docker enabled=yes state=started
- name: enable docker systemd memory accounting
command: systemctl set-property docker.service MemoryAccounting=yes
- name: enable docker systemd cpu accounting
command: systemctl set-property docker.service CPUAccounting=yes
- name: add kubernetes repository file
copy: src=kubernetes.repo dest=/etc/yum.repos.d/
- name: set necessary kernel parameter
copy: src=k8s_sysctl.conf dest=/etc/sysctl.d/k8s.conf
- name: apply kernel parameter
command: sysctl --system
- name: install k8s packages
yum: name=kubelet,kubeadm,kubectl,etcd state=present
- name: enable k8s services
service: name=kubelet enabled=yes state=stopped
- name: remove kubelet pki stuff
file: name=/var/lib/kubelet/pki state=absent
- name: stop firewalld and disable it
service: name=firewalld enabled=no state=stopped
- name: naughty selinux stopping us from using /var/lib/etcd, calico install-cni stops too
command: setenforce 0
- name: disable selinux permanently
replace:
dest: /etc/selinux/config
regexp: 'SELINUX=enforcing'
replace: 'SELINUX=disabled'
- name: etcd config file for systemd
template:
src: etcd.conf
dest: /etc/etcd/etcd.conf
- name: replace kubedns ip
replace:
dest: /etc/systemd/system/kubelet.service.d/10-kubeadm.conf
regexp: '10.96.0.10'
replace: '10.40.41.10'
backup: yes
- name: reload systemd units
command: systemctl daemon-reload
- hosts: k8s-masters
become: true
become_user: root
tasks:
- lineinfile:
name: /etc/fstab
state: absent
regexp: 'swap'
- name: disable swap for now (should be done initially if possible. k8s needs that)
command: swapoff -a
- name: create yaml file directory
file: name=/root/k8s-bootstrap state=directory
- name: place kubeadm config somewhere nice
template:
src: kubeadm_config.yaml
dest: /root/k8s-bootstrap
- name: create etcd pki directory
file: name=/etc/kubernetes/pki/etcd state=directory
- name: copy ca-csr.json to etcd pki directory
copy: src=ca-csr.json dest=/etc/kubernetes/pki/etcd/ca-csr.json
- name: copy ca-config.json to etcd pki directory
copy: src=ca-config.json dest=/etc/kubernetes/pki/etcd/ca-config.json
- name: copy client.json to etcd pki directory
copy: src=client.json dest=/etc/kubernetes/pki/etcd/client.json
- name: copy peer config.json to etcd dir
template: src=config.json dest=/etc/kubernetes/pki/etcd/config.json
- name: place cfssl on machines
copy: src=cfssl.tar dest=/tmp
- name: place cfssljson on machines
command: tar xfv /tmp/cfssl.tar -C /usr/local/bin/
- hosts: k8s-masters[0]
become: true
become_user: root
tasks:
# Note: "command:" doesn't work here, maybe cfssljson doesn't like it - didn't get deeper into this
- name: generate ca-certs
shell: /usr/local/bin/cfssl gencert -initca ca-csr.json | /usr/local/bin/cfssljson -bare ca -
args:
chdir: /etc/kubernetes/pki/etcd/
creates: /etc/kubernetes/pki/etcd/ca.pem
# Note: "command:" doesn't work here, maybe cfssljson doesn't like it - didn't get deeper into this, cfssl generates but fails with a warning...
- name: generate client certs
shell: /usr/local/bin/cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=client client.json | /usr/local/bin/cfssljson -bare client
ignore_errors: yes
args:
chdir: /etc/kubernetes/pki/etcd/
creates: /etc/kubernetes/pki/etcd/client.pem
- name: pull generated remote ca.pem for distribution
synchronize: src=/etc/kubernetes/pki/etcd/ca.pem dest=/tmp/etcpkifiles/ mode=pull
- name: pull generated remoteca-key.pem for distribution
synchronize: src=/etc/kubernetes/pki/etcd/ca-key.pem dest=/tmp/etcpkifiles/ mode=pull
- name: pull generated client.pem for distribution
synchronize: src=/etc/kubernetes/pki/etcd/client.pem dest=/tmp/etcpkifiles/ mode=pull
- name: pull generated client-key.pem for distribution
synchronize: src=/etc/kubernetes/pki/etcd/client-key.pem dest=/tmp/etcpkifiles/ mode=pull
- name: pull generated ca.pem for distribution
synchronize: src=/etc/kubernetes/pki/etcd/ca.pem dest=/tmp/etcpkifiles/ mode=pull
- hosts: k8s-masters[1],k8s-masters[2]
become: true
become_user: root
tasks:
- name: copy etcd pki files to other masters
synchronize: src=/tmp/etcpkifiles/ dest=/etc/kubernetes/pki/etcd/ mode=push
- name: copy peer config.json to etcd dir
template: src=config.json dest=/etc/kubernetes/pki/etcd/config.json
- hosts: k8s-masters
become: true
become_user: root
tasks:
- name: generate more certs...
shell: /usr/local/bin/cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=server config.json | /usr/local/bin/cfssljson -bare server
ignore_errors: yes
args:
chdir: /etc/kubernetes/pki/etcd/
creates: /etc/kubernetes/pki/etcd/server.pem
- name: generate more certs...2
shell: /usr/local/bin/cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=peer config.json | /usr/local/bin/cfssljson -bare peer
ignore_errors: yes
args:
chdir: /etc/kubernetes/pki/etcd/
creates: /etc/kubernetes/pki/etcd/peer.pem
- name: make certs readable by etcd
file: name=/etc/kubernetes/pki/etcd owner=etcd group=etcd recurse=yes
- name: start etcd here
service: name=etcd state=started
- hosts: k8s-masters[0]
become: true
become_user: root
tasks:
- name: stop kubelet for a moment...so kubeadm can start it :)
service: name=kubelet state=stopped
- name: run kubeadm init - yes, we need to wait here, otherwise kubeadm will NOT create the cluster-info configmap in kube-public namespace
shell: kubeadm init --config k8s-bootstrap/kubeadm_config.yaml chdir=/root ignore_errors=yes
#async: 30
#poll: 0
ignore_errors: yes
args:
creates: /etc/kubernetes/manifests/kube-apiserver.yaml
register: init_logi
- debug: var=init_logi
- name: copy file from remote ca.crt
synchronize: src=/etc/kubernetes/pki/ca.crt dest=/tmp/k8spkifiles/ mode=pull
- name: copy file from remote ca.key
synchronize: src=/etc/kubernetes/pki/ca.key dest=/tmp/k8spkifiles/ mode=pull
- name: copy file from remote sa.key
synchronize: src=/etc/kubernetes/pki/sa.key dest=/tmp/k8spkifiles/ mode=pull
- name: copy file from remote sa.pub
synchronize: src=/etc/kubernetes/pki/sa.pub dest=/tmp/k8spkifiles/ mode=pull
- hosts: k8s-masters
become: true
become_user: root
tasks:
- name: copry k8s pki files to other masters
synchronize: src=/tmp/k8spkifiles/ dest=/etc/kubernetes/pki/ mode=push
- hosts: k8s-masters[1],k8s-masters[2]
become: true
become_user: root
tasks:
- name: stop kubelet for a moment...
service: name=kubelet state=stopped
- name: run kubeadm init on the remaining masters
shell: kubeadm init --config k8s-bootstrap/kubeadm_config.yaml chdir=/root ignore_errors=yes
async: 30
poll: 0
args:
creates: /etc/kubernetes/manifests/kube-apiserver.yaml
ignore_errors: yes
- hosts: k8s-masters[0]
become: true
become_user: root
tasks:
- name: create base64 encoded version of etcd client-key.pem for usage in calico.yaml template
shell: cat /etc/kubernetes/pki/etcd/client-key.pem | base64 -w 0
register: calico_etcd_client_key_output
- name: create base64 encoded version of etcd client.pem for usage in calico.yaml template
shell: cat /etc/kubernetes/pki/etcd/client.pem | base64 -w 0
register: calico_etcd_client_pem_output
- name: create base64 encoded version of etcd ca.pem for usage in calico.yaml template
shell: cat /etc/kubernetes/pki/etcd/ca.pem | base64 -w 0
register: calico_etcd_ca_pem_output
- name: wait until all masters are up, then apply kube-proxy via kubeadm (tried this out, only works if all master are up)
shell: kubectl --kubeconfig /etc/kubernetes/admin.conf get pods -n kube-system | grep apiserver | wc -l
register: task_result
until: task_result.stdout == "3"
retries: 120
delay: 3
ignore_errors: yes
- name: need to taint the first master to get calico running
shell: kubectl --kubeconfig=/etc/kubernetes/admin.conf taint nodes --all node-role.kubernetes.io/master-
ignore_errors: yes
- name: place calico.yaml somewhere nice
template:
src: calico.yaml
dest: /root/k8s-bootstrap
- name: place calico-rbac.yaml somewhere nice
template:
src: calico-rbac.yaml
dest: /root/k8s-bootstrap
- name: start kube-proxy, it's an alpha addon in kubeadm 1.9
command: kubeadm alpha phase addon kube-proxy
- name: Deploy cni (calico)
command: kubectl create --kubeconfig=/etc/kubernetes/admin.conf -f /root/k8s-bootstrap/calico-rbac.yaml -f /root/k8s-bootstrap/calico.yaml