-
Notifications
You must be signed in to change notification settings - Fork 24
/
Copy pathocp4-upi-util
executable file
·1870 lines (1732 loc) · 58 KB
/
ocp4-upi-util
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
#!/bin/bash
# Copyright 2019-2020 Robert Krawitz/Red Hat
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
declare config_file=${OCP4_CONFIG_FILE:-}
declare ocp4_pull_secret=${OCP4_PULL_SECRET:-$HOME/.docker/config.json}
declare ocp4_public_key=${OCP4_PUBLIC_KEY:-$HOME/.ssh/id_rsa.pub}
declare command=$0
# The following can be overridden in the config file if needed.
declare bootstrap_prefix=192.168.222
declare bootstrap_ipaddr
declare -i pxe_port=81
declare cluster_cidr=10.128.0.0/14
declare cluster_host_prefix=23
declare cluster_network_type=OpenShiftSDN
declare cluster_service_network=172.30.0.0/16
declare cluster_hyperthreading=Enabled
declare cluster_domain=test.myocp4.com
declare cluster_basedomain
declare cluster_name
declare cluster_network_bridge_name=baremetal
declare cluster_install_dir=${KUBECONFIG:+${KUBECONFIG/\/auth\/kubeconfig/}}
declare filesystem_json=
declare -i masters_schedulable=-1
declare -i infra_count=0
declare -i master_as_infra=-1
declare -r cachedir="$HOME/.cache/ocp4_upi_install"
declare bootstrap_mac=52:54:00:f9:8e:41
declare -a mgmt_masters=()
declare -a master_macs=()
declare -a worker_macs=()
declare -a mgmt_workers=()
declare -a exclude_ifs=()
declare public_interface=
declare bare_metal_interface=
declare -i do_install_kata=0
declare -i do_install_cnv=0
declare -i apply_kata_workaround=1
declare platform
declare installer_image=
declare client_image=
declare local_toolsdir=
declare install_device=sda
declare -i install_retries=2 # Many bare metal nodes take a long time to POST
declare -i require_config_file=1
declare kata_install_label=
declare kata_config_name=example-kataconfig
declare -i update_packages=1
declare -A install_disks=()
declare -i processed_cmdline_vars=0
declare -A known_macaddrs=()
declare -i checked_config=0
declare -A needs_config_file=()
platform=$(uname -i)
# shellcheck disable=SC2155
declare -i start=$(date +%s)
# shellcheck disable=SC2155
declare -r __cr=$(echo -ne '\r')
# shellcheck disable=SC2155
declare -r __nl=$(echo -ne '\n')
# shellcheck disable=SC2155
declare -r command_help=$(expand <<'EOF'
A configuration file is required for all commands noted with `*'.
Commands:
* install <version> Install a baremetal cluster using UPI
* do_install <version> (deprecated) Same as install
install_cnv Install sandboxed containers into cluster
install_kata Install sandboxed containers into cluster
bootstrap_destroy Destroy any (virtual) bootstrap node
* setup_dnsmasq Set up dnsmasq/haproxy configuration
* setup_infra Set up an infra node
migrate_infra Migrate infra components to set up infra node
describe_operator <op> Describe the specified operator
list_operators List available operators
install_operator <op> Install the specified operator
The following commands all use IPMI:
* reset_all Reset all nodes
* reset_masters Reset all master nodes
* reset_workers Reset all worker nodes
* bios_all Boot all nodes into BIOS
* bios_masters Boot master nodes into BIOS
* bios_workers Boot worker nodes into BIOS
* bios_master <index> Boot specified master (by index) into BIOS
* bios_worker <index> Boot specified worker (by index) into BIOS
* pxe_all PXE boot all nodes
* pxe_masters PXE all master nodes
* pxe_workers PXE all worker nodes
* pxe_master <index> PXE specified master (by index)
* pxe_worker <index> PXE specified worker (by index)
* poweroff_all Power off all nodes
* poweroff_masters Power off all masters
* poweroff_workers Power off all workers
* poweroff_master <index> Power off specified master (by index)
* poweroff_worker <index> Power off specified woerker (by index)
* start_workers Start the worker nodes
EOF
)
declare -a simple_commands=()
function __setup_commands() {
local line
while IFS= read -r line ; do
if [[ $line = ' '* ]] ; then
local needs_config=${line:6:1}
line=${line:8}
line=${line%% *}
simple_commands+=("$line")
[[ ${needs_config:-} = '*' ]] && needs_config_file[$line]=1
fi
done <<< "$command_help"
}
function __fatal() {
echo "Fatal Error: $*" 1>&2
exit 1
}
function __doit() {
echo "$*" 1>&2
"$@"
}
function __warn() {
echo "Warning: $*" 1>&2
}
function __usage() {
cmdmsg=
if [[ ${command##*/} = ocp4-upi-util ]] ; then
cmdmsg=' command'
fi
cat 1>&2 <<EOF
Usage: $command [-c <configfile>] [options]$cmdmsg args...
$command_help
Options:
-k public key Public key file (default $ocp4_public_key)
-p pull secret Pull secret (default $ocp4_pull_secret)
-i installdir Install into specified directory
--var=val Override setting from the config file
The config file is bash syntax and is sourced by this installer.
All nodes, including the bootstrap node on which this command is run,
must have the same network topology. There must be one public interface
that is externally accessible (this is disabled on the master and worker
nodes) and one interface on a shared (presumably fast) network that is
isolated physically or virtually. The latter is also known as the
"bare metal interface".
Non-array settings may be provided from the environment.
Mandatory setttings:
master_macs Array of master MAC addresses (should be 1 or 3)
mgmt_masters Array of master management addresses (for IPMI)
worker_macs Array of worker MAC addresses
mgmt_workers Array of worker management addresses (for IPMI)
public_interface Name of external interface device on all nodes
bare_metal_interface Name of the bare metal interface
Options:
cluster_domain Domain name of the cluster
(default $cluster_domain)
pull_secret=<secret> Name of file containing the pull secret to be
used (default $ocp4_pull_secret).
public_key=<key> Name of public key file to be used
(default $ocp4_public_key).
IPMI_USER IPMI username (no default)
IPMI_PASSWORD IPMI password (no default)
exclude_ifs List of additional interfaces that should be
disabled on all nodes
infra_count Number of dedicated infrastructure nodes
(default 0)
master_as_infra<=node>
Use a master node as a dedicated infra node.
If no node is specified, the first master (0)
is used. -1 (default) means no infra node
used.
do_install_kata Install OpenShift sandboxed containers
apply_kata_workaround Apply 4.8 workaround for sandboxed containers
not adding additional CPUs correctly
masters_schedulable Allow user pods to be scheduled on master nodes
(default 1)
bootstrap_mac MAC address of bootstrap VM
(default 52:54:00:f9:8e:41)
cluster_host_prefix 32 - desired bare metal network size (default 23)
filesystem_json File containing code to modify the filesystem
layout
install_device Device (without /dev) to install to (default sda)
install_retries Number of times to retry waiting for bootstrap
to complete; needed for bare metal nodes that
take a long time to boot. Default 2 (90 minutes)
client_image Path to local client image .tgz to be used for oc
installer_image Path to local installer image .tgz
local_toolsdir Location of client_image and installer_image.
Must match the version to be installed.
update_packages Update local packages (default 1)
EOF
exit
}
# shellcheck disable=SC2206
declare -a default_exclude_ifs=(${EXCLUDE_IFS:-})
declare pub_if=${PUB_IF:-}
declare ip_base=192.168.222
declare -i host_base=20
declare -i max_host_count=71
function __bool() {
local OPTIND=0
local yes=1
local no=0
while getopts 'y:t:n:f:Y:T:' opt "$@" ; do
case "$opt" in
y|t) yes="$OPTARG" ;;
n|f) no="$OPTARG" ;;
Y|T) yes="$OPTARG"; no='' ;;
*) ;;
esac
done
local value
for value in "$@" ; do
case "${value,,}" in
''|1|y|yes|tru*) echo "$yes" ;;
*) echo "$no" ;;
esac
done
}
function __parse_size() {
local size
local echoarg=
if [[ $1 = '-n' ]] ; then
echoarg=-n
shift
fi
local sizes=$*
sizes=${sizes//,/ }
for size in $sizes ; do
if [[ $size =~ (-?[[:digit:]]+)([[:alpha:]]*) ]] ; then
local sizen=${BASH_REMATCH[1]}
local size_modifier=${BASH_REMATCH[2],,}
local -i size_multiplier=1
case "$size_modifier" in
''|b) size_multiplier=1 ;;
k|kb|kilobytes) size_multiplier=1000 ;;
ki|kib|kibibytes) size_multiplier=1024 ;;
m|mb|megabytes) size_multiplier=1000000 ;;
mi|mib|mebibytes) size_multiplier=1048576 ;;
g|gb|gigabytes) size_multiplier=1000000000 ;;
gi|gib|gibibytes) size_multiplier=1073741824 ;;
t|tb|terabytes) size_multiplier=1000000000000 ;;
ti|tib|tebibytes) size_multiplier=1099511627776 ;;
*) __fatal "Cannot parse size $size" ;;
esac
echo $echoarg "$((sizen*size_multiplier)) "
else
__fatal "Cannot parse size $size"
fi
done
}
function __parse_optvalues() {
echoarg=
if [[ $1 = '-n' ]] ; then
echoarg=-n
shift
fi
local args=$*
args=${args//,/ }
for arg in $args ; do
echo $echoarg "$arg"
done
}
function __parse_option() {
local option=$1
option=${option## }
option=${option%% }
if [[ $option =~ ^([^=]+)\ *=\ *([^\ ].*)? ]] ; then
option="${BASH_REMATCH[1]}=${BASH_REMATCH[2]}"
fi
[[ -n "$option" ]] || return
local optname
local optvalue
optname=${option%%=*}
optname=${optname,,}
optvalue=${option#*=}
noptname=${optname//-/_}
if [[ $option != *'='* ]] ; then
if [[ $noptname = "no_"* || $optname = "dont_"* || $noptname = "no-"* || $optname = "dont-"* ]] ; then
noptname=${noptname#dont_}
noptname=${noptname#no_}
noptname=${noptname#dont-}
noptname=${noptname#no-}
optvalue=0
else
optvalue=1
fi
fi
local noptname1=${noptname//_/}
echo "$noptname1 $noptname $optvalue"
}
function __process_option() {
local noptname
local noptname1
local optvalue
read -r noptname1 noptname optvalue <<< "$(__parse_option "$1")"
# shellcheck disable=SC1822
# shellcheck disable=SC2206
case "$noptname1" in
help*) __usage ;;
clusterdomain) cluster_domain=$optvalue ;;
ipmiuser) IPMI_USER=$optvalue ;;
ipmipassword) IPMI_PASSWORD=$optvalue ;;
excludeifs) exclude_ifs+=($optvalue) ;;
infracount) infra_count=$(__parse_size "$optvalue") ;;
masterasinfra) master_as_infra=$(__parse_size "$optvalue") ;;
doinstallkata) do_install_kata=$(__bool "$optvalue") ;;
applykatawork*) apply_kata_workaround=$(__bool "$optvalue") ;;
masterssch*) masters_schedulable=$(__bool "$optvalue") ;;
bootstrapmac) bootstrap_mac=$optvalue ;;
clusterhost*) cluster_host_prefix=$(__parse_size "$optvalue") ;;
clusterinst*) cluster_install_dir="$optvalue" ;;
*pub*key) ocp4_public_key="$optvalue" ;;
*pullsecret) ocp4_pull_secret="$optvalue" ;;
filesystemj*) filesystem_json=$optvalue ;;
installdev*) install_device=$optvalue ;;
installret*) install_retries=$(__parse_size "$optvalue") ;;
clientim*) client_image=$optvalue ;;
installerim*) installer_image=$optvalue ;;
localtools*) local_toolsdir=$optvalue ;;
updatepac*) update_packages=$(__bool "$optvalue") ;;
*) eval "$noptname=$optvalue" ;;
esac
}
function __process_cmdline_vars() {
if ((! processed_cmdline_vars)) ; then
local var=
for var in "${cmdline_vars[@]}" ; do
# Really should sanity check this, but this command is not
# intended to run with privilege higher than the command line
# that invoked it.
__process_option "$var"
done
processed_cmdline_vars=1
fi
}
function __store_option() {
local noptname
local noptname1
local optvalue
read -r noptname1 noptname optvalue <<< "$(__parse_option "$1")"
case "$noptname1" in
configfile) config_file="$optvalue" ;;
*) cmdline_vars+=("$1") ;;
esac
}
function __check_macaddr() {
local addr
for addr in "$@" ; do
[[ -z "${known_macaddrs[$addr]}" ]] || __fatal "Duplicate macaddr $addr"
[[ $addr =~ ^[0-9a-f]{2}(:[0-f]{2}){5} ]] || __fatal "Malformed address $addr"
done
}
function __check_config_file() {
((checked_config)) && return
local -i OPTIND=0
local opt
local -i do_check_config=1
while getopts 'yn' opt "$@" ; do
case "$opt" in
y) do_check_config=1 ;;
n) do_check_config=0 ;;
*) ;;
esac
done
if [[ -n "${config_file:-}" ]] || ((do_check_config)) ; then
if [[ -z "${config_file:-}" ]] ; then
# shellcheck disable=SC2016
if ((require_config_file)) ; then
__fatal 'Config file must be specified, either with -c or $OCP4_CONFIG_FILE'
fi
else
if [[ ! -r "$config_file" ]] ; then
__fatal "Cannot read config file $config_file"
fi
# shellcheck disable=SC1090
. "$config_file" || __fatal "Unable to process config file $config_file"
fi
checked_config=1
fi
__process_cmdline_vars
if ((do_check_config)) ; then
# Check configuration validity
(( ${#mgmt_masters[@]} == 1 || ${#mgmt_masters[@]} == 3 )) || __fatal "Configuration must have 1 or 3 masters, actual ${#mgmt_masters[@]}"
(( ${#mgmt_masters[@]} == ${#master_macs[@]} )) || __fatal "Configuration must have same number of mgmt_masters as master_macs"
(( infra_count == 0 || ${#worker_macs[@]} > infra_count )) || __fatal "Configuration must have at least 1 worker_macs in addition to infra nodes"
(( ${#worker_macs[@]} == ${#mgmt_workers[@]} )) || __fatal "Configuration must have as many worker_macs as mgmt_workers, actual ${#worker_macs[@]} and ${#mgmt_workers[@]}"
bootstrap_mac=${bootstrap_mac,,}
master_macs=("${master_macs[@],,}")
worker_macs=("${worker_macs[@],,}")
__check_macaddr "$bootstrap_mac"
__check_macaddr "${master_macs[@]}"
__check_macaddr "${worker_macs[@]}"
[[ -n "$public_interface" ]] || __fatal "public_interface must be specified"
[[ -n "$bare_metal_interface" ]] || __fatal "bare_metal_interface must be specified"
[[ -n "$cluster_domain" ]] || __fatal "cluster_domain must be specified"
[[ -n "$IPMI_USER" ]] || __warn "IPMI_USER is not specified; IPMI may not work correctly."
[[ -n "$IPMI_PASSWORD" ]] || __fatal "IPMI_PASSWORD is not specified; IPMI may not work correctly."
[[ -r "$ocp4_pull_secret" ]] || __fatal "No pull secret!"
[[ -r "$ocp4_public_key" ]] || __fatal "No public key!"
cluster_basedomain=${cluster_basedomain:-${cluster_domain#*.}}
cluster_name=${cluster_name:-${cluster_domain%%.*}}
bootstrap_ipaddr=${bootstrap_prefix}.1
fi
}
function __do_setup_dnsmasq_help() {
cat <<EOF
Usage: $0 [options] domain bootstrap_mac master0_mac [master_macs...] [workers_macs]
Options:
-e exclude Exclude the specified interface (may repeat)
-p pub_if Use the specified interface as the public IF
-N base Use the specified base network (default $ip_base)
-b host_base Start hosts at the specified number (default $host_base)
-n max_hosts Allow the specified maximum number of hosts (default $max_host_count)
-m masters Specify the number of masters (default $master_count)
-i infras Specify the number of infra nodes (default $infra_count)
At least $master_count MAC addrs must be specified
EOF
exit 1
}
function __generate_dnsmasq() {
local -i master_count=$1; shift
local -i worker_count=$1; shift
local -i infra_count=$1; shift
cat <<EOF
listen-address=${ip_base}.${prov_host}
bind-interfaces
strict-order
local=/${ocp4_domain}/
domain=${ocp4_domain}
expand-hosts
#except-interface=lo
#except-interface=virbr0
EOF
for int in "$pub_if" "${exclude_list[@]}" ; do
echo "#except-interface=$int"
done
cat <<EOF
#interface=${cluster_network_bridge_name}
#dhcp-range=${ip_base}.${host_base},${ip_base}.${host_max}
dhcp-range=192.168.222.0,static
dhcp-ignore=tag:!known
dhcp-no-override
dhcp-authoritative
dhcp-lease-max=41
dhcp-host=${bootstrap_mac},${ip_base}.${bootstrap_host},bootstrap
EOF
for ((i = 0; i < master_count; i++)) ; do
printf "dhcp-host=%s,${ip_base}.%d,master-%d\n" "$1" $((master_base+i)) "$i"
shift
done
i=0
for ((i = 0; i < infra_count; i++)) ; do
printf "dhcp-host=%s,${ip_base}.%d,infra-%d\n" "$1" $((infra_base+i)) "$i"
shift
done
i=0
for arg in "$@" ; do
printf "dhcp-host=%s,${ip_base}.%d,worker-%d\n" "$arg" $((worker_base+i)) "$i"
i=$((i+1))
done
tftproot=/var/lib/tftpboot
if [[ $platform = aarch64 ]] ; then
bootfile=BOOTAA64.EFI
else
bootfile=lpxelinux.0
fi
cat <<EOF
enable-tftp
tftp-root=$tftproot
dhcp-boot=$bootfile
address=/api.${ocp4_domain}/${ip_base}.${prov_host}
address=/api-int.${ocp4_domain}/${ip_base}.${prov_host}
address=/.apps.${ocp4_domain}/${ip_base}.${prov_host}
EOF
}
function __generate_service() {
local name=$1
local -i port=$2
local -i master_count=$3
local -i worker_count=$4
local -i infra_count=$5
if [[ $name = worker && $worker_count -eq 0 ]] ; then
name=master
fi
local -i i
case "$name" in
master)
cat <<EOF
server bootstrap ${ip_base}.$((bootstrap_host)):${port} check inter 1s backup
EOF
for ((i = 0; i < master_count; i++)) ; do
cat <<EOF
server master-${i} ${ip_base}.$((master_base+i)):${port} check inter 1s
EOF
done
;;
worker)
for ((i = 0; i < infra_count; i++)) ; do
cat <<EOF
server infra-${i} ${ip_base}.$((infra_base+i)):${port} check inter 1s
EOF
done
if ((masters_schedulable)) ; then
for ((i = 0; i < master_count; i++)) ; do
cat <<EOF
server master-${i} ${ip_base}.$((master_base+i)):${port} check inter 1s
EOF
done
fi
for ((i = 0; i < worker_count; i++)) ; do
cat <<EOF
server worker-${i} ${ip_base}.$((worker_base+i)):${port} check inter 1s
EOF
done
;;
*)
echo "Unknown node type $name" 1>&2
exit 1
;;
esac
}
function __generate_haproxy_1() {
local -i master_count=$1; shift
local -i worker_count=$1; shift
local -i infra_count=$1; shift
local LINE
while IFS= read -r 'LINE' ; do
if [[ $LINE = '# --- BEGIN OCP4-UPI ---' ]] ; then
break
fi
echo "$LINE"
done < /etc/haproxy/haproxy.cfg
cat <<EOF
# --- BEGIN OCP4-UPI ---
#-----------------
# OCP4-UPI CONFIG
#-----------------
frontend kapi
mode tcp
bind *:6443
default_backend kapi
frontend mc
mode tcp
bind *:22623
default_backend mc
frontend https
mode tcp
bind *:443
default_backend https
frontend http
mode http
bind *:80
default_backend http
frontend ingress
mode http
bind *:1936
default_backend ingress
backend kapi
mode tcp
balance roundrobin
$(__generate_service master 6443 "$master_count" "$worker_count" "$infra_count")
backend mc
mode tcp
balance roundrobin
$(__generate_service master 22623 "$master_count" "$worker_count" "$infra_count")
backend https
mode tcp
balance roundrobin
$(__generate_service worker 443 "$master_count" "$worker_count" "$infra_count")
backend http
mode http
balance roundrobin
$(__generate_service worker 80 "$master_count" "$worker_count" "$infra_count")
backend ingress
mode http
balance roundrobin
$(__generate_service worker 1936 "$master_count" "$worker_count" "$infra_count")
EOF
}
function __generate_haproxy() {
local output
# If something fails during generation of the haproxy
# configuration, we won't inadvertently wind up with a
# broken file.
output="$(__generate_haproxy_1 "$@")"
echo "$output"
}
function __do_setup_dnsmasq() {
local -i master_count=3
local -i infra_count=0
local -i worker_count=0
local -a exclude_list=()
OPTIND=0
while getopts "e:p:N:b:n:m:i:" opt "$@"; do
case "$opt" in
e) exclude_list+=("$OPTARG") ;;
p) pub_if=$OPTARG ;;
N) ip_base=$OPTARG ;;
b) host_base=$OPTARG ;;
n) max_host_count=$OPTARG ;;
m) master_count=$OPTARG ;;
i) infra_count=$OPTARG ;;
*) __do_setup_dnsmasq_help ;;
esac
done
(( ${#exclude_list[@]} )) || exclude_list=("${default_exclude_ifs[@]}")
shift $((OPTIND-1))
local ocp4_domain=$1
shift
[[ -z "$pub_if" ]] && __fatal "pub_if must be set to the public IP address on this node"
(( $# < master_count + infra_count + 1 )) && __do_setup_dnsmasq_help
[[ -f /etc/dnsmasq.conf ]] || __fatal "dnsmasq does not appear to be installed; please install!"
[[ -n "$ocp4_domain" ]] || __fatal "domain must be specified"
badaddr=0
for addr in "$@" ; do
if [[ ! $addr =~ ([0-9a-f]{2}:){5}[0-9a-f]{2} ]] ; then
echo "$addr is not a valid MAC"
badaddr=1
fi
done
(( badaddr )) && exit 1
local bootstrap_mac=$1
shift
[[ -d /etc/dnsmasq.d ]] && mkdir -p /etc/dnsmasq.d
if grep -q '^conf-dir=.*/etc/dnsmasq.d' /etc/dnsmasq.conf ; then
:
else
echo 'conf-dir=/etc/dnsmasq.d' >> /etc/dnsmasq.conf
fi
local prov_host=1
host_max=$((host_base+max_host_count-1))
bootstrap_host=$((host_base+max_host_count-1))
master_base=$((host_base+0))
infra_base=$((host_base+5))
worker_base=$((master_base+10))
worker_count=$(($# - master_count - infra_count))
__generate_dnsmasq "$master_count" "$worker_count" "$infra_count" "$@" > /etc/dnsmasq.d/ocp &&
__generate_haproxy "$master_count" "$worker_count" "$infra_count" "$@" > /etc/haproxy/haproxy.cfg.new &&
mv -f /etc/haproxy/haproxy.cfg.new /etc/haproxy/haproxy.cfg &&
rm -f /var/lib/dnsmasq/dnsmasq.leases &&
systemctl restart dnsmasq haproxy
}
function __generate_install_config() {
cat <<EOF
apiVersion: v1
baseDomain: ${cluster_basedomain}
compute:
- hyperthreading: ${cluster_hyperthreading}
name: worker
replicas: 0
controlPlane:
hyperthreading: ${cluster_hyperthreading}
name: master
replicas: ${#master_macs[@]}
metadata:
name: ${cluster_name}
networking:
clusterNetwork:
- cidr: ${cluster_cidr}
hostPrefix: ${cluster_host_prefix}
networkType: ${cluster_network_type}
serviceNetwork:
- ${cluster_service_network}
platform:
none: {}
pullSecret: '$(jq -c . < "$ocp4_pull_secret")'
sshKey: '$(head -1 "$ocp4_public_key")'
EOF
}
function __generate_pxelinux_cfg() {
local node_type=${1:-master}
local installdev=${2:-$install_device}
shift 2
cat <<EOF
DEFAULT pxeboot
TIMEOUT 20
PROMPT 0
LABEL pxeboot
KERNEL http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/rhcos-installer-kernel
APPEND rdblacklist=megaraid_sas ip=dhcp rd.neednet=1 initrd=http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/rhcos-installer-initramfs.img console=tty0 console=ttyS0 coreos.inst=yes coreos.inst.install_dev=${installdev} coreos.inst.image_url=http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/rhcos-metal-bios.raw.gz coreos.inst.ignition_url=http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/${node_type}.ign $*
SYSAPPEND 2
EOF
}
function __generate_grub_cfg() {
local node_type=${1:-master}
local installdev=${2:-$install_device}
shift 2
cat <<EOF
set timeout=10
menuentry 'Install CoreOS' {
linux rhcos-installer-kernel rdblacklist=megaraid_sas ip=dhcp coreos.inst.install_dev=${installdev} nomodeset rd.neednet=1 coreos.inst=yes coreos.inst.image_url=http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/rhcos-metal-bios.raw.gz coreos.inst.ignition_url=http://${bootstrap_ipaddr}:${pxe_port}/ocp4-upi/${node_type}.ign $*
initrd rhcos-installer-initramfs.img
}
EOF
}
function __get_installdev() {
local node_type=${1:-master}
local mac=${2:-}
if [[ $node_type = bootstrap ]] ; then
installdev=sda
else
installdev=${install_disks[${mac,,}]:-$install_device}
fi
if [[ ${installdev:0:1} != '/' ]] ; then
installdev="/dev/$installdev"
fi
echo "$installdev"
}
function __setup_tftpboot() {
local ntype=${1:-master}
shift 1
local -a macs=()
while [[ "$*" ]] ; do
local arg=$1
shift
if [[ $arg = '--' ]] ; then break; fi
macs+=("$arg")
done
local m
local installdev
if [[ $platform = aarch64 ]] ; then
for m in "${macs[@]}" ; do
local file="/var/lib/tftpboot/grub.cfg-01-${m//:/-}"
echo "Configuring $m as $ntype: $file"
installdev=$(__get_installdev "$ntype" "$m")
__generate_grub_cfg "$ntype" "$installdev" "$@" > "$file"
done
cp /boot/efi/EFI/BOOT/BOOTAA64.EFI /var/lib/tftpboot
cp /boot/efi/EFI/redhat/grubaa64.efi /var/lib/tftpboot
chmod 644 /var/lib/tftpboot/BOOTAA64.EFI /var/lib/tftpboot/grubaa64.efi
chmod 644 "$tftproot/$bootfile" "$tftproot/grubaa64.efi"
else
for m in "${macs[@]}" ; do
local file="/var/lib/tftpboot/pxelinux.cfg/01-${m//:/-}"
echo "Configuring $m as $ntype: $file"
installdev=$(__get_installdev "$ntype" "$m")
__generate_pxelinux_cfg "$ntype" "$installdev" "$@" > "$file"
done
cp -p /tftpboot/lpxelinux.0 /var/lib/tftpboot
cp -p /tftpboot/ldlinux.c32 /var/lib/tftpboot
fi
}
function __setup_bm_if() {
[[ -n "$bare_metal_interface" ]] || __fatal "No bare metal interface defined in bare_metal_interface"
local line
# Purge any other masters for our bare metal interface
while read -r line ; do
if [[ $line =~ ^[[:digit:]]+:\ +([^:]+):.*master ]] ; then
nmcli con del "${BASH_REMATCH[1]}"
fi
done <<< "$(ip addr show master "$cluster_network_bridge_name" 2>/dev/null || true)"
# Purge any stale instances of our bare metal interface
local name
local uuid
local type
local device
local rest
# shellcheck disable=SC2034
while read -r name uuid type device rest ; do
if [[ $name = "$cluster_network_bridge_name" ]] ; then
nmcli con del "$uuid"
fi
done <<< "$(nmcli con)"
nmcli con add type bridge ifname "${cluster_network_bridge_name}" con-name "${cluster_network_bridge_name}" ipv4.method manual ipv4.addr "${bootstrap_ipaddr}"/24 ipv4.dns "${bootstrap_ipaddr}" ipv4.dns-priority 10 autoconnect yes bridge.stp no
nmcli con add type bridge-slave autoconnect yes con-name "$bare_metal_interface" ifname "$bare_metal_interface" master "${cluster_network_bridge_name}"
nmcli con reload "$bare_metal_interface"
nmcli con reload "${cluster_network_bridge_name}"
nmcli con up "${cluster_network_bridge_name}"
nmcli con up "$bare_metal_interface"
}
function __setup_virt_net() {
if ! virsh net-info ocp4-upi >/dev/null 2>&1 ; then
tnetfile=$(mktemp net-ocp4-upi.XXXXXXXX)
cat > "$tnetfile" <<EOF
<network>
<name>ocp4-upi</name>
<forward mode='bridge'/>
<bridge name='$cluster_network_bridge_name'/>
</network>
EOF
virsh net-define "$tnetfile"
rm -f "$tnetfile"
virsh net-start ocp4-upi
virsh net-autostart ocp4-upi
fi
}
function __setup_iptables() {
# shellcheck disable=SC2155
local now=$(date)
echo "*** Disabling firewall"
systemctl stop firewalld
systemctl disable firewalld
cat > /etc/sysconfig/iptables <<EOF
# Generated by iptables-save v1.8.4 on ${now}
*filter
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
COMMIT
# Completed on ${now}
# Generated by iptables-save ${now}
*nat
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
-A POSTROUTING -s 192.168.222.0/24 -d 224.0.0.0/24 -j RETURN
-A POSTROUTING -s 192.168.222.0/24 -d 255.255.255.255/32 -j RETURN
-A POSTROUTING -s 192.168.222.0/24 ! -d 192.168.222.0/24 -p tcp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.222.0/24 ! -d 192.168.222.0/24 -p udp -j MASQUERADE --to-ports 1024-65535
-A POSTROUTING -s 192.168.222.0/24 ! -d 192.168.222.0/24 -j MASQUERADE
-A POSTROUTING -s 192.168.222.0/24 ! -d 192.168.222.0/24 -o enp2s0f0 -j MASQUERADE
COMMIT
# Completed on ${now}
# Generated by iptables-save v1.8.4 on ${now}
*mangle
:PREROUTING ACCEPT [0:0]
:INPUT ACCEPT [0:0]
:FORWARD ACCEPT [0:0]
:OUTPUT ACCEPT [0:0]
:POSTROUTING ACCEPT [0:0]
-A POSTROUTING -o virbr0 -p udp -m udp --dport 68 -j CHECKSUM --checksum-fill
COMMIT
# Completed on ${now}
EOF
iptables-restore < /etc/sysconfig/iptables
}
function __check_packages() {
if (( update_packages )) ; then
echo '*** Updating packages'
dnf -y groupinstall 'Virtualization Host' || yum module install virt
local -a required_rpms=()
local rpm
if [[ $platform = aarch64 ]] ; then
platform_packages=(shim-aa64 grub2-efi-aa64)
else
platform_packages=()
fi
for rpm in wget virt-install jq podman git python3-pyyaml qemu-kvm virt-manager virt-viewer xorg-x11-xauth xinetd syslinux-tftpboot haproxy httpd perl perl-JSON ipmitool python3-magic "${platform_packages[@]}" ; do
rpm --quiet -q "$rpm" || required_rpms+=("$rpm")
done
if [[ -n "${!required_rpms[*]}" ]] ; then
echo "Installing ${required_rpms[*]}"
dnf -y install "${required_rpms[@]}"
fi
fi
systemctl enable libvirtd
systemctl start libvirtd
}
function __pre_setup() {
cd ~
[[ -d "$cachedir" ]] || mkdir -p "$cachedir"
setenforce 0 || true
grep -q SELINUX=disabled /etc/selinux/config || sed -i s/^SELINUX=.*/SELINUX=disabled/ /etc/selinux/config
__check_packages
sed -i s/Listen\ 80/Listen\ ${pxe_port}/ /etc/httpd/conf/httpd.conf
[[ -f "$ocp4_public_key" ]] || ssh-keygen -t rsa -b 4096 -f "${ocp4_public_key%.pub}" -N "" -q
__setup_bm_if
__setup_iptables
local bootstrap_ipaddr_quoted=${bootstrap_ipaddr//./\\.}
grep -q "nameserver *${bootstrap_ipaddr_quoted}" /etc/resolv.conf ||
sed -i "/^search/a nameserver\ ${bootstrap_ipaddr}" /etc/resolv.conf
}
function __get_key() {
local data="$1"
shift
local key
for key in "$@" ; do
# shellcheck disable=SC2155
local answer="$(jq -r "($key)" <<< "$data")"
if [[ -n "$answer" && $answer != 'null' ]] ; then
echo "$answer"
return
fi
done
}
function __try_to_download() {
local -a registry_hosts=(openshift-release.apps.ci.l2s4.p1.openshiftapps.com openshift-release-artifacts.apps.ci.l2s4.p1.openshiftapps.com)
local target=$1; shift
local host
for host in "${registry_hosts[@]}" ; do
wget -P "$cachedir" "$@" "https://${host}/${target}" && return 0
done
return 1
}
function __fetch_tools_if_needed() {
local client_filename=openshift-client-linux${arch_infix}-${VERSION}.tar.gz
local installer_filename=openshift-install-linux${arch_infix}-${VERSION}.tar.gz
if [[ -d "$local_toolsdir" ]] ; then
if [[ -z "$installer_image" ]] ; then
installer_image="${local_toolsdir}/$installer_filename"
fi
if [[ -z "$client_image" ]] ; then
client_image="${local_toolsdir}/$client_filename"
fi
fi
if [[ -n "$installer_image" && -f "$installer_image" ]] ; then
cp -p "$installer_image" "$cachedir/$installer_filename"
fi
if [[ -n "$client_image" && -f "$client_image" ]] ; then
cp -p "$client_image" "$cachedir/$client_filename"
fi
if [[ -f "$cachedir/$client_filename" &&
-f "$cachedir/$installer_filename" ]] ; then
return 0
elif [[ -n "$(type -p oc)" ]] && (cd "$cachedir" && oc adm release extract --tools "$VERSION") ; then
return 0
else
[[ -f "$cachedir/$client_filename" ]] || __try_to_download "${VERSION}/$client_filename" -nv -N || {
echo "Can't download client for $VERSION"
return 1
}
[[ -f "$cachedir/$installer_filename" ]] || __try_to_download "${VERSION}/$installer_filename" -nv -N || {
echo "Can't download installer for $VERSION"
return 1
}
fi
}
function __do_setup_install() {
[[ -n "$1" ]] && VERSION=$1
if [[ -z "$VERSION" ]] ; then
__fatal "VERSION is not set!"
fi
cd "$HOME"
__pre_setup
[[ $VERSION =~ ([0-9]\.[0-9]+).* ]] && export RELEASE=${BASH_REMATCH[1]}
[[ -d "$cluster_install_dir" ]] && rm -rf "$cluster_install_dir"
mkdir "$cluster_install_dir"
if ((masters_schedulable < 0)) ; then
if ((${#worker_macs[@]} >= 1)) ; then