init setting for helper node ssh -tt -D 8801 -R 18801:10.147.17.89:5085 [email protected] ' bash -l -c byobu' # init setting for helper node<< EOF > ~/.ssh/configStrictHostKeyChecking no
UserKnownHostsFile=/dev/null
EOF ~ /.ssh/config
cat << EOF >> /etc/hosts127.0.0.1 registry.ocp4.redhat.ren nexus.ocp4.redhat.ren git.ocp4.redhat.ren
EOF ' server with gui' enable --now libvirtd
setup ntp server on helper node # setup ntp server on helper node# sed -i "s/#allow.*/allow 192.168.0.0\/16/" /etc/chrony.conf" s/#allow.*/allow all/" enable --now chronyd
chronyc tracking
# Reference ID : CA760182 (202.118.1.130)# Stratum : 2# Ref time (UTC) : Mon May 02 03:55:48 2022# System time : 0.000000530 seconds fast of NTP time# Last offset : -0.003027542 seconds# RMS offset : 0.003027542 seconds# Frequency : 36.009 ppm slow# Residual freq : +61.371 ppm# Skew : 25.290 ppm# Root delay : 0.016805194 seconds# Root dispersion : 0.002184978 seconds# Update interval : 0.8 seconds# Leap status : Normal# .-- Source mode '^' = server, '=' = peer, '#' = local clock.# / .- Source state '*' = current best, '+' = combined, '-' = not combined,# | / 'x' = may be in error, '~' = too variable, '?' = unusable.# || .- xxxx [ yyyy ] +/- zzzz# || Reachability register (octal) -. | xxxx = adjusted offset,# || Log2(Polling interval) --. | | yyyy = measured offset,# || \ | | zzzz = estimated error.# || | | \# MS Name/IP address Stratum Poll Reach LastRx Last sample# ===============================================================================# ^- time.cloudflare.com 3 6 17 41 +28ms[ +28ms] +/- 157ms# ^* 202.118.1.130 1 6 17 42 -9871ns[-3037us] +/- 8572us# ^- time.cloudflare.com 3 6 17 40 +35ms[ +35ms] +/- 162ms# ^- makaki.miuku.net 2 6 17 40 +46ms[ +46ms] +/- 110ms# .- Number of sample points in measurement set.# / .- Number of residual runs with same sign.# | / .- Length of measurement set (time).# | | / .- Est. clock freq error (ppm).# | | | / .- Est. error in freq.# | | | | / .- Est. offset.# | | | | | | On the -.# | | | | | | samples. \# | | | | | | |# Name/IP Address NP NR Span Frequency Freq Skew Offset Std Dev# ==============================================================================# time.cloudflare.com 4 3 7 -1249.574 60710.363 -54ms 8100us# 202.118.1.130 4 3 6 +61.371 5439.969 +3713us 581us# time.cloudflare.com 4 3 7 -3223.009 204771 -185ms 29ms# makaki.miuku.net 4 3 7 +6244.955 92563.305 +411ms 12ms# 200 OKcat << 'EOF ' > /data/kvm/bridge.sh#!/usr/bin/env bash
PUB_CONN='ens192'
PUB_IP='172.21.6.11/24'
PUB_GW='172.21.6.254'
PUB_DNS='172.21.1.1'
nmcli con down "$PUB_CONN"
nmcli con delete "$PUB_CONN"
nmcli con down baremetal
nmcli con delete baremetal
# RHEL 8.1 appends the word "System" in front of the connection,delete in case it exists
nmcli con down "System $PUB_CONN"
nmcli con delete "System $PUB_CONN"
nmcli connection add ifname baremetal type bridge con-name baremetal ipv4.method 'manual' \
ipv4.address "$PUB_IP" \
ipv4.gateway "$PUB_GW" \
ipv4.dns "$PUB_DNS"
nmcli con add type bridge-slave ifname "$PUB_CONN" master baremetal
nmcli con down "$PUB_CONN";pkill dhclient;dhclient baremetal
nmcli con up baremetal
EOF ' 192.168.7.11/24' # nmcli networking off; nmcli networking on mkdir -p /etc/crts/ && cd /etc/crts
# https://access.redhat.com/documentation/en-us/red_hat_codeready_workspaces/2.1/html/installation_guide/installing-codeready-workspaces-in-tls-mode-with-self-signed-certificates_crw" Local Red Hat Ren Signer" <( cat /etc/pki/tls/openssl.cnf \ <( printf ' [SAN]\nbasicConstraints=critical, CA:TRUE\nkeyUsage=keyCertSign, cRLSign, digitalSignature' ) )
openssl genrsa -out /etc/crts/redhat.ren.key 2048
openssl req -new -sha256 \
-key /etc/crts/redhat.ren.key \
-subj " /O=Local Red Hat Ren /CN=*.infra.redhat.ren" <( cat /etc/pki/tls/openssl.cnf \ <( printf " \n[SAN]\nsubjectAltName=DNS:*.infra.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth" ) ) \
-out /etc/crts/redhat.ren.csr
openssl x509 \
-req \
-sha256 \
-extfile <( printf " subjectAltName=DNS:*.infra.redhat.ren,DNS:*.redhat.ren\nbasicConstraints=critical, CA:FALSE\nkeyUsage=digitalSignature, keyEncipherment, keyAgreement, dataEncipherment\nextendedKeyUsage=serverAuth" ) dnf -y install git
cd /data
rm -rf /data/ocp4
# scripts can be found here:# https://github.com/wangzheng422/openshift4-shell # copy helper ansible project to /data/ocp4/ocp4-upi-helpernode-master" $( cat ~ /.ssh/id_rsa.pub) " ' {"auths":{"registry.redhat.io": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"registry.ocp4.redhat.ren:5443": {"auth": "ZHVtbXk6ZHVtbXk=","email": "noemail@localhost"},"' ${INSTALL_IMAGE_REGISTRY} ' ": {"auth": "' $( echo -n ' admin:shadowman' | openssl base64 ) ' ","email": "noemail@localhost"}}}' echo $PULL_SECRET
# 定义单节点集群的节点信息` printf ' 00:60:2F:%02X:%02X:%02X' ` echo ${SNO_IF_MAC} > /data/sno/sno.mac
cat << EOF > /data/ocp4/ocp4-upi-helpernode-master/vars.yaml---
ocp_version: 4.10.12
ssh_gen_key: false
staticips: true
bm_ipi: false
firewalld: false
dns_forward: true
iso:
iso_dl_url: "file:///data/ocp4/rhcos-live.x86_64.iso"
my_iso: "rhcos-live.iso"
helper:
name: "helper"
ipaddr: "${HELP_SERVER} "
networkifacename: "baremetal"
gateway: "${SNO_GW} "
netmask: "${SNO_NETMAST} "
dns:
domain: "redhat.ren"
clusterid: "ocp4"
forwarder1: "172.21.1.1"
forwarder2: "172.21.1.1"
api_vip: "${API_VIP} "
ingress_vip: "${INGRESS_VIP} "
bootstrap:
name: "bootstrap"
ipaddr: "${BOOTSTRAP_IP} "
interface: "enp1s0"
install_drive: "vda"
masters:
- name: "master-0"
ipaddr: "192.168.7.13"
interface: "enp1s0"
install_drive: "vda"
workers:
- name: "worker-0"
ipaddr: "192.168.7.16"
interface: "ens18f0"
install_drive: "sda"
- name: "worker-1"
ipaddr: "192.168.7.17"
interface: "eno2"
install_drive: "sda"
- name: "worker-2"
ipaddr: "192.168.7.18"
interface: "enp1s0"
install_drive: "sda"
others:
- name: "registry"
ipaddr: "192.168.7.103"
- name: "yum"
ipaddr: "172.21.6.103"
- name: "quay"
ipaddr: "172.21.6.103"
- name: "nexus"
ipaddr: "172.21.6.103"
- name: "git"
ipaddr: "172.21.6.103"
otherdomains:
- domain: "infra.redhat.ren"
hosts:
- name: "registry"
ipaddr: "192.168.7.11"
- name: "yum"
ipaddr: "192.168.7.11"
- name: "quay"
ipaddr: "192.168.7.11"
- name: "quaylab"
ipaddr: "192.168.7.11"
- name: "nexus"
ipaddr: "192.168.7.11"
- name: "git"
ipaddr: "192.168.7.11"
- domain: "${ACM_DEMO_MNGED_CLUSTER} .${SNO_BASE_DOMAIN} "
hosts:
- name: "api"
ipaddr: "${ACM_DEMO_MNGED_SNO_IP} "
- name: "api-int"
ipaddr: "${ACM_DEMO_MNGED_SNO_IP} "
- name: "${ACM_DEMO_MNGED_CLUSTER} -master"
ipaddr: "${ACM_DEMO_MNGED_SNO_IP} "
- name: "*.apps"
ipaddr: "${ACM_DEMO_MNGED_SNO_IP} "
- domain: "${SNO_CLUSTER_NAME} .${SNO_BASE_DOMAIN} "
hosts:
- name: "api"
ipaddr: "${SNO_IP} "
- name: "api-int"
ipaddr: "${SNO_IP} "
- name: "${SNO_CLUSTER_NAME} -master"
ipaddr: "${SNO_IP} "
- name: "*.apps"
ipaddr: "${SNO_IP} "
force_ocp_download: false
remove_old_config_files: false
ocp_client: "file:///data/ocp4/{{ ocp_version }}/openshift-client-linux-{{ ocp_version }}.tar.gz"
ocp_installer: "file:///data/ocp4/{{ ocp_version }}/openshift-install-linux-{{ ocp_version }}.tar.gz"
ppc64le: false
arch: 'x86_64'
chronyconfig:
enabled: true
content:
- server: "${NTP_SERVER} "
options: iburst
setup_registry: # don't worry about this, just leave it here
deploy: false
registry_image: docker.io/library/registry:2
local_repo: "ocp4/openshift4"
product_repo: "openshift-release-dev"
release_name: "ocp-release"
release_tag: "4.6.1-x86_64"
ocp_filetranspiler: "file:///data/ocp4/filetranspiler.tgz"
EOF cd /data/ocp4/ocp4-upi-helpernode-master
ansible-playbook -e @vars.yaml -e ' { staticips: true, bm_ipi: false }' # https://docs.openshift.com/container-platform/4.10/installing/disconnected_install/installing-mirroring-creating-registry.htmlcd /data/ocp4/clients
tar zvxf mirror-registry.tar.gz
./mirror-registry install -v \
--initPassword shadowman --initUser admin \
--quayHostname quaylab.infra.redhat.ren --quayRoot /data/quay \
--targetHostname quaylab.infra.redhat.ren \
--sslKey /etc/crts/redhat.ren.key --sslCert /etc/crts/redhat.ren.crt
--
# PLAY RECAP ******************************************************************************************************************************************************************************************[email protected] : ok=52 changed=20 unreachable=0 failed=0 skipped=7 rescued=0 ignored=0# INFO[2022-05-02 18:01:10] Quay installed successfully, permanent data is stored in /data/quay# INFO[2022-05-02 18:01:10] Quay is available at https://quaylab.infra.redhat.ren:8443 with credentials (admin, shadowman) # total 4.1G# -rw-r--r--. 1 root root 2.1G May 2 17:28 image-archive.tar# -rw-r--r--. 1 root root 3.4M Mar 9 21:52 pause.tar# drwxrwxr-x+ 3 root root 22 May 2 17:29 pg-data# -rw-r--r--. 1 root root 585M Mar 9 21:54 postgres.tar# drwxr-xr-x. 2 root root 90 May 2 18:00 quay-config# drwxr-xr-x. 2 root root 60 May 2 17:29 quay-rootCA# drwxrwxr-x+ 2 root root 6 May 2 17:29 quay-storage# -rw-r--r--. 1 root root 1.1G Mar 9 21:53 quay.tar# -rw-r--r--. 1 root root 430M Mar 9 21:54 redis.tar# to uninstall, do not use in setup# ./mirror-registry uninstall -v \# --autoApprove true --quayRoot /data/quay \# --targetHostname quaylab.infra.redhat.ren \# --# https://quaylab.infra.redhat.ren:8443/import ocp content into quay podman login quaylab.infra.redhat.ren:8443 --username admin --password shadowman
# oc image mirror -a /run/user/0/containers/auth.json --from-dir=/data/file.registry/ 'file://openshift/release:4.10.12-x86_64*' quaylab.infra.redhat.ren:8443/ocp4/openshift4' file://openshift/release:4.10.15-x86_64*' # # import nexus fscd /data/ccn
podman create --name swap quay.io/wangzheng422/qimgs:nexus-fs-image-2022-01-14-2155 ls
podman cp swap:/nexus-image.tgz - > /data/ccn/nexus-image.tgz.tar
podman rm -fv swap
tar vxf nexus-image.tgz.tar
tar zxf nexus-image.tgz
rm -f nexus-image.tgz*
chown -R 200 /data/ccn/nexus-image
# # run the nexus for image# podman run -d -p 8082:8081 -p 8083:8083 -it --name nexus-image -v /data/ccn/nexus-image:/nexus-data:Z docker.io/sonatype/nexus3:3.33.1# /root/container-local-registry.serviceenable --now container-nexus-image.service
systemctl status container-nexus-image.service
# get the admin password&& echo
# 84091bcd-c82f-44a3-8b7b-dfc90f5b7da1# open http://nexus.infra.redhat.ren:8082setup assisted install service ( AIS ) # https://github.com/openshift/assisted-service/blob/master/docs/user-guide/assisted-service-on-local.md# https://github.com/openshift/assisted-service/tree/master/deploy/podman# Version: 3.4.2# API Version: 3.4.2# Go Version: go1.16.12# Built: Wed Feb 2 07:59:28 2022# OS/Arch: linux/amd64cd /data/assisted-service/
export http_proxy=" http://127.0.0.1:18801" export https_proxy=${http_proxy}
wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/configmap.yml
wget https://raw.githubusercontent.com/openshift/assisted-service/master/deploy/podman/pod.yml
/bin/cp -f configmap.yml configmap.yml.bak
/bin/cp -f pod.yml pod.yml.bak
unset http_proxy
unset https_proxy
sed -i ' s/ SERVICE_BASE_URL:.*/ SERVICE_BASE_URL: "http:\/\/172.21.6.11:8090"/' | grep " machine-os " # machine-os 410.84.202204261500-0 Red Hat Enterprise Linux CoreOS<< EOF > /data/assisted-service/os_image.json[{
"openshift_version": "4.10",
"cpu_architecture": "x86_64",
"url": "http://172.21.6.11:8080/install/rhcos-live.x86_64.iso",
"rootfs_url": "http://172.21.6.11:8080/install/rootfs.img",
"version": "410.84.202204261500-0"
}]
EOF << EOF > /data/assisted-service/release.json[{
"openshift_version": "4.10",
"cpu_architecture": "x86_64",
"url": "quaylab.infra.redhat.ren/ocp4/openshift4:4.10.12-x86_64",
"version": "4.10.12",
"default": true
}]
EOF | python3 -c ' import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' | jq --arg OSIMAGE " $( jq -c . /data/assisted-service/os_image.json) " ' . | .data.OS_IMAGES = $OSIMAGE ' | jq --arg RELEASE_IMAGES " $( jq -c . /data/assisted-service/release.json) " ' . | .data.RELEASE_IMAGES = $RELEASE_IMAGES ' | python3 -c ' import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' > configmap.yml
cat pod.yml.bak \
| python3 -c ' import json, yaml, sys; print(json.dumps(yaml.load(sys.stdin)))' | jq ' .spec.containers[1].ports[0].hostPort = 8180 ' | python3 -c ' import yaml, sys; print(yaml.dump(yaml.load(sys.stdin), default_flow_style=False))' > pod.yml
# 启动本地assisted servicecd /data/assisted-service/
podman play kube --configmap configmap.yml pod.yml
# 注入离线镜像仓库的证书exec assisted-installer-service update-ca-trust
# 用以下命令，停止/删除本地assisted servicecd /data/assisted-service/
podman play kube --down pod.yml
# assisted service启动以后，会下载安装介质，我们看看下载占用的空间。exec assisted-installer-image-service du -h /data
# 1.1G /datavncpasswd
cat << EOF > ~/.vnc/configsession=gnome
securitytypes=vncauth,tlsvnc
# desktop=sandbox
geometry=1440x855
alwaysshared
EOF << EOF >> /etc/tigervnc/vncserver.users:1=root
EOF # 如果你想停掉vnc server，这么做# firewall-cmd --permanent --add-port=6001/tcp# firewall-cmd --permanent --add-port=5901/tcp# firewall-cmd --reload# connect vnc at port 5901# export DISPLAY=:1