Skip to content

Commit 89328dc

Browse files
bfourniebfournie
committed
Add version-specific operator validation for ISO_NO_REGISTRY mode
This commit adds a new validation function to verify that expected operators are installed after cluster deployment in ISO_NO_REGISTRY mode. Key features: - Version-specific operator lists for OCP 4.20 and 4.21 - Automatic version detection using openshift_version function - Clear error reporting for missing operators - 4.20 validates 7 operators - 4.21 validates 12 operators The validation runs after the API server is available and helps ensure that the virtualization bundle and other operators were properly installed during cluster creation.
1 parent 5901c75 commit 89328dc

1 file changed

Lines changed: 178 additions & 1 deletion

File tree

‎agent/agent_post_install_validation.sh‎

Lines changed: 178 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,176 @@ set -euxo pipefail
44
SCRIPTDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )"
55

66
source $SCRIPTDIR/common.sh
7+
source $SCRIPTDIR/release_info.sh
8+
9+
function validate_installed_manifests() {
10+
echo "Validating installed operator manifests..."
11+
12+
# Determine OCP version - manifest validation only supported in 4.21+
13+
ocp_version=$(openshift_version ${OCP_DIR})
14+
echo "Detected OpenShift version: ${ocp_version}"
15+
16+
major_version=$(echo "$ocp_version" | cut -d. -f1)
17+
minor_version=$(echo "$ocp_version" | cut -d. -f2)
18+
19+
# Skip validation for versions older than 4.21
20+
if [ "$major_version" -lt 4 ] || [ "$major_version" -eq 4 -a "$minor_version" -le 20 ]; then
21+
echo "Manifest validation not supported for version ${ocp_version}, skipping."
22+
return 0
23+
fi
24+
25+
# Get the ConfigMap as JSON
26+
if ! configmap_json=$(oc get configmap/olm-operator-manifests -n assisted-installer -o json 2>&1); then
27+
echo "ERROR: Could not find ConfigMap olm-operator-manifests in assisted-installer namespace"
28+
return 1
29+
fi
30+
31+
# Get all keys that end with .metadata.yaml
32+
metadata_keys=$(echo "$configmap_json" | jq -r '.data | keys[] | select(endswith(".metadata.yaml"))')
33+
34+
if [ -z "$metadata_keys" ]; then
35+
echo "ERROR: No metadata files found in ConfigMap"
36+
return 1
37+
fi
38+
39+
missing_resources=()
40+
41+
for metadata_key in $metadata_keys; do
42+
echo "Processing metadata: $metadata_key"
43+
44+
# Get the metadata content (plain YAML)
45+
set +x
46+
metadata_yaml=$(echo "$configmap_json" | jq -r ".data[\"$metadata_key\"]")
47+
set -x
48+
49+
# Extract manifest filenames (lines starting with "- " in YAML list)
50+
manifest_files=$(echo "$metadata_yaml" | grep "^- " | sed 's/^- //')
51+
52+
for manifest_file in $manifest_files; do
53+
echo " Validating manifest: $manifest_file"
54+
55+
# Get the base64-encoded manifest content and decode it
56+
set +x
57+
manifest_yaml=$(echo "$configmap_json" | jq -r ".data[\"$manifest_file\"]" | base64 -d)
58+
set -x
59+
60+
# Extract kind and name from the manifest YAML
61+
kind=$(echo "$manifest_yaml" | sed -n 's/^kind: *//p' | head -1 | tr -d '\r')
62+
name=$(echo "$manifest_yaml" | sed -n 's/^ name: *//p' | head -1 | tr -d '\r')
63+
64+
if [ -z "$kind" ] || [ -z "$name" ]; then
65+
echo " WARNING: Could not extract kind or name from $manifest_file"
66+
continue
67+
fi
68+
69+
# Try to get resources of this kind
70+
get_output=$(oc get "$kind" -A --no-headers 2>&1)
71+
get_exit_code=$?
72+
73+
if [ $get_exit_code -ne 0 ]; then
74+
# Check if it's because the resource type doesn't exist
75+
if echo "$get_output" | grep -q "error: the server doesn't have a resource type"; then
76+
missing_resources+=("$kind/$name (resource type '$kind' not available)")
77+
else
78+
# Some other error
79+
missing_resources+=("$kind/$name (error: $get_output)")
80+
fi
81+
else
82+
# Resource type exists, check if our specific resource is in the list
83+
# Check both column 1 (cluster-scoped) and column 2 (namespaced with -A)
84+
if ! echo "$get_output" | awk '{print $1, $2}' | grep -qw "$name"; then
85+
missing_resources+=("$kind/$name")
86+
else
87+
echo " Verified $kind/$name exists"
88+
fi
89+
fi
90+
done
91+
done
92+
93+
if [ ${#missing_resources[@]} -gt 0 ]; then
94+
echo "ERROR: The following expected resources are not applied:"
95+
for missing_res in "${missing_resources[@]}"; do
96+
echo " - $missing_res"
97+
done
98+
return 1
99+
else
100+
echo "SUCCESS: All expected operator manifests are applied."
101+
fi
102+
}
103+
104+
function validate_installed_operators() {
105+
echo "Validating installed operators..."
106+
107+
# Define expected operators per version
108+
expected_operators_4_20=(
109+
"cluster-kube-descheduler-operator"
110+
"fence-agents-remediation"
111+
"kubernetes-nmstate-operator"
112+
"kubevirt-hyperconverged"
113+
"local-storage-operator"
114+
"mtv-operator"
115+
"node-healthcheck-operator"
116+
"node-maintenance-operator"
117+
)
118+
119+
expected_operators_4_21=(
120+
"cluster-kube-descheduler-operator"
121+
"cluster-observability-operator"
122+
"fence-agents-remediation"
123+
"kubernetes-nmstate-operator"
124+
"kubevirt-hyperconverged"
125+
"local-storage-operator"
126+
"metallb-operator"
127+
"mtv-operator"
128+
"node-healthcheck-operator"
129+
"node-maintenance-operator"
130+
"numaresources-operator"
131+
"redhat-oadp-operator"
132+
"loki-operator"
133+
"cluster-logging"
134+
)
135+
136+
# Determine OCP version and select appropriate operator list
137+
ocp_version=$(openshift_version ${OCP_DIR})
138+
echo "Detected OpenShift version: ${ocp_version}"
139+
140+
case "${ocp_version}" in
141+
"4.20")
142+
expected_operators=("${expected_operators_4_20[@]}")
143+
;;
144+
"4.21")
145+
expected_operators=("${expected_operators_4_21[@]}")
146+
;;
147+
*)
148+
echo "Using 4.21 operator list as default"
149+
expected_operators=("${expected_operators_4_21[@]}")
150+
;;
151+
esac
152+
153+
# Get list of installed operators (just the names, first column)
154+
installed_operators=$(oc get operators -o custom-columns=NAME:.metadata.name --no-headers)
155+
156+
missing_operators=()
157+
for expected_op in "${expected_operators[@]}"; do
158+
if ! echo "$installed_operators" | grep -q "^${expected_op}\."; then
159+
missing_operators+=("$expected_op")
160+
fi
161+
done
162+
163+
if [ ${#missing_operators[@]} -gt 0 ]; then
164+
echo "ERROR: The following expected operators are not installed:"
165+
for missing_op in "${missing_operators[@]}"; do
166+
echo " - $missing_op"
167+
done
168+
echo ""
169+
echo "Installed operators:"
170+
oc get operators
171+
return 1
172+
else
173+
echo "SUCCESS: All expected operators are installed."
174+
oc get operators
175+
fi
176+
}
7177

8178
if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
9179
MAX_ATTEMPTS=120
@@ -12,7 +182,8 @@ if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
12182

13183
echo "Starting oc wait retry loop for API connection up to 60 minutes..."
14184
set +x # Disable debug tracing for cleaner output
15-
while ! oc wait clusterversion version --for=condition=Available=True --timeout=1s 2>/dev/null; do
185+
while ! (oc wait clusterversion version --for=condition=Available=True --timeout=2s 2>/dev/null && \
186+
oc wait clusterversion version --for=condition=Progressing=False --timeout=2s 2>/dev/null); do
16187
ATTEMPT_COUNT=$((ATTEMPT_COUNT + 1))
17188
if [ $ATTEMPT_COUNT -ge $MAX_ATTEMPTS ]; then
18189
echo ""
@@ -29,6 +200,12 @@ if [[ "${AGENT_E2E_TEST_BOOT_MODE}" == "ISO_NO_REGISTRY" ]]; then
29200
echo "SUCCESS: API server connection established and ClusterVersion is available."
30201
# Run subsequent commands after successful cluster setup
31202
oc get packagemanifests -n openshift-marketplace
203+
204+
# Validate expected operators are installed
205+
validate_installed_operators || exit 1
206+
207+
# Validate operator manifests are applied
208+
validate_installed_manifests || exit 1
32209
fi
33210

34211
installed_control_plane_nodes=$(oc get nodes --selector=node-role.kubernetes.io/master | grep -v AGE | wc -l)

0 commit comments

Comments
 (0)