From fb48468650991a789721ee233edd25f6e68e8353 Mon Sep 17 00:00:00 2001 From: Noam Manos Date: Tue, 1 Oct 2024 10:23:20 +0300 Subject: [PATCH] New KW `Wait For Deployment Replica To Be Ready` for oc_install.robot (#1867) * New KW `Wait For Deployment Replica To Be Ready` for oc_install.robot This Keyword waits for all Deployment pods with a specified label, to have its Replica-Set ready (desired pods running). Signed-off-by: manosnoam --- .../RHODS_OLM/install/oc_install.robot | 94 +++++++------------ ods_ci/tests/Resources/Common.robot | 9 +- ods_ci/tests/Resources/OCP.resource | 10 ++ 3 files changed, 47 insertions(+), 66 deletions(-) diff --git a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot index 6cbd83687..81eea02f0 100644 --- a/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot +++ b/ods_ci/tasks/Resources/RHODS_OLM/install/oc_install.robot @@ -80,10 +80,8 @@ Verify RHODS Installation Set Global Variable ${DASHBOARD_APP_NAME} ${PRODUCT.lower()}-dashboard Log Verifying RHODS installation console=yes Log To Console Waiting for all RHODS resources to be up and running - Wait For Pods Numbers 1 - ... namespace=${OPERATOR_NAMESPACE} - ... label_selector=name=${OPERATOR_NAME_LABEL} - ... timeout=2000 + Wait For Deployment Replica To Be Ready namespace=${OPERATOR_NAMESPACE} + ... label_selector=name=${OPERATOR_NAME_LABEL} timeout=2000s Wait For Pods Status namespace=${OPERATOR_NAMESPACE} timeout=1200 Log Verified ${OPERATOR_NAMESPACE} console=yes @@ -99,96 +97,68 @@ Verify RHODS Installation END Apply DataScienceCluster CustomResource dsc_name=${DSC_NAME} END + ${dashboard} = Is Component Enabled dashboard ${DSC_NAME} IF ("${UPDATE_CHANNEL}" == "stable" or "${UPDATE_CHANNEL}" == "beta") or "${dashboard}" == "true" # Needs to be removed ASAP IF "${PRODUCT}" == "ODH" - Log To Console "Waiting for 2 pods in ${APPLICATIONS_NAMESPACE}, label_selector=app=odh-dashboard" - Wait For Pods Numbers 2 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=odh-dashboard - ... timeout=1200 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=odh-dashboard timeout=1200s #This line of code is strictly used for the exploratory cluster to accommodate UI/UX team requests Add UI Admin Group To Dashboard Admin - ELSE - Log To Console "Waiting for 5 pods in ${APPLICATIONS_NAMESPACE}, label_selector=app=${DASHBOARD_APP_NAME}" - Wait For Pods Numbers 5 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=${DASHBOARD_APP_NAME} - ... timeout=1200 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=${DASHBOARD_APP_NAME} timeout=1200s END END + ${workbenches} = Is Component Enabled workbenches ${DSC_NAME} IF ("${UPDATE_CHANNEL}" == "stable" or "${UPDATE_CHANNEL}" == "beta") or "${workbenches}" == "true" - Log To Console "Waiting for 1 pod in ${APPLICATIONS_NAMESPACE}, label_selector=app=notebook-controller" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=notebook-controller - ... timeout=400 - Log To Console "Waiting for 1 pod in ${APPLICATIONS_NAMESPACE}, label_selector=app=odh-notebook-controller" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=odh-notebook-controller - ... timeout=400 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=notebook-controller timeout=400s + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=odh-notebook-controller timeout=400s END + ${modelmeshserving} = Is Component Enabled modelmeshserving ${DSC_NAME} IF ("${UPDATE_CHANNEL}" == "stable" or "${UPDATE_CHANNEL}" == "beta") or "${modelmeshserving}" == "true" - Log To Console "Waiting for 3 pods in ${APPLICATIONS_NAMESPACE}, label_selector=app=odh-model-controller" - Wait For Pods Numbers 3 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=odh-model-controller - ... timeout=400 - Log To Console "Waiting for 1 pod in ${APPLICATIONS_NAMESPACE}, label_selector=component=model-mesh-etcd" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=component=model-mesh-etcd - ... timeout=400 - Log To Console "Waiting for 3 pods in ${APPLICATIONS_NAMESPACE}, label_selector=app.kubernetes.io/name=modelmesh-controller" - Wait For Pods Numbers 3 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app.kubernetes.io/name=modelmesh-controller - ... timeout=400 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=odh-model-controller timeout=400s + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=component=model-mesh-etcd timeout=400s + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app.kubernetes.io/name=modelmesh-controller timeout=400s END + ${datasciencepipelines} = Is Component Enabled datasciencepipelines ${DSC_NAME} IF ("${UPDATE_CHANNEL}" == "stable" or "${UPDATE_CHANNEL}" == "beta") or "${datasciencepipelines}" == "true" - Log To Console "Waiting for 1 pod in ${APPLICATIONS_NAMESPACE}, label_selector=app.kubernetes.io/name=data-science-pipelines-operator" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app.kubernetes.io/name=data-science-pipelines-operator - ... timeout=400 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app.kubernetes.io/name=data-science-pipelines-operator timeout=400s END + ${modelregistry} = Is Component Enabled modelregistry ${DSC_NAME} IF "${modelregistry}" == "true" - Log To Console "Waiting for 1 pod in ${APPLICATIONS_NAMESPACE}, label_selector=app.kubernetes.io/part-of=model-registry-operator" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app.kubernetes.io/part-of=model-registry-operator - ... timeout=400 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app.kubernetes.io/part-of=model-registry-operator timeout=400s END + ${kserve} = Is Component Enabled kserve ${DSC_NAME} IF "${kserve}" == "true" - Log To Console "Waiting for 3 pods in ${APPLICATIONS_NAMESPACE}, label_selector=app=odh-model-controller" - Wait For Pods Numbers 3 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=app=odh-model-controller - ... timeout=400 - Log To Console "Waiting for 1 pods in ${APPLICATIONS_NAMESPACE}, label_selector=control-plane=kserve-controller-manager" - Wait For Pods Numbers 1 - ... namespace=${APPLICATIONS_NAMESPACE} - ... label_selector=control-plane=kserve-controller-manager - ... timeout=400 + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=app=odh-model-controller timeout=400s + Wait For Deployment Replica To Be Ready namespace=${APPLICATIONS_NAMESPACE} + ... label_selector=control-plane=kserve-controller-manager timeout=400s END IF ("${UPDATE_CHANNEL}" == "stable" or "${UPDATE_CHANNEL}" == "beta") or "${dashboard}" == "true" or "${workbenches}" == "true" or "${modelmeshserving}" == "true" or "${datasciencepipelines}" == "true" # robocop: disable - Log To Console "Waiting for pod status in ${APPLICATIONS_NAMESPACE}" + Log To Console Waiting for pod status in ${APPLICATIONS_NAMESPACE} Wait For Pods Status namespace=${APPLICATIONS_NAMESPACE} timeout=200 Log Verified Applications NS: ${APPLICATIONS_NAMESPACE} console=yes END # Monitoring stack only deployed for managed, as modelserving monitoring stack is no longer deployed IF "${cluster_type}" == "managed" - Log To Console "Waiting for pod status in ${MONITORING_NAMESPACE}" + Log To Console Waiting for pod status in ${MONITORING_NAMESPACE} Wait For Pods Status namespace=${MONITORING_NAMESPACE} timeout=600 Log Verified Monitoring NS: ${MONITORING_NAMESPACE} console=yes END diff --git a/ods_ci/tests/Resources/Common.robot b/ods_ci/tests/Resources/Common.robot index cba7f048f..6aeccc8db 100644 --- a/ods_ci/tests/Resources/Common.robot +++ b/ods_ci/tests/Resources/Common.robot @@ -437,11 +437,12 @@ Extract URLs From Text RETURN ${urls} Run And Verify Command - [Documentation] Run an oc delete command and log the output and errors - [Arguments] ${command} - ${result}= Run Process ${command} shell=yes - Log ${result.stdout}\n${result.stderr} console=True + [Documentation] Run and verify shell command + [Arguments] ${command} ${print_to_log}=${TRUE} + ${result}= Run Process ${command} shell=yes stderr=STDOUT + IF ${print_to_log} Log ${result.stdout} console=True Should Be True ${result.rc} == 0 + RETURN ${result.stdout} Run And Watch Command [Documentation] Run any shell command (including args) with optional: diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource index 464635b19..87aa8004e 100644 --- a/ods_ci/tests/Resources/OCP.resource +++ b/ods_ci/tests/Resources/OCP.resource @@ -124,6 +124,16 @@ Wait For Pods To Be Ready Length Should Be ${replicas} ${exp_replicas} END +Wait For Deployment Replica To Be Ready + [Documentation] Wait for Deployment of ${label_selector} in ${namespace} to have the Replica-Set Ready + [Arguments] ${namespace} ${label_selector} ${timeout}=600s + Log To Console Waiting for Deployment with label "${label_selector}" in Namespace "${namespace}", to have desired Replica-Set + ${output} = Wait Until Keyword Succeeds ${timeout} 3s Run And Verify Command + ... oc wait -n ${namespace} --for condition\=ready pod -l "${label_selector}" --timeout\=${timeout} + ... print_to_log=${FALSE} + Run And Verify Command + ... oc get deployment -l ${label_selector} -n ${namespace} -o json | jq -e '.status | .replicas \=\= .readyReplicas' + Wait For Pods To Succeed [Arguments] ${label_selector} ${namespace} ${timeout}=300s ${exp_replicas}=${NONE} Wait Until Keyword Succeeds ${timeout} 3s