From 0449e1594e216c96de89b1708947aa5775257ba8 Mon Sep 17 00:00:00 2001 From: Conor O'Malley Date: Mon, 14 Oct 2024 10:17:06 +0100 Subject: [PATCH 1/5] fix for ODS-2626 --- .../ModelServer.resource | 3 +- .../ODH/ODHDashboard/ODHModelServing.resource | 31 +++---------------- .../1005__model_serving_ovms_on_kserve.robot | 4 ++- 3 files changed, 10 insertions(+), 28 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 2333463d4..1426e3e5f 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -31,7 +31,7 @@ ${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]// *** Keywords *** Create Model Server [Documentation] Keyword to create a Model Server in a Data Science Project - [Arguments] ${no_replicas}=1 ${server_size}=Small ${ext_route}=${TRUE} + [Arguments] ${no_replicas}=1 ${server_size}=Small ${ext_route}=${FALSE} ... ${token}=${TRUE} ${runtime}=OpenVINO Model Server ${server_name}=Model Serving Test ... ${no_gpus}=0 ${existing_server}=${FALSE} Move To Tab Models @@ -57,6 +57,7 @@ Create Model Server END IF ${ext_route}==${TRUE} Enable External Serving Route + Wait Until Page Contains Element ${TOKEN_AUTH_CHECKBOX_XP} timeout=10s IF ${token}==${FALSE} Disable Token Authentication ELSE diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 66a86b725..75c500e7b 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -27,7 +27,7 @@ ${MS_TABLE_STATUS_FAILURE}= //span[contains(@class,"pf-v5-c-icon__content")] ${KSERVE_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Deploy model"] ${KSERVE_RUNTIME_DROPDOWN}= //button[@data-testid="serving-runtime-template-selection"] ${LLM_RESOURCES_DIRPATH}= tests/Resources/Files/llm -${DEPLOY_MODEL_BTN}= //button[contains(@data-testid,"deploy")] +${DEPLOY_MODEL_BTN}= //button[contains(@data-testid,"deploy-model-button")] ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] @@ -133,6 +133,7 @@ Select Framework [Arguments] ${framework} TRY Open Model framework (name - version) Options Menu + SeleniumLibrary.Wait Until Page Contains Element xpath://span[.="Model framework (name - version)"] Page Should Contain Element xpath://li/button[contains(., "${framework}")] Click Element xpath://li/button[contains(., "${framework}")] EXCEPT @@ -437,36 +438,14 @@ Deploy Model Via CLI Deploy Kserve Model Via UI #robocop: disable [Documentation] Deploys a model using the kserve/caikit runtime using the Data Science Projects UI [Arguments] ${model_name} ${serving_runtime} ${data_connection} ${path} - ... ${model_framework}=caikit ${replicas}=1 ${size}=Small ${no_gpus}=${0} + ... ${model_framework} ${replicas}=1 ${size}=Small ${no_gpus}=${0} ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} Move To Tab Models SeleniumLibrary.Click Button ${DEPLOY_MODEL_BTN} SeleniumLibrary.Wait Until Page Contains Element xpath=${KSERVE_MODAL_HEADER} Set Model Name ${model_name} - Set Model Server Runtime ${serving_runtime} Select Framework ${model_framework} - Set Replicas Number With Buttons ${replicas} - Set Server Size ${size} - IF ${public_endpoint} - Enable External Serving Route - # By default this also enables Token Authentication. Let's disable it and let it get re-enable by the next - # IF block if needed. - ${token_enabled}= Run Keyword And Return Status SeleniumLibrary.Checkbox Should Be Selected - ... ${TOKEN_AUTH_CHECKBOX_XP} - IF ${token_enabled} - Disable Token Authentication - ELSE - Log Token Authentication was supposed to be automatically enabled, but it wasn't level=ERROR - ... console=${True} - END - END - IF ${token} - Enable Token Authentication service_account_name=${service_account_name} - IF ${multi_token} - Add Another Service Account ${multi_service_account_name} - END - END Select Existing Data Connection ${data_connection} Set Folder Path ${path} IF ${no_gpus} > ${0} @@ -479,7 +458,7 @@ Deploy Kserve Model Via UI #robocop: disable Set Number of GPU With Buttons ${no_gpus} END Click Button Deploy - Wait Until Page Does Not Contain Element xpath=${KSERVE_MODAL_HEADER} timeout=60s + Wait Until Page Does Not Contain Element xpath=//button[@data-testid="modal-submit-button"] timeout=60s Set Model Server Runtime [Documentation] Opens the Serving runtime dropdown in the deploy model modal window for models @@ -487,7 +466,7 @@ Set Model Server Runtime [Arguments] ${runtime}=Caikit TGIS Page Should Contain Element ${KSERVE_RUNTIME_DROPDOWN} Click Element ${KSERVE_RUNTIME_DROPDOWN} - Click Element ${KSERVE_RUNTIME_DROPDOWN}/..//span[contains(text(),"${runtime}")] + Click Element ${KSERVE_RUNTIME_DROPDOWN}/..//*[@data-testid="ovms"] Get Kserve Inference Host Via UI [Documentation] Fetches the host of the model's URL from the Data Science Project UI diff --git a/ods_ci/tests/Tests/1000__model_serving/1005__model_serving_ovms_on_kserve.robot b/ods_ci/tests/Tests/1000__model_serving/1005__model_serving_ovms_on_kserve.robot index 302489341..29003c822 100644 --- a/ods_ci/tests/Tests/1000__model_serving/1005__model_serving_ovms_on_kserve.robot +++ b/ods_ci/tests/Tests/1000__model_serving/1005__model_serving_ovms_on_kserve.robot @@ -170,10 +170,12 @@ Create Openvino Models For Kserve # robocop: off=too-many-calls-in-keyword Recreate S3 Data Connection project_title=${new_project} dc_name=model-serving-connection ... aws_access_key=${S3.AWS_ACCESS_KEY_ID} aws_secret_access=${S3.AWS_SECRET_ACCESS_KEY} ... aws_bucket_name=ods-ci-s3 + Create Model Server token=${TRUE} server_name=test-server existing_server=${TRUE} + Wait Until Element Is Visible ${DEPLOY_MODEL_BTN} Deploy Kserve Model Via UI model_name=${model_name} serving_runtime=OpenVINO Model Server ... data_connection=model-serving-connection path=kserve-openvino-test/openvino-example-model ... model_framework=openvino_ir - Wait For Pods To Be Ready label_selector=serving.kserve.io/inferenceservice=${model_name} + Wait For Pods To Be Ready label_selector=name=modelmesh-serving-test-server ... namespace=${new_project} Verify Model Status ${model_name} success ${project_postfix}= Evaluate ${idx}+1 From 488cd19aff9087cca3a399bb5db2d98a9335164e Mon Sep 17 00:00:00 2001 From: Conor O'Malley Date: Mon, 14 Oct 2024 15:47:25 +0100 Subject: [PATCH 2/5] UI element path updates --- .../ODHDataScienceProject/ModelServer.resource | 2 +- .../ODH/ODHDashboard/ODHModelServing.resource | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 1426e3e5f..6a735ed0c 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -25,7 +25,7 @@ ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] ${ADD_SERVICE_ACCOUNT_BUTTON}= xpath://button[text()='Add a service account'] ${SERVICE_ACCOUNT_INPUT}= xpath://input[@data-testid='service-account-form-name'] ${REPLICAS_COUNT_XP}= xpath://section[@class='pf-v5-c-form__section']//span[@class='pf-v5-c-form-control']/input -${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]//*[@data-testid="project-selector-dropdown"] +${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]//*[@data-testid="project-selector-toggle"] *** Keywords *** diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 75c500e7b..4cac8bb1a 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -19,11 +19,9 @@ ${S3_REGION_DC_INPUT_XP}= xpath=//input[@aria-label="Field list AWS_DEF ${S3_BUCKET_DC_INPUT_XP}= xpath=//input[@aria-label="Field list AWS_S3_BUCKET"] ${S3_DEFAULT_BUCKET}= ods-ci-s3 ${MS_TABLE_PROJECT}= /../../td[@data-label="Project"] -${MS_TABLE_STATUS}= /../../td[@data-label="Status"] -${MS_TABLE_ENDPOINT}= /../../td[@data-label="Inference endpoint"] -${MS_TABLE_ENDPOINT_INPUT}= ${MS_TABLE_ENDPOINT}//div[@class="pf-v5-c-clipboard-copy"]//input -${MS_TABLE_STATUS_SUCCESS}= //span[contains(@class,"pf-v5-c-icon__content")][contains(@class,"pf-m-success")] -${MS_TABLE_STATUS_FAILURE}= //span[contains(@class,"pf-v5-c-icon__content")][contains(@class,"pf-m-danger")] +${MS_TABLE_ENDPOINT_INPUT}= /ancestor::tr//td[@data-label="Inference endpoint"]//button[@data-testid='internal-service-button'] +${MS_TABLE_STATUS_SUCCESS}= /ancestor::tr//td[@data-label="Status"]//span[contains(@class,"pf-m-success")] +${MS_TABLE_STATUS_FAILURE}= /ancestor::tr//td[@data-label="Status"]//span[contains(@class,"pf-m-danger")] ${KSERVE_MODAL_HEADER}= //header[@class="pf-v5-c-modal-box__header"]/h1[.="Deploy model"] ${KSERVE_RUNTIME_DROPDOWN}= //button[@data-testid="serving-runtime-template-selection"] ${LLM_RESOURCES_DIRPATH}= tests/Resources/Files/llm @@ -180,12 +178,12 @@ Verify Model Status [Arguments] ${model_name} ${expected_status}=success IF "${expected_status}"=="success" SeleniumLibrary.Wait Until Page Contains Element - ... //div[.="${model_name} "]${MS_TABLE_STATUS}${MS_TABLE_STATUS_SUCCESS} timeout=2m - Page Should Not Contain Element //div[.="${model_name} "]${MS_TABLE_STATUS}${MS_TABLE_STATUS_FAILURE} + ... //a[@data-testid='metrics-link-test-model' and text()='${model_name}']${MS_TABLE_STATUS_SUCCESS} timeout=2m + Page Should Not Contain Element //a[@data-testid='metrics-link-test-model' and text()='${model_name}']${MS_TABLE_STATUS_FAILURE} ELSE SeleniumLibrary.Wait Until Page Contains Element - ... //div[.="${model_name} "]${MS_TABLE_STATUS}${MS_TABLE_STATUS_FAILURE} timeout=2m - Page Should Not Contain Element //div[.="${model_name} "]${MS_TABLE_STATUS}${MS_TABLE_STATUS_SUCCESS} + ... //a[@data-testid='metrics-link-test-model' and text()='${model_name}']${MS_TABLE_STATUS_FAILURE} timeout=2m + Page Should Not Contain Element //a[@data-testid='metrics-link-test-model' and text()='${model_name}']${MS_TABLE_STATUS_SUCCESS} END Delete Model Via UI @@ -210,7 +208,7 @@ Get Model Route Via UI [Documentation] Grabs the serving route (URL) of an already deployed model from the Model Serving page. [Arguments] ${model_name} # TODO: Open model serving home page if needed? - ${route_xpath}= Set Variable xpath://div[.="${model_name} "]${MS_TABLE_ENDPOINT_INPUT} + ${route_xpath}= Set Variable //a[.="${model_name}"]${MS_TABLE_ENDPOINT_INPUT} ${loaded}= Run Keyword And Return Status SeleniumLibrary.Wait Until Page Contains Element ${route_xpath} timeout=15s IF ${loaded} == ${FALSE} Log message=Model Route was not loaded in UI (RHOAIENG-1919). Trying refreshing! level=WARN From c10b786b6b0510018526e1866fdf96ae4cb42c78 Mon Sep 17 00:00:00 2001 From: Conor O'Malley Date: Tue, 15 Oct 2024 11:06:35 +0100 Subject: [PATCH 3/5] address robocop checks --- .../ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource | 2 +- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource index 6a735ed0c..2a93203a8 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHDataScienceProject/ModelServer.resource @@ -25,7 +25,7 @@ ${TOKEN_AUTH_CHECKBOX_XP}= xpath://input[@id="alt-form-checkbox-auth"] ${ADD_SERVICE_ACCOUNT_BUTTON}= xpath://button[text()='Add a service account'] ${SERVICE_ACCOUNT_INPUT}= xpath://input[@data-testid='service-account-form-name'] ${REPLICAS_COUNT_XP}= xpath://section[@class='pf-v5-c-form__section']//span[@class='pf-v5-c-form-control']/input -${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]//*[@data-testid="project-selector-toggle"] +${PROJECT_SELECTOR_XP}= xpath://main[contains(@id, 'dashboard-page-main')]//*[@data-testid="project-selector-toggle"] *** Keywords *** diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 4cac8bb1a..ca9214c60 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -436,7 +436,7 @@ Deploy Model Via CLI Deploy Kserve Model Via UI #robocop: disable [Documentation] Deploys a model using the kserve/caikit runtime using the Data Science Projects UI [Arguments] ${model_name} ${serving_runtime} ${data_connection} ${path} - ... ${model_framework} ${replicas}=1 ${size}=Small ${no_gpus}=${0} + ... ${model_framework} ${no_gpus}=${0} ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} Move To Tab Models From b24708c3350f8f9147c213cc4dd5cbeec56f6399 Mon Sep 17 00:00:00 2001 From: Conor O'Malley Date: Tue, 15 Oct 2024 11:27:13 +0100 Subject: [PATCH 4/5] revert args removal --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index ca9214c60..4cac8bb1a 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -436,7 +436,7 @@ Deploy Model Via CLI Deploy Kserve Model Via UI #robocop: disable [Documentation] Deploys a model using the kserve/caikit runtime using the Data Science Projects UI [Arguments] ${model_name} ${serving_runtime} ${data_connection} ${path} - ... ${model_framework} ${no_gpus}=${0} + ... ${model_framework} ${replicas}=1 ${size}=Small ${no_gpus}=${0} ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} Move To Tab Models From 8143bf732679dc4899687763e1f799e193431af3 Mon Sep 17 00:00:00 2001 From: Conor O'Malley Date: Tue, 15 Oct 2024 13:14:51 +0100 Subject: [PATCH 5/5] revert model_framework arg --- .../Resources/Page/ODH/ODHDashboard/ODHModelServing.resource | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource index 4cac8bb1a..c1b9c70cc 100644 --- a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource +++ b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource @@ -436,7 +436,7 @@ Deploy Model Via CLI Deploy Kserve Model Via UI #robocop: disable [Documentation] Deploys a model using the kserve/caikit runtime using the Data Science Projects UI [Arguments] ${model_name} ${serving_runtime} ${data_connection} ${path} - ... ${model_framework} ${replicas}=1 ${size}=Small ${no_gpus}=${0} + ... ${model_framework}=caikit ${replicas}=1 ${size}=Small ${no_gpus}=${0} ... ${token}=${FALSE} ${multi_token}=${FALSE} ${multi_service_account_name}=default-name2 ... ${public_endpoint}=${TRUE} ${service_account_name}=${NONE} Move To Tab Models