Add TGIS test for fetching model from Minio object storage (#1231)

red-hat-data-services · Feb 29, 2024 · da97eb1 · da97eb1
1 parent 3862de5
commit da97eb1
Show file tree

Hide file tree

Showing 5 changed files with 209 additions and 1 deletion.
diff --git a/ods_ci/tests/Resources/CLI/Minio.resource b/ods_ci/tests/Resources/CLI/Minio.resource
@@ -0,0 +1,74 @@
+*** Settings ***
+Documentation       Collcetion of keywords to deploy and remove MinIO storage
+Library             OperatingSystem
+Resource            ../Common.robot
+
+
+*** Variables ***
+${MINIO_RESOURCES_DIRPATH}=    ods_ci/tests/Resources/Files/minio
+${MINIO_POD_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio.yaml
+${MINIO_SECRET_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio_secret.yaml
+${MINIO_SA_FILEPATH}=    ${MINIO_RESOURCES_DIRPATH}/minio_serviceaccount.yaml
+
+
+*** Keywords ***
+Deploy MinIO
+    [Documentation]    Deploys MiniIO pod, service and route using the given ${minio_image}
+    [Arguments]    ${minio_image}=quay.io/modh/ods-ci-minio-models@sha256:801f49a705e827283a9970d2e5a807fec8357f18457abd0f46d9797beabe8160
+    ...            ${namespace}=minio    ${podname}=ods-ci-minio
+    ...            ${service_name}=ods-ci-minio-srv    ${route_name}=ods-ci-minio-route
+    ${exists}=    Run Keyword And Return Status
+    ...    Check If Pod Exists    namespace=${namespace}    label_selector=app=minio    status_only=${FALSE}
+    IF    ${exists}
+        Log    message=Minio pod with name ${podname} in ${namespace} NS already present. Re-using existing one.
+    ELSE
+        ${key}    ${pw}=    Generate Minio Random Credentials
+        Set Test Variable    ${key}
+        Set Test Variable    ${pw}
+        Set Test Variable    ${podname}
+        Set Test Variable    ${minio_image}
+        Set Test Variable    ${service_name}
+        Set Test Variable    ${route_name}
+        Create File From Template    ${MINIO_POD_FILEPATH}    ${MINIO_RESOURCES_DIRPATH}/minio_filled.yaml
+        ${rc}=    Run And Return Rc    oc new-project ${namespace}
+        ${rc}    ${out}=    Run And Return Rc And Output
+        ...    oc apply -f ${MINIO_RESOURCES_DIRPATH}/minio_filled.yaml -n ${namespace}
+        Wait For Pods To Be Ready    label_selector=app=minio
+        ...    namespace=${namespace}
+    END
+    ${rc}    ${route}=    Run And Return Rc And Output
+    ...    oc get route ${route_name} -n ${namespace} --template={{.spec.host}}
+    Should Be Equal As Integers    ${rc}    ${0}
+    RETURN    ${route}
+
+Generate Minio Random Credentials
+    [Documentation]    Generates a random pair of keywords to be used as default MinIO user credentials
+    ${key}=    Generate Random String    chars=[NUMBERS][LETTERS]
+    ${pw}=    Generate Random String    chars=[NUMBERS][LETTERS]    length=12
+    RETURN    ${key}    ${pw}
+
+Get Minio Credentials
+    [Documentation]    Extracts the default MinIO user credentials from the pod env variables
+    [Arguments]    ${namespace}    ${podname}=ods-ci-minio
+    ${rc}    ${key}=    Run And Return Rc And Output
+    ...    oc get pod ${podname} -n ${namespace} -o jsonpath='{range .spec.containers[*].env[?(@.name=="MINIO_ACCESS_KEY")]}{@.value}{end}'    # robocop: disable
+    Should Be Equal As Integers    ${rc}    ${0}
+    ${rc}    ${pw}=    Run And Return Rc And Output
+    ...    oc get pod ${podname} -n ${namespace} -o jsonpath='{range .spec.containers[*].env[?(@.name=="MINIO_SECRET_KEY")]}{@.value}{end}'    # robocop: disable
+    Should Be Equal As Integers    ${rc}    ${0}
+    RETURN    ${key}    ${pw}
+
+Remove Minio
+    [Documentation]    Deletes the MinIO objects from the cluster
+    [Arguments]    ${namespace}
+    ${rc}    ${out}=    Run And Return Rc And Output    oc delete all -l app=minio -n ${namespace}
+    Should Be Equal As Integers    ${rc}    ${0}
+
+Clean Up Minio Namespace
+    [Documentation]    Removes MinIO objects and delete the OCP namespace
+    [Arguments]    ${namespace}
+    Remove Minio    ${namespace}
+    ${rc}    ${out}=    Run And Return Rc And Output    oc delete project ${namespace}
+    Should Be Equal As Integers    ${rc}    ${0}
+    ${rc}    ${out}=    Run And Return Rc And Output    oc wait --for=delete namespace ${namespace} --timeout=20s
+    Should Be Equal As Integers    ${rc}    ${0}
diff --git a/ods_ci/tests/Resources/CLI/ModelServing/llm.resource b/ods_ci/tests/Resources/CLI/ModelServing/llm.resource
@@ -55,9 +55,13 @@ Set Project And Runtime
     [Documentation]    Creates the DS Project (if not exists), creates the data connection for the models,
     ...                creates caikit runtime. This can be used as test setup
     [Arguments]    ${namespace}    ${enable_metrics}=${FALSE}    ${runtime}=caikit-tgis-runtime    ${protocol}=grpc
+    ...            ${access_key_id}=${S3.AWS_ACCESS_KEY_ID}    ${access_key}=${S3.AWS_SECRET_ACCESS_KEY}
+    ...            ${endpoint}=${MODELS_BUCKET.ENDPOINT}    ${verify_ssl}=${TRUE}
     Set Up Test OpenShift Project    test_ns=${namespace}
-    Create Secret For S3-Like Buckets    endpoint=${MODELS_BUCKET.ENDPOINT}
+    Create Secret For S3-Like Buckets    endpoint=${endpoint}
     ...    region=${MODELS_BUCKET.REGION}    namespace=${namespace}
+    ...    access_key_id=${access_key_id}    access_key=${access_key}
+    ...    verify_ssl=${verify_ssl}
     Deploy Serving Runtime    namespace=${namespace}    runtime=${runtime}    protocol=${protocol}
     IF   ${enable_metrics} == ${TRUE}
         Enable User Workload Monitoring
@@ -72,6 +76,7 @@ Create Secret For S3-Like Buckets
     ...            ${namespace}=${TEST_NS}    ${endpoint}=${S3.AWS_DEFAULT_ENDPOINT}
     ...            ${region}=${S3.AWS_DEFAULT_REGION}    ${access_key_id}=${S3.AWS_ACCESS_KEY_ID}
     ...            ${access_key}=${S3.AWS_SECRET_ACCESS_KEY}    ${use_https}=${USE_BUCKET_HTTPS}
+    ...            ${verify_ssl}=${TRUE}
     ${rc}    ${out}=    Run And Return Rc And Output    oc get secret ${name} -n ${namespace}
     IF    "${rc}" == "${0}"
         Log    message=Secret ${name} in ${namespace} NS already present. Skipping secret setup...
@@ -102,6 +107,11 @@ Create Secret For S3-Like Buckets
     Should Be Equal As Integers    ${rc}    ${0}
     Run Keyword And Ignore Error    Run    oc create -f ${LLM_RESOURCES_DIRPATH}/bucket_sa_filled.yaml -n ${namespace}
     Add Secret To Service Account    sa_name=${sa_name}    secret_name=${name}    namespace=${namespace}
+    IF    ${verify_ssl} == ${FALSE}
+        ${rc}    ${out}=    Run And Return Rc And Output
+        ...    oc annotate secret ${name} -n ${namespace} serving.kserve.io/s3-verifyssl='false' --overwrite
+        Should Be Equal As Integers    ${rc}    ${0}
+    END
 
 Compile Inference Service YAML
     [Documentation]    Prepare the Inference Service YAML file in order to deploy a model

diff --git a/ods_ci/tests/Resources/Files/minio/minio.yaml b/ods_ci/tests/Resources/Files/minio/minio.yaml
@@ -0,0 +1,60 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${service_name}
+  labels:
+    app: minio
+spec:
+  ports:
+    - name: minio-client-port
+      port: 9000
+      protocol: TCP
+      targetPort: 9000
+  selector:
+    app: minio
+---
+kind: Route
+apiVersion: route.openshift.io/v1
+metadata:
+  labels:
+    app: minio
+  name: ${route_name}
+spec:
+  to:
+    kind: Service
+    name: ${service_name}
+    weight: 100
+  port:
+    targetPort: minio-client-port
+  wildcardPolicy: None
+  tls:
+    termination: edge
+    insecureEdgeTerminationPolicy: Redirect
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: minio
+  name: ${podname}
+spec:
+  automountServiceAccountToken: false
+  containers:
+    - args:
+        - server
+        - /data1
+      env:
+        - name: MINIO_ACCESS_KEY
+          value:  ${key}
+        - name: MINIO_SECRET_KEY
+          value: ${pw}
+      image: ${minio_image}
+      imagePullPolicy: Always
+      name: minio
+      resources:
+        limits:
+          cpu: 250m
+          memory: 10Gi
+        requests:
+          cpu: 20m
+          memory: 400Mi
diff --git a/ods_ci/tests/Resources/RHOSi.resource b/ods_ci/tests/Resources/RHOSi.resource
@@ -38,6 +38,8 @@ Resource            Common.robot
 ...                         Login To OCP Using API
 ...                         Generate Thanos Token
 ...                         Set Thanos Credentials Variables
+...                         Generate Minio Random Credentials
+...                         Get Minio Credentials
 
 
 *** Keywords ***

diff --git a/...__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis_object_storages.robot b/...__ods_dashboard/420__model_serving/LLMs/422__model_serving_llm_tgis_object_storages.robot
@@ -0,0 +1,62 @@
+*** Settings ***
+Documentation     Collection of CLI tests to validate fetching models from different object storages
+...               in the scope of model serving stack for Large Language Models (LLM).
+...               These tests leverage on TGIS Standalone Serving Runtime
+Resource          ../../../../Resources/OCP.resource
+Resource          ../../../../Resources/CLI/ModelServing/llm.resource
+Resource          ../../../../Resources/CLI/Minio.resource
+Library           OpenShiftLibrary
+Suite Setup       Suite Setup
+Suite Teardown    RHOSi Teardown
+Test Tags         KServe
+
+
+*** Variables ***
+${MODEL_S3_DIR}=    flan-t5-small-hf
+${TEST_NS}=    tgis-storages
+${TGIS_RUNTIME_NAME}=    tgis-runtime
+
+
+*** Test Cases ***
+Verify User Can Serve And Query A Model From Minio
+    [Documentation]    Basic tests for preparing, deploying and querying a LLM model
+    ...                using Kserve and TGIS runtime fetching models from a MinIO server
+    [Tags]    Tier1    RHOAIENG-3490
+    ${minio_namespace}=    Set Variable    minio-models
+    ${minio_endpoint}=    Deploy MinIO    namespace=${minio_namespace}
+    ${key}    ${pw}=    Get Minio Credentials    namespace=${minio_namespace}
+    Set Project And Runtime    runtime=${TGIS_RUNTIME_NAME}     namespace=${TEST_NS}-minio
+    ...    access_key_id=${key}    access_key=${pw}
+    ...    endpoint=${minio_endpoint}
+    ...    verify_ssl=${FALSE}    # temporary
+    ${test_namespace}=    Set Variable     ${TEST_NS}-minio
+    ${model_name}=    Set Variable    flan-t5-small-hf
+    ${models_names}=    Create List    ${model_name}
+    ${storage_uri}=    Set Variable    s3://models/${MODEL_S3_DIR}/
+    Compile Inference Service YAML    isvc_name=${model_name}
+    ...    sa_name=${DEFAULT_BUCKET_SA_NAME}
+    ...    model_storage_uri=${storage_uri}
+    ...    model_format=pytorch    serving_runtime=${TGIS_RUNTIME_NAME}
+    Deploy Model Via CLI    isvc_filepath=${INFERENCESERVICE_FILLED_FILEPATH}
+    ...    namespace=${test_namespace}
+    Wait For Pods To Be Ready    label_selector=serving.kserve.io/inferenceservice=${model_name}
+    ...    namespace=${test_namespace}
+    Query Model Multiple Times    model_name=${model_name}    runtime=${TGIS_RUNTIME_NAME}
+    ...    inference_type=all-tokens    n_times=1
+    ...    namespace=${test_namespace}
+    Query Model Multiple Times    model_name=${model_name}    runtime=${TGIS_RUNTIME_NAME}
+    ...    inference_type=streaming    n_times=1
+    ...    namespace=${test_namespace}    validate_response=${FALSE}
+    [Teardown]    Run Keywords
+    ...    Clean Up Test Project    test_ns=${test_namespace}    isvc_names=${models_names}    wait_prj_deletion=${FALSE}
+    ...    AND
+    ...    Clean Up Minio Namespace    namespace=${minio_namespace}
+
+
+*** Keywords ***
+Suite Setup
+    [Documentation]
+    Skip If Component Is Not Enabled    kserve
+    RHOSi Setup
+    Load Expected Responses
+    Run    git clone https://github.com/IBM/text-generation-inference/