First round of automation for new serving stack (red-hat-data-service…

…s#867) * add checks for kserve deployment ootb Signed-off-by: bdattoma <[email protected]> * fix docs Signed-off-by: bdattoma <[email protected]> * set default ns Signed-off-by: bdattoma <[email protected]> * add operator install test 1 Signed-off-by: bdattoma <[email protected]> * rename test suite Signed-off-by: bdattoma <[email protected]> * add suite documentation Signed-off-by: bdattoma <[email protected]> * remove unused line from kserve checks Signed-off-by: bdattoma <[email protected]> * add kw to track ns under service mesh Signed-off-by: bdattoma <[email protected]> * star service mesh and serveless cr deployment Signed-off-by: bdattoma <[email protected]> * replace NS with sed Signed-off-by: bdattoma <[email protected]> * improve operators group creation Signed-off-by: bdattoma <[email protected]> * first working draft of deployment check Signed-off-by: bdattoma <[email protected]> * remove hardcoded ns from istio configs Signed-off-by: bdattoma <[email protected]> * add wait for pods using oc wait Signed-off-by: bdattoma <[email protected]> * add peer auth yaml Signed-off-by: bdattoma <[email protected]> * add knative gateways creation Signed-off-by: bdattoma <[email protected]> * set test project and caikit runtime + docs Signed-off-by: bdattoma <[email protected]> * add missing serveless CR yaml Signed-off-by: bdattoma <[email protected]> * add sleeps + kiali check + improve pod wait Signed-off-by: bdattoma <[email protected]> * fix test ns creation Signed-off-by: bdattoma <[email protected]> * label selector fixes for serverless Signed-off-by: bdattoma <[email protected]> * improve wait for pods Signed-off-by: bdattoma <[email protected]> --------- Signed-off-by: bdattoma <[email protected]>
lugi0 · Aug 2, 2023 · de276bd · de276bd
1 parent 817a515
commit de276bd
Show file tree

Hide file tree

Showing 14 changed files with 510 additions and 14 deletions.
diff --git a/ods_ci/tests/Resources/Files/llm/caikit_servingruntime.yaml b/ods_ci/tests/Resources/Files/llm/caikit_servingruntime.yaml
@@ -0,0 +1,30 @@
+apiVersion: serving.kserve.io/v1alpha1
+kind: ServingRuntime
+metadata:
+  name: caikit-runtime
+spec:
+  containers:
+  - env:
+    - name: RUNTIME_LOCAL_MODELS_DIR
+      value: /mnt/models
+    # Use this to enable CPU-only inferencing
+    - name: DTYPE_STR 
+      value: float32
+    # TODO: This will eventually point to the official image
+    # Note: This is currently the latest image, this can change frequently
+    image: quay.io/opendatahub/caikit-tgis-serving:pr-25
+    name: kserve-container
+    ports:
+    # Note, KServe only allows a single port, this is the gRPC port. Subject to change in the future
+    - containerPort: 8085
+      name: h2c
+      protocol: TCP
+    resources:
+      requests:
+        cpu: 4
+        memory: 8Gi
+  multiModel: false
+  supportedModelFormats:
+  # Note: this currently *only* supports caikit format models
+  - autoSelect: true
+    name: caikit
diff --git a/ods_ci/tests/Resources/Files/llm/gateways.yaml b/ods_ci/tests/Resources/Files/llm/gateways.yaml
@@ -0,0 +1,51 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    experimental.istio.io/disable-gateway-port-translation: "true"
+  name: knative-local-gateway
+  namespace: {{SERVICEMESH_CR_NS}}
+spec:
+  ports:
+    - name: http2
+      port: 80
+      protocol: TCP
+      targetPort: 8081
+  selector:
+    istio: ingressgateway
+  type: ClusterIP
+---
+apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: knative-ingress-gateway
+  namespace: {{SERVERLESS_CR_NS}}
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+    - hosts:
+        - '*'
+      port:
+        name: https
+        number: 443
+        protocol: HTTPS
+      tls:
+        credentialName: wildcard-certs
+        mode: SIMPLE
+---
+apiVersion: networking.istio.io/v1beta1
+kind: Gateway
+metadata:
+  name: knative-local-gateway
+  namespace: {{SERVERLESS_CR_NS}}
+spec:
+  selector:
+    istio: ingressgateway
+  servers:
+    - hosts:
+        - '*'
+      port:
+        name: http
+        number: 8081
+        protocol: HTTP
diff --git a/ods_ci/tests/Resources/Files/llm/knativeserving_istio.yaml b/ods_ci/tests/Resources/Files/llm/knativeserving_istio.yaml
@@ -0,0 +1,20 @@
+apiVersion: operator.knative.dev/v1beta1
+kind: KnativeServing
+metadata:
+  name: knative-serving
+  namespace: {{SERVERLESS_CR_NS}}
+  annotations:
+    serverless.openshift.io/default-enable-http2: "true"
+spec:
+  deployments:
+    - annotations:
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/rewriteAppHTTPProbers: "true"
+      name: activator
+    - annotations:
+        sidecar.istio.io/inject: "true"
+        sidecar.istio.io/rewriteAppHTTPProbers: "true"
+      name: autoscaler
+  ingress:
+    istio:
+      enabled: true
diff --git a/ods_ci/tests/Resources/Files/llm/peer_auth.yaml b/ods_ci/tests/Resources/Files/llm/peer_auth.yaml
@@ -0,0 +1,8 @@
+apiVersion: security.istio.io/v1beta1
+kind: PeerAuthentication
+metadata:
+  name: default
+  namespace: {{NAMESPACE}}
+spec:
+  mtls:
+    mode: STRICT
diff --git a/ods_ci/tests/Resources/Files/llm/smcp.yaml b/ods_ci/tests/Resources/Files/llm/smcp.yaml
@@ -0,0 +1,23 @@
+apiVersion: maistra.io/v2
+kind: ServiceMeshControlPlane
+metadata:
+  name: minimal
+  namespace: {{SERVICEMESH_CR_NS}}
+spec:
+  addons:
+    grafana:
+      enabled: false
+    kiali:
+      name: kiali
+      enabled: true
+    prometheus:
+      enabled: true
+    jaeger: 
+      name: jaeger
+  security:
+    dataPlane:
+      mtls: false # otherwise inference-graph will not work. We use PeerAuthentication resources to force mTLS
+    identity:
+      type: ThirdParty        
+  profiles:
+    - default
diff --git a/ods_ci/tests/Resources/Files/llm/smmr.yaml b/ods_ci/tests/Resources/Files/llm/smmr.yaml
@@ -0,0 +1,7 @@
+apiVersion: maistra.io/v1
+kind: ServiceMeshMemberRoll
+metadata:
+  name: default
+  namespace: {{SERVICEMESH_CR_NS}}
+spec:
+  members: []
diff --git a/ods_ci/tests/Resources/OCP.resource b/ods_ci/tests/Resources/OCP.resource
@@ -2,6 +2,8 @@
 Documentation  Set of Keywords for OCP checks
 Library    OperatingSystem
 Library    OpenShiftLibrary
+Library    Process
+Resource   Page/OCPDashboard/Pods/Pods.robot
 
 
 *** Keywords ***
@@ -97,3 +99,12 @@ Fetch CA Certificate If RHODS Is Self-Managed
     IF  ${self_managed}==${TRUE}
         Fetch Openshift CA Bundle
     END
+
+Wait For Pods To Be Ready
+    [Arguments]    ${label_selector}    ${namespace}    ${timeout}=300s
+    Wait Until Keyword Succeeds    ${timeout}    3s
+    ...    Check If Pod Exists    namespace=${namespace}    label_selector=${label_selector}
+    ...    status_only=${FALSE}
+    ${rc}    ${out}=    Run And Return Rc And Output
+    ...    oc wait --for=condition=ready pod -l ${label_selector} -n ${namespace} --timeout=${timeout}
+    Should Be Equal As Numbers    ${rc}    ${0}
diff --git a/ods_ci/tests/Resources/Page/OCPDashboard/Pods/Pods.robot b/ods_ci/tests/Resources/Page/OCPDashboard/Pods/Pods.robot
diff --git a/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource b/ods_ci/tests/Resources/Page/ODH/ODHDashboard/ODHModelServing.resource
@@ -243,3 +243,8 @@ Clean Up Model Serving Page
         ${name} =    SeleniumLibrary.Get Text   ${model}
         Delete Model Via UI    ${name}
     END
+
+Add Namespace To ServiceMeshMemberRoll
+    [Arguments]    ${namespace}    ${servicemesh_ns}=istio-system
+    ${rc}    ${out}=    Run And Return Rc And Output
+    ...    oc patch smmr/default -n ${servicemesh_ns} --type='json' -p="[{'op': 'add', 'path': '/spec/members/-', 'value': \"${namespace}\"}]"
diff --git a/ods_ci/tests/Resources/Page/Operators/ISVs.resource b/ods_ci/tests/Resources/Page/Operators/ISVs.resource
@@ -15,7 +15,8 @@ Install ISV Operator From OperatorHub Via CLI    # robocop: disable
     [Documentation]    Installs the given operator via CLI
     [Arguments]    ${operator_name}    ${subscription_name}    ${namespace}=openshift-operators
     ...            ${channel}=stable    ${catalog_source_name}=certified-operators
-    ...            ${cs_namespace}=openshift-marketplace    ${operator_group_target_ns}=${NONE}
+    ...            ${cs_namespace}=openshift-marketplace    ${operator_group_name}=${NONE}
+    ...            ${operator_group_ns}=${NONE}    ${operator_group_target_ns}=${NONE}
     ${operator_sub_filepath}=    Set Variable    ${FILES_RESOURCES_DIRPATH}/${operator_name}-sub.yaml
     Copy File    ${SUBSCRIPTION_YAML_TEMPLATE_FILEPATH}    ${operator_sub_filepath}
     ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<SUBSCRIPTION_NAME>/${subscription_name}/g" ${operator_sub_filepath}    # robocop: disable
@@ -25,26 +26,27 @@ Install ISV Operator From OperatorHub Via CLI    # robocop: disable
     ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<CATALOG_SOURCE>/${catalog_source_name}/g" ${operator_sub_filepath}    # robocop: disable
     ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<CS_NAMESPACE>/${cs_namespace}/g" ${operator_sub_filepath}    # robocop: disable
     Oc Apply    kind=Subscription    src=${operator_sub_filepath}
-    IF    "${operator_group_target_ns}" != "${NONE}"
-        Create Operator Group    operator_name=${operator_name}
-        ...    namespace=${namespace}    target_namespace=${operator_group_target_ns}
+    IF    "${operator_group_name}" != "${NONE}"
+        Create Operator Group    name=${operator_group_name}
+        ...    namespace=${operator_group_ns}    target_namespace=${operator_group_target_ns}
+        ...    operator_name=${operator_name}
     END
 
 Create Operator Group
     [Documentation]    Creates the Operator Group object which might be needed by an operator.
     ...                It currently supports creating OperatorGroups with only 1 targeted namespace
-    [Arguments]    ${operator_name}    ${namespace}    ${target_namespace}
+    [Arguments]    ${name}    ${namespace}    ${target_namespace}    ${operator_name}
     ${operator_group_filepath}=    Set Variable    ${FILES_RESOURCES_DIRPATH}/${operator_name}-group.yaml    # robocop: disable
     Copy File    ${OPERATORGROUP_YAML_TEMPLATE_FILEPATH}    ${operator_group_filepath}
-    ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<BASE_NAME>/${operator_name}/g" ${operator_group_filepath}    # robocop: disable
+    ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<BASE_NAME>/${name}/g" ${operator_group_filepath}    # robocop: disable
     ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<NAMESPACE>/${namespace}/g" ${operator_group_filepath}    # robocop: disable
     ${rc}    ${out}=    Run And Return Rc And Output    sed -i "s/<TARGET_NAMESPACE>/${target_namespace}/g" ${operator_group_filepath}    # robocop: disable
     Oc Apply    kind=OperatorGroup    src=${operator_group_filepath}    api_version=v1
 
 Operator Subscription Last Condition Should Be
     [Documentation]    Checks the subscription status matches the expected status
     [Arguments]    ${type}    ${status}    ${reason}
-    ...    ${subcription_name}    ${namespace}
+    ...    ${subcription_name}    ${namespace}=openshift-operators
     ${rc}    ${out}=    Run And Return Rc And Output
     ...    oc get subscription ${subcription_name} -n ${namespace} -ojson | jq '.status.conditions | last | select(.type=="${type}" and .status=="${status}" and .reason=="${reason}")'    # robocop: disable
     Should Be Equal As Integers    ${rc}     ${0}
@@ -53,7 +55,7 @@ Operator Subscription Last Condition Should Be
 Wait Until Operator Subscription Last Condition Is    # robocop: disable
     [Documentation]    Keeps checking until the subscription status matches the expected status
     [Arguments]    ${type}    ${status}    ${reason}
-    ...    ${subcription_name}    ${namespace}
+    ...    ${subcription_name}    ${namespace}=openshift-operators
     ...    ${retry}=20    ${retry_interval}=3s
     Wait Until Keyword Succeeds    ${retry} times    ${retry_interval}    Operator Subscription Last Condition Should Be    # robocop: disable
     ...    type=${type}     status=${status}
@@ -112,7 +114,7 @@ Delete Custom Resource
 
 Uninstall ISV Operator From OperatorHub Via CLI
     [Documentation]    Uninstalls the given operator via CLI
-    [Arguments]    ${subscription_name}    ${namespace}
+    [Arguments]    ${subscription_name}    ${namespace}=openshift-operators
     ${rc}    ${current_csv_name}=    Run And Return Rc And Output
     ...    oc get subscription ${subscription_name} -n ${namespace} -ojson | jq '.status.currentCSV' | tr -d '"'    # robocop: disable
     Oc Delete    kind=Subscription    name=${subscription_name}    namespace=${namespace}

diff --git a/ods_ci/tests/Tests/100__deploy/100__installation/110__model_serving_watsonx.robot b/ods_ci/tests/Tests/100__deploy/100__installation/110__model_serving_watsonx.robot
@@ -0,0 +1,101 @@
+*** Settings ***
+Documentation       Post install test cases that verify OCP KServe resources and objects
+Library             OpenShiftLibrary
+Resource            ../../../Resources/ODS.robot
+Resource            ../../../Resources/Page/ODH/JupyterHub/HighAvailability.robot
+# Resource            ../../../Resources/RHOSi.resource
+# Suite Setup         RHOSi Setup
+# Suite Teardown      RHOSi Teardown
+
+
+*** Variables ***
+${KSERVE_NS}=    kserve    # will be replaced by redhat-ods-applications
+
+*** Test Cases ***
+Verify KServe Is Shipped
+    [Documentation]    Verify KServe Is Shipped And Enabled Within ODS
+    [Tags]    WatsonX
+    ...       ODS-2325
+    @{kserve_pods_info} =    Fetch KServe Pods
+    @{kserve_services_info} =    Fetch KServe Controller Services
+    @{kserve_wh_services_info} =    Fetch KServe Webhook Services
+    Verify KServe Deployment
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    status.phase    Running    @{kserve_pods_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    status.conditions[2].status    True    @{kserve_pods_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    spec.ports[0].port    8443    @{kserve_services_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    spec.ports[0].protocol    TCP    @{kserve_services_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    spec.ports[0].port    443    @{kserve_wh_services_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Be Equal As Strings    spec.ports[0].protocol    TCP    @{kserve_wh_services_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Match Regexp    spec.clusterIP    ^((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])(\.(?!$)|$)){4}$
+    ...    @{kserve_services_info}
+    Run Keyword And Continue On Failure
+    ...    OpenShift Resource Field Value Should Match Regexp    spec.clusterIP    ^((25[0-5]|(2[0-4]|1[0-9]|[1-9]|)[0-9])(\.(?!$)|$)){4}$
+    ...    @{kserve_wh_services_info}
+    Wait Until Keyword Succeeds    10 times  5s    Verify KServe ReplicaSets Info
+    ServingRuntime CustomResourceDefinition Should Exist
+    InferenceService CustomResourceDefinition Should Exist
+
+
+*** Keywords ***
+Fetch KServe Pods
+    [Documentation]    Fetches information from KServe pods
+    ...    Args:
+    ...        None
+    ...    Returns:
+    ...        kserve_pods_info(list(dict)): KServe pods selected by label and namespace
+    @{kserve_pods_info} =    Oc Get    kind=Pod    api_version=v1    namespace=${KSERVE_NS}
+    ...    label_selector=app.kubernetes.io/part-of=kserve
+    RETURN    @{kserve_pods_info}
+
+Fetch KServe Controller Services
+    [Documentation]    Fetches information from KServe services
+    ...    Args:
+    ...        None
+    ...    Returns:
+    ...        kserve_services_info(list(dict)): KServe services selected by name and namespace
+    @{kserve_services_info} =    Oc Get    kind=Service    api_version=v1    label_selector=control-plane=kserve-controller-manager
+    ...    namespace=${KSERVE_NS}
+    RETURN    @{kserve_services_info}
+
+Fetch KServe Webhook Services
+    [Documentation]    Fetches information from KServe services
+    ...    Args:
+    ...        None
+    ...    Returns:
+    ...        kserve_services_info(list(dict)): KServe services selected by name and namespace
+    @{kserve_wh_services_info} =    Oc Get    kind=Service    api_version=v1    name=kserve-webhook-server-service
+    ...    namespace=${KSERVE_NS}
+    RETURN    @{kserve_wh_services_info}
+
+Verify KServe ReplicaSets Info
+    [Documentation]    Fetches and verifies information from KServe replicasets
+    @{kserve_replicasets_info} =    Oc Get    kind=ReplicaSet    api_version=v1    namespace=${KSERVE_NS}
+    ...    label_selector=app.kubernetes.io/part-of=kserve
+    OpenShift Resource Field Value Should Be Equal As Strings    status.readyReplicas
+    ...    1    @{kserve_replicasets_info}
+    OpenShift Resource Field Value Should Be Equal As Strings    status.replicas
+    ...    1    @{kserve_replicasets_info}
+
+Verify Kserve Deployment
+    [Documentation]  Verifies RHODS KServe deployment
+    @{kserve} =  Oc Get    kind=Pod    namespace=${KSERVE_NS}    api_version=v1
+    ...    label_selector=app.kubernetes.io/part-of=kserve
+    ${containerNames} =    Create List    manager
+    Verify Deployment    ${kserve}    1    1    ${containerNames}
+
+ServingRuntime CustomResourceDefinition Should Exist
+    [Documentation]    Checks that the ServingRuntime CRD is present
+    ${sr_crd}=    Oc Get    kind=CustomResourceDefinition    field_selector=metadata.name=servingruntimes.serving.kserve.io
+    Should Not Be Empty    ${sr_crd}
+
+InferenceService CustomResourceDefinition Should Exist
+    [Documentation]    Checks that the InferenceService CRD is present
+    ${is_crd}=    Oc Get    kind=CustomResourceDefinition    field_selector=metadata.name=inferenceservices.serving.kserve.io
+    Should Not Be Empty    ${is_crd}