Merge branch 'master' into fix/create_project_btn

lugi0 · Sep 20, 2024 · dae1585 · dae1585
2 parents 02fd543 + 3c6eb92
commit dae1585
Show file tree

Hide file tree

Showing 33 changed files with 1,096 additions and 106 deletions.
diff --git a/ods_ci/libs/DataSciencePipelinesAPI.py b/ods_ci/libs/DataSciencePipelinesAPI.py
@@ -11,10 +11,10 @@
 
 class DataSciencePipelinesAPI:
     # init should not have a call to external system, otherwise dry-run will fail
-    def __init__(self):
+    def __init__(self, sleep_time: int = 45):
         self.route = ""
         self.sa_token = None
-        self.sleep_time = 45
+        self.sleep_time = sleep_time
 
     @keyword
     def login_and_wait_dsp_route(

diff --git a/ods_ci/libs/DataSciencePipelinesKfp.py b/ods_ci/libs/DataSciencePipelinesKfp.py
diff --git a/ods_ci/tasks/Resources/Provisioning/Hive/AWS/aws-cluster.yaml b/ods_ci/tasks/Resources/Provisioning/Hive/AWS/aws-cluster.yaml
@@ -17,6 +17,9 @@ items:
           name: aws-creds
     installConfigSecretTemplateRef:
       name: aws-sno-install-config
+    installerEnv:
+      - name: OPENSHIFT_INSTALL_SKIP_HOSTCRYPT_VALIDATION
+        value: 'true'
     size: 0
     skipMachinePools: true
 - apiVersion: v1

diff --git a/ods_ci/tasks/Resources/Provisioning/Hive/AZURE/azure-cluster.yaml b/ods_ci/tasks/Resources/Provisioning/Hive/AZURE/azure-cluster.yaml
@@ -19,6 +19,9 @@ items:
           name: azure-creds
     installConfigSecretTemplateRef:
       name: azure-sno-install-config
+    installerEnv:
+      - name: OPENSHIFT_INSTALL_SKIP_HOSTCRYPT_VALIDATION
+        value: 'true'
     size: 0
     skipMachinePools: true
 - apiVersion: v1

diff --git a/ods_ci/tasks/Resources/Provisioning/Hive/GCP/gcp-cluster.yaml b/ods_ci/tasks/Resources/Provisioning/Hive/GCP/gcp-cluster.yaml
@@ -17,6 +17,9 @@ items:
           name: gcp-creds
     installConfigSecretTemplateRef:
       name: gcp-sno-install-config
+    installerEnv:
+      - name: OPENSHIFT_INSTALL_SKIP_HOSTCRYPT_VALIDATION
+        value: 'true'
     size: 0
     skipMachinePools: true
 - apiVersion: v1

diff --git a/ods_ci/tasks/Resources/Provisioning/Hive/IBM/ibmcloud-cluster.yaml b/ods_ci/tasks/Resources/Provisioning/Hive/IBM/ibmcloud-cluster.yaml
@@ -75,3 +75,6 @@ items:
         name: ibmcloud-sno-install-config
       manifestsSecretRef:
         name: ${infrastructure_configurations}[hive_cluster_name]-manifests
+      installerEnv:
+        - name: OPENSHIFT_INSTALL_SKIP_HOSTCRYPT_VALIDATION
+          value: 'true'
diff --git a/ods_ci/tasks/Resources/Provisioning/Hive/OSP/hive_osp_cluster_template.yaml b/ods_ci/tasks/Resources/Provisioning/Hive/OSP/hive_osp_cluster_template.yaml
@@ -9,16 +9,19 @@ items:
   spec:
     baseDomain: ${infrastructure_configurations}[base_domain]
     imageSetRef:
-      name: ${infrastructure_configurations}[image_set] # see line 100
+      name: ${infrastructure_configurations}[image_set] # see line 102
     inventory:
-      - name: ${infrastructure_configurations}[hive_cluster_name]-conf # see line 72
+      - name: ${infrastructure_configurations}[hive_cluster_name]-conf # see line 74
     platform:
       openstack:
         cloud: ${infrastructure_configurations}[osp_cloud_name]
         credentialsSecretRef:
-          name: ${infrastructure_configurations}[hive_cluster_name]-cred # see line 91
+          name: ${infrastructure_configurations}[hive_cluster_name]-cred # see line 93
     installConfigSecretTemplateRef:
-      name: ${infrastructure_configurations}[hive_cluster_name]-sec # see line 30
+      name: ${infrastructure_configurations}[hive_cluster_name]-sec # see line 32
+    installerEnv:
+      - name: OPENSHIFT_INSTALL_SKIP_HOSTCRYPT_VALIDATION
+        value: 'true'
     size: 1
     maxSize: 1
     runningCount: 1

diff --git a/ods_ci/tests/Resources/CLI/DataSciencePipelines/DataSciencePipelinesBackend.resource b/ods_ci/tests/Resources/CLI/DataSciencePipelines/DataSciencePipelinesBackend.resource
@@ -2,9 +2,10 @@
 Documentation    Collection of keywords to interact with Data Science Pipelines via CLI
 Library          OperatingSystem
 Library          String
+Library          ../../../../libs/DataSciencePipelinesAPI.py
+Library          ../../../../libs/DataSciencePipelinesKfp.py
 Resource         ../../../Resources/OCP.resource
 
-
 *** Variables ***
 ${DSPA_PATH}=      tests/Resources/Files/pipeline-samples/v2/dspa
 
@@ -134,3 +135,77 @@ Create Secret With Pipelines Object Storage Information
     [Arguments]    ${namespace}    ${object_storage_access_key}    ${object_storage_secret_key}
     Run    oc create secret generic dashboard-dspa-secret -n ${namespace} --from-literal=AWS_ACCESS_KEY_ID=${object_storage_access_key} --from-literal=AWS_SECRET_ACCESS_KEY=${object_storage_secret_key}
     Run    oc label secret dashboard-dspa-secret -n ${namespace} opendatahub.io/dashboard=true
+
+
+Import Pipeline And Create Run
+    [Documentation]
+    [Arguments]    ${namespace}           ${username}                ${password}
+    ...    ${pipeline_name}               ${pipeline_description}    ${pipeline_package_path}
+    ...    ${pipeline_run_name}           ${pipeline_run_params}
+    ...    ${experiment_name}=Default     ${experiment_description}=${EMPTY}
+
+    DataSciencePipelinesKfp.Setup Client    user=${username}    pwd=${password}    project=${namespace}
+
+    ${pipeline_id}    ${pipeline_version_id}=    DataSciencePipelinesKfp.Upload Pipeline
+    ...   pipeline_package_path=${pipeline_package_path}
+    ...   pipeline_name=${pipeline_name}
+    ...   description=${pipeline_description}
+    ...   namespace=${namespace}
+
+    ${experiment_id}=    DataSciencePipelinesKfp.Create Experiment    name=${experiment_name}
+    ...    description=${experiment_description}    namespace=${namespace}
+
+    ${pipeline_run_id}=    DataSciencePipelinesKfp.Run Pipeline
+    ...   experiment_id=${experiment_id}
+    ...   job_name=${pipeline_run_name}
+    ...   pipeline_id=${pipeline_id}
+    ...   version_id=${pipeline_version_id}
+    ...   params=${pipeline_run_params}
+
+    RETURN    ${pipeline_id}    ${pipeline_version_id}    ${pipeline_run_id}    ${experiment_id}
+
+Wait For Run Completion And Verify Status
+    [Documentation]
+    [Arguments]    ${namespace}           ${username}                  ${password}
+    ...    ${pipeline_run_id}             ${pipeline_run_timeout}=180
+    ...    ${pipeline_run_expected_status}="SUCCEEDED"
+
+    DataSciencePipelinesKfp.Setup Client    user=${username}    pwd=${password}    project=${namespace}
+
+    ${pipeline_run_status}=    DataSciencePipelinesKfp.Wait For Run Completion    run_id=${pipeline_run_id}
+    ...    timeout=${pipeline_run_timeout}     sleep_duration=${5}
+
+    IF   "${pipeline_run_status}" != "${pipeline_run_expected_status}"
+        ${error_msg}=    Catenate    Expected pipeline status was ${pipeline_run_expected_status} but pipeline run
+        ...    finished with status=${pipeline_run_status}
+        Fail    ${error_msg}
+    END
+
+    RETURN    ${pipeline_run_status}
+
+Delete Pipeline And Related Resources
+    [Documentation]  Deletes a pipeline, all it's versions and runs depending on the parameters
+    ...    If ${delete_versions}=${TRUE}, deletes all versions
+    ...    If ${delete_runs}=${TRUE}, deletes pipeline runs in ${experiment_id}
+    ...    If ${experiment_id} is not provided, Default experiment will be used
+    [Tags]    robot:recursive-continue-on-failure
+    [Arguments]    ${namespace}           ${username}                ${password}
+    ...    ${pipeline_id}                 ${experiment_id}=${EMPTY}
+    ...    ${delete_versions}=${TRUE}     ${delete_runs}=${TRUE}
+
+    DataSciencePipelinesKfp.Setup Client    user=${username}    pwd=${password}    project=${namespace}
+
+    IF    ${delete_runs}==True
+        DataSciencePipelinesKfp.Delete All Runs For Pipeline
+        ...    namespace=${namespace}    pipeline_id=${pipeline_id}    experiment_id=${experiment_id}
+    END
+
+    # TODO: delete recurent jobs
+
+    IF    ${delete_versions}==True
+        DataSciencePipelinesKfp.Delete All Pipeline Versions    ${pipeline_id}
+    END
+
+    DataSciencePipelinesKfp.Delete Pipeline    ${pipeline_id}
+
+
diff --git a/ods_ci/tests/Resources/Files/pipeline-samples/v2/pip_index_url/hello_world_pip_index_url.py b/ods_ci/tests/Resources/Files/pipeline-samples/v2/pip_index_url/hello_world_pip_index_url.py
@@ -13,31 +13,44 @@
 modify the yaml file to use PIP_TRUSTED_HOST.
 
 """
+
 from kfp import compiler, dsl
 from kfp import kubernetes
 
-common_base_image = "registry.redhat.io/ubi8/python-39@sha256:3523b184212e1f2243e76d8094ab52b01ea3015471471290d011625e1763af61"
+common_base_image = (
+    "registry.redhat.io/ubi8/python-39@sha256:3523b184212e1f2243e76d8094ab52b01ea3015471471290d011625e1763af61"
+)
 
 
-@dsl.component(base_image=common_base_image, pip_index_urls=['$PIP_INDEX_URL'], pip_trusted_hosts=['$PIP_TRUSTED_HOST'])
+@dsl.component(
+    base_image=common_base_image,
+    packages_to_install=["pyfiglet==1.0.2"],
+    pip_index_urls=["$PIP_INDEX_URL"],
+    pip_trusted_hosts=["$PIP_TRUSTED_HOST"],
+)
 def print_message(message: str):
     import os
+    from pyfiglet import Figlet
+
     """Prints a message"""
     print("------------------------------------------------------------------")
     print(message)
-    print('pip_index_url:' + os.environ['PIP_INDEX_URL'])
-    print('pip_trusted_host:' + os.environ['PIP_TRUSTED_HOST'])
+    print("pip_index_url:" + os.environ["PIP_INDEX_URL"])
+    print("pip_trusted_host:" + os.environ["PIP_TRUSTED_HOST"])
     print("------------------------------------------------------------------")
 
+    f = Figlet(font="slant")
+    print(f.renderText(message))
+
 
 @dsl.pipeline(name="hello-world-pipeline", description="Pipeline that prints a hello message")
 def hello_world_pipeline(message: str = "Hello world"):
     print_message_task = print_message(message=message)
     print_message_task.set_caching_options(False)
     kubernetes.use_config_map_as_env(
         print_message_task,
-        config_map_name='ds-pipeline-custom-env-vars',
-        config_map_key_to_env={'pip_index_url': 'PIP_INDEX_URL', 'pip_trusted_host': 'PIP_TRUSTED_HOST'}
+        config_map_name="ds-pipeline-custom-env-vars",
+        config_map_key_to_env={"pip_index_url": "PIP_INDEX_URL", "pip_trusted_host": "PIP_TRUSTED_HOST"},
     )
 
 

diff --git a/...Resources/Files/pipeline-samples/v2/pip_index_url/hello_world_pip_index_url_compiled.yaml b/...Resources/Files/pipeline-samples/v2/pip_index_url/hello_world_pip_index_url_compiled.yaml
@@ -26,7 +26,9 @@ deploymentSpec:
           \ python3 -m ensurepip --user || apt-get install python3-pip\nfi\n\nPIP_DISABLE_PIP_VERSION_CHECK=1\
           \ python3 -m pip install --quiet --no-warn-script-location --index-url $PIP_INDEX_URL\
           \ --trusted-host $PIP_TRUSTED_HOST 'kfp==2.9.0' '--no-deps' 'typing-extensions>=3.7.4,<5;\
-          \ python_version<\"3.9\"' && \"$0\" \"$@\"\n"
+          \ python_version<\"3.9\"'  &&  python3 -m pip install --quiet --no-warn-script-location\
+          \ --index-url $PIP_INDEX_URL --trusted-host $PIP_TRUSTED_HOST 'pyfiglet==1.0.2'\
+          \ && \"$0\" \"$@\"\n"
         - sh
         - -ec
         - 'program_path=$(mktemp -d)
@@ -38,11 +40,12 @@ deploymentSpec:
 
           '
         - "\nimport kfp\nfrom kfp import dsl\nfrom kfp.dsl import *\nfrom typing import\
-          \ *\n\ndef print_message(message: str):\n    import os\n    \"\"\"Prints\
-          \ a message\"\"\"\n    print(\"------------------------------------------------------------------\"\
-          )\n    print(message)\n    print('pip_index_url:' + os.environ['PIP_INDEX_URL'])\n\
-          \    print('pip_trusted_host:' + os.environ['PIP_TRUSTED_HOST'])\n    print(\"\
-          ------------------------------------------------------------------\")\n\n"
+          \ *\n\ndef print_message(message: str):\n    import os\n    from pyfiglet\
+          \ import Figlet\n\n    \"\"\"Prints a message\"\"\"\n    print(\"------------------------------------------------------------------\"\
+          )\n    print(message)\n    print(\"pip_index_url:\" + os.environ[\"PIP_INDEX_URL\"\
+          ])\n    print(\"pip_trusted_host:\" + os.environ[\"PIP_TRUSTED_HOST\"])\n\
+          \    print(\"------------------------------------------------------------------\"\
+          )\n\n    f = Figlet(font=\"slant\")\n    print(f.renderText(message))\n\n"
         image: registry.redhat.io/ubi8/python-39@sha256:3523b184212e1f2243e76d8094ab52b01ea3015471471290d011625e1763af61
 pipelineInfo:
   description: Pipeline that prints a hello message