forked from openshift-psap/topsail
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
WatsonX-Serving: keep working on the scale test (openshift-psap#13)
- Loading branch information
Showing
6 changed files
with
126 additions
and
59 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,69 +1,114 @@ | ||
| --- | ||
| - name: Create the src directory | ||
| file: | ||
| path: "{{ artifact_extra_logs_dir }}/src" | ||
| state: directory | ||
| mode: '0755' | ||
|
|
||
| - name: Apply the DaemonSet template | ||
| template: | ||
| src: "{{ cluster_preload_image_ds_template }}" | ||
| dest: "{{ artifact_extra_logs_dir }}/preload_daemonset.yaml" | ||
| dest: "{{ artifact_extra_logs_dir }}/src/preload_daemonset.yaml" | ||
| mode: 0400 | ||
|
|
||
| - name: Delete the DaemonSet, it it exists | ||
| command: | ||
| oc delete -f "{{ artifact_extra_logs_dir }}/preload_daemonset.yaml" | ||
| oc delete -f "{{ artifact_extra_logs_dir }}/src/preload_daemonset.yaml" | ||
| --ignore-not-found | ||
|
|
||
| - name: Create the DaemonSet | ||
| command: | ||
| oc create -f "{{ artifact_extra_logs_dir }}/preload_daemonset.yaml" | ||
| - name: Get the number of nodes matching the label | ||
| shell: | ||
| set -o pipefail; | ||
| oc get nodes -oname | ||
| -l{{ cluster_preload_image_node_selector_key }}={{ cluster_preload_image_node_selector_value }} | ||
| | wc -l | ||
| register: number_target_nodes | ||
|
|
||
| - name: Fail if there is no target node | ||
| fail: msg="The node selector '{{ cluster_preload_image_node_selector_key }}={{ cluster_preload_image_node_selector_value }}' does not match any node" | ||
| when: number_target_nodes.stdout == "0" | ||
|
|
||
| - name: Preload the image | ||
| block: | ||
| - name: Get the number of desired Pods | ||
| - name: Create the DaemonSet | ||
| command: | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| '-ojsonpath={.status.desiredNumberScheduled}' | ||
| -n {{ cluster_preload_image_namespace }} | ||
| register: desiredNumberScheduled_cmd | ||
| oc create -f "{{ artifact_extra_logs_dir }}/src/preload_daemonset.yaml" | ||
|
|
||
| # --- | ||
|
|
||
| - name: Wait for the pods to be scheduled | ||
| - name: Wait for the desired number to be populated | ||
| command: | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| '-ojsonpath={.status.currentNumberScheduled}' | ||
| '-ojsonpath={.status.desiredNumberScheduled}' | ||
| -n {{ cluster_preload_image_namespace }} | ||
| register: desiredNumberScheduled_cmd | ||
| retries: 3 | ||
| delay: 15 | ||
| until: desiredNumberScheduled_cmd or "0" | int > 0 | ||
|
|
||
| - name: Wait for the pods to be scheduled | ||
| shell: | ||
| set -o pipefail; | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| -ojson | ||
| -n {{ cluster_preload_image_namespace }} | ||
| register: currentNumberScheduled_cmd | ||
| | jq '.status.desiredNumberScheduled - .status.currentNumberScheduled' | ||
| register: currentNumberNotScheduled_cmd | ||
| retries: 3 | ||
| delay: 15 | ||
| until: currentNumberScheduled_cmd.stdout >= desiredNumberScheduled_cmd.stdout | ||
| until: currentNumberNotScheduled_cmd.stdout | int == 0 | ||
|
|
||
| - name: Wait for the pods to be ready | ||
| command: | ||
| shell: | ||
| set -o pipefail; | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| '-ojsonpath={.status.numberReady}' | ||
| -ojson | ||
| -n {{ cluster_preload_image_namespace }} | ||
| register: numberReady_cmd | ||
| | jq '.status.desiredNumberScheduled - .status.numberReady' | ||
| register: currentNumberNotReady_cmd | ||
| retries: 20 | ||
| delay: 30 | ||
| until: numberReady_cmd.stdout == desiredNumberScheduled_cmd.stdout | ||
| until: currentNumberNotReady_cmd.stdout | int == 0 | ||
|
|
||
| - name: Get the final desired number | ||
| command: | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| '-ojsonpath={.status.desiredNumberScheduled}' | ||
| -n {{ cluster_preload_image_namespace }} | ||
| register: finalDesiredNumberScheduled_cmd | ||
|
|
||
| - name: Fail if the image has been preloaded on 0 nodes | ||
| fail: msg="The node selector '{{ cluster_preload_image_node_selector_key }}={{ cluster_preload_image_node_selector_value }}' did not match any node" | ||
| when: desiredNumberScheduled_cmd.stdout == "0" | ||
| when: finalDesiredNumberScheduled_cmd.stdout | int == 0 | ||
|
|
||
| always: | ||
| - name: Get the description of the preload Pods | ||
| shell: | ||
| oc describe pods -l name={{ cluster_preload_image_name }} | ||
| -n {{ cluster_preload_image_namespace }} | ||
| > "{{ artifact_extra_logs_dir }}/preload_pods.descr" | ||
| > "{{ artifact_extra_logs_dir }}/pods.descr" | ||
|
|
||
| - name: Get the status of the preload Pods | ||
| shell: | ||
| oc get pods -l name={{ cluster_preload_image_name }} | ||
| -owide | ||
| -n {{ cluster_preload_image_namespace }} | ||
| > "{{ artifact_extra_logs_dir }}/preload_pods.status" | ||
| > "{{ artifact_extra_logs_dir }}/pods.status" | ||
|
|
||
| - name: Get the yaml of the daemonset | ||
| shell: | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| -oyaml | ||
| -n {{ cluster_preload_image_namespace }} | ||
| > "{{ artifact_extra_logs_dir }}/daemonset.yaml" | ||
|
|
||
| - name: Get the status of the daemonset | ||
| shell: | ||
| oc get ds/{{ cluster_preload_image_name }} | ||
| -n {{ cluster_preload_image_namespace }} | ||
| > "{{ artifact_extra_logs_dir }}/daemonset.status" | ||
|
|
||
| - name: Delete the DaemonSet, it it exists | ||
| command: | ||
| oc delete -f "{{ artifact_extra_logs_dir }}/preload_daemonset.yaml" | ||
| oc delete -f "{{ artifact_extra_logs_dir }}/src/preload_daemonset.yaml" --ignore-not-found | ||
| failed_when: false |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters