Add upgrade tests with separate persistent vars file (#576)

* Add upgrade tests with separate vars file * Fetch template in upgrade test
azimuth-cloud · Jul 16, 2024 · 773d466 · 773d466
1 parent 2ae193c
commit 773d466
Show file tree

Hide file tree

Showing 7 changed files with 143 additions and 42 deletions.
diff --git a/roles/generate_tests/defaults/main.yml b/roles/generate_tests/defaults/main.yml
@@ -1,5 +1,11 @@
 ---
 
+# The path for the test vars file
+generate_tests_vars_file: "{{ (ansible_env.HOME, 'testvars.yaml') | path_join }}"
+
+# The template to use for generating the test vars file
+generate_tests_vars_template: vars.yaml
+
 # The directory to generate the test suite files in
 # Use a directory in $HOME by default
 generate_tests_suite_directory: "{{ (ansible_env.HOME, 'test-suite') | path_join }}"
@@ -50,6 +56,8 @@ generate_tests_suite_template: suite.robot
 # The name for the suite
 generate_tests_suite_name: Azimuth Tests
 
+# Indicates whether upgrade tests should be generated
+generate_tests_include_upgrade_tests: false
 
 #-----
 # Settings for the CaaS test suite
@@ -265,6 +273,7 @@ generate_tests_kubernetes_apps_default_test_tags: [kubernetes, kubeapps]
 # The timeouts to apply to the setup, verify and teardown of the Kubernetes cluster
 generate_tests_kubernetes_apps_setup_timeout: "15 minutes"
 generate_tests_kubernetes_apps_verify_timeout: "30 minutes"
+generate_tests_kubernetes_apps_upgrade_timeout: "15 minutes"
 generate_tests_kubernetes_apps_teardown_timeout: "15 minutes"
 
 # The timeout to apply to tests in the suite

diff --git a/roles/generate_tests/tasks/main.yml b/roles/generate_tests/tasks/main.yml
@@ -11,6 +11,32 @@
 # This is done to avoid hiding the console output as the tests progress
 #####
 
+- name: Check if test vars file exists
+  stat:
+    path: "{{ generate_tests_vars_file }}"
+  register: generate_tests_vars_file_stat
+
+- block:
+    - name: Read the existing variables
+      slurp:
+        src: "{{ generate_tests_vars_file }}"
+      register: generate_tests_slurp_vars
+
+    - name: Set fact for existing test vars
+      set_fact:
+        generate_tests_existing_vars: >-
+          {{-
+            generate_tests_slurp_vars.content |
+              b64decode |
+              from_yaml
+          }}
+  when: generate_tests_vars_file_stat.stat.exists
+
+- name: Generate test vars
+  template:
+    src: "{{ generate_tests_vars_template }}"
+    dest: "{{ generate_tests_vars_file }}"
+
 - name: Delete test suite directory if it exists
   file:
     path: "{{ generate_tests_suite_directory }}"

diff --git a/roles/generate_tests/templates/caas_test_case.robot b/roles/generate_tests/templates/caas_test_case.robot
@@ -1,9 +1,7 @@
-{% set test_case_cluster_name_prefix = test_case.cluster_name_prefix | default('test', True) %}
-{% set test_case_cluster_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) %}
-{% set test_case_cluster_name = test_case_cluster_name_prefix ~ "-" ~ test_case_cluster_name_suffix %}
+{% set test_case_name = test_case.name | default(test_case.cluster_type, True) %}
 {% set test_case_tags = test_case.tags | default([]) %}
 
-Create {{ test_case.name | default(test_case.cluster_type, True) }}
+Create {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["create"]) | join('  ') }}
 {% if test_case.create_timeout is defined and test_case.create_timeout %}
     [Timeout]  {{ test_case.create_timeout }}
@@ -20,14 +18,24 @@ Create {{ test_case.name | default(test_case.cluster_type, True) }}
 {% endif %}
 {% endfor %}
 {% endif %}
-    ${cluster} =  Create Cluster  {{ test_case_cluster_name }}  ${ctype.name}  &{params}
+    ${cluster} =  Create Cluster  ${caas.cluster_names['{{ test_case_name }}']}  ${ctype.name}  &{params}
 
-Verify {{ test_case.name | default(test_case.cluster_type, True) }}
+{% if generate_tests_include_upgrade_tests %}
+Upgrade {{ test_case_name }}
+    [Tags]  {{ (test_case_tags + ["upgrade"]) | join('  ') }}
+{% if test_case.upgrade_timeout is defined and test_case.upgrade_timeout %}
+    [Timeout]  {{ test_case.upgrade_timeout }}
+{% endif %}
+    ${cluster} =  Find Cluster By Name  ${caas.cluster_names['{{ test_case_name }}']}
+    Patch Cluster  ${cluster.id}
+{% endif %}
+
+Verify {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["verify"]) | join('  ') }}
 {% if test_case.verify_timeout is defined and test_case.verify_timeout %}
     [Timeout]  {{ test_case.verify_timeout }}
 {% endif %}
-    ${cluster} =  Find Cluster By Name  {{ test_case_cluster_name }}
+    ${cluster} =  Find Cluster By Name  ${caas.cluster_names['{{ test_case_name }}']}
     ${cluster} =  Wait For Cluster Ready  ${cluster.id}
 {% if test_case.services is defined and test_case.services %}
 {% for service in test_case.services %}
@@ -39,10 +47,10 @@ Verify {{ test_case.name | default(test_case.cluster_type, True) }}
 {% endfor %}
 {% endif %}
 
-Delete {{ test_case.name | default(test_case.cluster_type, True) }}
+Delete {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["delete"]) | join('  ') }}
 {% if test_case.delete_timeout is defined and test_case.delete_timeout %}
     [Timeout]  {{ test_case.delete_timeout }}
 {% endif %}
-    ${cluster} =  Find Cluster By Name  {{ test_case_cluster_name }}
+    ${cluster} =  Find Cluster By Name  ${caas.cluster_names['{{ test_case_name }}']}
     Delete Cluster  ${cluster.id}
diff --git a/roles/generate_tests/templates/kubernetes_apps_suite.robot b/roles/generate_tests/templates/kubernetes_apps_suite.robot
@@ -7,15 +7,6 @@ Test Tags       {{ generate_tests_kubernetes_apps_default_test_tags | join("  ")
 Test Timeout    {{ generate_tests_kubernetes_apps_default_test_timeout }}
 
 
-*** Variables ***
-
-${apps_cluster_name}  {{
-    generate_tests_kubernetes_apps_k8s_name_prefix ~
-    "-" ~
-    lookup('community.general.random_string', length = 5, upper = false, special = false)
-}}
-
-
 *** Test Cases ***
 
 Setup Apps Kubernetes Cluster
@@ -38,7 +29,7 @@ Setup Apps Kubernetes Cluster
     ${worker_size} =  Find Smallest Size With Resources  min_cpus=2  min_ram=4096  min_disk=20
 {% endif %}
     ${config} =  New Kubernetes Config
-    ...  name=${apps_cluster_name}
+    ...  name=${kubeapps.cluster_name}
     ...  template=${template.id}
     ...  control_plane_size=${cp_size.id}
     ${config} =  Add Node Group To Kubernetes Config  ${config}
@@ -47,21 +38,28 @@ Setup Apps Kubernetes Cluster
     ...  count={{ generate_tests_kubernetes_apps_k8s_worker_count }}
     ${cluster} =  Create Kubernetes Cluster  ${config}
 
+{% if generate_tests_include_upgrade_tests %}
+Upgrade Apps Kubernetes Cluster
+    [Tags]  appscluster  upgrade
+    [Timeout]  {{ generate_tests_kubernetes_apps_upgrade_timeout }}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubeapps.cluster_name}
+    ${template} =  Find Kubernetes Cluster Template For Upgrade  ${cluster.template.id}
+    Upgrade Kubernetes Cluster  ${cluster.id}  ${template.id}
+{% endif %}
 
 Verify Apps Kubernetes Cluster
-    [tags]  appscluster  verify
+    [Tags]  appscluster  verify
     [Timeout]  {{ generate_tests_kubernetes_apps_verify_timeout }}
-    ${cluster} =  Find Kubernetes Cluster By Name  ${apps_cluster_name}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubeapps.cluster_name}
     ${cluster} =  Wait For Kubernetes Cluster Ready  ${cluster.id}
 
 {% for test_case in generate_tests_kubernetes_apps_test_cases %}
 {% include (test_case.template | default(generate_tests_kubernetes_apps_test_case_template, True)) %}
 
-
 {% endfor %}
 
 Teardown Apps Kubernetes Cluster
     [Tags]  appscluster  delete
     [Timeout]  {{ generate_tests_kubernetes_apps_teardown_timeout }}
-    ${cluster} =  Find Kubernetes Cluster By Name  ${apps_cluster_name}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubeapps.cluster_name}
     Delete Kubernetes Cluster  ${cluster.id}
diff --git a/roles/generate_tests/templates/kubernetes_apps_test_case.robot b/roles/generate_tests/templates/kubernetes_apps_test_case.robot
@@ -1,29 +1,40 @@
-{% set test_case_app_name_prefix = test_case.app_name_prefix | default('testapp', True) %}
-{% set test_case_app_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) %}
-{% set test_case_app_name = test_case_app_name_prefix ~ "-" ~ test_case_app_name_suffix %}
+{% set test_case_name = test_case.name | default(test_case.app_template, True) %}
 {% set test_case_tags = test_case.tags | default([]) %}
 
-Create {{ test_case.name | default(test_case.app_template, True) }}
+Create {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["create"]) | join('  ') }}
 {% if test_case.create_timeout is defined and test_case.create_timeout %}
     [Timeout]  {{ test_case.create_timeout }}
 {% endif %}
-    ${cluster} =  Find Kubernetes Cluster By Name  ${apps_cluster_name}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubeapps.cluster_name}
     ${template} =  Fetch Kubernetes App Template  {{ test_case.app_template }}
     ${latest} =  Get Latest Version For Kubernetes App Template  ${template}
     ${defaults} =  Get Defaults For Kubernetes App Template Version  ${latest}
     ${app} =  Create Kubernetes App
-    ...  {{ test_case_app_name }}
+    ...  ${kubeapps.app_names['{{ test_case_name }}']}
     ...  ${template.id}
     ...  ${cluster.id}
     ...  ${defaults}
 
-Verify {{ test_case.name | default(test_case.app_template, True) }}
+{% if generate_tests_include_upgrade_tests %}
+Upgrade {{ test_case_name }}
+    [Tags]  {{ (test_case_tags + ["upgrade"]) | join('  ') }}
+{% if test_case.upgrade_timeout is defined and test_case.upgrade_timeout %}
+    [Timeout]  {{ test_case.upgrade_timeout }}
+{% endif %}
+    ${app} =  Find Kubernetes App By Name  ${kubeapps.app_names['{{ test_case_name }}']}
+    ${template} =  Fetch Kubernetes App Template  ${app.template.id}
+    ${latest} =  Get Latest Version For Kubernetes App Template  ${template}
+    ${defaults} =  Get Defaults For Kubernetes App Template Version  ${latest}
+    Update Kubernetes App  ${app.id}  ${latest}  ${defaults}
+{% endif %}
+
+Verify {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["verify"]) | join('  ') }}
 {% if test_case.verify_timeout is defined and test_case.verify_timeout %}
     [Timeout]  {{ test_case.verify_timeout }}
 {% endif %}
-    ${app} =  Find Kubernetes App By Name  {{ test_case_app_name }}
+    ${app} =  Find Kubernetes App By Name  ${kubeapps.app_names['{{ test_case_name }}']}
     ${app} =  Wait For Kubernetes App Deployed  ${app.id}
 {% if test_case.services is defined and test_case.services %}
 {% for service in test_case.services %}
@@ -35,10 +46,10 @@ Verify {{ test_case.name | default(test_case.app_template, True) }}
 {% endfor %}
 {% endif %}
 
-Delete {{ test_case.name | default(test_case.app_template, True) }}
+Delete {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["delete"]) | join('  ') }}
 {% if test_case.delete_timeout is defined and test_case.delete_timeout %}
     [Timeout]  {{ test_case.delete_timeout }}
 {% endif %}
-    ${app} =  Find Kubernetes App By Name  {{ test_case_app_name }}
+    ${app} =  Find Kubernetes App By Name  ${kubeapps.app_names['{{ test_case_name }}']}
     Delete Kubernetes App  ${app.id}
diff --git a/roles/generate_tests/templates/kubernetes_test_case.robot b/roles/generate_tests/templates/kubernetes_test_case.robot
@@ -1,9 +1,7 @@
-{% set test_case_cluster_name_prefix = test_case.cluster_name_prefix | default('testk8s', True) %}
-{% set test_case_cluster_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) %}
-{% set test_case_cluster_name = test_case_cluster_name_prefix ~ "-" ~ test_case_cluster_name_suffix %}
+{% set test_case_name = test_case.name | default(test_case.kubernetes_template, True) %}
 {% set test_case_tags = test_case.tags | default([]) %}
 
-Create {{ test_case.name | default(test_case.kubernetes_template, True) }}
+Create {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["create"]) | join('  ') }}
 {% if test_case.create_timeout is defined and test_case.create_timeout %}
     [Timeout]  {{ test_case.create_timeout }}
@@ -20,7 +18,7 @@ Create {{ test_case.name | default(test_case.kubernetes_template, True) }}
     ${worker_size} =  Find Smallest Size With Resources  min_cpus=2  min_ram=4096  min_disk=20
 {% endif %}
     ${config} =  New Kubernetes Config
-    ...  name={{ test_case_cluster_name }}
+    ...  name=${kubernetes.cluster_names['{{ test_case_name }}']}
     ...  template=${template.id}
     ...  control_plane_size=${cp_size.id}
     ${config} =  Add Node Group To Kubernetes Config  ${config}
@@ -35,12 +33,23 @@ Create {{ test_case.name | default(test_case.kubernetes_template, True) }}
 {% endif %}
     ${cluster} =  Create Kubernetes Cluster  ${config}
 
-Verify {{ test_case.name | default(test_case.kubernetes_template, True) }}
+{% if generate_tests_include_upgrade_tests %}
+Upgrade {{ test_case_name }}
+    [Tags]  {{ (test_case_tags + ["upgrade"]) | join('  ') }}
+{% if test_case.upgrade_timeout is defined and test_case.upgrade_timeout %}
+    [Timeout]  {{ test_case.upgrade_timeout }}
+{% endif %}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubernetes.cluster_names['{{ test_case_name }}']}
+    ${template} =  Find Kubernetes Cluster Template For Upgrade  ${cluster.template.id}
+    Upgrade Kubernetes Cluster  ${cluster.id}  ${template.id}
+{% endif %}
+
+Verify {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["verify"]) | join('  ') }}
 {% if test_case.verify_timeout is defined and test_case.verify_timeout %}
     [Timeout]  {{ test_case.verify_timeout }}
 {% endif %}
-    ${cluster} =  Find Kubernetes Cluster By Name  {{ test_case_cluster_name }}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubernetes.cluster_names['{{ test_case_name }}']}
     ${cluster} =  Wait For Kubernetes Cluster Ready  ${cluster.id}
 {% if test_case.dashboard_enabled is not defined or test_case.dashboard_enabled %}
     ${dashboard} =  Get Kubernetes Cluster Service Url  ${cluster}  dashboard
@@ -53,10 +62,10 @@ Verify {{ test_case.name | default(test_case.kubernetes_template, True) }}
     Wait Until Page Title Contains  Grafana
 {% endif %}
 
-Delete {{ test_case.name | default(test_case.kubernetes_template, True) }}
+Delete {{ test_case_name }}
     [Tags]  {{ (test_case_tags + ["delete"]) | join('  ') }}
 {% if test_case.delete_timeout is defined and test_case.delete_timeout %}
     [Timeout]  {{ test_case.delete_timeout }}
 {% endif %}
-    ${cluster} =  Find Kubernetes Cluster By Name  {{ test_case_cluster_name }}
+    ${cluster} =  Find Kubernetes Cluster By Name  ${kubernetes.cluster_names['{{ test_case_name }}']}
     Delete Kubernetes Cluster  ${cluster.id}
diff --git a/roles/generate_tests/templates/vars.yaml b/roles/generate_tests/templates/vars.yaml
@@ -0,0 +1,40 @@
+caas:
+  cluster_names:
+    {% for test_case in generate_tests_caas_test_cases -%}
+    {%- set test_case_name = test_case.name | default(test_case.cluster_type, True) -%}
+    {%- set test_case_cluster_name_prefix = test_case.cluster_name_prefix | default('test', True) -%}
+    {%- set test_case_cluster_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) -%}
+    {%- set test_case_cluster_name = test_case_cluster_name_prefix ~ "-" ~ test_case_cluster_name_suffix -%}
+    {{ test_case_name }}: {{
+      generate_tests_existing_vars.caas.cluster_names[test_case_name] | default(test_case_cluster_name)
+    }}
+    {% endfor %}
+
+kubernetes:
+  cluster_names:
+    {% for test_case in generate_tests_kubernetes_test_cases -%}
+    {%- set test_case_name = test_case.name | default(test_case.kubernetes_template, True) -%}
+    {%- set test_case_cluster_name_prefix = test_case.cluster_name_prefix | default('testk8s', True) -%}
+    {%- set test_case_cluster_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) -%}
+    {%- set test_case_cluster_name = test_case_cluster_name_prefix ~ "-" ~ test_case_cluster_name_suffix -%}
+    {{ test_case_name }}: {{
+      generate_tests_existing_vars.kubernetes.cluster_names[test_case_name] | default(test_case_cluster_name)
+    }}
+    {% endfor %}
+
+kubeapps:
+  {% set kubeapps_cluster_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) -%}
+  cluster_name: {{
+    generate_tests_existing_vars.kubeapps.cluster_name |
+      default(generate_tests_kubernetes_apps_k8s_name_prefix ~ "-" ~ kubeapps_cluster_name_suffix)
+  }}
+  app_names:
+    {% for test_case in generate_tests_kubernetes_apps_test_cases -%}
+    {%- set test_case_name = test_case.name | default(test_case.app_template, True) -%}
+    {%- set test_case_app_name_prefix = test_case.app_name_prefix | default('testapp', True) -%}
+    {%- set test_case_app_name_suffix = lookup('community.general.random_string', length = 5, upper = false, special = false) -%}
+    {%- set test_case_app_name = test_case_app_name_prefix ~ "-" ~ test_case_app_name_suffix -%}
+    {{ test_case_name }}: {{
+      generate_tests_existing_vars.kubeapps.app_names[test_case_name] | default(test_case_app_name)
+    }}
+    {% endfor %}