From aefe924185c0b1075030e057a22fad3a770935e5 Mon Sep 17 00:00:00 2001 From: Oscar Barrios Date: Tue, 1 Mar 2022 13:58:54 +0100 Subject: [PATCH] QA: New pipeline to generate test coverage of a Cucumber test suite (#491) --- ...yuni-master-dev-acceptance-tests-sonarqube | 143 ++++++++++++++ .../tf_files/Uyuni-Master-tests-coverage.tf | 177 ++++++++++++++++++ 2 files changed, 320 insertions(+) create mode 100644 jenkins_pipelines/environments/uyuni-master-dev-acceptance-tests-sonarqube create mode 100644 terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf diff --git a/jenkins_pipelines/environments/uyuni-master-dev-acceptance-tests-sonarqube b/jenkins_pipelines/environments/uyuni-master-dev-acceptance-tests-sonarqube new file mode 100644 index 000000000..4404c94d1 --- /dev/null +++ b/jenkins_pipelines/environments/uyuni-master-dev-acceptance-tests-sonarqube @@ -0,0 +1,143 @@ +node('sumaform-cucumber') { + properties([ + buildDiscarder(logRotator(numToKeepStr: '1', daysToKeepStr: '1', artifactNumToKeepStr: '1')), + disableConcurrentBuilds(), + parameters([ + string(name: 'cucumber_gitrepo', defaultValue: 'https://github.com/uyuni-project/uyuni.git', description: 'Testsuite Git Repository'), + string(name: 'cucumber_ref', defaultValue: 'master', description: 'Testsuite Git reference (branch, tag...)'), + string(name: 'tf_file', defaultValue: 'susemanager-ci/terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf', description: 'Path to the tf file to be used'), + string(name: 'sumaform_gitrepo', defaultValue: 'https://github.com/uyuni-project/sumaform.git', description: 'Sumaform Git Repository'), + string(name: 'sumaform_ref', defaultValue: 'master', description: 'Sumaform Git reference (branch, tag...)'), + choice(name: 'sumaform_backend', choices: ['libvirt', 'aws'], description: 'Sumaform backend to be used (see https://github.com/uyuni-project/sumaform#backend-choice)'), + choice(name: 'terraform_bin', choices: ['/usr/bin/terraform'], description: 'Terraform binary path'), + choice(name: 'terraform_bin_plugins', choices: ['/usr/bin'], description: 'Terraform plugins path'), + string(name: 'terracumber_gitrepo', defaultValue: 'https://github.com/uyuni-project/terracumber.git', description: 'Terracumber Git Repository'), + string(name: 'terracumber_ref', defaultValue: 'master', description: 'Terracumber Git ref (branch, tag...)'), + booleanParam(name: 'terraform_init', defaultValue: true, description: 'Call terraform init (needed if modules are added or changes)'), + booleanParam(name: 'terraform_taint', defaultValue: true, description: 'Call terraform taint (so the resources, except volumes, are recreated)'), + booleanParam(name: 'use_previous_terraform_state', defaultValue: true, description: 'Use previous Terraform state'), + booleanParam(name: 'must_deploy', defaultValue: false, description: 'Deploy'), + choice(name: 'rake_namespace', choices: ['cucumber', 'parallel'], description: 'Choose [parallel] (Clients and some features will run in parallel) or [cucumber] (all sequential)'), + extendedChoice(name: 'functional_scopes', multiSelectDelimiter: ',', quoteValue: false, saveJSONParameterToFile: false, type: 'PT_CHECKBOX', visibleItemCount: 30, value: '@scope_smdba,@scope_spacecmd,@scope_spacewalk_utils,@scope_visualization,@scope_notification_message,@scope_virtual_host_manager,@scope_subscription_matching,@scope_formulas,@scope_sp_migration,@scope_cve_audit,@scope_onboarding,@scope_content_lifecycle_management,@scope_res,@scope_recurring_actions,@scope_maintenance_windows,@scope_cluster_management,@scope_building_container_images,@scope_kubernetes_integration,@scope_openscap,@scope_ubuntu,@scope_action_chains,@scope_salt_ssh,@scope_tomcat,@scope_changing_software_channels,@scope_monitoring,@scope_salt,@scope_cobbler,@scope_sumatoolbox,@scope_virtualization,@scope_hub,@scope_retail,@scope_configuration_channels,@scope_content_staging,@scope_proxy,@scope_traditional_client,@scope_xmlrpc,@scope_power_management,@scope_retracted_patches', description: 'Choose the functional scopes that you want to test') + ]) + ]) + + timestamps { + // Init path env variables + env.resultdir = "${WORKSPACE}/results" + env.resultdirbuild = "${resultdir}/${BUILD_NUMBER}" + + // The junit plugin doesn't affect full paths + junit_resultdir = "results/${BUILD_NUMBER}/results_junit" + env.common_params = "--outputdir ${resultdir} --tf ${params.tf_file} --gitfolder ${resultdir}/sumaform --terraform-bin ${params.terraform_bin}" + + // Start pipeline + deployed = false + try { + stage('Clone terracumber, susemanager-ci and sumaform') { + // Create a directory for to place the directory with the build results (if it does not exist) + sh "mkdir -p ${resultdir}" + git url: params.terracumber_gitrepo, branch: params.terracumber_ref + dir("susemanager-ci") { + checkout scm + } + // Clone sumaform + sh "set +x; source /home/jenkins/.credentials set -x; ./terracumber-cli ${common_params} --gitrepo ${params.sumaform_gitrepo} --gitref ${params.sumaform_ref} --runstep gitsync" + + // Restore Terraform states from artifacts + if (params.use_previous_terraform_state) { + copyArtifacts projectName: currentBuild.projectName, selector: specific("${currentBuild.previousBuild.number}") + } + } + + stage('Deploy') { + if(params.must_deploy) { + // Provision the environment + if (params.terraform_init) { + env.TERRAFORM_INIT = '--init' + } else { + env.TERRAFORM_INIT = '' + } + env.TERRAFORM_TAINT = '' + if (params.terraform_taint) { + switch(params.sumaform_backend) { + case "libvirt": + env.TERRAFORM_TAINT = " --taint '.*(domain|main_disk).*'"; + break; + case "aws": + env.TERRAFORM_TAINT = " --taint '.*(host).*'"; + break; + default: + println("ERROR: Unknown backend ${params.sumaform_backend}"); + sh "exit 1"; + break; + } + } + sh "set +x; source /home/jenkins/.credentials set -x; export TF_VAR_CUCUMBER_GITREPO=${params.cucumber_gitrepo}; export TF_VAR_CUCUMBER_BRANCH=${params.cucumber_ref}; export TERRAFORM=${params.terraform_bin}; export TERRAFORM_PLUGINS=${params.terraform_bin_plugins}; ./terracumber-cli ${common_params} --logfile ${resultdirbuild}/sumaform.log ${env.TERRAFORM_INIT} ${env.TERRAFORM_TAINT} --sumaform-backend ${params.sumaform_backend} --runstep provision" + deployed = true + } + } + + stage('Enable JaCoCo Agent') { + def url = "https://search.maven.org/remotecontent?filepath=org/jacoco/org.jacoco.agent/0.8.7/org.jacoco.agent-0.8.7-runtime.jar" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER wget ${url} -O /tmp/jacocoagent.jar'" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER \"echo -javaagent:\\/tmp\\/jacocoagent.jar=output=tcpserver,address=*,port=6300 >> /etc/sysconfig/tomcat\"'" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER systemctl restart tomcat.service'" + } + + stage('Sanity Check') { + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:sanity_check'" + } + + stage('Core - Setup') { + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:core'" + } + + stage('Dump Test Coverage results') { + def url = "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jacoco.cli-0.8.7-nodeps.jar" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER wget ${url} -O /tmp/jacococli.jar'" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER java -jar /tmp/jacococli.jar dump --address localhost --destfile /tmp/jacoco.exec --port 6300 --reset'" + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER java -jar jacococli.jar report /tmp/jacoco.exec --html /srv/www/htdocs/pub/jacoco-cucumber-report --xml /srv/www/htdocs/pub/jacoco-cucumber-report.xml --classfiles /srv/tomcat/webapps/rhn/WEB-INF/lib'" + } + } + + finally { + stage('Save TF state') { + archiveArtifacts artifacts: "results/sumaform/terraform.tfstate, results/sumaform/.terraform/**/*" + } + + stage('Get results') { + def error = 0 + if (deployed) { + try { + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:finishing'" + } catch(Exception ex) { + println("ERROR: rake cucumber:finishing failed") + error = 1 + } + try { + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake utils:generate_test_report'" + } catch(Exception ex) { + println("ERROR: rake utils:generate_test_repor failed") + error = 1 + } + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep getresults" + publishHTML( target: [ + allowMissing: true, + alwaysLinkToLastBuild: false, + keepAll: true, + reportDir: "${resultdirbuild}/cucumber_report/", + reportFiles: 'cucumber_report.html', + reportName: "TestSuite Report"] + ) + junit allowEmptyResults: true, testResults: "${junit_resultdir}/*.xml" + } + // Send email + sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/mail.log --runstep mail" + // Clean up old results + sh "./clean-old-results -r ${resultdir}" + sh "exit ${error}" + } + } + } +} diff --git a/terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf b/terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf new file mode 100644 index 000000000..5cd7e2c50 --- /dev/null +++ b/terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf @@ -0,0 +1,177 @@ +// Mandatory variables for terracumber +variable "URL_PREFIX" { + type = string + default = "https://ci.suse.de/view/Manager/view/Uyuni/job/uyuni-master-dev-acceptance-tests-sonarqube" +} + +// Not really used as this is for --runall parameter, and we run cucumber step by step +variable "CUCUMBER_COMMAND" { + type = string + default = "export PRODUCT='Uyuni' && run-testsuite" +} + +variable "CUCUMBER_GITREPO" { + type = string + default = "https://github.com/uyuni-project/uyuni.git" +} + +variable "CUCUMBER_BRANCH" { + type = string + default = "master" +} + +variable "CUCUMBER_RESULTS" { + type = string + default = "/root/spacewalk/testsuite" +} + +variable "MAIL_SUBJECT" { + type = string + default = "Results Uyuni-Master-test-coverage $status: $tests scenarios ($failures failed, $errors errors, $skipped skipped, $passed passed)" +} + +variable "MAIL_TEMPLATE" { + type = string + default = "../mail_templates/mail-template-jenkins.txt" +} + +variable "MAIL_SUBJECT_ENV_FAIL" { + type = string + default = "Results Uyuni-Master: Environment setup failed" +} + +variable "MAIL_TEMPLATE_ENV_FAIL" { + type = string + default = "../mail_templates/mail-template-jenkins-env-fail.txt" +} + +variable "MAIL_FROM" { + type = string + default = "galaxy-ci@suse.de" +} + +variable "MAIL_TO" { + type = string + default = "galaxy-ci@suse.de" +} + +// sumaform specific variables +variable "SCC_USER" { + type = string +} + +variable "SCC_PASSWORD" { + type = string +} + +variable "GIT_USER" { + type = string + default = null // Not needed for master, as it is public +} + +variable "GIT_PASSWORD" { + type = string + default = null // Not needed for master, as it is public +} + +terraform { + required_version = "1.0.10" + required_providers { + libvirt = { + source = "dmacvicar/libvirt" + version = "0.6.3" + } + } +} + +provider "libvirt" { + uri = "qemu+tcp://hyperion.mgr.prv.suse.net/system" +} + +module "cucumber_testsuite" { + source = "./modules/cucumber_testsuite" + + product_version = "uyuni-master" + + // Cucumber repository configuration for the controller + git_username = var.GIT_USER + git_password = var.GIT_PASSWORD + git_repo = var.CUCUMBER_GITREPO + branch = var.CUCUMBER_BRANCH + + cc_username = var.SCC_USER + cc_password = var.SCC_PASSWORD + + images = ["opensuse152o", "opensuse153o", "sles15sp2o", "sles15sp3o"] + + use_avahi = false + name_prefix = "suma-pr8-" + domain = "mgr.prv.suse.net" + from_email = "root@suse.de" + + no_auth_registry = "registry.mgr.suse.de" + auth_registry = "registry.mgr.suse.de:5000/cucutest" + auth_registry_username = "cucutest" + auth_registry_password = "cucusecret" + git_profiles_repo = "https://github.com/uyuni-project/uyuni.git#:testsuite/features/profiles/internal_nue" + + server_http_proxy = "http-proxy.mgr.suse.de:3128" + + host_settings = { + controller = { + provider_settings = { + mac = "aa:b2:92:04:00:7c" + } + } + server = { + provider_settings = { + mac = "aa:b2:92:04:00:7d" + memory = 10240 + } + } + proxy = { + provider_settings = { + mac = "aa:b2:92:04:00:7e" + } + additional_packages = [ "venv-salt-minion" ] + install_salt_bundle = true + } + suse-client = { + image = "sles15sp2o" + name = "cli-sles15" + provider_settings = { + mac = "aa:b2:92:04:00:7f" + } + additional_packages = [ "venv-salt-minion" ] + install_salt_bundle = true + } + suse-minion = { + image = "sles15sp2o" + name = "min-sles15" + provider_settings = { + mac = "aa:b2:92:04:00:80" + } + additional_packages = [ "venv-salt-minion" ] + install_salt_bundle = true + } + suse-sshminion = { + image = "sles15sp2o" + name = "minssh-sles15" + provider_settings = { + mac = "aa:b2:92:04:00:81" + } + additional_packages = [ "venv-salt-minion" ] + install_salt_bundle = true + } + } + provider_settings = { + pool = "ssd" + network_name = null + bridge = "br1" + additional_network = "192.168.108.0/24" + } +} + +output "configuration" { + value = module.cucumber_testsuite.configuration +}