-
Notifications
You must be signed in to change notification settings - Fork 23
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
QA: New pipeline to generate test coverage of a Cucumber test suite (#…
…491)
- Loading branch information
Showing
2 changed files
with
320 additions
and
0 deletions.
There are no files selected for viewing
143 changes: 143 additions & 0 deletions
143
jenkins_pipelines/environments/uyuni-master-dev-acceptance-tests-sonarqube
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,143 @@ | ||
| node('sumaform-cucumber') { | ||
| properties([ | ||
| buildDiscarder(logRotator(numToKeepStr: '1', daysToKeepStr: '1', artifactNumToKeepStr: '1')), | ||
| disableConcurrentBuilds(), | ||
| parameters([ | ||
| string(name: 'cucumber_gitrepo', defaultValue: 'https://github.com/uyuni-project/uyuni.git', description: 'Testsuite Git Repository'), | ||
| string(name: 'cucumber_ref', defaultValue: 'master', description: 'Testsuite Git reference (branch, tag...)'), | ||
| string(name: 'tf_file', defaultValue: 'susemanager-ci/terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf', description: 'Path to the tf file to be used'), | ||
| string(name: 'sumaform_gitrepo', defaultValue: 'https://github.com/uyuni-project/sumaform.git', description: 'Sumaform Git Repository'), | ||
| string(name: 'sumaform_ref', defaultValue: 'master', description: 'Sumaform Git reference (branch, tag...)'), | ||
| choice(name: 'sumaform_backend', choices: ['libvirt', 'aws'], description: 'Sumaform backend to be used (see https://github.com/uyuni-project/sumaform#backend-choice)'), | ||
| choice(name: 'terraform_bin', choices: ['/usr/bin/terraform'], description: 'Terraform binary path'), | ||
| choice(name: 'terraform_bin_plugins', choices: ['/usr/bin'], description: 'Terraform plugins path'), | ||
| string(name: 'terracumber_gitrepo', defaultValue: 'https://github.com/uyuni-project/terracumber.git', description: 'Terracumber Git Repository'), | ||
| string(name: 'terracumber_ref', defaultValue: 'master', description: 'Terracumber Git ref (branch, tag...)'), | ||
| booleanParam(name: 'terraform_init', defaultValue: true, description: 'Call terraform init (needed if modules are added or changes)'), | ||
| booleanParam(name: 'terraform_taint', defaultValue: true, description: 'Call terraform taint (so the resources, except volumes, are recreated)'), | ||
| booleanParam(name: 'use_previous_terraform_state', defaultValue: true, description: 'Use previous Terraform state'), | ||
| booleanParam(name: 'must_deploy', defaultValue: false, description: 'Deploy'), | ||
| choice(name: 'rake_namespace', choices: ['cucumber', 'parallel'], description: 'Choose [parallel] (Clients and some features will run in parallel) or [cucumber] (all sequential)'), | ||
| extendedChoice(name: 'functional_scopes', multiSelectDelimiter: ',', quoteValue: false, saveJSONParameterToFile: false, type: 'PT_CHECKBOX', visibleItemCount: 30, value: '@scope_smdba,@scope_spacecmd,@scope_spacewalk_utils,@scope_visualization,@scope_notification_message,@scope_virtual_host_manager,@scope_subscription_matching,@scope_formulas,@scope_sp_migration,@scope_cve_audit,@scope_onboarding,@scope_content_lifecycle_management,@scope_res,@scope_recurring_actions,@scope_maintenance_windows,@scope_cluster_management,@scope_building_container_images,@scope_kubernetes_integration,@scope_openscap,@scope_ubuntu,@scope_action_chains,@scope_salt_ssh,@scope_tomcat,@scope_changing_software_channels,@scope_monitoring,@scope_salt,@scope_cobbler,@scope_sumatoolbox,@scope_virtualization,@scope_hub,@scope_retail,@scope_configuration_channels,@scope_content_staging,@scope_proxy,@scope_traditional_client,@scope_xmlrpc,@scope_power_management,@scope_retracted_patches', description: 'Choose the functional scopes that you want to test') | ||
| ]) | ||
| ]) | ||
|
|
||
| timestamps { | ||
| // Init path env variables | ||
| env.resultdir = "${WORKSPACE}/results" | ||
| env.resultdirbuild = "${resultdir}/${BUILD_NUMBER}" | ||
|
|
||
| // The junit plugin doesn't affect full paths | ||
| junit_resultdir = "results/${BUILD_NUMBER}/results_junit" | ||
| env.common_params = "--outputdir ${resultdir} --tf ${params.tf_file} --gitfolder ${resultdir}/sumaform --terraform-bin ${params.terraform_bin}" | ||
|
|
||
| // Start pipeline | ||
| deployed = false | ||
| try { | ||
| stage('Clone terracumber, susemanager-ci and sumaform') { | ||
| // Create a directory for to place the directory with the build results (if it does not exist) | ||
| sh "mkdir -p ${resultdir}" | ||
| git url: params.terracumber_gitrepo, branch: params.terracumber_ref | ||
| dir("susemanager-ci") { | ||
| checkout scm | ||
| } | ||
| // Clone sumaform | ||
| sh "set +x; source /home/jenkins/.credentials set -x; ./terracumber-cli ${common_params} --gitrepo ${params.sumaform_gitrepo} --gitref ${params.sumaform_ref} --runstep gitsync" | ||
|
|
||
| // Restore Terraform states from artifacts | ||
| if (params.use_previous_terraform_state) { | ||
| copyArtifacts projectName: currentBuild.projectName, selector: specific("${currentBuild.previousBuild.number}") | ||
| } | ||
| } | ||
|
|
||
| stage('Deploy') { | ||
| if(params.must_deploy) { | ||
| // Provision the environment | ||
| if (params.terraform_init) { | ||
| env.TERRAFORM_INIT = '--init' | ||
| } else { | ||
| env.TERRAFORM_INIT = '' | ||
| } | ||
| env.TERRAFORM_TAINT = '' | ||
| if (params.terraform_taint) { | ||
| switch(params.sumaform_backend) { | ||
| case "libvirt": | ||
| env.TERRAFORM_TAINT = " --taint '.*(domain|main_disk).*'"; | ||
| break; | ||
| case "aws": | ||
| env.TERRAFORM_TAINT = " --taint '.*(host).*'"; | ||
| break; | ||
| default: | ||
| println("ERROR: Unknown backend ${params.sumaform_backend}"); | ||
| sh "exit 1"; | ||
| break; | ||
| } | ||
| } | ||
| sh "set +x; source /home/jenkins/.credentials set -x; export TF_VAR_CUCUMBER_GITREPO=${params.cucumber_gitrepo}; export TF_VAR_CUCUMBER_BRANCH=${params.cucumber_ref}; export TERRAFORM=${params.terraform_bin}; export TERRAFORM_PLUGINS=${params.terraform_bin_plugins}; ./terracumber-cli ${common_params} --logfile ${resultdirbuild}/sumaform.log ${env.TERRAFORM_INIT} ${env.TERRAFORM_TAINT} --sumaform-backend ${params.sumaform_backend} --runstep provision" | ||
| deployed = true | ||
| } | ||
| } | ||
|
|
||
| stage('Enable JaCoCo Agent') { | ||
| def url = "https://search.maven.org/remotecontent?filepath=org/jacoco/org.jacoco.agent/0.8.7/org.jacoco.agent-0.8.7-runtime.jar" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER wget ${url} -O /tmp/jacocoagent.jar'" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER \'sed -i \'s/.\$/ -javaagent:\\/tmp\\/jacocoagent.jar=output=tcpserver,address=*,port=6300\" /\' /etc/sysconfig/tomcat\''" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER systemctl restart tomcat.service'" | ||
| } | ||
|
|
||
| stage('Sanity Check') { | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:sanity_check'" | ||
| } | ||
|
|
||
| stage('Core - Setup') { | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:core'" | ||
| } | ||
|
|
||
| stage('Dump Test Coverage results') { | ||
| def url = "https://repo1.maven.org/maven2/org/jacoco/org.jacoco.cli/0.8.7/org.jacoco.cli-0.8.7-nodeps.jar" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER wget ${url} -O /tmp/jacococli.jar'" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER java -jar /tmp/jacococli.jar dump --address localhost --destfile /tmp/jacoco.exec --port 6300 --reset'" | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'ssh \$SERVER java -jar jacococli.jar report /tmp/jacoco.exec --html /srv/www/htdocs/pub/jacoco-cucumber-report --xml /srv/www/htdocs/pub/jacoco-cucumber-report.xml --classfiles /srv/tomcat/webapps/rhn/WEB-INF/lib'" | ||
| } | ||
| } | ||
|
|
||
| finally { | ||
| stage('Save TF state') { | ||
| archiveArtifacts artifacts: "results/sumaform/terraform.tfstate, results/sumaform/.terraform/**/*" | ||
| } | ||
|
|
||
| stage('Get results') { | ||
| def error = 0 | ||
| if (deployed) { | ||
| try { | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake cucumber:finishing'" | ||
| } catch(Exception ex) { | ||
| println("ERROR: rake cucumber:finishing failed") | ||
| error = 1 | ||
| } | ||
| try { | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; rake utils:generate_test_report'" | ||
| } catch(Exception ex) { | ||
| println("ERROR: rake utils:generate_test_repor failed") | ||
| error = 1 | ||
| } | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/testsuite.log --runstep getresults" | ||
| publishHTML( target: [ | ||
| allowMissing: true, | ||
| alwaysLinkToLastBuild: false, | ||
| keepAll: true, | ||
| reportDir: "${resultdirbuild}/cucumber_report/", | ||
| reportFiles: 'cucumber_report.html', | ||
| reportName: "TestSuite Report"] | ||
| ) | ||
| junit allowEmptyResults: true, testResults: "${junit_resultdir}/*.xml" | ||
| } | ||
| // Send email | ||
| sh "./terracumber-cli ${common_params} --logfile ${resultdirbuild}/mail.log --runstep mail" | ||
| // Clean up old results | ||
| sh "./clean-old-results -r ${resultdir}" | ||
| sh "exit ${error}" | ||
| } | ||
| } | ||
| } | ||
| } |
177 changes: 177 additions & 0 deletions
177
terracumber_config/tf_files/Uyuni-Master-tests-coverage.tf
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,177 @@ | ||
| // Mandatory variables for terracumber | ||
| variable "URL_PREFIX" { | ||
| type = string | ||
| default = "https://ci.suse.de/view/Manager/view/Uyuni/job/uyuni-master-dev-acceptance-tests-sonarqube" | ||
| } | ||
|
|
||
| // Not really used as this is for --runall parameter, and we run cucumber step by step | ||
| variable "CUCUMBER_COMMAND" { | ||
| type = string | ||
| default = "export PRODUCT='Uyuni' && run-testsuite" | ||
| } | ||
|
|
||
| variable "CUCUMBER_GITREPO" { | ||
| type = string | ||
| default = "https://github.com/uyuni-project/uyuni.git" | ||
| } | ||
|
|
||
| variable "CUCUMBER_BRANCH" { | ||
| type = string | ||
| default = "master" | ||
| } | ||
|
|
||
| variable "CUCUMBER_RESULTS" { | ||
| type = string | ||
| default = "/root/spacewalk/testsuite" | ||
| } | ||
|
|
||
| variable "MAIL_SUBJECT" { | ||
| type = string | ||
| default = "Results Uyuni-Master-test-coverage $status: $tests scenarios ($failures failed, $errors errors, $skipped skipped, $passed passed)" | ||
| } | ||
|
|
||
| variable "MAIL_TEMPLATE" { | ||
| type = string | ||
| default = "../mail_templates/mail-template-jenkins.txt" | ||
| } | ||
|
|
||
| variable "MAIL_SUBJECT_ENV_FAIL" { | ||
| type = string | ||
| default = "Results Uyuni-Master: Environment setup failed" | ||
| } | ||
|
|
||
| variable "MAIL_TEMPLATE_ENV_FAIL" { | ||
| type = string | ||
| default = "../mail_templates/mail-template-jenkins-env-fail.txt" | ||
| } | ||
|
|
||
| variable "MAIL_FROM" { | ||
| type = string | ||
| default = "[email protected]" | ||
| } | ||
|
|
||
| variable "MAIL_TO" { | ||
| type = string | ||
| default = "[email protected]" | ||
| } | ||
|
|
||
| // sumaform specific variables | ||
| variable "SCC_USER" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "SCC_PASSWORD" { | ||
| type = string | ||
| } | ||
|
|
||
| variable "GIT_USER" { | ||
| type = string | ||
| default = null // Not needed for master, as it is public | ||
| } | ||
|
|
||
| variable "GIT_PASSWORD" { | ||
| type = string | ||
| default = null // Not needed for master, as it is public | ||
| } | ||
|
|
||
| terraform { | ||
| required_version = "1.0.10" | ||
| required_providers { | ||
| libvirt = { | ||
| source = "dmacvicar/libvirt" | ||
| version = "0.6.3" | ||
| } | ||
| } | ||
| } | ||
|
|
||
| provider "libvirt" { | ||
| uri = "qemu+tcp://hyperion.mgr.prv.suse.net/system" | ||
| } | ||
|
|
||
| module "cucumber_testsuite" { | ||
| source = "./modules/cucumber_testsuite" | ||
|
|
||
| product_version = "uyuni-master" | ||
|
|
||
| // Cucumber repository configuration for the controller | ||
| git_username = var.GIT_USER | ||
| git_password = var.GIT_PASSWORD | ||
| git_repo = var.CUCUMBER_GITREPO | ||
| branch = var.CUCUMBER_BRANCH | ||
|
|
||
| cc_username = var.SCC_USER | ||
| cc_password = var.SCC_PASSWORD | ||
|
|
||
| images = ["opensuse152o", "opensuse153o", "sles15sp2o", "sles15sp3o"] | ||
|
|
||
| use_avahi = false | ||
| name_prefix = "suma-pr8-" | ||
| domain = "mgr.prv.suse.net" | ||
| from_email = "[email protected]" | ||
|
|
||
| no_auth_registry = "registry.mgr.suse.de" | ||
| auth_registry = "registry.mgr.suse.de:5000/cucutest" | ||
| auth_registry_username = "cucutest" | ||
| auth_registry_password = "cucusecret" | ||
| git_profiles_repo = "https://github.com/uyuni-project/uyuni.git#:testsuite/features/profiles/internal_nue" | ||
|
|
||
| server_http_proxy = "http-proxy.mgr.suse.de:3128" | ||
|
|
||
| host_settings = { | ||
| controller = { | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:7c" | ||
| } | ||
| } | ||
| server = { | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:7d" | ||
| memory = 10240 | ||
| } | ||
| } | ||
| proxy = { | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:7e" | ||
| } | ||
| additional_packages = [ "venv-salt-minion" ] | ||
| install_salt_bundle = true | ||
| } | ||
| suse-client = { | ||
| image = "sles15sp2o" | ||
| name = "cli-sles15" | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:7f" | ||
| } | ||
| additional_packages = [ "venv-salt-minion" ] | ||
| install_salt_bundle = true | ||
| } | ||
| suse-minion = { | ||
| image = "sles15sp2o" | ||
| name = "min-sles15" | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:80" | ||
| } | ||
| additional_packages = [ "venv-salt-minion" ] | ||
| install_salt_bundle = true | ||
| } | ||
| suse-sshminion = { | ||
| image = "sles15sp2o" | ||
| name = "minssh-sles15" | ||
| provider_settings = { | ||
| mac = "aa:b2:92:04:00:81" | ||
| } | ||
| additional_packages = [ "venv-salt-minion" ] | ||
| install_salt_bundle = true | ||
| } | ||
| } | ||
| provider_settings = { | ||
| pool = "ssd" | ||
| network_name = null | ||
| bridge = "br1" | ||
| additional_network = "192.168.108.0/24" | ||
| } | ||
| } | ||
|
|
||
| output "configuration" { | ||
| value = module.cucumber_testsuite.configuration | ||
| } |