Merge pull request #295 from dimagi/QA-5485-script-addition-automatio…

…n-of-tests-related-to-the-es-upgrade-work

QA-5485 added testcases for first 7 tabs in the testplan

.github/workflows/es-tests.yml

@@ -0,0 +1,204 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: Elastic Search Tests
+
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+  schedule:
+    - cron: '0 0 * * 0'
+  workflow_dispatch:
+    inputs:
+      environment:
+        description: 'Environment to run tests against'
+        required: true
+        default: 'staging'
+        type: choice
+        options:
+          - staging
+          - production
+
+jobs:
+  set_matrix:
+    runs-on: ubuntu-latest
+    outputs:
+      matrix: ${{ steps.set-matrix-manual.outputs.matrix || steps.set-matrix-default.outputs.matrix }}
+    steps:
+      - id: set-matrix-manual
+        if: ${{ github.event_name == 'workflow_dispatch' }}
+        run: |
+          echo "::set-output name=matrix::{\"environment\": [\"${{ inputs.environment }}\"]}"
+      - id: set-matrix-default
+        if: ${{ !contains(github.event_name , 'dispatch') }}
+        run: |
+          echo "::set-output name=matrix::{\"environment\": [\"staging\"]}"
+  build:
+    needs: set_matrix
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJSON(needs.set_matrix.outputs.matrix) }}
+    concurrency:
+      group: ${{ github.workflow }}-${{ matrix.environment }}
+      cancel-in-progress: true
+    name: ES Test on '${{ matrix.environment }}'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.9
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r ElasticSearchTests/requires.txt
+
+      - name: ES Test with pytest
+        env:
+          DIMAGIQA_ENV: ${{ matrix.environment }}  #${{ secrets.DIMAGIQA_URL }}
+          DIMAGIQA_LOGIN_USERNAME: ${{ secrets.DIMAGIQA_LOGIN_USERNAME }}
+          DIMAGIQA_LOGIN_PASSWORD: ${{ secrets.DIMAGIQA_LOGIN_PASSWORD }}
+          DIMAGIQA_MAIL_USERNAME: ${{ secrets.DIMAGIQA_MAIL_USERNAME }}
+          DIMAGIQA_MAIL_PASSWORD: ${{ secrets.DIMAGIQA_MAIL_PASSWORD }}
+          DIMAGIQA_BS_USER: ${{ secrets.DIMAGIQA_BS_USER }}
+          DIMAGIQA_BS_KEY: ${{ secrets.DIMAGIQA_BS_KEY }}
+          DIMAGIQA_STAGING_AUTH_KEY: ${{ secrets.DIMAGIQA_STAGING_AUTH_KEY }}
+          DIMAGIQA_PROD_AUTH_KEY: ${{ secrets.DIMAGIQA_PROD_AUTH_KEY }}
+          DIMAGIQA_INVITED_WEBUSER_PASSWORD: ${{ secrets.DIMAGIQA_INVITED_WEBUSER_PASSWORD }}
+          DIMAGIQA_IMAP_PASSWORD: ${{secrets.DIMAGIQA_IMAP_PASSWORD}}
+        run: |
+          echo "client_payload: ${{ toJson(github.event.client_payload) }}"
+          echo "matrix environment: ${{ matrix.environment }}"
+          echo "NOW=$(date +'%m-%d %H:%M')" >> $GITHUB_ENV
+          echo ${{env.NOW}}
+          pytest -v --rootdir= ElasticSearchTests/testCases -n 4 --dist=loadfile --reruns 1 --html=es_report_${{ matrix.environment }}.html
+
+      - name: Set email vars
+        if: ${{ failure() }}
+        id: configure_email
+        uses: actions/github-script@v6
+        env:
+            JOB_STATUS: ${{ job.status }}
+            CC_ENV: ${{ matrix.environment }}
+        with:
+          script: |
+            const { promises: fs } = require('fs')
+
+            const {JOB_STATUS, NOW, CC_ENV, GITHUB_HEAD_REF} = process.env
+            const prefix = `[${CC_ENV}] ES Tests - ${JOB_STATUS.toUpperCase()} - Run #${context.runNumber}`
+            const suffix = `at ${NOW}`
+
+            let subject = `${prefix} on "deploy_success" ${suffix}`
+            if (context.eventName !== "repository_dispatch") {
+              subject = `${prefix} on branch "${GITHUB_HEAD_REF}" ${suffix}`
+            }
+
+            let bodyFile = './common_utilities/mail_templates/email_pass.md'
+            if (JOB_STATUS !== 'success') {
+              bodyFile = './common_utilities/mail_templates/email_fail.md'
+            }
+
+            let actionRunLink = context.payload.repository.html_url + `/actions/runs/${context.runId}`
+            let testSuite = 'ES Test'
+            let bodyContent = await fs.readFile(bodyFile, 'utf8')
+            bodyContent = bodyContent.replace(/{{actionRunLink}}/g, actionRunLink)
+                .replace(/{{runNumber}}/g, context.runNumber)
+                .replace(/{{environment}}/g, CC_ENV)
+                .replace(/{{testSuite}}/g, testSuite)
+
+            let receivers = '[email protected]'
+            if (context.eventName !== "pull_request" || context.eventName !== "push") {
+              receivers = '[email protected]'
+            }
+
+            return {
+              "subject": subject,
+              "body": bodyContent,
+              "reference": Math.random().toString(36).substr(2),  // used to prevent threading of similar emails
+              "receivers": receivers
+            }
+
+      - name: Send Result Email
+        if: ${{ failure() }}
+        uses: dawidd6/action-send-mail@v3
+        with:
+          server_address: smtp.gmail.com
+          server_port: 465
+          username: ${{secrets.DIMAGIQA_MAIL_USERNAME}}
+          password: ${{secrets.DIMAGIQA_MAIL_PASSWORD}}
+          subject: ${{ fromJSON(steps.configure_email.outputs.result).subject }}
+          to: ${{ fromJSON(steps.configure_email.outputs.result).receivers }}
+          from: <${{secrets.DIMAGIQA_MAIL_USERNAME}}>
+          html_body: ${{ fromJSON(steps.configure_email.outputs.result).body }}
+          convert_markdown: true
+          attachments: ${{ github.workspace }}/es_report_${{ matrix.environment }}.html
+          in_reply_to: ${{ fromJSON(steps.configure_email.outputs.result).reference }}
+
+      - name: Post to Slack channel on Failure
+        id: slack
+        uses: slackapi/[email protected]
+        if: failure()
+        with:
+          payload: |
+            {
+                "blocks": [
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": " Konnichiwa :alphabet-yellow-q::alphabet-yellow-a: 👋 \n*${{ github.workflow }}* were just triggered!"
+                        }
+                    },
+                    {
+                        "type": "context",
+                        "elements": [
+                            {
+                                "type": "mrkdwn",
+                                "text": "*Environment: *\n ${{ matrix.environment }}  \n"
+                            },
+                          {
+                                "type": "mrkdwn",
+                                "text": " "
+                            },
+                            {
+                                "type": "mrkdwn",
+                                "text": "*Status: *\n ${{ job.status }}  :x:"
+                            }
+                        ]
+                    },
+                    {
+                        "type": "section",
+                        "text": {
+                            "type": "mrkdwn",
+                            "text": "Here's the corresponding workflow execution :arrow_right::arrow_right:"
+                        },
+                        "accessory": {
+                            "type": "button",
+                            "text": {
+                                "type": "plain_text",
+                                "text": "View on Github",
+                                "emoji": true
+                            },
+                            "value": "click_me_123",
+                            "url": "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}",
+                            "action_id": "button-action",
+                            "style": "danger"
+                        }
+                    }
+                ]
+            }
+        env:
+          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL_SMOKE }}
+          SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
+
+      - name: Archive test results
+        if: ${{ success() || failure() }}
+        uses: actions/upload-artifact@v2
+        with:
+          name: test-result-reports-${{ matrix.environment }}
+          path: ${{ github.workspace }}/es_report_${{ matrix.environment }}.html
+          retention-days: 2

.gitignore

@@ -234,5 +234,9 @@ Features/Lookuptable/settings.cfg
 Features/Lookuptable/report.html
 Features/Lookuptable/testCases/report.html
 
+#Ignoring compliled files of ElasticSearchTests
+ElasticSearchTests/settings.cfg
+ElasticSearchTests/report.html
+ElasticSearchTests/testCases/report.html
 
 

ElasticSearchTests/README.md

@@ -0,0 +1,60 @@
+## Elastic Search Test Script
+
+ES tests ensure that the Elastic Search functionalities are working and there are no showstoppers in the ES upgrades deployed to environments.\
+The automated tests comprises of [these tests.](https://docs.google.com/spreadsheets/d/1ahqBcHWlp1YoxwkIpSc_zV25PjY4V5j6GW4jb3LOyuc/edit#gid=1461812550)
+
+## Executing Scripts
+
+### <ins> On Local Machine </ins>
+
+#### Setting up test environment
+
+```sh
+
+# create and activate a virtualenv using your preferred method. Example:
+python -m venv venv
+source venv/bin/activate
+
+
+# install requirements
+pip install -r requires.txt
+
+```
+
+[More on setting up virtual environments](https://confluence.dimagi.com/display/GTD/QA+and+Python+Virtual+Environments)
+
+
+#### Running Tests
+
+
+ -   Copy `settings-sample.cfg` to `settings.cfg` and populate `settings.cfg` for
+the environment you want to test.
+- Run tests using pytest command like:
+
+```sh
+
+# To execute all the test cases 
+pytest -v --rootdir= ElasticSearchTests/testCases
+
+```
+- You could also pass the following arguments
+  - ` -n 3 --dist=loadfile` - This will run the tests parallelly in 3 instances. The number of reruns is configurable.
+  - ` --reruns 1` - This will re-run the tests once in case of failures.The number of reruns is configurable too.
+
+### <ins> Trigger Manually on Gitaction </ins>
+
+<img align="right" width="400" src="https://github.com/dimagi/dimagi-qa/assets/67914792/002fbfd3-2512-4e12-a8ea-e57f93f5a615" alt="clone this repository" />
+
+To manually trigger the script,
+  - Go to [Gitactions](https://github.com/dimagi/dimagi-qa/actions/)
+  - Select the desired workflow, here [ES Tests action](https://github.com/dimagi/dimagi-qa/actions/workflows/es-tests.yml)
+  - Run workflow
+  - Select workflow as ```master```
+  - Select the environment as desired
+  - Run!
+
+If you are a part of the QA team, you'll receive emails for the result of the run after it's complete. 
+
+<img align="right" width="400" src="https://user-images.githubusercontent.com/67914792/168756705-88e4b330-b05a-4df2-a60c-7d45e8a2d002.PNG" alt="clone this repository" />
+
+Besides, you should be able to find the zipped results in the **Artifacts** section, of the corresponding run (after it's complete).

ElasticSearchTests/requires.txt

@@ -0,0 +1,20 @@
+## Stores information about all the libraries, modules, and packages that are used in this project.
+
+flake8>=3.8.4
+pandas>=1.2.2
+pytest
+pytest-html
+py
+openpyxl
+selenium == 4.11.0
+pytest-rerunfailures
+pytest-xdist
+pytest-xdist[psutil]
+pytest-order
+python-dateutil
+natsort
+pyotp
+imap-tools
+beautifulsoup4
+html5lib
+requests

ElasticSearchTests/settings-sample.cfg

@@ -0,0 +1,19 @@
+[default]
+# This is the environment url of commcare
+url = https://www.commcarehq.org/
+# Login username of the webuser
+login_username =
+# Login password of the webuser
+login_password =
+# This is a preconfigured browsertack username for mobile tests
+bs_user =
+# This is a preconfigured browsertack key used for mobile tests
+bs_key =
+# This is a preconfigured authentication key used for 2FA tests on staging
+staging_auth_key =
+# This is a preconfigured authentication key used for 2FA tests on prod
+prod_auth_key =
+# This is a preconfigured password for the user used to test web user invitation tests
+invited_webuser_password =
+# This is a preconfigured password for the email account used for saved reports verification
+imap_password =

ElasticSearchTests/settings.cfg

@@ -0,0 +1,7 @@
+[default]
+url = https://staging.commcarehq.org/
+login_username = [email protected]
+login_password = pass@123
+staging_auth_key = "DRCAKX6AG3V3Q7JWV27KIPOWVMAYJ3R4"
+prod_auth_key = "YRJSX3BARTDVYUUQKIQUX3EIKQUZZS5P"
+imap_password = gpzwiwkpseyhjoub

ElasticSearchTests/testCases/conftest.py

@@ -0,0 +1,72 @@
+import os
+
+from configparser import ConfigParser
+from pathlib import Path
+from common_utilities.fixtures import *
+
+""""This file provides fixture functions for driver initialization"""
+
+global driver
+
+
+@pytest.fixture(scope="session")
+def environment_settings_hq():
+    """Load settings from os.environ
+
+            Names of environment variables:
+                DIMAGIQA_URL
+                DIMAGIQA_LOGIN_USERNAME
+                DIMAGIQA_LOGIN_PASSWORD
+                DIMAGIQA_MAIL_USERNAME
+                DIMAGIQA_MAIL_PASSWORD
+
+            See https://docs.github.com/en/actions/reference/encrypted-secrets
+            for instructions on how to set them.
+            """
+    settings = {}
+    for name in ["url", "login_username", "login_password", "mail_username",
+                 "mail_password", "bs_user", "bs_key", "staging_auth_key", "prod_auth_key", "invited_webuser_password", "imap_password"]:
+
+        var = f"DIMAGIQA_{name.upper()}"
+        if var in os.environ:
+            settings[name] = os.environ[var]
+    if "url" not in settings:
+        env = os.environ.get("DIMAGIQA_ENV") or "staging"
+        subdomain = "www" if env == "production" else env
+        # updates the url with the project domain while testing in CI
+        project = "a/qa-automation-prod" if env == "production" else "a/qa-automation"
+        settings["url"] = f"https://{subdomain}.commcarehq.org/{project}"
+    return settings
+
+
+@pytest.fixture(scope="session", autouse=True)
+def settings(environment_settings_hq):
+    if os.environ.get("CI") == "true":
+        settings = environment_settings_hq
+        settings["CI"] = "true"
+        if any(x not in settings for x in ["url", "login_username", "login_password",
+                                           "mail_username", "mail_password", "bs_user", "bs_key", "staging_auth_key",
+                                           "prod_auth_key", "invited_webuser_password", "imap_password"]):
+            lines = environment_settings_hq.__doc__.splitlines()
+            vars_ = "\n  ".join(line.strip() for line in lines if "DIMAGIQA_" in line)
+            raise RuntimeError(
+                f"Environment variables not set:\n  {vars_}\n\n"
+                "See https://docs.github.com/en/actions/reference/encrypted-secrets "
+                "for instructions on how to set them."
+            )
+        return settings
+    path = Path(__file__).parent.parent / "settings.cfg"
+    if not path.exists():
+        raise RuntimeError(
+            f"Not found: {path}\n\n"
+            "Copy settings-sample.cfg to settings.cfg and populate "
+            "it with values for the environment you want to test."
+        )
+    settings = ConfigParser()
+    settings.read(path)
+    # updates the url with the project domain while testing in local
+    if settings["default"]["url"] == "https://www.commcarehq.org/":
+        settings["default"]["url"] = f"{settings['default']['url']}a/qa-automation-prod"
+    else:
+        settings["default"]["url"] = f"{settings['default']['url']}a/qa-automation"
+    return settings["default"]