Add E2E test of /chat on UIUC.chat in Production, via Github Actions (

#210)

* Add nightwatch test, works on local in headless mode

* Ignore type requirement errors

* Do simpler chrome args

* Add github action for Nightwatch e2e test

* Make the test wait for buttons to be visible before clicking them

* Rename files, only run test every minutes and not on push

.github/workflows/e2e-prod-status-test.yml

@@ -0,0 +1,22 @@
+name: E2E Test
+on:
+  schedule:
+    - cron: '7,37 * * * *' # Runs at 7 and 37 minutes past every hour
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: '18'
+      - name: Install dependencies
+        run: npm install
+      - name: Install Chrome
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y chromium-browser
+      - name: Run Nightwatch e2e tests on Prod
+        run: npx nightwatch nightwatch/e2e-test-of-uiucchat-prod.js

nightwatch.conf.js

@@ -0,0 +1,134 @@
+// Refer to the online docs for more details:
+// https://nightwatchjs.org/gettingstarted/configuration/
+//
+
+//  _   _  _         _      _                     _          _
+// | \ | |(_)       | |    | |                   | |        | |
+// |  \| | _   __ _ | |__  | |_ __      __  __ _ | |_   ___ | |__
+// | . ` || | / _` || '_ \ | __|\ \ /\ / / / _` || __| / __|| '_ \
+// | |\  || || (_| || | | || |_  \ V  V / | (_| || |_ | (__ | | | |
+// \_| \_/|_| \__, ||_| |_| \__|  \_/\_/   \__,_| \__| \___||_| |_|
+//             __/ |
+//            |___/
+
+module.exports = {
+  // An array of folders (excluding subfolders) where your tests are located;
+  // if this is not specified, the test source must be passed as the second argument to the test runner.
+  src_folders: ['nightwatch'],
+
+  // See https://nightwatchjs.org/guide/concepts/page-object-model.html
+  page_objects_path: [],
+
+  // See https://nightwatchjs.org/guide/extending-nightwatch/adding-custom-commands.html
+  custom_commands_path: [],
+
+  // See https://nightwatchjs.org/guide/extending-nightwatch/adding-custom-assertions.html
+  custom_assertions_path: [],
+
+  // See https://nightwatchjs.org/guide/extending-nightwatch/adding-plugins.html
+  plugins: [],
+
+  // See https://nightwatchjs.org/guide/concepts/test-globals.html
+  globals_path: '',
+
+  webdriver: {
+    start_process: true,
+    server_path: require('chromedriver').path,
+    port: 9515,
+  },
+
+  test_workers: {
+    enabled: true,
+  },
+
+  test_settings: {
+    default: {
+      disable_error_log: false,
+      launch_url: 'https://www.uiuc.chat/',
+
+      screenshots: {
+        enabled: false,
+        path: 'screens',
+        on_failure: true,
+      },
+
+      desiredCapabilities: {
+        browserName: 'chrome',
+        'goog:chromeOptions': {
+          args: [
+            '--headless=new',
+            '--no-sandbox',
+            '--disable-dev-shm-usage',
+            '--disable-gpu',
+          ],
+        },
+      },
+    },
+
+    chrome: {
+      desiredCapabilities: {
+        browserName: 'chrome',
+        'goog:chromeOptions': {
+          // More info on Chromedriver: https://sites.google.com/a/chromium.org/chromedriver/
+          args: [
+            //'--no-sandbox',
+            //'--ignore-certificate-errors',
+            //'--allow-insecure-localhost',
+            //'--headless=new'
+          ],
+        },
+      },
+
+      webdriver: {
+        start_process: true,
+        server_path: '',
+        cli_args: [
+          // --verbose
+        ],
+      },
+    },
+
+    ////////////////////////////////////////////////////////////////////////////////////////
+    // Configuration for using remote Selenium service or a cloud-based testing provider.  |
+    //                                                                                     |
+    // Please set the hostname and port of your remote selenium-server or cloud-provider   |
+    // (by setting the following properties in the configuration below):                   |
+    // - `selenium.host`                                                                   |
+    // - `selenium.port`                                                                   |
+    //                                                                                     |
+    // If you are using a cloud provider such as CrossBrowserTesting, LambdaTests, etc.,   |
+    // please set the username and access_key by setting the below environment variables:  |
+    // - REMOTE_USERNAME                                                                   |
+    // - REMOTE_ACCESS_KEY                                                                 |
+    // (.env files are supported)                                                          |
+    ////////////////////////////////////////////////////////////////////////////////////////
+    remote: {
+      // Info on all the available options with "selenium":
+      // https://nightwatchjs.org/guide/configuration/settings.html#selenium-server-settings
+      selenium: {
+        start_process: false,
+        server_path: '',
+        host: '<remote-hostname>',
+        port: 4444,
+      },
+
+      username: '${REMOTE_USERNAME}',
+      access_key: '${REMOTE_ACCESS_KEY}',
+
+      webdriver: {
+        keep_alive: true,
+        start_process: false,
+      },
+    },
+
+    'remote.chrome': {
+      extends: 'remote',
+      desiredCapabilities: {
+        browserName: 'chrome',
+        'goog:chromeOptions': {
+          w3c: true,
+        },
+      },
+    },
+  },
+}

nightwatch/e2e-test-of-uiucchat-prod.js

@@ -0,0 +1,57 @@
+/* eslint-disable @typescript-eslint/ban-ts-comment */
+// @ts-nocheck -- this is our only js file so far, just don't bother with types.
+describe('test-ece-2', function () {
+  before(function (browser) {
+    browser.options.desiredCapabilities['goog:chromeOptions'] = {
+      args: [
+        '--headless=new',
+        '--no-sandbox',
+        '--disable-dev-shm-usage',
+        '--disable-gpu',
+      ],
+    }
+  })
+
+  it('tests test-ece-2', function (browser) {
+    browser.timeouts('implicit', 10000)
+
+    browser
+      .windowRect({ width: 955, height: 1045 })
+      .navigateTo('https://www.uiuc.chat/ece120/chat')
+      .pause(5000)
+      .click('textarea')
+      .setValue('textarea', 'hi')
+      .perform(function () {
+        const actions = this.actions({ async: true })
+
+        return actions.keyDown(this.Keys.ENTER)
+      })
+      .perform(function () {
+        const actions = this.actions({ async: true })
+
+        return actions.keyUp(this.Keys.ENTER)
+      })
+      // Add waitForElementVisible before clicking
+      .waitForElementVisible(
+        'div.bg-gray-50\\/50 div.dark\\:prose-invert > div > div.w-full > div',
+        15000,
+      )
+      .click(
+        'div.bg-gray-50\\/50 div.dark\\:prose-invert > div > div.w-full > div',
+      )
+      .pause(5000)
+      .getText(
+        'div.bg-gray-50\\/50 div.dark\\:prose-invert > div > div.w-full > div',
+        function (result) {
+          console.log('Text content:', result.value)
+          // this.assert.ok(result.value.length > 0, 'Response text should not be empty');
+          const responseText = String(result.value)
+          this.assert.ok(
+            responseText.length > 0,
+            'Response text should not be empty',
+          )
+        },
+      )
+      .end()
+  })
+})

nightwatch/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "../tsconfig",
+  "compilerOptions": {
+    "target": "esnext",
+    "module": "commonjs",
+    "esModuleInterop": true,
+    "allowSyntheticDefaultImports": true,
+    "moduleResolution": "node",
+    "resolveJsonModule": true,
+    "skipLibCheck": true,
+    "allowJs": true,
+    "forceConsistentCasingInFileNames": true,
+    "noEmit": true
+  },
+  "ts-node": {
+    "transpileOnly": true
+  },
+  "include": ["."]
+}