test: E2E playwright tests for web app

.devcontainer/devcontainer.json

@@ -33,7 +33,8 @@
         "esbenp.prettier-vscode",
         "humao.rest-client",
         "GitHub.copilot",
-        "runem.lit-plugin"
+        "runem.lit-plugin",
+        "ms-playwright.playwright"
       ]
     }
   },
@@ -42,7 +43,7 @@
   "forwardPorts": [3000, 3001, 5173],
 
   // Use 'postCreateCommand' to run commands after the container is created.
-  "postCreateCommand": "npm install",
+  "postCreateCommand": "./.devcontainer/postCreateCommand.sh",
 
   // Set minimal host requirements for the container.
   "hostRequirements": {

.devcontainer/postCreateCommand.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+echo "Installing dependencies..."
+npm install
+
+echo "Installing playwright browsers..."
+npx playwright install --with-deps

.github/workflows/playwright.yml

@@ -0,0 +1,27 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-node@v3
+        with:
+          node-version: 18
+      - name: Install dependencies
+        run: npm ci
+      - name: Install Playwright Browsers
+        run: npm run install:playwright
+      - name: Run Playwright tests
+        run: npm run test:playwright
+      - uses: actions/upload-artifact@v3
+        if: failure()
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.gitignore

@@ -20,3 +20,6 @@ dist/
 
 # misc
 TODO
+/test-results/
+/playwright-report/
+/playwright/.cache/

package.json

@@ -12,14 +12,16 @@
     "start:search": "npm run dev --workspace=search",
     "start:indexer": "npm run dev --workspace=indexer",
     "test": "npm run test -ws --if-present",
+    "test:playwright": "npx playwright test",
     "build": "npm run build -ws --if-present",
     "clean": "npm run clean -ws --if-present",
     "docker:build": "npm run docker:build -ws --if-present",
     "format": "prettier --list-different --write .",
     "format:check": "prettier --check .",
     "lint": "eslint .",
     "lint:fix": "eslint --fix .",
-    "prepare": "simple-git-hooks || echo 'simple-git-hooks install skipped'"
+    "prepare": "simple-git-hooks || echo 'simple-git-hooks install skipped'",
+    "install:playwright": "npx playwright install --with-deps"
   },
   "keywords": [],
   "author": "Microsoft",
@@ -28,7 +30,9 @@
     "packages/*"
   ],
   "devDependencies": {
+    "@playwright/test": "^1.39.0",
     "@tapjs/nock": "^3.1.13",
+    "@types/node": "^18.15.3",
     "concurrently": "^8.2.1",
     "eslint-config-shared": "^1.0.0",
     "lint-staged": "^14.0.1",

packages/chat-component/src/main.ts

@@ -368,6 +368,7 @@ export class ChatComponent extends LitElement {
                 <a
                   class="items__link"
                   href="${this.apiUrl}/content/${citation.text}"
+                  data-testid="citation"
                   target="_blank"
                   rel="noopener noreferrer"
                   >${citation.ref}. ${citation.text}</a
@@ -418,6 +419,7 @@ export class ChatComponent extends LitElement {
                   <button
                     title="${globalConfig.RESET_CHAT_BUTTON_TITLE}"
                     class="button chat__header--button"
+                    data-testid="chat-reset-button"
                     @click="${this.resetCurrentChat}"
                   >
                     <span class="chat__header--span">${globalConfig.RESET_CHAT_BUTTON_TITLE}</span>
@@ -435,6 +437,7 @@ export class ChatComponent extends LitElement {
                                 <button
                                   title="${globalConfig.SHOW_THOUGH_PROCESS_BUTTON_LABEL_TEXT}"
                                   class="button chat__header--button"
+                                  data-testid="chat-show-thought-process"
                                   @click="${this.showThoughtProcess}"
                                   ?disabled="${this.isShowingThoughtProcess || !this.canShowThoughtProcess}"
                                 >
@@ -520,6 +523,7 @@ export class ChatComponent extends LitElement {
                               role="button"
                               href="#"
                               class="defaults__button"
+                              data-testid="default-question"
                               @click="${(event: Event) => this.handleDefaultPromptClick(prompt, event)}"
                             >
                               ${prompt}
@@ -540,6 +544,7 @@ export class ChatComponent extends LitElement {
             <div class="chatbox__container container-col container-row">
               <input
                 class="chatbox__input"
+                data-testid="question-input"
                 id="question-input"
                 placeholder="${globalConfig.CHAT_INPUT_PLACEHOLDER}"
                 aria-labelledby="chatbox-label"
@@ -553,6 +558,7 @@ export class ChatComponent extends LitElement {
               />
               <button
                 class="chatbox__button"
+                data-testid="submit-question-button"
                 @click="${this.handleUserChatSubmit}"
                 title="${globalConfig.CHAT_BUTTON_LABEL_TEXT}"
                 ?disabled="${this.isDisabled}"
@@ -583,11 +589,12 @@ export class ChatComponent extends LitElement {
         </section>
         ${this.isShowingThoughtProcess
           ? html`
-              <aside class="aside">
+              <aside class="aside" data-testid="aside-thought-process">
                 <div class="aside__header">
                   <button
                     title="${globalConfig.HIDE_THOUGH_PROCESS_BUTTON_LABEL_TEXT}"
                     class="button chat__header--button"
+                    data-testid="chat-hide-thought-process"
                     @click="${this.hideThoughtProcess}"
                   >
                     <span class="chat__header--span">${globalConfig.HIDE_THOUGH_PROCESS_BUTTON_LABEL_TEXT}</span>

packages/webapp/src/components/SettingsButton/SettingsButton.tsx

@@ -10,7 +10,11 @@ interface Props {
 
 export const SettingsButton = ({ className, onClick }: Props) => {
   return (
-    <button className={`${styles.container} ${className ?? ''}`} onClick={onClick}>
+    <button
+      className={`${styles.container} ${className ?? ''}`}
+      onClick={onClick}
+      data-testid="button__developer-settings"
+    >
       <Settings24Regular />
       <Text>{'Developer settings'}</Text>
     </button>

playwright.config.ts

@@ -0,0 +1,82 @@
+import { defineConfig, devices } from '@playwright/test';
+
+/**
+ * Read environment variables from file.
+ * https://github.com/motdotla/dotenv
+ */
+// require('dotenv').config();
+
+const useLocalServer = true;
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  testDir: './tests',
+  testMatch: '*.spec.ts',
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 1 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: 'html',
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    baseURL: useLocalServer ? 'http://localhost:5173' : process.env.WEBAPP_URI,
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'chromium',
+      use: { ...devices['Desktop Chrome'] },
+    },
+
+    {
+      name: 'firefox',
+      use: { ...devices['Desktop Firefox'] },
+    },
+
+    {
+      name: 'webkit',
+      use: { ...devices['Desktop Safari'] },
+    },
+
+    /* Test against mobile viewports. */
+    // {
+    //   name: 'Mobile Chrome',
+    //   use: { ...devices['Pixel 5'] },
+    // },
+    // {
+    //   name: 'Mobile Safari',
+    //   use: { ...devices['iPhone 12'] },
+    // },
+
+    /* Test against branded browsers. */
+    // {
+    //   name: 'Microsoft Edge',
+    //   use: { ...devices['Desktop Edge'], channel: 'msedge' },
+    // },
+    // {
+    //   name: 'Google Chrome',
+    //   use: { ...devices['Desktop Chrome'], channel: 'chrome' },
+    // },
+  ],
+
+  /* Run your local dev server before starting the tests */
+  webServer: useLocalServer
+    ? {
+        command: 'npm run start:webapp',
+        url: 'http://localhost:5173',
+        reuseExistingServer: !process.env.CI,
+      }
+    : undefined,
+});