test: add end-to-end tests with Playwright (#516)

* chore: add a workspace for e2e testing

* chore: add a containerized playwright setup

* chore: rename .devcontainer.json

* chore: move devcontainer.json

* test: depend on 'latest' for now

* ci: fix linter findings

* test: fix incorrect workspaceFolder

* test: fix workspaceFolder by using ${containerWorkspaceFolder}

* test: different strategy to move workspace folder

* test: hard-code workspaceFolder

* test: add authentication flow for GitHub

* refactor: remove unused import

* chore: add npm test

* ci: add acceptance tests

* ci: trying to run setup

* test: enable running tests from VS Code extension

* chore: add required extension

* ci: try to dynamically choose the image again

* ci: fix starting from incorrect path

* ci: inject env secrets

* ci: create codespace under test

* ci: fix linter findings

* ci: get codespace going

* ci: rtfm

* test: inspect host environment before starting codespace

* ci: communicate image version via codespaces secret

* ci: test if token already has correct scopes

* ci: able to start codespace, yay

* chore: fix linter findings

* chore: first working test

* chore: fix linter findings

* ci: add harden-runner action

* test: use a page object model paradigm

* test: mark the beforeEach as slow because of first-time-start

* test: make start-up more robust

* test: different timeout strategy

* ci: fail on create codespace and clean-up afterwards

* test: more timeout

* test: give the build more time to finish

* test: increase timeout even more

* ci: move acceptance test to the right place

* ci: correct issues

* test: add more tests and make them robust

* test: use proper test.describe

* test: increase robustness

* test: enable discovery of code smells

* docs: fix badges

* chore: reset IMAGE_VERSION after job completion

* chore: update feature desktop-lite

---------

Co-authored-by: ronj <[email protected]>

.devcontainer/cpp-test/devcontainer.json

@@ -0,0 +1,11 @@
+{
+  "image": "ghcr.io/philips-software/amp-devcontainer-cpp:${localEnv:IMAGE_VERSION}",
+  "workspaceFolder": "/workspaces/amp-devcontainer/.devcontainer/cpp/e2e/workspace",
+  "customizations": {
+    "vscode": {
+      "settings": {
+        "cmake.automaticReconfigure": false
+      }
+    }
+  }
+}

.devcontainer/cpp/devcontainer.json

@@ -3,12 +3,20 @@
     "dockerfile": "Dockerfile",
     "context": "../.."
   },
+  "forwardPorts": [6080],
   "remoteEnv": {
-    "CONTAINER_FLAVOR": "cpp"
+    "CONTAINER_FLAVOR": "cpp",
+    "NODE_EXTRA_CA_CERTS": "/usr/local/share/ca-certificates/Cisco_Umbrella_Root_CA.crt"
   },
   "mounts": [
     "source=/var/run/docker.sock,target=/var/run/docker.sock,type=bind"
   ],
+  "features": {
+    "ghcr.io/devcontainers/features/desktop-lite:1.2.4": {},
+    "ghcr.io/devcontainers/features/github-cli:1.0.13": {},
+    "ghcr.io/devcontainers/features/node:1.5.0": {}
+  },
+  "postCreateCommand": "npm install && npx playwright install --with-deps",
   "customizations": {
     "vscode": {
       "settings": {
@@ -23,6 +31,7 @@
         "[email protected]",
         "[email protected]",
         "[email protected]",
+        "[email protected]",
         "[email protected]",
         "[email protected]",
         "[email protected]",

.devcontainer/cpp/e2e/playwright.config.ts

@@ -0,0 +1,29 @@
+import { defineConfig, devices } from '@playwright/test';
+import path from 'path';
+
+require('dotenv').config();
+
+export const STORAGE_STATE = path.join(__dirname, 'playwright/.auth/user.json');
+
+export default defineConfig({
+  testDir: './tests',
+  fullyParallel: false,
+  forbidOnly: !!process.env.CI,
+  retries: process.env.CI ? 2 : 0,
+  workers: 1,
+  reporter: 'list',
+  use: {
+    trace: 'on-first-retry'
+  },
+  projects: [
+    { name: 'setup', testMatch: '**/*.setup.ts' },
+    {
+      name: 'chromium',
+      use: {
+        ...devices['Desktop Chrome'],
+        storageState: STORAGE_STATE
+      },
+      dependencies: ['setup']
+    }
+  ]
+});

.devcontainer/cpp/e2e/tests/authentication.setup.ts

@@ -0,0 +1,29 @@
+import { test as setup } from '@playwright/test';
+import * as OTPAuth from 'otpauth';
+import { STORAGE_STATE } from '../playwright.config';
+
+setup('authenticate', async ({ page }) => {
+  await page.goto('https://github.com/login');
+  await page.getByLabel('Username or email address').fill(process.env.GITHUB_USER!);
+  await page.getByLabel('Password').fill(process.env.GITHUB_PASSWORD!);
+  await page.getByRole('button', { name: 'Sign in', exact: true }).click();
+
+  let totp = new OTPAuth.TOTP({
+    issuer: 'GitHub',
+    label: 'GitHub',
+    algorithm: 'SHA1',
+    digits: 6,
+    period: 30,
+    secret: process.env.GITHUB_TOTP_SECRET!
+  });
+
+  let code = totp.generate();
+  await page.getByPlaceholder('XXXXXX').fill(code);
+
+  // Wait until the page receives the cookies.
+  //
+  // Sometimes login flow sets cookies in the process of several redirects.
+  // Wait for the final URL to ensure that the cookies are actually set.
+  await page.waitForURL('https://github.com/');
+  await page.context().storageState({ path: STORAGE_STATE });
+});

.devcontainer/cpp/e2e/tests/codespace.pom.ts

@@ -0,0 +1,103 @@
+import { test, expect, type Page, type Locator } from '@playwright/test';
+
+type CommandAndPrompt = {
+  command: string,
+  prompt?: string
+};
+
+export class CodespacePage {
+  readonly page: Page;
+  readonly outputPanel: Locator;
+  readonly terminal: Locator;
+
+  constructor(page: Page) {
+    this.page = page;
+    this.outputPanel = page.locator('[id="workbench.panel.output"]');
+    this.terminal = page.locator('.terminal-widget-container').first();
+  }
+
+  async goto() {
+    await this.page.goto('https://' + process.env.CODESPACE_NAME + '.github.dev');
+  }
+
+  /**
+   * Wait for the extensions to be active in the Codespace.
+   *
+   * This method is used to verify that the extensions in `extensions` are active in the Codespace.
+   * Used when waiting for the Codespace to be ready for testing. As the
+   * extensions are typically activated last, before the Codespace is ready for use.
+   *
+   * **Usage**
+   *
+   * ```ts
+   * const codespace = new CodespacePage(page);
+   * await codespace.areExtensionsActive(['SonarLint', 'CMake', 'Live Share', 'GitHub Pull Requests']);
+   * ```
+   *
+   * @param extensions - The list of extensions to wait for.
+   */
+  async areExtensionsActive(extensions: string[]) {
+    test.setTimeout(3 * 60 * 1000);
+
+    for (const plugin of extensions) {
+      await expect(this.page.getByRole('tab', { name: plugin }).locator('a')).toBeVisible({ timeout: 5 * 60 * 1000 });
+    }
+
+    await expect(this.page.getByRole('button', { name: 'Activating Extensions...' })).toBeHidden();
+  }
+
+  /**
+   * Executes the given commands in the terminal.
+   *
+   * **Usage**
+   *
+   * ```ts
+   * const codespace = new CodespacePage(page);
+   * await codespace.executeInTerminal('git clean -fdx');
+   * ```
+   *
+   * @param commands - The commands to execute in the terminal. It can be a single command or an array of commands.
+   */
+  async executeInTerminal(commands: string | string[]) {
+    await this.page.keyboard.press('Control+Shift+`');
+    await expect(this.page.locator('.terminal-wrapper.active')).toBeVisible();
+
+    for (const command of Array.isArray(commands) ? [...commands + 'exit'] : [commands, 'exit']) {
+      await this.terminal.pressSequentially(command);
+      await this.terminal.press('Enter');
+    }
+  }
+
+  /**
+   * Executes the given commands in the command palette.
+   *
+   * This method waits for `prompt` to appear and then types `command` and presses Enter.
+   * When no prompt is given the default prompt is used.
+   *
+   * @param commands - The commands to execute in the command palette. It can be a single command or an array of commands.
+   */
+  async executeFromCommandPalette(commands: CommandAndPrompt | CommandAndPrompt[]) {
+    await this.page.keyboard.press('Control+Shift+P');
+
+    for (const command of Array.isArray(commands) ? commands : [commands]) {
+      let prompt = this.page.getByPlaceholder(command.prompt || 'Type the name of a command to run');
+
+      await prompt.pressSequentially(command.command);
+      await prompt.press('Enter');
+    }
+  }
+
+  /**
+   * Opens the tab with the given name.
+   *
+   * @param name - The name of the tab to open.
+   */
+  async openTabByName(name: string) {
+    await this.page.getByRole('tab', { name: name }).locator('a').click();
+  }
+
+  async openFileInEditor(name: string) {
+    await this.page.getByRole('treeitem', { name: name }).locator('a').click();
+    await expect(this.page.locator('[id="workbench.parts.editor"]')).toContainText(name);
+  }
+}

.devcontainer/cpp/e2e/tests/smoke.spec.ts

@@ -0,0 +1,39 @@
+import { test, expect } from '@playwright/test';
+import { CodespacePage } from './codespace.pom';
+
+test.beforeEach(async ({ page }) => {
+  const codespace = new CodespacePage(page);
+
+  await codespace.goto();
+  await codespace.areExtensionsActive(['Testing', 'SonarLint', 'CMake', 'Live Share', 'GitHub Pull Requests']);
+  await codespace.executeInTerminal('git clean -fdx');
+});
+
+test.describe('CMake', () => {
+  test('should succesfully build without selecting configuration', async ({ page }) => {
+    const codespace = new CodespacePage(page);
+
+    await page.getByRole('button', { name: 'Build the selected target' }).click();
+    await page.getByLabel('host, Build for host').locator('a').click();
+    await expect(codespace.outputPanel).toContainText('Build finished with exit code 0', { timeout: 5 * 60 * 1000 });
+  });
+
+  test('should succesfully build after selecting configuration', async ({ page }) => {
+    const codespace = new CodespacePage(page);
+
+    await codespace.openTabByName('CMake');
+    await expect(page.getByRole('treeitem', { name: 'Change Configure Preset' })).toContainText('[No Configure Preset Selected]');
+    await expect(page.getByRole('treeitem', { name: 'Change Build Preset' })).toContainText('[No Build Preset Selected]');
+
+    await codespace.executeFromCommandPalette([{ command: 'CMake: Select Configure Preset' },
+                                               { command: 'host', prompt: 'Select a configure preset' }]);
+    await expect(page.getByRole('treeitem', { name: 'Change Configure Preset' })).toContainText('host');
+
+    await codespace.executeFromCommandPalette([{ command: 'CMake: Select Build Preset' },
+                                               { command: 'host-Release', prompt: 'Select a build preset' }]);
+    await expect(page.getByRole('treeitem', { name: 'Change Build Preset' })).toContainText('host-Release');
+
+    await page.getByRole('button', { name: 'Build the selected target' }).click();
+    await expect(codespace.outputPanel).toContainText('Build finished with exit code 0', { timeout: 5 * 60 * 1000 });
+  });
+});

.devcontainer/cpp/e2e/workspace/CMakeLists.txt

@@ -0,0 +1,5 @@
+cmake_minimum_required(VERSION 3.24)
+
+project(e2e-test VERSION 1.0.0)
+
+add_executable(e2e-test main.cpp)

.devcontainer/cpp/e2e/workspace/CMakePresets.json

@@ -0,0 +1,42 @@
+{
+  "version": 3,
+  "configurePresets": [
+    {
+      "name": "defaults",
+      "hidden": true,
+      "binaryDir": "${sourceDir}/build/${presetName}",
+      "generator": "Ninja Multi-Config",
+      "cacheVariables": {
+        "CMAKE_CONFIGURATION_TYPES": "Debug;Release;RelWithDebInfo;MinSizeRel"
+      }
+    },
+    {
+      "name": "host",
+      "displayName": "host",
+      "description": "Build for host",
+      "inherits": "defaults"
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "host-Debug",
+      "configuration": "Debug",
+      "configurePreset": "host"
+    },
+    {
+      "name": "host-Release",
+      "configuration": "Release",
+      "configurePreset": "host"
+    },
+    {
+      "name": "host-RelWithDebInfo",
+      "configuration": "RelWithDebInfo",
+      "configurePreset": "host"
+    },
+    {
+      "name": "host-MinSizeRel",
+      "configuration": "MinSizeRel",
+      "configurePreset": "host"
+    }
+  ]
+}

.devcontainer/cpp/e2e/workspace/main.cpp

@@ -0,0 +1,12 @@
+#include <iostream>
+
+void SmellyFunction()
+{
+    auto array = new int[10];
+}
+
+int main()
+{
+    std::cout << "Hello World!" << std::endl;
+    return 0;
+}

.github/dependabot.yml

@@ -5,7 +5,7 @@ updates:
   - package-ecosystem: github-actions
     directory: /
     schedule:
-      interval: daily
+      interval: weekly
     groups:
       github-actions:
         update-types:
@@ -16,8 +16,16 @@ updates:
   - package-ecosystem: docker
     directory: .devcontainer
     schedule:
-      interval: daily
+      interval: weekly
+  - package-ecosystem: devcontainers
+    directory: .devcontainer
+    schedule:
+      interval: weekly
+  - package-ecosystem: npm
+    directory: /
+    schedule:
+      interval: weekly
   - package-ecosystem: pip
     directory: .devcontainer
     schedule:
-      interval: daily
+      interval: weekly

.github/workflows/acceptance-test.yml

@@ -0,0 +1,52 @@
+---
+name: Acceptance Test
+
+on:
+  workflow_call:
+    inputs:
+      flavor:
+        required: true
+        type: string
+
+concurrency:
+  group: ${{ github.workflow }}
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: step-security/harden-runner@0d381219ddf674d61a7572ddd19d7941e271515c # v9.0.1
+        with:
+          egress-policy: audit
+      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        with:
+          persist-credentials: false
+      - uses: actions/setup-node@1e60f620b9541d16bece96c5465dc8ee9832be0b # v4.0.3
+        with:
+          node-version: 20
+      - run: npm ci
+      - run: npx playwright install --with-deps
+      # Create a GitHub Codespace and communicate the image version via a Codespace secret (should be a Codespace environment variable).
+      # This secret is used by devcontainer.json, as such it is a resource that should not be used concurrently.
+      - run: |
+          set -Eeuo pipefail
+          gh secret set -a codespaces IMAGE_VERSION --body "pr-${{ github.event.pull_request.number }}"
+          echo CODESPACE_NAME="$(gh codespace create -R "${{ github.repository }}" -b "$HEAD_REF" -m basicLinux32gb --devcontainer-path ".devcontainer/${{ inputs.flavor }}-test/devcontainer.json" --idle-timeout 10m --retention-period 1h)" >> "$GITHUB_ENV"
+        env:
+          GH_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}
+          HEAD_REF: ${{ github.head_ref }}
+      - run: cd .devcontainer/${{ inputs.flavor }}/e2e && npm test
+        env:
+          GITHUB_USER: ${{ secrets.TEST_GITHUB_USER }}
+          GITHUB_PASSWORD: ${{ secrets.TEST_GITHUB_PASSWORD }}
+          GITHUB_TOTP_SECRET: ${{ secrets.TEST_GITHUB_TOTP_SECRET }}
+      - run: |
+          set -Eeuo pipefail
+          gh codespace delete --force --codespace "$CODESPACE_NAME"
+          gh secret set -a codespaces IMAGE_VERSION --body "latest"
+        if: always()
+        env:
+          GH_TOKEN: ${{ secrets.TEST_GITHUB_TOKEN }}