From c705d6e53ec2f2154e1096012c5179223b3ab8de Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 12 Oct 2026 14:17:14 +0200 Subject: [PATCH 01/11] chore(tests-playwright): backport new structure and updated test setup from the template --- .agents/skills/write-tests/SKILL.md | 239 +++++++++++++++ .github/workflows/qa.yml | 173 +++++++++-- .../content/__tests__/fetchPage-5xx.test.ts | 91 ++++++ docs/commands.md | 12 +- package.json | 9 +- pnpm-lock.yaml | 3 + qa/tests/README.md | 183 +++++++---- qa/tests/playwright/.env.example | 7 +- qa/tests/playwright/.gitignore | 2 +- qa/tests/playwright/axe/axe-test.ts | 219 -------------- qa/tests/playwright/axe/axe.spec.ts | 120 ++++++++ qa/tests/playwright/e2e/test_example.spec.ts | 11 - qa/tests/playwright/helpers/fixtures.ts | 30 ++ qa/tests/playwright/helpers/flatten-urls.ts | 16 + .../playwright/helpers/get-sitemap-links.ts | 12 +- .../helpers/urls-all-components-page.json | 1 - qa/tests/playwright/helpers/urls.json | 77 ++++- qa/tests/playwright/package.json | 14 +- qa/tests/playwright/perfo/lhci.ts | 161 +++++++++- .../playwright/perfo/lighthouse-history.md | 0 qa/tests/playwright/playwright.config.ts | 123 ++++++-- qa/tests/playwright/seo/seo.spec.ts | 283 +++++++++++++++++- qa/tests/playwright/tsconfig.json | 8 +- .../visual/compare-snapshots-docker.sh | 71 +++++ .../visual/update-snapshots-docker.sh | 72 +++++ qa/tests/playwright/visual/visual.spec.ts | 83 ++++- turbo.json | 22 +- 27 files changed, 1640 insertions(+), 402 deletions(-) create mode 100644 .agents/skills/write-tests/SKILL.md create mode 100644 apps/ui/src/lib/strapi-api/content/__tests__/fetchPage-5xx.test.ts delete mode 100644 qa/tests/playwright/axe/axe-test.ts create mode 100644 qa/tests/playwright/axe/axe.spec.ts delete mode 100644 qa/tests/playwright/e2e/test_example.spec.ts create mode 100644 qa/tests/playwright/helpers/fixtures.ts create mode 100644 qa/tests/playwright/helpers/flatten-urls.ts delete mode 100644 qa/tests/playwright/helpers/urls-all-components-page.json create mode 100644 qa/tests/playwright/perfo/lighthouse-history.md create mode 100755 qa/tests/playwright/visual/compare-snapshots-docker.sh create mode 100755 qa/tests/playwright/visual/update-snapshots-docker.sh diff --git a/.agents/skills/write-tests/SKILL.md b/.agents/skills/write-tests/SKILL.md new file mode 100644 index 00000000..869aba28 --- /dev/null +++ b/.agents/skills/write-tests/SKILL.md @@ -0,0 +1,239 @@ +--- +name: write-tests +description: "Add or extend automated tests using the repo's existing conventions. Covers Vitest (apps/ui, apps/strapi) and Playwright (qa/tests/playwright: e2e smoke/mock, visual, axe, seo). Triggers: write test, add test, cover X with tests, reproduce with a test, test this, e2e test, playwright test." +argument-hint: "[file-or-feature-to-cover]" +--- + +# Write Tests + +Add or extend tests using the repo's existing conventions. Do not introduce new test frameworks. Do not invent new helpers when an existing one fits. + +Reference: `qa/tests/README.md` (Playwright suite setup, commands, visual/CI workflow). + +## Decide the test layer first + +Pick before writing. Wrong layer = wasted test. + +| Goal | Layer | Where | +| ---------------------------------------------------------------------------------- | ------------------------------ | -------------------------------------------------------------------------------------------------------------------------------- | +| Pure function / utility / hook (no network, no DOM) | **Vitest unit** (`apps/ui`) | `apps/ui/src/**/__tests__/*.test.ts` | +| Strapi content types / services against a booted instance | **Vitest** (`apps/strapi`) | `apps/strapi/tests/**/*.test.ts` | +| Critical-path browser flow against the real app/backend (POM) | **Playwright e2e/smoke** | spec: `qa/tests/playwright/e2e/smoke/*.spec.ts`, page object: `qa/tests/playwright/helpers/pages/Page.ts` | +| UI behavior that needs a specific/edge-case backend response (POM + route mocking) | **Playwright e2e/mock** | spec: `qa/tests/playwright/e2e/mock/*.spec.ts`, same page objects, network stubbed via `qa/tests/playwright/helpers/fixtures.ts` | +| Pure HTTP check (status/redirect/JSON), no browser needed | **Playwright e2e** (`request`) | `qa/tests/playwright/e2e/smoke/*.spec.ts` — no POM, use the `request` fixture directly | +| Visual regression | **Playwright visual** | `qa/tests/playwright/visual/*.spec.ts` | +| Accessibility | **Playwright axe** | `qa/tests/playwright/axe/*.spec.ts` | +| SEO meta, head, robots | **Playwright seo** | `qa/tests/playwright/seo/*.spec.ts` | + +Unit > integration > e2e. Push tests down the pyramid when behavior allows. + +## Phase 1 — Locate convention + +Before writing, read **one existing test in the same layer** to mirror style: + +- UI unit ref: `apps/ui/src/lib/__tests__/dates.test.ts` +- Strapi ref: `apps/strapi/tests/app.test.ts` (boots Strapi via `tests/helpers/strapi.ts`) +- Playwright e2e/smoke ref: `qa/tests/playwright/e2e/smoke/pages.spec.ts` (page object: `helpers/pages/ContentPage.ts`) +- Playwright e2e/mock ref: `qa/tests/playwright/e2e/mock/media-fallback.spec.ts` (route stubbing), `preview-refresh.spec.ts` (stubbed cross-origin window) +- Playwright visual ref: `qa/tests/playwright/visual/visual.spec.ts` +- Playwright axe ref: `qa/tests/playwright/axe/axe.spec.ts` +- Playwright seo ref: `qa/tests/playwright/seo/seo.spec.ts` + +Match imports, helper usage, describe/it nesting, naming. + +## Phase 2 — Write the test + +### Vitest (UI + Strapi) + +- Import from `vitest`: `describe`, `it`, `expect`, `beforeAll`, `afterAll`. +- `globals: true` already set — but **still import explicitly** (matches existing code). +- File pattern: `*.test.ts`. UI include: `src/**/*.test.ts` (place in a `__tests__/` folder next to the source). Strapi include: `tests/**/*.test.{js,ts}`. +- Path alias `@` resolves to `apps/ui/src` (UI tests only). +- Strapi tests boot a real instance with `setupStrapi()` / `teardownStrapi()` from `apps/strapi/tests/helpers/strapi.ts` in `beforeAll` / `afterAll` — mirror `app.test.ts`, including its timeouts. `pnpm test:ci` (CI) excludes Strapi, so these run locally only. + +### Playwright — e2e (Page Object Model) + +Any e2e spec that drives the browser (navigation, clicks, assertions on rendered content) uses the **Page Object Model**, whether it's a smoke or a mock spec. Every page under test gets one POM class in `qa/tests/playwright/helpers/pages/Page.ts`, shared by both its smoke and mock specs — a page has one POM, not one per test kind. + +Pure HTTP checks (status code, redirect location, JSON body via the `request` fixture — no DOM involved) skip the POM entirely. + +#### Smoke vs mock — which folder + +- **`e2e/smoke/`** — drives the browser against the real running app and its real backend. Small, critical-path set: does the page load, does the core flow complete. +- **`e2e/mock/`** — drives the browser but stubs the network layer with [Playwright's route mocking](https://playwright.dev/docs/mock), via the shared fixture in `qa/tests/playwright/helpers/fixtures.ts` (built with [Playwright's test-fixtures pattern](https://playwright.dev/docs/test-fixtures)). Use it for backend states that are hard or slow to produce for real — a specific error shape, a dropped connection, data that would need seeding — not to re-test something the real backend already exercises in a smoke spec. If a scenario doesn't need a specific stubbed response, it belongs in `smoke/`, not `mock/`. + +`qa/tests/playwright/helpers/fixtures.ts` exports `mockTest` (a `@playwright/test` `test` extended with a `mockJson` fixture — fulfills a route with a JSON body/status, no real request made) and re-exports `expect`. Import `mockTest as test` from there instead of `@playwright/test` in a mock spec. For a non-JSON case (e.g. simulating a dropped connection), call `page.route(url, (route) => route.abort())` directly. + +Only browser-side requests can be mocked. Data the Next.js server fetches from Strapi during SSR never passes through the browser, so it can't be stubbed with `page.route` — cover those states in Vitest instead. + +#### POM class template + +```typescript +import type { Locator, Page } from "@playwright/test" + +export class ExamplePage { + private readonly page: Page + + readonly heading: Locator + readonly editButtons: Locator + readonly editFooButton: Locator // editButtons.nth(0) + + constructor(page: Page) { + this.page = page + this.heading = page.getByRole("heading", { name: "Example", exact: true }) + this.editButtons = page.getByRole("button", { name: "Edit", exact: true }) + this.editFooButton = this.editButtons.nth(0) + } + + async goTo() { + const response = await this.page.goto("/example", { + waitUntil: "domcontentloaded", + }) + await this.page.waitForLoadState("networkidle") + + return response + } +} +``` + +Rules: + +- `import type { Locator, Page }` as a top-level type import, not an inline `{ type Locator }` specifier. +- Multiple elements with the same accessible name → `.nth(0)`, `.nth(1)`, exposed as named locators (not raw indices in the spec). +- Dialog locators are scoped: `page.getByRole("dialog").getByRole(...)`, never a bare `page.getByRole(...)` that could match content behind the dialog. +- POM classes do NOT import `expect` and do NOT define assertion methods — they only expose locators and navigation/interaction methods. Assertions live in the spec. +- `goTo()` returns the navigation `Response` when the spec needs to assert on it (e.g. a smoke test checking `response?.ok()`); otherwise it can return `void`. + +#### Spec file template — smoke + +Specs live in `e2e/smoke/` or `e2e/mock/`, so the import path back to `helpers/` is `../../helpers/...`. + +```typescript +import { expect, test } from "@playwright/test" + +import { ExamplePage } from "../../helpers/pages/ExamplePage" + +test.describe("Example page", () => { + let examplePage: ExamplePage + + test.beforeEach(async ({ page }) => { + examplePage = new ExamplePage(page) + await examplePage.goTo() + }) + + test("shows page heading", async () => { + await expect(examplePage.heading).toBeVisible() + }) +}) +``` + +Use `test.beforeEach` to construct the POM and navigate once a file has more than one test. For a single-test file, skip `beforeEach` and construct the POM directly inside the one `test(...)` instead. + +- Navigation is handled by `goTo()` on the POM — never call `page.goto()` directly in a new spec. +- Prefer role-based locators (`getByRole`) over CSS selectors — more resilient to markup changes. +- Do not hardcode ports/hosts — `baseURL` comes from `BASE_URL` (set in `playwright.config.ts`'s `use.baseURL`). + +#### Spec file template — mock + +Same POM, imported alongside `mockTest as test` and `expect` from the fixtures file instead of `@playwright/test` directly, so `mockJson` is available on the test context: + +```typescript +import { expect, mockTest as test } from "../../helpers/fixtures" +import { ExamplePage } from "../../helpers/pages/ExamplePage" + +test.describe("Example page — mocked backend", () => { + test("shows an error when the backend rejects the request", async ({ + page, + mockJson, + }) => { + await mockJson( + "**/api/example", + { message: "Something went wrong" }, + { status: 500 } + ) + + const examplePage = new ExamplePage(page) + await examplePage.goTo() + + await expect(page.getByText("Something went wrong")).toBeVisible() + }) +}) +``` + +Set up the route mock **before** navigating/triggering the request it intercepts — a mock registered after the request has already fired does nothing. + +#### Checklist when adding a new e2e page + +1. Create `qa/tests/playwright/helpers/pages/Page.ts` with locators + a `goTo()` method. One POM, reused by both its smoke and mock specs. +2. Decide smoke, mock, or both (see "Smoke vs mock — which folder" above). +3. Create `qa/tests/playwright/e2e/smoke/.spec.ts` and/or `qa/tests/playwright/e2e/mock/.spec.ts`, importing the POM as `../../helpers/pages/Page`. +4. Keep assertions in the spec, not the POM. +5. If the flow doesn't touch the DOM at all (pure status/redirect/JSON check), skip the POM and use the `request` fixture directly instead. + +### Playwright — visual, axe, seo + +- **Page lists**: all three (and Lighthouse) read their paths from `qa/tests/playwright/helpers/urls.json`, keyed by suite and environment (`seo`/`axe`/`visual`/`perfo` → `dev`/`staging`/`prod`). The page smoke spec reads `smoke` (main pages) and `templates` (one example per page template). To cover a new page, add it there — no spec change needed. +- **Visual**: if no baseline snapshot exists yet for a page/browser combination, the spec creates one and skips comparison for that run instead of failing — inspect a newly created baseline before committing it. Update existing baselines only when the change is intentional: `pnpm tests:playwright:visual:update` (local OS) or `pnpm tests:playwright:visual:docker:update` (Linux, commit-ready — matches what CI compares against). +- **Axe**: add per-page exceptions via `GLOBAL_WARNING_RULE_IDS` / `PATH_CONFIGS` in `axe.spec.ts` rather than skipping a page outright — a known, unfixable violation becomes a warning, not a silent gap. +- **SEO**: mirror `seo.spec.ts`'s per-page `test.describe` blocks (Title, Meta description, Robots, Canonical, H1, Heading hierarchy, Structured data, Open Graph, hreflang). Production-only checks gate on `isProductionBaseUrl()` (strapi.io). +- Do not hardcode ports/hosts — always resolve via `BASE_URL` / `baseURL`. + +## Phase 3 — Naming + structure + +- One `describe` per unit-under-test (function, component, route). +- One `it` per behavior. Behavior phrased as outcome: `it("returns null when input is empty")`, not `it("test 1")`. +- AAA: arrange, act, assert. Separate with a blank line if it aids reading. +- No console.log left behind. No `.only` / `.skip` committed. +- Follow the repo code style: braces on every `if`/`else`, no one-line branches. + +## Phase 4 — Run + +Run only the relevant package, not the whole monorepo, while iterating. + +| Test | Command | +| ---------------------------------------------- | ------------------------------------------------------------------- | +| UI unit (all) | `pnpm test:ui` | +| Strapi (all) | `pnpm test:strapi` | +| Single Vitest file | `pnpm --filter @repo/ui exec vitest run ` (or `@repo/strapi`) | +| Vitest watch | `pnpm --filter @repo/ui test:watch` | +| Playwright e2e (smoke + mock) | `pnpm tests:playwright:e2e:test` | +| Playwright e2e smoke only | `pnpm tests:playwright:e2e:smoke` | +| Playwright e2e mock only | `pnpm tests:playwright:e2e:mock` | +| Playwright e2e interactive | `pnpm tests:playwright:e2e:test:interactive` | +| Playwright visual | `pnpm tests:playwright:visual` | +| Playwright visual (Docker, Linux-matched) | `pnpm tests:playwright:visual:docker` | +| Playwright axe | `pnpm tests:playwright:axe` | +| Playwright seo | `pnpm tests:playwright:seo` | +| Update visual snapshots (local OS) | `pnpm tests:playwright:visual:update` | +| Update visual snapshots (Docker, commit-ready) | `pnpm tests:playwright:visual:docker:update` | +| LHCI performance collect | `pnpm tests:lhci:perfo` | +| Lint the QA package | `pnpm -F @repo/tests-playwright lint` | + +Playwright requires a running app (`BASE_URL` in `qa/tests/playwright/.env`). If e2e fails with connection refused, ask the user to start the app — never launch dev servers in the background yourself. + +## Phase 5 — Verify before reporting done + +1. Run the new test — it must pass. +2. Run the file's full suite — no regressions. +3. If reproducing a bug: first confirm the test **fails** without the fix, then apply the fix, then confirm it **passes**. Do not skip the failing step. +4. Snapshot updates: never update without inspecting the diff visually. + +## Report + +Print: + +``` +Tests added: +Files: +Layer: vitest-ui | vitest-strapi | playwright-e2e-smoke | playwright-e2e-mock | playwright-visual | playwright-axe | playwright-seo +Result: pass +Command: +``` + +## Notes + +- Do not add new dev dependencies for testing without explicit user request. +- Do not introduce Jest, Mocha, Cypress, Cucumber, or other competing frameworks. POM is the required pattern for every new e2e page-interaction spec. +- Coverage reports are configured in `vitest.config.ts` — don't reconfigure per-test. +- Playwright `MOBILE_VIEWPORTS_TESTING_ENABLED=true` env var opts in mobile e2e projects (see `playwright.config.ts`). +- Visual snapshots are platform-sensitive — generated on macOS may diff on CI. Note in PR if you regenerated locally. diff --git a/.github/workflows/qa.yml b/.github/workflows/qa.yml index 3c830b35..901f1cb2 100644 --- a/.github/workflows/qa.yml +++ b/.github/workflows/qa.yml @@ -1,20 +1,28 @@ -name: QA (manual) +name: QA run-name: >- [${{ + github.event_name == 'schedule' && 'SCHEDULED' || inputs.run_e2e && 'E2E' || inputs.run_axe && 'AXE' || inputs.run_seo && 'SEO' || - inputs.run_lhci_perfo && 'Perfo' + inputs.run_lhci_perfo && 'Perfo' || + inputs.run_visual && 'Visual' }}${{ - inputs.run_e2e && inputs.run_axe && ', AXE' || '' + github.event_name != 'schedule' && inputs.run_e2e && inputs.run_axe && ', AXE' || '' }}${{ - (inputs.run_e2e || inputs.run_axe) && inputs.run_seo && ', SEO' || '' + github.event_name != 'schedule' && (inputs.run_e2e || inputs.run_axe) && inputs.run_seo && ', SEO' || '' }}${{ - (inputs.run_e2e || inputs.run_axe || inputs.run_seo) && inputs.run_lhci_perfo && ', Perfo' || '' - }}] QA – ${{ inputs.base_url }} + github.event_name != 'schedule' && (inputs.run_e2e || inputs.run_axe || inputs.run_seo) && inputs.run_lhci_perfo && ', Perfo' || '' + }}${{ + github.event_name != 'schedule' && (inputs.run_e2e || inputs.run_axe || inputs.run_seo || inputs.run_lhci_perfo) && inputs.run_visual && ', Visual' || '' + }}] QA – ${{ inputs.base_url || vars.BASE_URL || 'missing-base-url' }} on: + # Uncomment the schedule trigger to run tests automatically on weekdays. + # schedule: + # Runs at 00:00 UTC on weekdays (02:00 CEST). + # - cron: "0 0 * * 1-5" workflow_dispatch: inputs: run_e2e: @@ -33,10 +41,15 @@ on: description: LHCI Performance tests type: boolean default: false + run_visual: + description: Visual tests + type: boolean + default: false base_url: - description: Base URL to test against (must include protocol) + description: "Base URL to test against (must include protocol). Leave empty to use the default BASE_URL variable." type: string - required: true + required: false + default: "" concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -48,6 +61,7 @@ permissions: env: CI: true NPM_CONFIG_IGNORE_SCRIPTS: "true" + BASE_URL: ${{ inputs.base_url || vars.BASE_URL }} jobs: e2e: @@ -58,13 +72,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7.0.1 - name: Setup pnpm with cache uses: ./.github/actions/setup-pnpm - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v6.1.0 with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('pnpm-lock.yaml') }} @@ -76,10 +90,11 @@ jobs: - name: Run Playwright tests env: - BASE_URL: ${{ inputs.base_url }} + BASE_URL: ${{ env.BASE_URL }} run: pnpm tests:playwright:e2e:test - - uses: actions/upload-artifact@v4.6.2 + - name: Publish Playwright report + uses: actions/upload-artifact@v7.0.1 if: failure() with: name: playwright-report @@ -87,7 +102,7 @@ jobs: retention-days: 7 if-no-files-found: warn - - uses: actions/upload-artifact@v4.6.2 + - uses: actions/upload-artifact@v7.0.1 if: failure() with: name: playwright-traces @@ -103,13 +118,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7.0.1 - name: Setup pnpm with cache uses: ./.github/actions/setup-pnpm - name: Cache Playwright browsers - uses: actions/cache@v4 + uses: actions/cache@v6.1.0 with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('pnpm-lock.yaml') }} @@ -117,19 +132,22 @@ jobs: ${{ runner.os }}-playwright- - name: Install Playwright Browsers - run: pnpm -F @repo/tests-playwright exec playwright install --with-deps + run: pnpm -F @repo/tests-playwright exec playwright install --with-deps chromium - name: Run Axe tests env: - BASE_URL: ${{ inputs.base_url }} + BASE_URL: ${{ env.BASE_URL }} run: pnpm tests:playwright:axe - name: Pack Axe reports if: failure() run: | - tar -czf axe-reports-${{ github.run_id }}.tgz -C qa/tests/playwright/axe/reports . + tar -czf axe-reports-${{ github.run_id }}.tgz \ + -C qa/tests/playwright \ + playwright-report - - uses: actions/upload-artifact@v4.6.2 + - name: Publish Axe report + uses: actions/upload-artifact@v7.0.1 if: failure() with: name: axe-report @@ -145,13 +163,13 @@ jobs: steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7.0.1 - name: Setup pnpm with cache uses: ./.github/actions/setup-pnpm - name: Cache Playwright browsers - uses: actions/cache@v5.0.2 + uses: actions/cache@v6.1.0 with: path: ~/.cache/ms-playwright key: ${{ runner.os }}-playwright-${{ hashFiles('pnpm-lock.yaml') }} @@ -159,21 +177,22 @@ jobs: ${{ runner.os }}-playwright- - name: Install Playwright Browsers - run: pnpm -F @repo/tests-playwright exec playwright install --with-deps + run: pnpm -F @repo/tests-playwright exec playwright install --with-deps chromium - name: Run Playwright SEO tests env: - BASE_URL: ${{ inputs.base_url }} - run: pnpm tests:playwright:seo --project=seo + BASE_URL: ${{ env.BASE_URL }} + run: pnpm tests:playwright:seo - name: Pack SEO reports - if: always() + if: failure() run: | tar -czf seo-reports-${{ github.run_id }}.tgz \ -C qa/tests/playwright \ - playwright-report test-results + playwright-report - - uses: actions/upload-artifact@v4.6.2 + - name: Publish SEO report + uses: actions/upload-artifact@v7.0.1 if: failure() with: name: seo-report @@ -181,30 +200,124 @@ jobs: retention-days: 7 if-no-files-found: warn + visual: + if: ${{ inputs.run_visual }} + name: Playwright Visual tests + runs-on: ubuntu-latest + timeout-minutes: 15 + + steps: + - name: Checkout code + uses: actions/checkout@v7.0.1 + + - name: Setup pnpm with cache + uses: ./.github/actions/setup-pnpm + + - name: Resolve Playwright version + id: playwright_version + run: | + VERSION="$(node -p "require('./qa/tests/playwright/node_modules/@playwright/test/package.json').version")" + echo "version=$VERSION" >> "$GITHUB_OUTPUT" + + - name: Check for baseline snapshots + id: baseline_check + run: | + if find qa/tests/playwright/visual/visual.spec.ts-snapshots -type f -name '*.png' 2>/dev/null | grep -q .; then + echo "Baseline snapshots found." + echo "has_baselines=true" >> "$GITHUB_OUTPUT" + else + echo "No baseline snapshots found under qa/tests/playwright/visual/visual.spec.ts-snapshots - skipping visual regression run." + echo "has_baselines=false" >> "$GITHUB_OUTPUT" + fi + + - name: Cache Playwright Docker image + id: docker-cache + if: steps.baseline_check.outputs.has_baselines == 'true' + uses: actions/cache@v6.1.0 + with: + path: /tmp/docker-playwright-cache + key: docker-playwright-v${{ steps.playwright_version.outputs.version }}-noble + + - name: Load or pull Playwright Docker image + if: steps.baseline_check.outputs.has_baselines == 'true' + run: | + IMAGE="mcr.microsoft.com/playwright:v${{ steps.playwright_version.outputs.version }}-noble" + CACHE_FILE="/tmp/docker-playwright-cache/playwright.tar" + if [[ -f "$CACHE_FILE" ]]; then + echo "Loading Docker image from cache..." + docker load -i "$CACHE_FILE" + else + echo "Pulling Docker image..." + docker pull "$IMAGE" + mkdir -p /tmp/docker-playwright-cache + docker save "$IMAGE" -o "$CACHE_FILE" + fi + + - name: Run Visual tests + if: steps.baseline_check.outputs.has_baselines == 'true' + env: + BASE_URL: ${{ env.BASE_URL }} + run: pnpm tests:playwright:visual:docker + + - name: Pack Visual reports + if: failure() + run: | + tar -czf visual-reports-${{ github.run_id }}.tgz \ + -C qa/tests/playwright \ + playwright-report + + - name: Publish Visual report + uses: actions/upload-artifact@v7.0.1 + if: failure() + with: + name: visual-report + path: visual-reports-${{ github.run_id }}.tgz + retention-days: 7 + if-no-files-found: warn + lhci_perfo: if: ${{ inputs.run_lhci_perfo }} name: LHCI Performance tests runs-on: ubuntu-latest timeout-minutes: 15 + permissions: + contents: write + pull-requests: write steps: - name: Checkout code - uses: actions/checkout@v4 + uses: actions/checkout@v7.0.1 - name: Setup pnpm with cache uses: ./.github/actions/setup-pnpm - name: Run LHCI Performance tests env: - BASE_URL: ${{ inputs.base_url }} + BASE_URL: ${{ env.BASE_URL }} run: pnpm tests:lhci:perfo + - name: Open PR with Lighthouse trend history + if: success() + uses: peter-evans/create-pull-request@v8.1.1 + with: + add-paths: qa/tests/playwright/perfo/lighthouse-history.md + commit-message: "chore(tests-playwright): record Lighthouse trend results" + branch: qa/lighthouse-trend-update + delete-branch: true + title: "chore(qa): update Lighthouse trend history" + labels: QA, auto-generated + body: | + Automated update of the Lighthouse performance trend log (`qa/tests/playwright/perfo/lighthouse-history.md`) from a manual QA workflow run. + + Run: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + - name: Pack LHCI reports if: always() run: | tar -czf lhci-perfo-reports-${{ github.run_id }}.tgz \ -C qa/tests/playwright/perfo/.lighthouseci . - - uses: actions/upload-artifact@v4.6.2 + - name: Publish LHCI report + uses: actions/upload-artifact@v7.0.1 if: always() with: name: perfo-report diff --git a/apps/ui/src/lib/strapi-api/content/__tests__/fetchPage-5xx.test.ts b/apps/ui/src/lib/strapi-api/content/__tests__/fetchPage-5xx.test.ts new file mode 100644 index 00000000..6822f1f2 --- /dev/null +++ b/apps/ui/src/lib/strapi-api/content/__tests__/fetchPage-5xx.test.ts @@ -0,0 +1,91 @@ +import { beforeEach, describe, expect, it, vi } from "vitest" + +// `content/server.ts` is server-only; stub the guard so it can load under vitest. +vi.mock("server-only", () => ({})) + +vi.mock("next/headers", () => ({ + draftMode: async () => ({ isEnabled: false }), +})) + +vi.mock("@/lib/strapi-api/request-auth", () => ({ + createStrapiAuthHeader: async () => ({}), +})) + +vi.mock("@/lib/env-vars", () => ({ + getEnvVar: (name: string) => + name === "STRAPI_URL" ? "https://strapi.example.test" : undefined, +})) + +// Keep the non-blocking logger quiet but observable. +const logNonBlockingError = vi.fn() +vi.mock("@/lib/logging", () => ({ + logNonBlockingError: (...args: unknown[]) => logNonBlockingError(...args), +})) + +const { fetchPage } = await import("@/lib/strapi-api/content/server") + +/** Strapi Cloud during the incident: HTML error page, not a JSON API response. */ +const gatewayTimeoutHtml = () => + new Response( + "502 Bad Gateway502", + { status: 504, headers: { "content-type": "text/html; charset=utf-8" } } + ) + +/** A plain JSON 500 from Strapi itself. */ +const internalServerErrorJson = () => + Response.json( + { error: { name: "InternalServerError", status: 500 } }, + { + status: 500, + headers: { "content-type": "application/json" }, + } + ) + +/** A healthy Strapi that simply has no page at that path. */ +const emptyCollectionOk = () => + Response.json( + { data: [], meta: { pagination: { total: 0 } } }, + { + status: 200, + headers: { "content-type": "application/json" }, + } + ) + +describe("fetchPage when Strapi is unavailable", () => { + beforeEach(() => { + logNonBlockingError.mockClear() + vi.unstubAllGlobals() + }) + + it("propagates a 504 HTML gateway error instead of reporting 'no page'", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => gatewayTimeoutHtml()) + ) + + // The render must fail so ISR serves the last good page. + // Today this resolves to `undefined`, the caller calls notFound(), + // and Next caches a 404 for the full revalidate window. + await expect(fetchPage("/enterprise-terms", "en")).rejects.toThrow() + }) + + it("propagates a 500 JSON error instead of reporting 'no page'", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => internalServerErrorJson()) + ) + + await expect(fetchPage("/enterprise-terms", "en")).rejects.toThrow() + }) + + it("still returns empty data when Strapi is healthy and the page truly does not exist", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => emptyCollectionOk()) + ) + + // This path MUST keep working: a real 404 has to stay a 404. + const response = await fetchPage("/no-such-page", "en") + expect(response?.data ?? null).toBeNull() + }) +}) diff --git a/docs/commands.md b/docs/commands.md index 66437a57..ff24283c 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -46,7 +46,17 @@ This updates `@repo/strapi-types`. Forgetting causes silent type mismatches. ## Testing ```bash -pnpm tests:playwright:e2e:test # Playwright E2E tests +pnpm tests:playwright:e2e:test # Playwright E2E tests (smoke + mock) +pnpm tests:playwright:e2e:smoke # Playwright E2E, smoke only +pnpm tests:playwright:e2e:mock # Playwright E2E, mocked backend only pnpm tests:playwright:e2e:test:interactive # Playwright UI mode pnpm tests:playwright:axe # Accessibility tests (axe-core) +pnpm tests:playwright:seo # SEO tests +pnpm tests:playwright:visual # Visual regression tests +pnpm tests:playwright:visual:update # Update visual snapshots (local OS) +pnpm tests:playwright:visual:docker # Visual regression via Docker (CI-compatible) +pnpm tests:playwright:visual:docker:update # Update Linux snapshots via Docker +pnpm tests:lhci:perfo # Lighthouse CI performance + trend history ``` + +See [qa/tests/README.md](../qa/tests/README.md) for the QA suite setup. diff --git a/package.json b/package.json index c5863548..5a7ccbfb 100644 --- a/package.json +++ b/package.json @@ -26,11 +26,16 @@ "test:strapi": "turbo run test --filter=@repo/strapi", "test:ui": "turbo run test --filter=@repo/ui", "tests:lhci:perfo": "turbo run lhci-perfo-check --filter=@repo/tests-playwright", - "tests:playwright:axe": "turbo run axe-test --filter=@repo/tests-playwright", + "tests:playwright:axe": "turbo run playwright-axe-test --filter=@repo/tests-playwright", + "tests:playwright:e2e:mock": "turbo run playwright-e2e-mock-tests --filter=@repo/tests-playwright", + "tests:playwright:e2e:smoke": "turbo run playwright-e2e-smoke-tests --filter=@repo/tests-playwright", "tests:playwright:e2e:test": "turbo run playwright-e2e-tests --filter=@repo/tests-playwright", "tests:playwright:e2e:test:interactive": "turbo run playwright-interactive-mode --filter=@repo/tests-playwright", - "tests:playwright:seo": "pnpm -F @repo/tests-playwright exec playwright test --project=seo", + "tests:playwright:seo": "turbo run playwright-seo-tests --filter=@repo/tests-playwright", "tests:playwright:visual": "turbo run playwright-visual-tests --filter=@repo/tests-playwright", + "tests:playwright:visual:docker": "turbo run playwright-visual-tests-docker --filter=@repo/tests-playwright", + "tests:playwright:visual:docker:update": "turbo run playwright-visual-update-snapshots-docker --filter=@repo/tests-playwright", + "tests:playwright:visual:update": "turbo run playwright-visual-update-tests --filter=@repo/tests-playwright", "turbo": "turbo build lint:fix typecheck; pnpm run format", "typecheck": "turbo run typecheck" }, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f768dc9c..f359df6e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -529,6 +529,9 @@ importers: '@axe-core/playwright': specifier: ^4.12.1 version: 4.12.1(playwright-core@1.61.0) + axe-core: + specifier: ~4.12.1 + version: 4.12.1 get-sitemap-links: specifier: ^1.1.0 version: 1.1.0 diff --git a/qa/tests/README.md b/qa/tests/README.md index 2c74a21c..8d18cec7 100644 --- a/qa/tests/README.md +++ b/qa/tests/README.md @@ -2,15 +2,16 @@ ## 🐞 Automated Testing Suite -This directory contains an automated testing suite designed to ensure quality across functionality, accessibility, performance, and SEO. +This directory contains an automated testing suite designed to ensure quality across functionality, accessibility, performance, and SEO. It validates the deployed or local UI through a browser. -It can include different testing frameworks and tools depending on project needs, for example: - -- **End-to-End (E2E) Tests** with tools like [Playwright](https://playwright.dev/) or [Cypress](https://www.cypress.io/) -- **Accessibility Tests** using [AXE-Core](https://classic.yarnpkg.com/en/package/@axe-core/playwright) -- **SEO Tests** for automated validation of search engine optimizations -- **Visual Tests** for automated visual validation -- **Performance Tests** using [LighthouseCI](https://github.com/GoogleChrome/lighthouse-ci/tree/main) +| Area | Tooling | Location | +| ------------- | ------------------------------------------------------------------------ | ------------------------------- | +| E2E — smoke | [Playwright](https://playwright.dev/) | `qa/tests/playwright/e2e/smoke` | +| E2E — mock | Playwright | `qa/tests/playwright/e2e/mock` | +| Accessibility | Playwright + [axe-core](https://github.com/dequelabs/axe-core) | `qa/tests/playwright/axe` | +| SEO | Playwright | `qa/tests/playwright/seo` | +| Visual | Playwright screenshots | `qa/tests/playwright/visual` | +| Performance | [Lighthouse CI](https://github.com/GoogleChrome/lighthouse-ci/tree/main) | `qa/tests/playwright/perfo` | --- @@ -20,21 +21,58 @@ It can include different testing frameworks and tools depending on project needs qa/ ├── tests/ │ └── playwright/ ← Playwright-based tests (workspace package) -│ ├── e2e/ ← End-to-End tests +│ ├── e2e/ +│ │ ├── smoke/ ← Critical-path flows against the real app/backend +│ │ └── mock/ ← Same flows, with the network layer stubbed │ ├── axe/ ← Accessibility tests │ ├── seo/ ← SEO tests -│ ├── visual/ ← Visual tests -│ ├── perfo/ ← Performance tests using LighthouseCI -│ ├── helpers/ ← Shared test utilities/helpers +│ ├── visual/ ← Visual regression tests + Docker scripts +│ ├── perfo/ ← Lighthouse CI performance tests + trend history +│ ├── helpers/ ← Shared utilities (page objects, fixtures, page lists) │ ├── .env.example ← Example environment variables file │ ├── package.json ← Test dependencies & scripts for this package │ ├── playwright.config.ts ← Playwright configuration │ ├── tsconfig.json ← TypeScript configuration for this package ``` -### Playwright +### Page lists per environment + +`helpers/urls.json` holds the pages each suite checks, split by environment: + +```json +{ + "smoke": { "dev": ["/", "/pricing"], "staging": [...], "prod": [...] }, + "templates": { + "blog-post": { "dev": ["/blog/..."], "staging": [...], "prod": [...] }, + "user-story": { "dev": ["/user-stories/airbus"], "staging": [...], "prod": [...] } + }, + "seo": { "dev": ["/"], "staging": ["/"], "prod": ["/"] }, + "axe": { "dev": ["/"], "staging": ["/"], "prod": ["/"] }, + "visual": { "dev": ["/"], "staging": ["/"], "prod": ["/"] }, + "perfo": { "dev": ["/"], "staging": ["/"], "prod": ["/"] } +} +``` + +Each suite gets its full page list (every environment, deduped) via `helpers/flatten-urls.ts`. To cover a new page, add it to the right suite's list. It's picked up automatically, no spec changes needed. + +`smoke` lists the main pages and `templates` holds one example page per page template (blog post, blog category, user story, feature, solution). Both are used by `e2e/smoke/pages.spec.ts`. When a template example is removed from the CMS, replace it with another page of the same template. + +Leave out pages that redirect to `community.strapi.io` (e.g. `/integrations`). They aren't served by this repo. + +### E2E: smoke vs mock + +Both kinds share the same page objects from `helpers/pages/` (one Page Object Model class per page). + +- **`e2e/smoke/`**: drives a real browser against the real running app and its real backend. Kept small and critical-path: does the page load, does the core flow complete. `pages.spec.ts` checks that every main and template page returns a non-error status, renders its main content, and throws no uncaught browser errors (e.g. a crashing client component). Errors that don't break the page, like React hydration mismatches (#418), are listed in `IGNORED_PAGE_ERRORS` in the spec. They don't fail the test, but show up as a warning on it in the HTML report. The navbar and footer aren't checked there, because some pages intentionally have a different one or none. `homepage.spec.ts` checks the main navbar (visible, with links) and the main footer on the homepage instead. +- **`e2e/mock/`**: drives the browser the same way, but stubs network responses with [Playwright's route mocking](https://playwright.dev/docs/mock) via the `mockTest` fixture in `helpers/fixtures.ts`. Use it for backend states that are hard to reproduce for real (a specific error response, a dropped connection). + - `media-fallback.spec.ts`: Strapi images respond with 404, and the page must show the fallback image (`ImageWithFallback`) instead of broken images. + - `preview-refresh.spec.ts`: a stubbed Strapi admin window embeds the page in an iframe and posts a `strapiUpdate` message. The page must refresh when the message comes from the Strapi origin, and ignore it from any other origin. Needs `STRAPI_URL` (see below), otherwise it's skipped. -#### 🔧 Environment Variables +--- + +## Playwright + +### 🔧 Environment Variables Playwright tests require a `.env` file. @@ -44,11 +82,11 @@ Playwright tests require a `.env` file. ``` 2. Update the `BASE_URL` value to point to the environment you want to test. ```bash - BASE_URL=http://localhost:3000 + BASE_URL=http://localhost:3000 ``` - `BASE_URL` defines the base URL of the application under test and is used as the starting point for all Playwright test suites. + `BASE_URL` defines the base URL of the application under test and is used as the starting point for all suites. -#### 🌎 Install Playwright Browsers +### 🌎 Install Playwright Browsers Before running tests for the first time, install the required browsers: @@ -60,64 +98,99 @@ pnpm -F @repo/tests-playwright exec playwright install --with-deps cd qa/tests/playwright && pnpm exec playwright install --with-deps ``` -#### �📱 Mobile Viewport Testing +### 📱 Mobile Viewport Testing -Mobile browser projects can be enabled via an environment variable. +Mobile browser projects (Pixel 7, iPhone 15) are disabled by default. Enable them for E2E runs via: ```env MOBILE_VIEWPORTS_TESTING_ENABLED=true ``` -#### ▶️ Running Tests +### 🧩 Strapi URL for the preview mock test + +Set `STRAPI_URL` to the Strapi URL the app under test is configured with. Only `e2e/mock/preview-refresh.spec.ts` uses it, and it's skipped when the variable is empty (also in CI). + +```env +STRAPI_URL=https://cms.example.com +``` + +### ▶️ Running Tests -All test commands are run from the **project root**, not from inside the `qa/` folder. +All commands are run from the **project root**: -- **Playwright - Run E2E tests (headless)** +```bash +pnpm tests:playwright:e2e:test # E2E, headless (smoke + mock) +pnpm tests:playwright:e2e:smoke # E2E, smoke only +pnpm tests:playwright:e2e:mock # E2E, mock only +pnpm tests:playwright:e2e:test:interactive # E2E, UI mode +pnpm tests:playwright:axe # Accessibility checks +pnpm tests:playwright:seo # SEO checks +pnpm tests:playwright:visual # Visual regression checks +pnpm tests:playwright:visual:update # Update visual snapshots (local OS) +pnpm tests:playwright:visual:docker # Visual regression checks via Docker (CI-compatible) +pnpm tests:playwright:visual:docker:update # Update Linux snapshots via Docker +pnpm tests:lhci:perfo # Lighthouse CI performance checks +``` + +Each suite runs as its own Playwright project: E2E specs on `chromium`, `firefox` and `webkit`; `axe` and `seo` on Chromium only; visual on `visual-chromium`, `visual-firefox` and `visual-webkit`. + +--- + +## ♿ Accessibility Tests + +`axe/axe.spec.ts` runs axe-core via `@axe-core/playwright`, one test per page from `helpers/urls.json`. Any violation fails the test, unless it's configured as a warning: + +- A known, not-fixed-today violation on a specific page → add its rule ID to that page's `warningRuleIds` in `PATH_CONFIGS`. +- An element axe should ignore entirely on a page → that page's `excludeSelectors`. +- A violation that applies everywhere → `GLOBAL_WARNING_RULE_IDS` / `GLOBAL_EXCLUDE_SELECTORS`. - ```bash - pnpm run tests:playwright:e2e:test - ``` +## 🔍 SEO Tests -- **Playwright - Run E2E tests in interactive mode (headed)** +`seo/seo.spec.ts` checks title, meta description, robots, canonical URL, H1 and heading hierarchy, structured data (JSON-LD), Open Graph tags and hreflang, one `test.describe` block per page from `helpers/urls.json`. - ```bash - pnpm run tests:playwright:e2e:test:interactive - ``` +The robots `noindex` check runs only when `BASE_URL` points to production (`strapi.io`). Other environments render `noindex` on purpose, so it skips itself there. -- **Playwright - Run AXE tests** +## 📸 Visual Regression Tests - ```bash - pnpm run tests:playwright:axe - ``` +Visual tests compare full-page screenshots against committed baseline images. -- **Playwright - Run SEO tests** +| Browser | Local | Docker / CI | +| --------------- | ----- | ----------- | +| Chromium | ✅ | ✅ | +| Firefox | ✅ | ✅ | +| WebKit (Safari) | ✅ | ❌ | - ```bash - pnpm run tests:playwright:seo - ``` +WebKit on Linux produces blank or incorrectly rendered screenshots, so it runs locally on macOS only. -- **Playwright - Run Visual tests** +**Baselines must be generated on Linux.** macOS and Linux render fonts differently, so snapshots made locally would fail on CI. Only `linux` snapshots are committed; `darwin` ones are gitignored. - ```bash - pnpm tests:playwright:visual - ``` +Snapshot names encode the environment and page: `{env-slug}-{page}-{browser}.png`, where `env-slug` comes from the `BASE_URL` hostname. Each environment keeps its own baselines. The first run on an environment creates the baselines and passes; failures only happen on later runs when something changed visually. -- **LighthouseCI - Run Perfo tests** - ```bash - pnpm tests:lhci:perfo - ``` +**Workflow:** -#### Additional info for visual tests +1. Generate Linux baselines locally (requires Docker Desktop running): + ```bash + pnpm tests:playwright:visual:docker:update + ``` +2. Review the new files in `qa/tests/playwright/visual/visual.spec.ts-snapshots/linux/`, then commit them with the related UI change. +3. Optionally verify the comparison locally before pushing: + ```bash + pnpm tests:playwright:visual:docker + ``` -**Playwright – Running visual regression tests** +## ⚡ Lighthouse Performance Tests + +`pnpm tests:lhci:perfo` runs `lhci collect` against every URL in the `perfo` list of `helpers/urls.json`. Raw reports go to `qa/tests/playwright/perfo/.lighthouseci/` (gitignored). + +Each run also appends its category scores (performance, accessibility, best practices, SEO) to `qa/tests/playwright/perfo/lighthouse-history.md`, which is committed. It has one `## ` section per page, so a page's trend stays together. + +--- -1. **Initial run (baseline creation)** - Run the visual tests for the first time. - This initial run will generate the **baseline (golden) screenshots**, which serve as the reference for future comparisons. +## 🤖 CI (GitHub Actions) -2. **Deploy the application** - Deploy the application version you want to validate visually. +`.github/workflows/qa.yml` is triggered manually: **Actions → QA → Run workflow**. Pick the branch, check the suites to run (E2E, AXE, SEO, LHCI Performance, Visual) and optionally set `base_url`. An empty `base_url` falls back to the `BASE_URL` repository variable. -3. **Comparison run** - Run the visual tests again. - Playwright will generate new screenshots and **compare them against the previously created baseline** to detect visual changes or regressions. +- Jobs run in parallel. Reports and traces are uploaded as artifacts on failure (Lighthouse reports always). +- The visual job runs in the Playwright Docker image and skips itself when no baseline snapshots are committed. +- On success, the Lighthouse job opens (or updates) a PR with the updated `lighthouse-history.md`. This needs **Settings → Actions → General → Allow GitHub Actions to create and approve pull requests** enabled. +- A weekday cron schedule is included but commented out. To enable it, each job's `if:` also needs `|| github.event_name == 'schedule'`. diff --git a/qa/tests/playwright/.env.example b/qa/tests/playwright/.env.example index d993be73..8bdbcfe5 100644 --- a/qa/tests/playwright/.env.example +++ b/qa/tests/playwright/.env.example @@ -1,9 +1,12 @@ # Base URL of the application # Example: # - Local development: http://localhost:3000 -# - Development: https://dev.example.com # - Staging: https://staging.example.com # - Production: https://www.example.com BASE_URL=http://localhost:3000 # Enable testing against mobile viewports (true / false) -MOBILE_VIEWPORTS_TESTING_ENABLED=false \ No newline at end of file +MOBILE_VIEWPORTS_TESTING_ENABLED=false + +# Strapi URL the app under test is configured with (its STRAPI_URL). +# Used by the preview-refresh mock test, which is skipped when unset. +STRAPI_URL= diff --git a/qa/tests/playwright/.gitignore b/qa/tests/playwright/.gitignore index 99b309a4..f420b530 100644 --- a/qa/tests/playwright/.gitignore +++ b/qa/tests/playwright/.gitignore @@ -8,4 +8,4 @@ node_modules/ .env /axe/reports/ /perfo/.lighthouseci/ -/visual/visual.spec.ts-snapshots \ No newline at end of file +/visual/visual.spec.ts-snapshots/darwin/ \ No newline at end of file diff --git a/qa/tests/playwright/axe/axe-test.ts b/qa/tests/playwright/axe/axe-test.ts deleted file mode 100644 index c39a35a2..00000000 --- a/qa/tests/playwright/axe/axe-test.ts +++ /dev/null @@ -1,219 +0,0 @@ -import fs from "fs" -import path from "path" - -import AxeBuilder from "@axe-core/playwright" -import { chromium } from "@playwright/test" -import dotenv from "dotenv" -import urls from "helpers/urls.json" - -import type { AxeResults, NodeResult, Result } from "axe-core" - -dotenv.config({ path: path.resolve(__dirname, "../.env"), override: true }) - -// import { fetchSitemap } from "../helpers/get-sitemap-links" - -// Rule IDs that should be treated as warnings instead of errors -const WARNING_RULE_IDS = new Set() - -type SiteOutcome = { - url: string - hasErrors: boolean - hasWarnings: boolean - analysisFailed: boolean -} - -function formatViolationsSection( - title: string, - violations: Result[], - icon: string -): string { - if (violations.length === 0) return `${title}\n———\nNone\n` - - const blocks = violations.map((v: Result) => { - const nodes = v.nodes?.length - ? v.nodes - .map((n: NodeResult, idx: number) => { - const target = Array.isArray(n.target) ? n.target.join(", ") : "" - return ` ${idx + 1}. ${n.html}\n target: ${target}` - }) - .join("\n") - : " (no affected nodes listed by axe)" - - return [ - `${icon} ${v.id} - ${v.help}`, - `Impact: ${v.impact ?? "unknown"}`, - `Description: ${v.description}`, - `Help: ${v.helpUrl}`, - `Affected Nodes:\n${nodes}`, - ].join("\n") - }) - - return `${title}\n———\n${blocks.join("\n\n")}\n` -} - -;(async () => { - const baseUrl = process.env.BASE_URL - if (!baseUrl) { - console.error("Missing BASE_URL environment variable") - process.exit(1) - } - - // If the sitemap is available, we can fetch pages directly from there - // await fetchSitemap(baseUrl + "/sitemap.xml", "--json") - // const sitesPath = path.join(__dirname, "../helpers/sites.json") - // const sites: string[] = JSON.parse(fs.readFileSync(sitesPath, "utf-8")) - - const PATHS = [...urls] - - if (!Array.isArray(urls) || urls.length === 0) { - console.error("No sites found in sites.json") - process.exit(1) - } - - // For now, we use the URLs defined in helpers/urls.json - const sites: string[] = PATHS.map((p) => new URL(p, baseUrl).toString()) - - const runTimestamp = new Date() - .toISOString() - .replace(/[-:]/g, "") - .replace(/\..+/, "") - - const reportDir = path.resolve("axe/reports", `axe_test_run_${runTimestamp}`) - fs.mkdirSync(reportDir, { recursive: true }) - - const reportFilePath = path.join(reportDir, `report_axe_${runTimestamp}.txt`) - fs.writeFileSync( - reportFilePath, - `AXE accessibility REPORT (run summary)\nTimestamp: ${runTimestamp}\nTotal pages: ${sites.length}\n\n` - ) - - const siteOutcomes: SiteOutcome[] = [] - const browser = await chromium.launch() - - try { - for (const site of sites) { - const context = await browser.newContext() - const page = await context.newPage() - - try { - await page.goto(site, { waitUntil: "load", timeout: 20000 }) - - const results: AxeResults = await new AxeBuilder({ page }).analyze() - const allViolations: Result[] = results.violations ?? [] - - const warningViolations = allViolations.filter((v) => - WARNING_RULE_IDS.has(v.id) - ) - const errorViolations = allViolations.filter( - (v) => !WARNING_RULE_IDS.has(v.id) - ) - - if (errorViolations.length > 0) { - console.error( - `❌ Errors found for ${site} (${errorViolations.length} error(s), ${warningViolations.length} warning(s))` - ) - } else if (warningViolations.length > 0) { - console.warn( - `🟠 Warnings found for ${site} (${warningViolations.length})` - ) - } else { - console.log(`✅ No violations for ${site}`) - } - - const headerPerPage = [ - `Page: ${site}`, - `Total: ${allViolations.length} | Errors: ${errorViolations.length} | Warnings: ${warningViolations.length}`, - ``, - ].join("\n") - - const errorsSection = formatViolationsSection( - "ERRORS (must fix)", - errorViolations, - "🔴" - ) - const warningsSection = formatViolationsSection( - "WARNINGS (FYI, out of our control)", - warningViolations, - "🟠" - ) - - fs.appendFileSync( - reportFilePath, - [headerPerPage, errorsSection, warningsSection, ""].join("\n") - ) - - siteOutcomes.push({ - url: site, - hasErrors: errorViolations.length > 0, - hasWarnings: warningViolations.length > 0, - analysisFailed: false, - }) - } catch (error: unknown) { - const message = error instanceof Error ? error.message : String(error) - console.error(`❌ Error checking ${site}:`, message) - fs.appendFileSync( - reportFilePath, - [ - `AXE accessibility REPORT`, - `Timestamp: ${runTimestamp}`, - `Page: ${site}`, - `ERROR running analysis: ${message}`, - ``, - ].join("\n") - ) - siteOutcomes.push({ - url: site, - hasErrors: false, - hasWarnings: false, - analysisFailed: true, - }) - } finally { - await context.close() - } - } - } finally { - await browser.close() - } - - const analysisFailedSites = siteOutcomes.filter((s) => s.analysisFailed) - const sitesWithErrors = siteOutcomes.filter( - (s) => s.hasErrors && !s.analysisFailed - ) - const sitesWithOnlyWarnings = siteOutcomes.filter( - (s) => !s.hasErrors && s.hasWarnings && !s.analysisFailed - ) - const cleanSites = siteOutcomes.filter( - (s) => !s.hasErrors && !s.hasWarnings && !s.analysisFailed - ) - - if (sitesWithErrors.length > 0) { - console.error( - `\n❌ Accessibility errors found on ${sitesWithErrors.length} site(s):` - ) - sitesWithErrors.forEach(({ url }) => console.error(`- ${url}`)) - } - - if (sitesWithOnlyWarnings.length > 0) { - console.warn( - `\n🟠 Accessibility warnings on ${sitesWithOnlyWarnings.length} site(s):` - ) - sitesWithOnlyWarnings.forEach(({ url }) => console.warn(`- ${url}`)) - } - - if (cleanSites.length > 0) { - console.log( - `\n✅ ${cleanSites.length} site(s) passed with no accessibility violations.` - ) - } - - if (analysisFailedSites.length > 0) { - console.error( - `\n⚠️ Analysis failed on ${analysisFailedSites.length} site(s):` - ) - analysisFailedSites.forEach(({ url }) => console.error(`- ${url}`)) - } - - process.exit( - sitesWithErrors.length > 0 || analysisFailedSites.length > 0 ? 1 : 0 - ) -})() diff --git a/qa/tests/playwright/axe/axe.spec.ts b/qa/tests/playwright/axe/axe.spec.ts new file mode 100644 index 00000000..5c817a9b --- /dev/null +++ b/qa/tests/playwright/axe/axe.spec.ts @@ -0,0 +1,120 @@ +import AxeBuilder from "@axe-core/playwright" +import { expect, test } from "@playwright/test" +import type { AxeResults, Result } from "axe-core" + +import { flattenUrls } from "../helpers/flatten-urls" + +import urls from "../helpers/urls.json" + +// Rule IDs that should be treated as warnings instead of errors on all pages. +const GLOBAL_WARNING_RULE_IDS = new Set() + +// Selectors to exclude from axe analysis on all pages. +const GLOBAL_EXCLUDE_SELECTORS: string[] = [] + +// Per-path configuration for excluding selectors or treating rule IDs as warnings. +const PATH_CONFIGS: Record< + string, + { excludeSelectors?: string[]; warningRuleIds?: string[] } +> = {} + +const PATHS = flattenUrls(urls.axe) + +test.describe("AXE accessibility", () => { + test.beforeAll(() => { + if (PATHS.length === 0) { + throw new Error("No sites found in urls.json") + } + }) + + for (const pathname of PATHS) { + test(`Check ${pathname}`, async ({ page, baseURL }) => { + const resolvedBaseUrl = baseURL ?? process.env.BASE_URL + + expect( + resolvedBaseUrl, + "Missing BASE_URL environment variable" + ).toBeTruthy() + + const siteUrl = new URL(pathname, resolvedBaseUrl) + const site = siteUrl.href + const config = PATH_CONFIGS[pathname] + const warningRuleIds = new Set([ + ...GLOBAL_WARNING_RULE_IDS, + ...(config?.warningRuleIds ?? []), + ]) + const excludeSelectors = [ + ...GLOBAL_EXCLUDE_SELECTORS, + ...(config?.excludeSelectors ?? []), + ] + + await page.goto(site, { + waitUntil: "domcontentloaded", + timeout: 20000, + }) + await page.waitForLoadState("networkidle") + + const results: AxeResults = await excludeSelectors + .reduce( + (builder, selector) => builder.exclude(selector), + new AxeBuilder({ page }) + ) + .analyze() + + const allViolations = results.violations ?? [] + + const warningViolations: Result[] = allViolations.filter((v) => + warningRuleIds.has(v.id) + ) + const errorViolations: Result[] = allViolations.filter( + (v) => !warningRuleIds.has(v.id) + ) + + if (errorViolations.length > 0) { + console.error( + `❌ Errors found for ${site} (${errorViolations.length} error(s), ${warningViolations.length} warning(s))` + ) + } else if (warningViolations.length > 0) { + console.warn( + `🟠 Warnings found for ${site} (${warningViolations.length})` + ) + } else { + // eslint-disable-next-line no-console + console.log(`✅ No violations for ${site}`) + } + + expect( + errorViolations.length, + [ + `Accessibility errors on ${site}`, + `Total: ${allViolations.length} | Errors: ${errorViolations.length} | Warnings: ${warningViolations.length}`, + "", + ...errorViolations.flatMap((v) => [ + `🔴 ${v.id} — ${v.help} (impact: ${v.impact ?? "unknown"})`, + ` ${v.helpUrl}`, + ...v.nodes.flatMap((n, i) => [ + ` ${i + 1}. ${n.html}`, + ` target: ${Array.isArray(n.target) ? n.target.join(", ") : String(n.target)}`, + ]), + "", + ]), + ...(warningViolations.length > 0 + ? [ + "WARNINGS (FYI, out of our control)", + "---", + ...warningViolations.flatMap((v) => [ + `🟠 ${v.id} — ${v.help} (impact: ${v.impact ?? "unknown"})`, + ` ${v.helpUrl}`, + ...v.nodes.flatMap((n, i) => [ + ` ${i + 1}. ${n.html}`, + ` target: ${Array.isArray(n.target) ? n.target.join(", ") : String(n.target)}`, + ]), + "", + ]), + ] + : []), + ].join("\n") + ).toBe(0) + }) + } +}) diff --git a/qa/tests/playwright/e2e/test_example.spec.ts b/qa/tests/playwright/e2e/test_example.spec.ts deleted file mode 100644 index 7da84413..00000000 --- a/qa/tests/playwright/e2e/test_example.spec.ts +++ /dev/null @@ -1,11 +0,0 @@ -import { expect, test } from "@playwright/test" - -test.describe("Example E2E Test", () => { - test("Should navigate to the homepage and check title on the page", async ({ - page, - }) => { - await page.goto("/") - await page.waitForLoadState("networkidle") - await expect(page).toHaveTitle(/.+/) - }) -}) diff --git a/qa/tests/playwright/helpers/fixtures.ts b/qa/tests/playwright/helpers/fixtures.ts new file mode 100644 index 00000000..d4c1d377 --- /dev/null +++ b/qa/tests/playwright/helpers/fixtures.ts @@ -0,0 +1,30 @@ +import { test as base } from "@playwright/test" + +type MockFixtures = { + /** + * Fulfills every request matching `url` with a JSON response, without + * hitting the real backend. See https://playwright.dev/docs/mock + */ + mockJson: ( + url: string | RegExp, + body: unknown, + options?: { status?: number } + ) => Promise +} + +export const mockTest = base.extend({ + mockJson: async ({ page }, use) => { + // Playwright's fixture callback parameter, not a React hook. + // eslint-disable-next-line react-hooks/rules-of-hooks + await use(async (url, body, options = {}) => { + await page.route(url, (route) => + route.fulfill({ + status: options.status ?? 200, + json: body, + }) + ) + }) + }, +}) + +export { expect } from "@playwright/test" diff --git a/qa/tests/playwright/helpers/flatten-urls.ts b/qa/tests/playwright/helpers/flatten-urls.ts new file mode 100644 index 00000000..fb0cc039 --- /dev/null +++ b/qa/tests/playwright/helpers/flatten-urls.ts @@ -0,0 +1,16 @@ +type UrlTree = string | UrlTree[] | { [key: string]: UrlTree } + +/** + * Recursively collects every string leaf from urls.json, however deeply it's + * nested (env, brand, region, ...), and dedupes the result. + */ +export function flattenUrls(node: UrlTree): string[] { + if (typeof node === "string") { + return [node] + } + if (Array.isArray(node)) { + return node.flatMap(flattenUrls) + } + + return Array.from(new Set(Object.values(node).flatMap(flattenUrls))) +} diff --git a/qa/tests/playwright/helpers/get-sitemap-links.ts b/qa/tests/playwright/helpers/get-sitemap-links.ts index a534bf5e..10be33bc 100644 --- a/qa/tests/playwright/helpers/get-sitemap-links.ts +++ b/qa/tests/playwright/helpers/get-sitemap-links.ts @@ -1,5 +1,6 @@ -import fs from "fs" -import path from "path" +/* eslint-disable no-console */ +import fs from "node:fs" +import path from "node:path" import GetSitemapLinks from "get-sitemap-links" @@ -10,20 +11,23 @@ export async function fetchSitemap(url: string, format: "--json" | "--txt") { if (validLinks.length === 0) { console.warn(`Sitemap at ${url} is empty or could not be parsed.`) + return } if (format === "--json") { const filePath = path.join(__dirname, "sites.json") - fs.writeFileSync(filePath, JSON.stringify(validLinks, null, 2), "utf-8") + fs.writeFileSync(filePath, JSON.stringify(validLinks, null, 2), "utf8") console.log(`Wrote ${validLinks.length} links to ${filePath}`) + return filePath } if (format === "--txt") { const filePath = path.join(__dirname, "sites.txt") - fs.writeFileSync(filePath, validLinks.join("\n"), "utf-8") + fs.writeFileSync(filePath, validLinks.join("\n"), "utf8") console.log(`Wrote ${validLinks.length} links to ${filePath}`) + return filePath } } catch (error) { diff --git a/qa/tests/playwright/helpers/urls-all-components-page.json b/qa/tests/playwright/helpers/urls-all-components-page.json deleted file mode 100644 index 41ca9b1a..00000000 --- a/qa/tests/playwright/helpers/urls-all-components-page.json +++ /dev/null @@ -1 +0,0 @@ -["/"] diff --git a/qa/tests/playwright/helpers/urls.json b/qa/tests/playwright/helpers/urls.json index 41ca9b1a..e8f50b8c 100644 --- a/qa/tests/playwright/helpers/urls.json +++ b/qa/tests/playwright/helpers/urls.json @@ -1 +1,76 @@ -["/"] +{ + "smoke": { + "staging": [ + "/", + "/pricing-cloud", + "/enterprise", + "/features", + "/solutions", + "/blog", + "/user-stories", + "/about-us", + "/contact-sales", + "/careers", + "/enterprise-terms", + "/terms", + "/privacy", + "/terms-of-use", + "/security" + ], + "prod": [ + "/", + "/pricing-cloud", + "/enterprise", + "/features", + "/solutions", + "/blog", + "/user-stories", + "/about-us", + "/contact-sales", + "/careers", + "/enterprise-terms", + "/terms", + "/privacy", + "/terms-of-use", + "/security" + ] + }, + "templates": { + "blog-post": { + "staging": ["/blog/content-type-folders-new-media-library"], + "prod": ["/blog/content-type-folders-new-media-library"] + }, + "blog-category": { + "staging": ["/blog/categories/tutorials"], + "prod": ["/blog/categories/tutorials"] + }, + "user-story": { + "staging": ["/user-stories/airbus"], + "prod": ["/user-stories/airbus"] + }, + "feature": { + "staging": ["/features/dynamic-zone"], + "prod": ["/features/dynamic-zone"] + }, + "solution": { + "staging": ["/solutions/corporate-website-cms"], + "prod": ["/solutions/corporate-website-cms"] + } + }, + "seo": { + "staging": ["/"], + "prod": ["/"] + }, + "axe": { + "staging": ["/"], + "prod": ["/"] + }, + "visual": { + "staging": ["/"], + "prod": ["/"] + }, + "perfo": { + "staging": ["/"], + "prod": ["/"] + } +} diff --git a/qa/tests/playwright/package.json b/qa/tests/playwright/package.json index 2d539d30..4e6c4e08 100644 --- a/qa/tests/playwright/package.json +++ b/qa/tests/playwright/package.json @@ -7,15 +7,23 @@ "author": "", "main": "index.js", "scripts": { - "axe-test": "tsx axe/axe-test.ts", "lhci-perfo-check": "tsx perfo/lhci.ts", + "lint": "eslint -c ../../../eslint.config.mjs './**/*.{js,ts,jsx,tsx,cjs,mjs}'", + "lint:fix": "pnpm lint --fix", + "playwright-axe-test": "playwright test --project=axe", + "playwright-e2e-mock-tests": "playwright test e2e/mock --pass-with-no-tests", + "playwright-e2e-smoke-tests": "playwright test e2e/smoke", "playwright-e2e-tests": "playwright test e2e", "playwright-interactive-mode": "playwright test e2e --ui", - "playwright-seo-tests": "playwright test seo/seo.spec.ts", - "playwright-visual-tests": "playwright test visual/visual.spec.ts" + "playwright-seo-tests": "playwright test --project=seo", + "playwright-visual-tests": "playwright test --project=visual-chromium --project=visual-firefox --project=visual-webkit", + "playwright-visual-tests-docker": "bash visual/compare-snapshots-docker.sh visual-chromium visual-firefox", + "playwright-visual-update-snapshots-docker": "bash visual/update-snapshots-docker.sh visual-chromium visual-firefox", + "playwright-visual-update-tests": "playwright test --project=visual-chromium --project=visual-firefox --project=visual-webkit --update-snapshots" }, "dependencies": { "@axe-core/playwright": "^4.12.1", + "axe-core": "~4.12.1", "get-sitemap-links": "^1.1.0" }, "devDependencies": { diff --git a/qa/tests/playwright/perfo/lhci.ts b/qa/tests/playwright/perfo/lhci.ts index 402f03c3..d67f87a7 100644 --- a/qa/tests/playwright/perfo/lhci.ts +++ b/qa/tests/playwright/perfo/lhci.ts @@ -1,29 +1,39 @@ +/* eslint-disable no-console */ import "dotenv/config" import { spawnSync } from "node:child_process" import fs from "node:fs" import path from "node:path" +import { flattenUrls } from "../helpers/flatten-urls" + import urls from "../helpers/urls.json" const BASE_URL = process.env.BASE_URL if (!BASE_URL) { - console.error("Missing BASE_URL environment variable") - process.exit(1) + throw new Error("Missing BASE_URL environment variable") } -if (!Array.isArray(urls) || urls.length === 0) { - console.error("No sites found in sites.json") - process.exit(1) +const perfoUrls = flattenUrls(urls.perfo) + +if (perfoUrls.length === 0) { + throw new Error("No sites found in urls.json") } -const fullUrls = urls.map((p) => `${BASE_URL}${p}`) +const fullUrls = perfoUrls.map((p) => `${BASE_URL}${p}`) const CWD = path.resolve("perfo") +const LHCI_OUTPUT_DIR = path.join(CWD, ".lighthouseci") +const HISTORY_FILE = path.join(CWD, "lighthouse-history.md") fs.mkdirSync(CWD, { recursive: true }) +// Start from a clean output directory so this run's report doesn't mix with +// leftovers from a previous local run. +fs.rmSync(LHCI_OUTPUT_DIR, { recursive: true, force: true }) +fs.mkdirSync(LHCI_OUTPUT_DIR, { recursive: true }) + const args = [ "lhci", "collect", @@ -31,9 +41,146 @@ const args = [ "--numberOfRuns=1", ] +// eslint-disable-next-line sonarjs/no-os-command-from-path const result = spawnSync("pnpm", args, { stdio: "inherit", cwd: CWD, }) -process.exit(result.status ?? 1) +if (result.status !== 0) { + throw new Error(`LHCI failed with status ${result.status ?? 1}`) +} + +interface LighthouseResult { + requestedUrl: string + fetchTime: string + categories: Record +} + +interface HistoryEntry { + date: string + url: string + performance: number | null + accessibility: number | null + bestPractices: number | null + seo: number | null +} + +function toScore(value: number | null | undefined): number | null { + return typeof value === "number" ? Math.round(value * 100) : null +} + +function buildHistoryEntries(): HistoryEntry[] { + const reportFiles = fs + .readdirSync(LHCI_OUTPUT_DIR) + .filter((file) => /^lhr-.*\.json$/.test(file)) + + return reportFiles.map((file) => { + const lhr: LighthouseResult = JSON.parse( + fs.readFileSync(path.join(LHCI_OUTPUT_DIR, file), "utf8") + ) + + return { + date: lhr.fetchTime.slice(0, 10), + url: lhr.requestedUrl, + performance: toScore(lhr.categories.performance?.score), + accessibility: toScore(lhr.categories.accessibility?.score), + bestPractices: toScore(lhr.categories["best-practices"]?.score), + seo: toScore(lhr.categories.seo?.score), + } + }) +} + +const ROW_HEADER = + "| Date | Performance | Accessibility | Best Practices | SEO |" +const ROW_SEPARATOR = "| --- | --- | --- | --- | --- |" + +interface Section { + url: string + rows: string[] +} + +function formatCell(value: number | null): string { + return value === null ? "–" : String(value) +} + +function formatRow(entry: HistoryEntry): string { + const cells = [ + entry.date, + formatCell(entry.performance), + formatCell(entry.accessibility), + formatCell(entry.bestPractices), + formatCell(entry.seo), + ] + + return `| ${cells.join(" | ")} |` +} + +// Parses the file's per-URL "## " sections back into their row lines, so +// a new run's results land next to that page's existing history instead of +// just being appended at the end of the file. +function parseSections(content: string): Section[] { + const sections: Section[] = [] + let current: Section | null = null + + for (const line of content.split("\n")) { + const heading = /^## (.+)$/.exec(line) + + if (heading) { + current = { url: heading[1], rows: [] } + sections.push(current) + continue + } + + if (!current || !line.startsWith("|")) { + continue + } + if (line === ROW_HEADER || line === ROW_SEPARATOR) { + continue + } + + current.rows.push(line) + } + + return sections +} + +function serializeSections(sections: Section[]): string { + return ( + sections + .map((section) => + [ + `## ${section.url}`, + "", + ROW_HEADER, + ROW_SEPARATOR, + ...section.rows, + ].join("\n") + ) + .join("\n\n") + "\n" + ) +} + +const entries = buildHistoryEntries() + +const existingContent = fs.existsSync(HISTORY_FILE) + ? fs.readFileSync(HISTORY_FILE, "utf8") + : "" + +const sections = parseSections(existingContent) + +for (const entry of entries) { + const section = sections.find((s) => s.url === entry.url) + + if (section) { + section.rows.push(formatRow(entry)) + } else { + sections.push({ url: entry.url, rows: [formatRow(entry)] }) + } +} + +fs.writeFileSync(HISTORY_FILE, serializeSections(sections)) + +console.log(`\nLighthouse trend (${entries.length} page(s)):`) +console.table(entries) +console.log(`Updated ${path.relative(process.cwd(), HISTORY_FILE)}`) diff --git a/qa/tests/playwright/perfo/lighthouse-history.md b/qa/tests/playwright/perfo/lighthouse-history.md new file mode 100644 index 00000000..e69de29b diff --git a/qa/tests/playwright/playwright.config.ts b/qa/tests/playwright/playwright.config.ts index c11a58be..d72a3feb 100644 --- a/qa/tests/playwright/playwright.config.ts +++ b/qa/tests/playwright/playwright.config.ts @@ -1,9 +1,11 @@ import fs from "node:fs" import path from "node:path" -import { defineConfig, devices } from "@playwright/test" +import { defineConfig, devices, type Project } from "@playwright/test" import dotenv from "dotenv" +import { cookieConsentStorageState } from "./helpers/cookie-consent" + const envPath = path.resolve(__dirname, ".env") if (fs.existsSync(envPath)) { dotenv.config({ path: envPath, override: false }) @@ -12,7 +14,7 @@ if (fs.existsSync(envPath)) { const mobileViewportsEnabled = process.env.MOBILE_VIEWPORTS_TESTING_ENABLED === "true" -const projects: any[] = [ +const baseProjects: Project[] = [ { name: "chromium", use: { ...devices["Desktop Chrome"] }, @@ -23,47 +25,111 @@ const projects: any[] = [ }, { name: "webkit", - use: { ...devices["Desktop Safari"] }, + use: { + ...devices["Desktop Safari"], + // Desktop Safari defaults to deviceScaleFactor: 2, overriding to match Chromium/Firefox + deviceScaleFactor: 1, + }, }, ] -if (mobileViewportsEnabled) { - projects.push( - // Android - { - name: "Mobile Chrome (Pixel 7)", - use: { ...devices["Pixel 7"] }, - }, - // iOS - { - name: "Mobile Safari (iPhone 15)", - use: { ...devices["iPhone 15"] }, - } - ) -} +const mobileProjects: Project[] = mobileViewportsEnabled + ? [ + // Android + { + name: "Mobile Chrome (Pixel 7)", + use: { ...devices["Pixel 7"] }, + }, + // iOS + { + name: "Mobile Safari (iPhone 15)", + use: { ...devices["iPhone 15"] }, + }, + ] + : [] -projects.push({ - name: "seo", - testMatch: ["seo/**/*.spec.ts"], - retries: 0, - use: { - ...devices["Desktop Chrome"], - trace: "off", - screenshot: "off", - video: "off", +const specializedProjects: Project[] = [ + { + name: "visual-chromium", + testMatch: ["visual/**/*.spec.ts"], + retries: 0, + use: { + ...devices["Desktop Chrome"], + trace: "off", + video: "off", + screenshot: "off", + }, }, -}) + { + name: "visual-firefox", + testMatch: ["visual/**/*.spec.ts"], + retries: 0, + use: { + ...devices["Desktop Firefox"], + trace: "off", + video: "off", + screenshot: "off", + }, + }, + { + name: "visual-webkit", + testMatch: ["visual/**/*.spec.ts"], + retries: 0, + use: { + ...devices["Desktop Safari"], + deviceScaleFactor: 1, + trace: "off", + video: "off", + screenshot: "off", + }, + }, + { + name: "seo", + testMatch: ["seo/**/*.spec.ts"], + retries: 1, + use: { + ...devices["Desktop Chrome"], + trace: "off", + screenshot: "off", + video: "off", + }, + }, + { + name: "axe", + testMatch: ["axe/**/*.spec.ts"], + workers: 1, + retries: 1, + use: { + ...devices["Desktop Chrome"], + trace: "off", + screenshot: "off", + video: "off", + }, + }, +] + +// Browser projects run only e2e specs - axe, seo and visual have their own +// projects above, so a plain `playwright test` doesn't run them per browser. +const e2eProjects: Project[] = [...baseProjects, ...mobileProjects].map( + (project) => ({ ...project, testMatch: ["e2e/**/*.spec.ts"] }) +) + +const projects: Project[] = [...e2eProjects, ...specializedProjects] export default defineConfig({ testDir: ".", + snapshotPathTemplate: + "{snapshotDir}/{testFileDir}/{testFileName}-snapshots/{platform}/{arg}{-projectName}{ext}", fullyParallel: true, forbidOnly: !!process.env.CI, retries: process.env.CI ? 2 : 0, workers: process.env.CI ? 3 : undefined, - reporter: "html", + reporter: [["html", { open: process.env.CI ? "never" : "on-failure" }]], use: { baseURL: process.env.BASE_URL, + // Closes the cookie consent banner up front (necessary cookies only) + storageState: cookieConsentStorageState(process.env.BASE_URL), // Uncomment if credentials are needed // httpCredentials: { @@ -75,6 +141,5 @@ export default defineConfig({ screenshot: "only-on-failure", video: "retain-on-failure", }, - projects, }) diff --git a/qa/tests/playwright/seo/seo.spec.ts b/qa/tests/playwright/seo/seo.spec.ts index 84b2132a..1200a897 100644 --- a/qa/tests/playwright/seo/seo.spec.ts +++ b/qa/tests/playwright/seo/seo.spec.ts @@ -1,13 +1,34 @@ -import { expect, test } from "@playwright/test" -import urls from "helpers/urls.json" +import { expect, test, type Page } from "@playwright/test" -const PATHS = [...urls] +import { flattenUrls } from "../helpers/flatten-urls" -async function expectAttrNonEmpty( - page: import("@playwright/test").Page, - selector: string, - attr: string -) { +import urls from "../helpers/urls.json" + +const PATHS = flattenUrls(urls.seo) + +// Non-production environments render noindex on purpose (see getMetaRobots in +// the UI app), so production-only checks run against these hosts only. +const PRODUCTION_HOSTNAMES = new Set(["strapi.io", "www.strapi.io"]) + +function isProductionBaseUrl(url: string): boolean { + try { + return PRODUCTION_HOSTNAMES.has(new URL(url).hostname) + } catch { + return false + } +} + +function normalizePath(url: string): string { + try { + const parsedUrl = new URL(url) + + return parsedUrl.pathname.replace(/\/$/, "") || "/" + } catch { + return url + } +} + +async function expectAttrNonEmpty(page: Page, selector: string, attr: string) { const el = page.locator(selector) const count = await el.count() @@ -63,12 +84,95 @@ for (const path of PATHS) { ].join("\n") ).not.toBe("") }) + + test("should have reasonable length (10–70 chars)", async ({ page }) => { + const title = (await page.title()).trim() + + expect( + title.length, + [ + "Page title is too short (min 10 chars)", + `URL: ${page.url()}`, + `Current title: "${title}"`, + `Length: ${title.length}`, + ].join("\n") + ).toBeGreaterThanOrEqual(10) + + expect( + title.length, + [ + "Page title is too long (max 70 chars)", + `URL: ${page.url()}`, + `Current title: "${title}"`, + `Length: ${title.length}`, + ].join("\n") + ).toBeLessThanOrEqual(70) + }) }) test.describe("Meta description", () => { test("should exist and be non-empty", async ({ page }) => { await expectAttrNonEmpty(page, "meta[name='description']", "content") }) + + test("should have reasonable length (50–160 chars)", async ({ page }) => { + const content = ( + (await page + .locator("meta[name='description']") + .first() + .getAttribute("content")) ?? "" + ).trim() + + expect( + content.length, + [ + "Meta description is too short (min 50 chars)", + `URL: ${page.url()}`, + `Current description: "${content}"`, + `Length: ${content.length}`, + ].join("\n") + ).toBeGreaterThanOrEqual(50) + + expect( + content.length, + [ + "Meta description is too long (max 160 chars)", + `URL: ${page.url()}`, + `Current description: "${content}"`, + `Length: ${content.length}`, + ].join("\n") + ).toBeLessThanOrEqual(160) + }) + }) + + test.describe("Robots", () => { + test("should not have noindex directive", async ({ page }) => { + const baseUrl = process.env.BASE_URL + + test.skip( + !baseUrl || !isProductionBaseUrl(baseUrl), + "Robots noindex check runs only against production (strapi.io)" + ) + + const robots = page.locator("meta[name='robots']") + + if ((await robots.count()) === 0) { + return + } + + const content = ( + (await robots.first().getAttribute("content")) ?? "" + ).toLowerCase() + + expect( + content, + [ + "robots meta must not contain noindex", + `URL: ${page.url()}`, + `Content: "${content}"`, + ].join("\n") + ).not.toContain("noindex") + }) }) test.describe("Canonical tag", () => { @@ -115,6 +219,37 @@ for (const path of PATHS) { ].join("\n") ).toBeTruthy() }) + + test("should self-reference the current page", async ({ page }) => { + const hasHreflang = + (await page.locator("link[rel='alternate'][hreflang]").count()) > 0 + + test.skip( + hasHreflang, + "Canonical self-reference skipped on multilingual pages — canonical may point to a language variant" + ) + + const href = + (await page + .locator("link[rel='canonical']") + .first() + .getAttribute("href")) ?? "" + const currentUrl = page.url() + + const canonicalPath = normalizePath(href) + const currentPath = normalizePath(currentUrl) + + expect( + canonicalPath, + [ + "Canonical path does not match current page path", + `URL: ${currentUrl}`, + `Canonical: "${href}"`, + `Canonical path: "${canonicalPath}"`, + `Current path: "${currentPath}"`, + ].join("\n") + ).toBe(currentPath) + }) }) test.describe("H1 heading", () => { @@ -139,7 +274,7 @@ for (const path of PATHS) { ) } - const text = (await h1.first().innerText()).trim() + const text = ((await h1.first().textContent()) ?? "").trim() if (!text) { throw new Error( @@ -166,8 +301,8 @@ for (const path of PATHS) { const heading = headings.nth(i) const tagName = await heading.evaluate((el) => el.tagName) - const level = parseInt(tagName[1], 10) - const text = (await heading.innerText()).trim() + const level = Number(tagName[1]) + const text = ((await heading.textContent()) ?? "").trim() expect( level, @@ -218,13 +353,11 @@ for (const path of PATHS) { ) ).toBeGreaterThan(0) - for (let i = 0; i < contents.length; i++) { - const content = contents[i] - + for (const [i, content] of contents.entries()) { try { JSON.parse(content) } catch (e) { - const snippet = content.trim().slice(0, 200).replace(/\s+/g, " ") + const snippet = content.trim().slice(0, 200).replaceAll(/\s+/g, " ") const message = e instanceof Error ? e.message : String(e) throw new Error( @@ -234,11 +367,67 @@ for (const path of PATHS) { `Index: ${i}`, `Error: ${message}`, `Snippet: ${snippet}`, - ].join("\n") + ].join("\n"), + { cause: e } ) } } }) + + test("JSON-LD schemas should have @context and @type", async ({ + page, + }) => { + const ld = page.locator("script[type='application/ld+json']") + const contents = await ld.allTextContents() + + for (const [i, content] of contents.entries()) { + let data: unknown + try { + data = JSON.parse(content) + } catch { + return + } + + const schemas = Array.isArray(data) ? data : [data] + + for (const [j, schema] of schemas.entries()) { + const s = schema as Record + + expect( + s["@context"], + [ + "JSON-LD missing @context", + `URL: ${page.url()}`, + `Script index: ${i}, schema index: ${j}`, + ].join("\n") + ).toBeTruthy() + + expect( + s["@type"], + [ + "JSON-LD missing @type", + `URL: ${page.url()}`, + `Script index: ${i}, schema index: ${j}`, + ].join("\n") + ).toBeTruthy() + } + } + }) + + test("JSON-LD should be present in raw HTML (not only JS-injected)", async ({ + page, + }) => { + const response = await page.request.get(page.url()) + const html = await response.text() + + expect( + html, + [ + "JSON-LD not found in raw HTML — it may be injected only by JavaScript", + `URL: ${page.url()}`, + ].join("\n") + ).toContain('type="application/ld+json"') + }) }) test.describe("Open Graph", () => { @@ -251,6 +440,68 @@ for (const path of PATHS) { "meta[property='og:description']", "content" ) + await expectAttrNonEmpty(page, "meta[property='og:image']", "content") + await expectAttrNonEmpty(page, "meta[property='og:url']", "content") + await expectAttrNonEmpty(page, "meta[property='og:type']", "content") + }) + }) + + test.describe("hreflang", () => { + test("hreflang tags should be valid if present", async ({ page }) => { + const alternates = page.locator("link[rel='alternate'][hreflang]") + + if ((await alternates.count()) === 0) { + return + } + + const xDefaultCount = await page + .locator("link[rel='alternate'][hreflang='x-default']") + .count() + + expect( + xDefaultCount, + ["x-default hreflang tag is missing", `URL: ${page.url()}`].join("\n") + ).toBeGreaterThan(0) + + const entries = await alternates.evaluateAll((links) => + links.map((l) => ({ + hreflang: l.getAttribute("hreflang") ?? "", + href: l.getAttribute("href") ?? "", + })) + ) + + for (const { hreflang, href } of entries) { + expect( + href, + [ + "hreflang href must be absolute", + `URL: ${page.url()}`, + `hreflang: "${hreflang}"`, + `href: "${href}"`, + ].join("\n") + ).toMatch(/^https?:\/\//) + } + + const currentPath = normalizePath(page.url()) + + const selfRef = entries.find(({ href }) => { + try { + return normalizePath(href) === currentPath + } catch { + return false + } + }) + + expect( + selfRef, + [ + "Current page path is not represented in hreflang tags", + `URL: ${page.url()}`, + `Current path: "${currentPath}"`, + "hreflang entries:", + ...entries.map(({ hreflang, href }) => ` ${hreflang}: ${href}`), + ].join("\n") + ).toBeTruthy() }) }) }) diff --git a/qa/tests/playwright/tsconfig.json b/qa/tests/playwright/tsconfig.json index 796b1a7e..4cd533f3 100644 --- a/qa/tests/playwright/tsconfig.json +++ b/qa/tests/playwright/tsconfig.json @@ -1,12 +1,14 @@ { "compilerOptions": { - "target": "es5", + "target": "ESNext", + "module": "ESNext", + "moduleResolution": "bundler", "lib": ["ESNext", "dom"], "sourceMap": false, "types": ["@playwright/test", "node"], - "baseUrl": "./", "resolveJsonModule": true, - "esModuleInterop": true + "esModuleInterop": true, + "noEmit": true }, "include": ["**/*.ts"] } diff --git a/qa/tests/playwright/visual/compare-snapshots-docker.sh b/qa/tests/playwright/visual/compare-snapshots-docker.sh new file mode 100755 index 00000000..6f9e66d4 --- /dev/null +++ b/qa/tests/playwright/visual/compare-snapshots-docker.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash +# Runs visual regression comparison inside a Linux Docker container against +# the committed baseline snapshots. Use this to verify locally before pushing. +# +# Usage: +# ./compare-snapshots-docker.sh # all browsers (default) +# ./compare-snapshots-docker.sh visual-chromium # specific browser +# ./compare-snapshots-docker.sh visual-chromium visual-firefox # multiple browsers +# BASE_URL=https://example.com ./compare-snapshots-docker.sh + +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLAYWRIGHT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# Match the Docker image to the actually installed @playwright/test version +PLAYWRIGHT_VERSION="$(node -e "console.log(require('$PLAYWRIGHT_DIR/node_modules/@playwright/test/package.json').version)")" +if [[ -z "$PLAYWRIGHT_VERSION" ]]; then + echo "ERROR: Could not resolve installed @playwright/test version. Run 'pnpm install' first." + exit 1 +fi +if [[ $# -eq 0 ]]; then + PROJECTS=("visual-chromium" "visual-firefox") +else + PROJECTS=("$@") +fi + +if [[ -z "${BASE_URL:-}" ]]; then + ENV_FILE="$PLAYWRIGHT_DIR/.env" + if [[ -f "$ENV_FILE" ]]; then + BASE_URL="$(grep -E '^BASE_URL=' "$ENV_FILE" | head -1 | cut -d'=' -f2-)" || true + fi +fi + +if [[ -z "${BASE_URL:-}" ]]; then + echo "ERROR: BASE_URL is not set. Set it via environment variable or in .env file." + exit 1 +fi + +if ! docker info > /dev/null 2>&1; then + echo "ERROR: Docker is not running. Start Docker Desktop and try again." + exit 1 +fi + +PROJECT_FLAGS=() +for p in "${PROJECTS[@]}"; do + PROJECT_FLAGS+=("--project=$p") +done + +echo "Playwright: v${PLAYWRIGHT_VERSION}" +echo "BASE_URL: $BASE_URL" +echo "Browsers: ${PROJECTS[*]}" +echo "" + +docker run --rm \ + --ipc=host \ + -v "$PLAYWRIGHT_DIR:/playwright" \ + -v "playwright_node_modules:/playwright/node_modules" \ + -w /playwright \ + -e BASE_URL="$BASE_URL" \ + -e CI=true \ + "mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" \ + bash -c "npm install --silent --no-package-lock && npx playwright test visual/visual.spec.ts ${PROJECT_FLAGS[*]}" +EXIT_CODE=$? + +if [[ $EXIT_CODE -ne 0 ]] && [[ -z "${CI:-}" ]]; then + echo "" + echo "Tests failed — opening Playwright report..." + cd "$PLAYWRIGHT_DIR" && pnpm exec playwright show-report +fi + +exit $EXIT_CODE diff --git a/qa/tests/playwright/visual/update-snapshots-docker.sh b/qa/tests/playwright/visual/update-snapshots-docker.sh new file mode 100755 index 00000000..458a2fa1 --- /dev/null +++ b/qa/tests/playwright/visual/update-snapshots-docker.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Generates Linux-compatible visual regression baseline snapshots using the official +# Playwright Docker image. Run this locally whenever UI changes require new baselines, +# then commit the generated snapshots so GitHub CI can compare against them. +# +# Usage: +# ./update-snapshots-docker.sh # all browsers (default) +# ./update-snapshots-docker.sh visual-chromium # specific browser +# ./update-snapshots-docker.sh visual-chromium visual-firefox # multiple browsers +# BASE_URL=https://example.com ./update-snapshots-docker.sh + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +PLAYWRIGHT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +# Match the Docker image to the actually installed @playwright/test version +PLAYWRIGHT_VERSION="$(node -e "console.log(require('$PLAYWRIGHT_DIR/node_modules/@playwright/test/package.json').version)")" +if [[ -z "$PLAYWRIGHT_VERSION" ]]; then + echo "ERROR: Could not resolve installed @playwright/test version. Run 'pnpm install' first." + exit 1 +fi +if [[ $# -eq 0 ]]; then + PROJECTS=("visual-chromium" "visual-firefox") +else + PROJECTS=("$@") +fi + +# Load BASE_URL from .env if not already set in environment +if [[ -z "${BASE_URL:-}" ]]; then + ENV_FILE="$PLAYWRIGHT_DIR/.env" + if [[ -f "$ENV_FILE" ]]; then + BASE_URL="$(grep -E '^BASE_URL=' "$ENV_FILE" | head -1 | cut -d'=' -f2-)" || true + fi +fi + +if [[ -z "${BASE_URL:-}" ]]; then + echo "ERROR: BASE_URL is not set. Set it via environment variable or in .env file." + exit 1 +fi + +if ! docker info > /dev/null 2>&1; then + echo "ERROR: Docker is not running. Start Docker Desktop and try again." + exit 1 +fi + +PROJECT_FLAGS=() +for p in "${PROJECTS[@]}"; do + PROJECT_FLAGS+=("--project=$p") +done + +echo "Playwright: v${PLAYWRIGHT_VERSION}" +echo "BASE_URL: $BASE_URL" +echo "Browsers: ${PROJECTS[*]}" +echo "" + +docker run --rm \ + --ipc=host \ + -v "$PLAYWRIGHT_DIR:/playwright" \ + -v "playwright_node_modules:/playwright/node_modules" \ + -w /playwright \ + -e BASE_URL="$BASE_URL" \ + -e CI=true \ + "mcr.microsoft.com/playwright:v${PLAYWRIGHT_VERSION}-noble" \ + bash -c "npm install --silent --no-package-lock && npx playwright test visual/visual.spec.ts --update-snapshots ${PROJECT_FLAGS[*]}" + +echo "" +echo "Snapshots saved to:" +echo " qa/tests/playwright/visual/visual.spec.ts-snapshots/" +echo "" +echo "Next steps:" +echo " git add qa/tests/playwright/visual/visual.spec.ts-snapshots/" +echo " git commit -m 'test: update Linux baseline snapshots'" diff --git a/qa/tests/playwright/visual/visual.spec.ts b/qa/tests/playwright/visual/visual.spec.ts index 55000cd7..9dedf2b4 100644 --- a/qa/tests/playwright/visual/visual.spec.ts +++ b/qa/tests/playwright/visual/visual.spec.ts @@ -1,8 +1,11 @@ -import fs from "fs" +/* eslint-disable sonarjs/no-commented-code */ +import fs from "node:fs" import { expect, test } from "@playwright/test" -import urls from "../helpers/urls-all-components-page.json" +import { flattenUrls } from "../helpers/flatten-urls" + +import urls from "../helpers/urls.json" const COMPARE_TIMEOUT = 20000 @@ -15,11 +18,25 @@ function prettifySlug(url: string): string { .replace(/^\//, "") .replace(/\/$/, "") .replace(/\.html$/, "") - .replace(/\//g, "-") + .replaceAll("/", "-") +} + +function getEnvSlug(baseUrl: string | undefined): string { + if (!baseUrl) { + return "unknown" + } + try { + const url = new URL(baseUrl) + const hostname = url.hostname.replace(/^www\./, "") + + return hostname.replaceAll(".", "-") + } catch { + return "unknown" + } } test.describe("Visual Regression", () => { - for (const url of urls) { + for (const url of flattenUrls(urls.visual)) { test(`Compare snapshot for ${url}`, async ({ page }, testInfo) => { // abort non essential requests to speed up tests await page.route("**/*", (route, request) => { @@ -32,24 +49,46 @@ test.describe("Visual Regression", () => { } }) + // Optional: enable this on projects that respect prefers-reduced-motion. + // It can make visual snapshots more stable for animated sections and carousels. + // Uncomment this to try it when snapshots are flaky because of motion. + // await page.emulateMedia({ reducedMotion: "reduce" }) + // Use baseURL from Playwright config await page.goto(url) await page.waitForLoadState("networkidle") + // Optional: uncomment this to trigger lazy-loaded assets before screenshots. + /* + const height = await page.evaluate(() => document.body.scrollHeight) + const singleScroll = 500 + const scrolls = Math.ceil(height / singleScroll) + + for (let i = 0; i < scrolls; i++) { + await page.mouse.wheel(0, singleScroll) + await page.waitForTimeout(100) + } + + await page.evaluate(() => window.scrollTo(0, 0)) + */ + await page.addStyleTag({ content: ` * { animation: none !important; transition: none !important; } #logo-carousel { visibility: hidden !important; } + ::-webkit-scrollbar { display: none !important; } + * { scrollbar-width: none !important; } `, }) - for (const el of await page.locator("img, video").elementHandles()) { - await el.evaluate((el) => - (el as HTMLElement).style.setProperty("opacity", "0", "important") - ) - } + await page.locator("img, video").evaluateAll((els) => { + for (const el of els) { + ;(el as HTMLElement).style.setProperty("opacity", "0", "important") + } + }) - const snapshotName = `${prettifySlug(url)}.png` + const envSlug = getEnvSlug(process.env.BASE_URL) + const snapshotName = `${envSlug}-${prettifySlug(url)}.png` const baselinePath = testInfo.snapshotPath(snapshotName) const baselineExists = fs.existsSync(baselinePath) const browserName = testInfo.project.name @@ -57,7 +96,7 @@ test.describe("Visual Regression", () => { // Baseline screenshots do not exist - create them and skip comparison if (!baselineExists) { console.warn( - `Baseline snapshot missing for "${url}" and browser ${browserName} - creating: ${snapshotName} and skipping comparison for this run.` + `🟡 Baseline snapshot missing for "${url}" and browser ${browserName} - creating: ${snapshotName}` ) await page.screenshot({ @@ -65,15 +104,27 @@ test.describe("Visual Regression", () => { fullPage: true, }) + console.warn( + `📷 Baseline snapshot created for "${url}" and browser ${browserName} - skipping comparison for this run.` + ) + return } // Baseline screenshot exists - compare with current screenshot - await expect(page).toHaveScreenshot(snapshotName, { - fullPage: true, - threshold: 0.02, // 0 = pixel perfect, higher = more tolerant - timeout: COMPARE_TIMEOUT, - }) + try { + await expect(page).toHaveScreenshot(snapshotName, { + fullPage: true, + threshold: 0.2, // 0 = pixel perfect, higher = more tolerant + maxDiffPixelRatio: 0.01, // 1% of pixels can be different + timeout: COMPARE_TIMEOUT, + }) + } catch (error) { + console.warn( + `⚠️ Snapshot comparison failed for "${url}" and browser ${browserName} - snapshot: ${snapshotName}` + ) + throw error + } }) } }) diff --git a/turbo.json b/turbo.json index f6cab1ab..8957088c 100644 --- a/turbo.json +++ b/turbo.json @@ -92,7 +92,7 @@ "dependsOn": ["^build"], "cache": true }, - "axe-test": { + "playwright-axe-test": { "cache": false, "persistent": false }, @@ -104,6 +104,14 @@ "cache": false, "persistent": false }, + "playwright-e2e-smoke-tests": { + "cache": false, + "persistent": false + }, + "playwright-e2e-mock-tests": { + "cache": false, + "persistent": false + }, "playwright-seo-tests": { "cache": false, "persistent": false @@ -115,6 +123,18 @@ "playwright-visual-tests": { "cache": false, "persistent": false + }, + "playwright-visual-update-tests": { + "cache": false, + "persistent": false + }, + "playwright-visual-tests-docker": { + "cache": false, + "persistent": false + }, + "playwright-visual-update-snapshots-docker": { + "cache": false, + "persistent": false } }, "ui": "tui" From 29f50edd51e53fc2713f8e8889e0b19ac42e7709 Mon Sep 17 00:00:00 2001 From: Anna Date: Mon, 12 Oct 2026 14:21:29 +0200 Subject: [PATCH 02/11] test(tests-playwright): add new tests --- .../e2e/mock/media-fallback.spec.ts | 45 +++++++ .../e2e/mock/preview-refresh.spec.ts | 123 ++++++++++++++++++ qa/tests/playwright/e2e/smoke/blog.spec.ts | 66 ++++++++++ .../playwright/e2e/smoke/homepage.spec.ts | 24 ++++ qa/tests/playwright/e2e/smoke/pages.spec.ts | 69 ++++++++++ qa/tests/playwright/e2e/smoke/pricing.spec.ts | 30 +++++ qa/tests/playwright/helpers/cookie-consent.ts | 45 +++++++ qa/tests/playwright/helpers/pages/BlogPage.ts | 63 +++++++++ .../playwright/helpers/pages/ContentPage.ts | 35 +++++ qa/tests/playwright/helpers/pages/HomePage.ts | 27 ++++ .../playwright/helpers/pages/PricingPage.ts | 43 ++++++ 11 files changed, 570 insertions(+) create mode 100644 qa/tests/playwright/e2e/mock/media-fallback.spec.ts create mode 100644 qa/tests/playwright/e2e/mock/preview-refresh.spec.ts create mode 100644 qa/tests/playwright/e2e/smoke/blog.spec.ts create mode 100644 qa/tests/playwright/e2e/smoke/homepage.spec.ts create mode 100644 qa/tests/playwright/e2e/smoke/pages.spec.ts create mode 100644 qa/tests/playwright/e2e/smoke/pricing.spec.ts create mode 100644 qa/tests/playwright/helpers/cookie-consent.ts create mode 100644 qa/tests/playwright/helpers/pages/BlogPage.ts create mode 100644 qa/tests/playwright/helpers/pages/ContentPage.ts create mode 100644 qa/tests/playwright/helpers/pages/HomePage.ts create mode 100644 qa/tests/playwright/helpers/pages/PricingPage.ts diff --git a/qa/tests/playwright/e2e/mock/media-fallback.spec.ts b/qa/tests/playwright/e2e/mock/media-fallback.spec.ts new file mode 100644 index 00000000..5a6085cc --- /dev/null +++ b/qa/tests/playwright/e2e/mock/media-fallback.spec.ts @@ -0,0 +1,45 @@ +// Strapi media is rendered through ImageWithFallback (apps/ui), which swaps a +// failed image for FALLBACK_IMAGE_PATH. A missing or unreachable asset isn't +// something a real environment produces on demand, so it's stubbed here. +import { expect, mockTest as test } from "../../helpers/fixtures" +import { + ContentPage, + FALLBACK_IMAGE_PATH, +} from "../../helpers/pages/ContentPage" + +// Pages that render Strapi images in their main content +const PATHS = ["/", "/user-stories/airbus"] + +test.describe("Media — mocked asset responses", () => { + for (const path of PATHS) { + test(`${path} shows the fallback image when media fails to load`, async ({ + page, + }) => { + const pageErrors: string[] = [] + page.on("pageerror", (error) => pageErrors.push(error.message)) + + // Every image except the fallback itself responds as a missing asset + await page.route( + (url) => !url.pathname.endsWith(FALLBACK_IMAGE_PATH), + (route) => { + if (route.request().resourceType() === "image") { + return route.fulfill({ status: 404 }) + } + + return route.fallback() + } + ) + + const contentPage = new ContentPage(page) + const response = await contentPage.goTo(path) + + expect(response?.ok()).toBe(true) + await expect(contentPage.main).toBeVisible() + await expect(contentPage.fallbackImages.first()).toBeVisible() + expect( + pageErrors, + `Uncaught browser errors on ${path}:\n${pageErrors.join("\n")}` + ).toEqual([]) + }) + } +}) diff --git a/qa/tests/playwright/e2e/mock/preview-refresh.spec.ts b/qa/tests/playwright/e2e/mock/preview-refresh.spec.ts new file mode 100644 index 00000000..a7a162f3 --- /dev/null +++ b/qa/tests/playwright/e2e/mock/preview-refresh.spec.ts @@ -0,0 +1,123 @@ +// The Strapi admin embeds the site in an iframe for live preview and posts a +// `strapiUpdate` message when content changes. StrapiPreviewListener (apps/ui) +// then calls router.refresh(), but only for messages from the Strapi origin. +// The admin window is stubbed with a route-fulfilled page, so no real Strapi +// admin is needed. +import type { Page, Request } from "@playwright/test" + +import { expect, mockTest as test } from "../../helpers/fixtures" + +// Must match STRAPI_URL of the app under test +const STRAPI_URL = process.env.STRAPI_URL +// Loopback, so the embedded app stays a secure context (the listener hashes +// the origin with crypto.subtle, which is missing in insecure contexts and +// would make the "ignores" test pass for the wrong reason) +const UNTRUSTED_ORIGIN = "http://127.0.0.1:4321" +const PREVIEW_PATH = "/" + +// router.refresh() re-fetches the current route as an RSC payload. Prefetches +// use the same header, so they're excluded. +function isRouterRefresh(request: Request, page: Page) { + const headers = request.headers() + + return ( + request.frame() !== page.mainFrame() && + headers["rsc"] === "1" && + !Object.keys(headers).some((name) => name.includes("prefetch")) + ) +} + +async function openInAdminWindow( + page: Page, + adminOrigin: string, + previewUrl: string +) { + const adminUrl = new URL("/qa-preview-host", adminOrigin).href + + await page.route(adminUrl, (route) => + route.fulfill({ + contentType: "text/html", + body: ``, + }) + ) + + await page.goto(adminUrl) + await expect(page.frameLocator("#preview").getByRole("main")).toBeVisible() + + // The message listener is attached once React hydrates, which is when it + // adds its internal fiber key to (rendered by the app's layout) + const iframe = await page.locator("#preview").elementHandle() + const frame = await iframe?.contentFrame() + + if (!frame) { + throw new Error("Preview iframe has no frame") + } + + await frame.waitForFunction(() => + Object.keys(document.body).some((key) => key.startsWith("__reactFiber")) + ) +} + +async function postStrapiUpdate(page: Page) { + await page.evaluate(() => { + const iframe = document.querySelector("#preview") + iframe?.contentWindow?.postMessage({ type: "strapiUpdate" }, "*") + }) +} + +test.describe("Strapi preview — mocked admin window", () => { + test.skip(!STRAPI_URL, "STRAPI_URL is not set in qa/tests/playwright/.env") + + test.beforeEach(async ({ context, browserName }) => { + // Loads the whole app inside an iframe - close to the default timeout + // against a local dev server + test.slow() + + // The stubbed admin page has no real address, so Chromium treats it as + // public and blocks it from framing a local app (e.g. localhost:3000) + if (browserName === "chromium") { + await context.grantPermissions(["local-network-access"]) + } + }) + + test("refreshes the page on an update message from Strapi", async ({ + page, + baseURL, + }) => { + const previewUrl = new URL(PREVIEW_PATH, baseURL).href + await openInAdminWindow(page, new URL(STRAPI_URL!).origin, previewUrl) + + const refresh = page.waitForRequest( + (request) => isRouterRefresh(request, page), + { timeout: 10_000 } + ) + await postStrapiUpdate(page) + + await expect( + refresh, + "Page didn't refresh after a strapiUpdate message from the Strapi origin" + ).resolves.toBeTruthy() + }) + + test("ignores update messages from other origins", async ({ + page, + baseURL, + }) => { + const previewUrl = new URL(PREVIEW_PATH, baseURL).href + await openInAdminWindow(page, UNTRUSTED_ORIGIN, previewUrl) + + const refreshes: string[] = [] + page.on("request", (request) => { + if (isRouterRefresh(request, page)) { + refreshes.push(request.url()) + } + }) + + await postStrapiUpdate(page) + // Nothing to wait for when the message is ignored - give a refresh time + // to show up before asserting there was none + await page.waitForTimeout(3000) + + expect(refreshes).toEqual([]) + }) +}) diff --git a/qa/tests/playwright/e2e/smoke/blog.spec.ts b/qa/tests/playwright/e2e/smoke/blog.spec.ts new file mode 100644 index 00000000..ec82153b --- /dev/null +++ b/qa/tests/playwright/e2e/smoke/blog.spec.ts @@ -0,0 +1,66 @@ +import { expect, test } from "@playwright/test" + +import { BlogPage } from "../../helpers/pages/BlogPage" + +// The first page loads 10 posts (featured + list). Asserting on a lower number +// and on no concrete titles keeps the test stable as new posts get published. +const MIN_VISIBLE_POSTS = 5 + +test.describe("Blog page", () => { + let blogPage: BlogPage + + test.beforeEach(async ({ page }) => { + blogPage = new BlogPage(page) + const response = await blogPage.goTo() + + expect(response?.ok()).toBe(true) + }) + + test("shows the category navigation", async () => { + await expect(blogPage.categoryNavigation).toBeVisible() + await expect(blogPage.categoryLinks.first()).toBeVisible() + }) + + test("shows the featured post", async () => { + await expect(blogPage.featuredPostLink).toBeVisible() + await expect(blogPage.featuredPostLink).toHaveText(/\S/) + }) + + test("shows the post list with titles and publish dates", async () => { + await expect(blogPage.postList).toBeVisible() + await expect + .poll(() => blogPage.postRows.count()) + .toBeGreaterThanOrEqual(MIN_VISIBLE_POSTS) + + for (const postRow of await blogPage.postRows.all()) { + const titleLink = blogPage.getPostTitleLink(postRow) + + await expect(titleLink).toBeVisible() + await expect(titleLink, "Post title is empty").toHaveText(/\S/) + await expect(titleLink).toHaveAttribute("href", /^\/blog\/[^/]+$/) + // e.g. "October 14, 2026" + await expect(blogPage.getPostPublishDate(postRow)).toHaveText( + /^[A-Z][a-z]+ \d{1,2}, \d{4}$/ + ) + } + }) + + test("loads more posts", async () => { + const initialCount = await blogPage.postRows.count() + + await blogPage.loadMorePosts() + + await expect + .poll(() => blogPage.postRows.count()) + .toBeGreaterThan(initialCount) + }) + + test("opens the featured post", async ({ page }) => { + const postPath = await blogPage.featuredPostLink.getAttribute("href") + + await blogPage.openFeaturedPost() + + await expect(page).toHaveURL(new RegExp(`${postPath}$`)) + await expect(page.getByRole("heading", { level: 1 })).toBeVisible() + }) +}) diff --git a/qa/tests/playwright/e2e/smoke/homepage.spec.ts b/qa/tests/playwright/e2e/smoke/homepage.spec.ts new file mode 100644 index 00000000..08ca3c38 --- /dev/null +++ b/qa/tests/playwright/e2e/smoke/homepage.spec.ts @@ -0,0 +1,24 @@ +import { expect, test } from "@playwright/test" + +import { HomePage } from "../../helpers/pages/HomePage" + +// The main navbar and footer are checked on the homepage only - other pages +// can intentionally use a different one or none (see pages.spec.ts) +test.describe("Homepage layout", () => { + let homePage: HomePage + + test.beforeEach(async ({ page }) => { + homePage = new HomePage(page) + await homePage.goTo() + }) + + test("shows the main navbar with links", async () => { + await expect(homePage.navbar).toBeVisible() + await expect(homePage.navbarLinks.first()).toBeVisible() + }) + + test("shows the main footer", async () => { + await expect(homePage.footer).toBeVisible() + await expect(homePage.footer).not.toBeEmpty() + }) +}) diff --git a/qa/tests/playwright/e2e/smoke/pages.spec.ts b/qa/tests/playwright/e2e/smoke/pages.spec.ts new file mode 100644 index 00000000..f9d123c5 --- /dev/null +++ b/qa/tests/playwright/e2e/smoke/pages.spec.ts @@ -0,0 +1,69 @@ +import { expect, test, type Page } from "@playwright/test" + +import { flattenUrls } from "../../helpers/flatten-urls" +import { ContentPage } from "../../helpers/pages/ContentPage" + +import urls from "../../helpers/urls.json" + +const MAIN_PATHS = flattenUrls(urls.smoke) +const TEMPLATES = Object.entries(urls.templates).map(([name, paths]) => ({ + name, + paths: flattenUrls(paths), +})) + +// Browser errors that don't break the page, so they don't fail the test. +// They're still listed as a warning on the test in the report. +const IGNORED_PAGE_ERRORS = [ + // Hydration mismatch - React recovers by re-rendering on the client + /Minified React error #418/, +] + +async function expectPageRenders(page: Page, path: string) { + // Uncaught exceptions in the browser (e.g. a crashing client component) + const pageErrors: string[] = [] + page.on("pageerror", (error) => { + if (IGNORED_PAGE_ERRORS.some((pattern) => pattern.test(error.message))) { + test.info().annotations.push({ + type: "warning", + description: `Ignored browser error on ${path}: ${error.message}`, + }) + + return + } + + pageErrors.push(error.message) + }) + + const contentPage = new ContentPage(page) + const response = await contentPage.goTo(path) + + expect( + response?.status(), + `Unexpected HTTP status for ${path} (final URL: ${page.url()})` + ).toBeLessThan(400) + + await expect(contentPage.main, "Main content is missing").toBeVisible() + + expect( + pageErrors, + `Uncaught browser errors on ${path}:\n${pageErrors.join("\n")}` + ).toEqual([]) +} + +test.describe("Main pages", () => { + for (const path of MAIN_PATHS) { + test(`${path} renders`, async ({ page }) => { + await expectPageRenders(page, path) + }) + } +}) + +test.describe("Template pages", () => { + for (const { name, paths } of TEMPLATES) { + for (const path of paths) { + test(`${name}: ${path} renders`, async ({ page }) => { + await expectPageRenders(page, path) + }) + } + } +}) diff --git a/qa/tests/playwright/e2e/smoke/pricing.spec.ts b/qa/tests/playwright/e2e/smoke/pricing.spec.ts new file mode 100644 index 00000000..3687b5ba --- /dev/null +++ b/qa/tests/playwright/e2e/smoke/pricing.spec.ts @@ -0,0 +1,30 @@ +import { expect, test } from "@playwright/test" + +import { PricingPage } from "../../helpers/pages/PricingPage" + +test.describe("Pricing page", () => { + let pricingPage: PricingPage + + test.beforeEach(async ({ page }) => { + pricingPage = new PricingPage(page) + const response = await pricingPage.goTo() + + expect(response?.ok()).toBe(true) + }) + + test("shows plan cards with sign-up links", async () => { + await expect(pricingPage.signUpLinks.first()).toBeVisible() + }) + + test("shows the plan comparison table", async () => { + await expect(pricingPage.comparisonTable).toBeVisible() + }) + + test("switches plan prices between yearly and monthly billing", async () => { + await pricingPage.selectYearlyBilling() + await expect(pricingPage.yearlyBillingNotes.first()).toBeVisible() + + await pricingPage.selectMonthlyBilling() + await expect(pricingPage.yearlyBillingNotes).toHaveCount(0) + }) +}) diff --git a/qa/tests/playwright/helpers/cookie-consent.ts b/qa/tests/playwright/helpers/cookie-consent.ts new file mode 100644 index 00000000..0e98e977 --- /dev/null +++ b/qa/tests/playwright/helpers/cookie-consent.ts @@ -0,0 +1,45 @@ +/** + * Pre-set Cookiebot consent ("Use necessary cookies only"), so the cookie + * banner never shows up and covers page content in tests. + * + * `stamp: '-1'` is a placeholder - Cookiebot doesn't need a real consent ID + * from its server to accept the cookie. + */ +export function cookieConsentStorageState(baseUrl: string | undefined) { + if (!baseUrl) { + return + } + + const { hostname, protocol } = new URL(baseUrl) + const value = [ + "{stamp:'-1'", + "necessary:true", + "preferences:false", + "statistics:false", + "marketing:false", + "method:'explicit'", + "ver:2", + `utc:${Date.now()}`, + "region:'cz'}", + ] + .join(",") + // Same encoding Cookiebot uses for the cookie value + .replaceAll("'", "%27") + .replaceAll(",", "%2C") + + return { + cookies: [ + { + name: "CookieConsent", + value, + domain: hostname, + path: "/", + expires: -1, + httpOnly: false, + secure: protocol === "https:", + sameSite: "Lax" as const, + }, + ], + origins: [], + } +} diff --git a/qa/tests/playwright/helpers/pages/BlogPage.ts b/qa/tests/playwright/helpers/pages/BlogPage.ts new file mode 100644 index 00000000..bd545535 --- /dev/null +++ b/qa/tests/playwright/helpers/pages/BlogPage.ts @@ -0,0 +1,63 @@ +import type { Locator, Page } from "@playwright/test" + +export class BlogPage { + private readonly page: Page + + readonly categoryNavigation: Locator + readonly categoryLinks: Locator + readonly featuredPostLink: Locator + readonly postList: Locator + readonly postRows: Locator + readonly loadMoreButton: Locator + + constructor(page: Page) { + this.page = page + const main = page.getByRole("main") + + this.categoryNavigation = main.getByRole("navigation").first() + this.categoryLinks = this.categoryNavigation.locator( + 'a[href^="/blog/categories/"]' + ) + // The featured post's title is the only level-2 heading linking a post + this.featuredPostLink = main + .getByRole("heading", { level: 2 }) + .getByRole("link") + .first() + // Rows have no role or test id - a row is the element whose direct child + // is the post title link (/blog/). The featured post's link sits in + // a heading, and category/author links go to /blog/categories/ and /user/. + this.postRows = main.locator( + 'div:has(> a[href^="/blog/"]:not([href^="/blog/categories/"]))' + ) + this.postList = this.postRows.first().locator("xpath=..") + this.loadMoreButton = main.getByRole("button", { + name: "Load More Articles", + exact: true, + }) + } + + async goTo() { + const response = await this.page.goto("/blog", { + waitUntil: "domcontentloaded", + }) + await this.page.waitForLoadState("networkidle") + + return response + } + + getPostTitleLink(postRow: Locator) { + return postRow.locator(":scope > a") + } + + getPostPublishDate(postRow: Locator) { + return postRow.locator(":scope > span") + } + + async loadMorePosts() { + await this.loadMoreButton.click() + } + + async openFeaturedPost() { + await this.featuredPostLink.click() + } +} diff --git a/qa/tests/playwright/helpers/pages/ContentPage.ts b/qa/tests/playwright/helpers/pages/ContentPage.ts new file mode 100644 index 00000000..039960f2 --- /dev/null +++ b/qa/tests/playwright/helpers/pages/ContentPage.ts @@ -0,0 +1,35 @@ +import type { Locator, Page } from "@playwright/test" + +// Must match FALLBACK_IMAGE_PATH in apps/ui/src/lib/constants.ts +export const FALLBACK_IMAGE_PATH = "/images/broken-image.png" + +/** + * Any page rendered by the catch-all page route (`[[...rest]]`) from a Strapi + * `page` entry. Main pages and template pages (blog posts, user stories, ...) + * always render their content in
. The navbar and footer can differ or be + * missing per page, so they aren't part of this object. + */ +export class ContentPage { + private readonly page: Page + + readonly main: Locator + readonly fallbackImages: Locator + + constructor(page: Page) { + this.page = page + this.main = page.getByRole("main") + // Images that failed to load and were swapped by ImageWithFallback + this.fallbackImages = this.main.locator( + `img[src*="${FALLBACK_IMAGE_PATH}"]` + ) + } + + async goTo(path: string) { + const response = await this.page.goto(path, { + waitUntil: "domcontentloaded", + }) + await this.page.waitForLoadState("networkidle") + + return response + } +} diff --git a/qa/tests/playwright/helpers/pages/HomePage.ts b/qa/tests/playwright/helpers/pages/HomePage.ts new file mode 100644 index 00000000..f399ca6b --- /dev/null +++ b/qa/tests/playwright/helpers/pages/HomePage.ts @@ -0,0 +1,27 @@ +import type { Locator, Page } from "@playwright/test" + +export class HomePage { + private readonly page: Page + + readonly navbar: Locator + readonly navbarLinks: Locator + readonly footer: Locator + + constructor(page: Page) { + this.page = page + // The footer has its own nested