chore(playwright): introduce playwright & screenshot test

.devcontainer/devcontainer.json

@@ -14,14 +14,16 @@
   // Uses array syntax to skip the shell: https://containers.dev/implementors/json_reference/#formatting-string-vs-array-properties
   "updateContentCommand": ["trunk", "install"],
   // Install npm dependencies within the container
-  // Uses array syntax to skip the shell: https://containers.dev/implementors/json_reference/#formatting-string-vs-array-properties
-  "postCreateCommand": ["npm", "ci"],
+  // Uses array syntax to skip default shell. Instead makes explicit sh call:
+  // https://containers.dev/implementors/json_reference/#formatting-string-vs-array-properties
+  "postCreateCommand": ["sh", "-c", "npm ci && npm run pw:install"],
   "customizations": {
     "vscode": {
       "extensions": [
         "astro-build.astro-vscode",
-        "unifiedjs.vscode-mdx",
-        "trunk.io"
+        "ms-playwright.playwright",
+        "trunk.io",
+        "unifiedjs.vscode-mdx"
       ]
     }
   }

.github/workflows/playwright.yml

@@ -0,0 +1,41 @@
+name: Playwright tests
+
+permissions:
+  contents: read
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+
+jobs:
+  test:
+    # timeout-minutes: 10
+    # Lets get a baseline for how long this actually takes
+    # Runs in <1m on my local machine...
+    timeout-minutes: 30
+    runs-on: ubuntu-latest
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@c6295a65d1254861815972266d5933fd6e532bdf # v2.11.1
+        with:
+          egress-policy: audit
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          cache: npm
+          node-version: lts/*
+      - name: Install dependencies
+        run: npm ci
+      - name: Install Playwright Browsers
+        run: npm run pw:install
+      - name: Run Playwright tests
+        run: npm run pw:run
+      - uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
+        if: ${{ !cancelled() }}
+        with:
+          name: playwright-report
+          path: playwright-report/
+          retention-days: 30

.gitignore

@@ -22,6 +22,12 @@ pnpm-debug.log*
 # macOS-specific files
 .DS_Store
 
+# Playwright
+/test-results/
+/playwright-report/
+/blob-report/
+/playwright/.cache/
+
 # Visual Studio Code specific files
 .vscode
 .idea

CONTRIBUTING.md

@@ -14,6 +14,47 @@ root of the project, from a terminal:
 | `npm run astro ...`       | Run CLI commands like `astro add`, `astro check` |
 | `npm run astro -- --help` | Get help using the Astro CLI                     |
 
+## Tests
+
+We use [Playwright](https://playwright.dev/) for integration testing. The tests
+are located in the [`./tests`](./tests) directory.
+
+_These instructions assume you are already running in the provided devcontainer_.
+
+### Run Playwright locally
+
+Playwright will automatically detect if your local dev server is already
+running. **If your local dev server isn't running, Playwright will build and
+preview the site for testing.**
+
+To open Playwright in UI mode:
+
+```sh
+npm run pw:open
+```
+
+To run Playwright tests in the terminal:
+
+```sh
+npm run pw:run
+```
+
+### Updating screenshots
+
+We use screenshot testing to ensure visual consistency. If you make intentional
+changes you will likely need to update the stored screenshots. To do so run:
+
+```sh
+npm run pw:run -- --update-snapshots changed
+```
+
+This will update only the screenshots for tests that have changed. On rare
+occasions you may want to update all screenshots, which you can do with:
+
+```sh
+npm run pw:run -- --update-snapshots all
+```
+
 ## Adding content
 
 ### Add a framework

astro.config.mjs → astro.config.mts

@@ -8,6 +8,24 @@ import { defineConfig, envField } from "astro/config";
 import arcjet from "@arcjet/astro";
 import starlightLinksValidator from "starlight-links-validator";
 import { main as sidebar } from "./src/lib/sidebars";
+import type { AstroUserConfig } from "astro";
+
+/*
+ * @astrojs/vercel does not support local previews without installing and
+ * authenticating the vercel cli. In order to avoid that in CI and for local
+ * dev we use this environment variable to force usage of the @astrojs/node
+ * adapter. It is a good enough approximation for our use case.
+ */
+
+let adapter: AstroUserConfig["adapter"] = vercelAdapter();
+const isLocalPreview = process.env.ASTRO_FORCE_NODE_ADAPTER === "1";
+if (isLocalPreview) {
+  console.warn("Using @astrojs/node adapter due to ASTRO_FORCE_NODE_ADAPTER=1");
+  const { default: nodeAdapter } = await import("@astrojs/node");
+  adapter = nodeAdapter({
+    mode: "standalone",
+  });
+}
 
 const jsoncString = fs.readFileSync(
   new URL(`./src/lib/code-dark.json`, import.meta.url),
@@ -22,6 +40,7 @@ const ajThemeLight = ExpressiveCodeTheme.fromJSONString(jsoncStringLight);
 
 // https://astro.build/config
 export default defineConfig({
+  adapter,
   env: {
     schema: {
       PUBLIC_POSTHOG_KEY: envField.string({
@@ -44,7 +63,6 @@ export default defineConfig({
     },
   },
   site: "https://docs.arcjet.com",
-  adapter: vercelAdapter(),
   output: "server",
   // This is a fix for https://github.com/withastro/astro/issues/8297
   vite: {

package.json

@@ -3,12 +3,14 @@
   "version": "0.0.1",
   "type": "module",
   "scripts": {
-    "prepare": "patch-package && npm --workspaces run check",
-    "dev": "astro dev --host",
-    "start": "astro dev",
+    "astro": "astro",
     "build": "astro check --minimumSeverity warning && astro build",
+    "dev": "astro dev --host",
+    "prepare": "patch-package && npm --workspaces run check",
     "preview": "astro preview",
-    "astro": "astro"
+    "pw:install": "playwright install --with-deps chromium --only-shell",
+    "pw:open": "playwright test --ui-host=0.0.0.0",
+    "pw:run": "playwright test"
   },
   "dependencies": {
     "@ai-sdk/openai": "1.3.24",
@@ -98,6 +100,7 @@
     "rollup": "npm:@rollup/wasm-node"
   },
   "devDependencies": {
+    "@playwright/test": "1.57.0",
     "@repo/astro": "workspace:*",
     "@repo/fastify": "workspace:*",
     "@repo/nuxt": "workspace:*",
@@ -107,7 +110,8 @@
     "@types/node": "22.18.13",
     "@types/react": "19.2.2",
     "@types/react-dom": "19.2.2",
-    "patch-package": "8.0.1"
+    "patch-package": "8.0.1",
+    "playwright": "1.57.0"
   },
   "workspaces": [
     "snippets/*"

playwright.config.ts

@@ -0,0 +1,58 @@
+import {
+  defineConfig,
+  devices,
+  type PlaywrightTestConfig,
+} from "@playwright/test";
+
+let baseURL: string | undefined = process.env.PLAYWRIGHT_BASE_URL;
+let webServer: PlaywrightTestConfig["webServer"] = undefined;
+
+if (!baseURL) {
+  baseURL = "http://localhost:4321";
+  webServer = {
+    command:
+      "npm run astro build && npm run astro -- preview --port 4321 --host 0.0.0.0",
+    env: {
+      // We need an Arcjet key for the dev server to run, but it doesn't
+      // actually need to be valid for the tests to run.
+      ARCJET_KEY: "ajkey_dummy",
+      // See astro.config.mts for context.
+      ASTRO_FORCE_NODE_ADAPTER: "1",
+    },
+    port: 4321,
+    reuseExistingServer: !process.env.CI,
+  };
+}
+
+/**
+ * See https://playwright.dev/docs/test-configuration.
+ */
+export default defineConfig({
+  forbidOnly: !!process.env.CI,
+  fullyParallel: true,
+  projects: [
+    {
+      name: "chromium",
+      use: {
+        ...devices["Desktop Chrome"],
+        launchOptions: {
+          args: [
+            "--font-render-hinting=none",
+            "--disable-lcd-text",
+            "--disable-font-subpixel-positioning",
+          ],
+        },
+      },
+    },
+  ],
+  reporter: [["html", { host: "0.0.0.0" }]],
+  retries: process.env.CI ? 1 : 0,
+  testDir: "./tests",
+  use: {
+    baseURL,
+    headless: true,
+    trace: "on-first-retry",
+  },
+  workers: process.env.CI ? 2 : undefined,
+  webServer,
+});