Skip to content

Commit 1691e33

Browse files
cdransfcdransf
authored
feat(testing): add Playwright accessibility testing POC for Badge and Status Light (#5835)
* test: add Playwright accessibility testing POC for Badge and Status Light Implements automated accessibility testing using two complementary approaches: 1. ARIA Snapshot Testing - Captures and validates accessibility tree structure - Detects unintended changes to component semantics - Serves as living documentation of expected a11y structure 2. aXe Rule Validation - Automated WCAG 2.0/2.1 Level A/AA compliance checking - Excludes best-practice rules (focused on component testing) - Validates color contrast, ARIA attributes, and more Test Coverage (14/14 passing, ~6s runtime): - Badge: default, icons, semantic variants, color contrast - Status Light: sizes (s/m/l), disabled state, color contrast Key Implementation Details: - Integrated with existing Storybook stories (no duplication) - Element visibility waits (reliable, fast) - WCAG-only scanning (appropriate for isolated components) - HTML report generation for debugging Files Added: - 1st-gen/playwright.config.ts - Playwright configuration - 1st-gen/test/playwright-a11y/aria-snapshots.spec.ts - ARIA tests - 1st-gen/test/playwright-a11y/axe-validation.spec.ts - aXe tests - 1st-gen/test/playwright-a11y/README.md - Documentation - RFC_A11Y_TESTING.md - Comprehensive RFC with scaling plan - README.A11Y.md - Quick start guide Usage: yarn test:a11y # Run all accessibility tests yarn test:a11y:ui # Open Playwright UI for debugging yarn test:a11y:1st # Run only 1st gen yarn test:a11y:2nd # Run only 2nd gen * fix(a11y-testing-spike): resolve issue with ts + tests in build * fix(a11y-testing-spike): resolve file name conflict * fix(a11y-testing-spike): updates to align with barebones * chore(a11y-testing-spike): move implementation to 2nd gen, move docs
1 parent 6b887f2 commit 1691e33

27 files changed

+1924
-1240
lines changed

‎.gitignore‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,14 @@ chromatic-build-*.xml
137137
chromatic-diagnostics*.json
138138
chromatic.config.json
139139

140+
# Playwright test reports (generated)
141+
1st-gen/test/playwright-a11y/report/
142+
2nd-gen/test/playwright-a11y/report/
143+
144+
# Playwright test results (generated)
145+
playwright-report/
146+
test-results/
147+
140148
# yarn
141149
.pnp.*
142150
.yarn/*

‎.stylelintignore‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,6 @@ HEADER
88
!*.css
99
1st-gen/projects/example-project/dist
1010
1st-gen/tools/styles
11-
1st-gen/spectrum-*.css
11+
1st-gen/spectrum-*.css
12+
1st-gen/test/playwright-a11y/report/
13+
2nd-gen/test/playwright-a11y/report/

‎1st-gen/package.json‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,11 @@
6262
"storybook:quick": "run-p build:watch storybook:run",
6363
"storybook:run": "web-dev-server --config wds-storybook.config.js",
6464
"test": "yarn test:focus unit",
65+
"test:a11y": "playwright test --config=../2nd-gen/playwright.a11y.config.ts",
66+
"test:a11y:1st": "playwright test --config=../2nd-gen/playwright.a11y.config.ts --project=1st-gen",
67+
"test:a11y:2nd": "playwright test --config=../2nd-gen/playwright.a11y.config.ts --project=2nd-gen",
68+
"test:a11y:report": "playwright show-report ../2nd-gen/test/playwright-a11y/report",
69+
"test:a11y:ui": "playwright test --config=../2nd-gen/playwright.a11y.config.ts --ui",
6570
"test:bench": "yarn build:tests && node test/benchmark/cli.js",
6671
"test:changed": "node ./scripts/test-changes.js",
6772
"test:ci": "yarn test:start",
@@ -87,10 +92,11 @@
8792
"react/*"
8893
],
8994
"devDependencies": {
95+
"@axe-core/playwright": "^4.11.0",
9096
"@changesets/changelog-github": "0.5.1",
9197
"@changesets/cli": "2.29.7",
9298
"@commitlint/cli": "19.8.1",
93-
"@commitlint/config-conventional": "19.8.1",
99+
"@commitlint/config-conventional": "^19.8.1",
94100
"@custom-elements-manifest/analyzer": "0.10.6",
95101
"@geometricpanda/storybook-addon-badges": "2.0.5",
96102
"@lit/react": "1.0.8",
@@ -167,7 +173,7 @@
167173
"jsonc-eslint-parser": "2.4.1",
168174
"latest-version": "9.0.0",
169175
"lightningcss": "1.30.1",
170-
"lint-staged": "16.2.6",
176+
"lint-staged": "^16.1.2",
171177
"lit": "^2.5.0 || ^3.1.3",
172178
"lit-analyzer": "2.0.3",
173179
"lit-html": "^2.4.0 || ^3.1.3",
@@ -181,7 +187,7 @@
181187
"prettier-plugin-package": "1.4.0",
182188
"pretty-bytes": "7.1.0",
183189
"re-template-tag": "2.0.1",
184-
"replace-in-file": "8.3.0",
190+
"replace-in-file": "^8.3.0",
185191
"rimraf": "6.0.1",
186192
"rollup": "4.52.2",
187193
"sinon": "17.0.2",

‎1st-gen/packages/badge/test/badge.a11y.spec.ts‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/**
2+
* Copyright 2025 Adobe. All rights reserved.
3+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License. You may obtain a copy
5+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under
8+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9+
* OF ANY KIND, either express or implied. See the License for the specific language
10+
* governing permissions and limitations under the License.
11+
*/
12+
13+
import { expect, test } from '@playwright/test';
14+
import AxeBuilder from '@axe-core/playwright';
15+
import { gotoStory } from '../../../../2nd-gen/test/a11y-helpers.js';
16+
17+
/**
18+
* Accessibility tests for Badge component
19+
*
20+
* Tests both ARIA snapshot structure and aXe WCAG compliance
21+
*/
22+
23+
test.describe('Badge - ARIA Snapshots', () => {
24+
test('should have correct accessibility tree for default badge', async ({
25+
page,
26+
}) => {
27+
const badge = await gotoStory(page, 'badge--default', 'sp-badge');
28+
const snapshot = await badge.ariaSnapshot();
29+
30+
expect(snapshot).toBeTruthy();
31+
await expect(badge).toMatchAriaSnapshot();
32+
});
33+
34+
test('should handle badge with icon', async ({ page }) => {
35+
const badge = await gotoStory(page, 'badge--icons', 'sp-badge');
36+
const snapshot = await badge.ariaSnapshot();
37+
38+
expect(snapshot).toBeTruthy();
39+
});
40+
41+
test('should maintain accessibility with semantic variants', async ({
42+
page,
43+
}) => {
44+
await gotoStory(page, 'badge--semantic', 'sp-badge');
45+
const badges = page.locator('sp-badge');
46+
47+
const count = await badges.count();
48+
expect(count).toBeGreaterThan(0);
49+
50+
// Verify each badge is accessible
51+
for (let i = 0; i < count; i++) {
52+
const badge = badges.nth(i);
53+
const snapshot = await badge.ariaSnapshot();
54+
expect(snapshot).toBeTruthy();
55+
}
56+
});
57+
});
58+
59+
test.describe('Badge - aXe Validation', () => {
60+
test('should not have accessibility violations - default', async ({
61+
page,
62+
}) => {
63+
await gotoStory(page, 'badge--default', 'sp-badge');
64+
65+
const results = await new AxeBuilder({ page })
66+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
67+
.analyze();
68+
69+
expect(results.violations).toEqual([]);
70+
});
71+
72+
test('should not have violations - semantic variants', async ({ page }) => {
73+
await gotoStory(page, 'badge--semantic', 'sp-badge');
74+
75+
const results = await new AxeBuilder({ page })
76+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
77+
.analyze();
78+
79+
expect(results.violations).toEqual([]);
80+
});
81+
82+
test('should not have violations - with icon', async ({ page }) => {
83+
await gotoStory(page, 'badge--icons', 'sp-badge');
84+
85+
const results = await new AxeBuilder({ page })
86+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
87+
.analyze();
88+
89+
expect(results.violations).toEqual([]);
90+
});
91+
92+
test('should verify color contrast', async ({ page }) => {
93+
await gotoStory(page, 'badge--semantic', 'sp-badge');
94+
95+
const results = await new AxeBuilder({ page })
96+
.withRules(['color-contrast'])
97+
.analyze();
98+
99+
expect(results.violations).toEqual([]);
100+
});
101+
});

‎1st-gen/packages/badge/test/badge.a11y.spec.ts-snapshots/Badge---ARIA-Snapshots-should-have-correct-accessibility-tree-for-default-badge-1.aria.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- text: Badge

‎1st-gen/packages/status-light/test/status-light.a11y.spec.ts‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,111 @@
1+
/**
2+
* Copyright 2025 Adobe. All rights reserved.
3+
* This file is licensed to you under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License. You may obtain a copy
5+
* of the License at http://www.apache.org/licenses/LICENSE-2.0
6+
*
7+
* Unless required by applicable law or agreed to in writing, software distributed under
8+
* the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
9+
* OF ANY KIND, either express or implied. See the License for the specific language
10+
* governing permissions and limitations under the License.
11+
*/
12+
13+
import { expect, test } from '@playwright/test';
14+
import AxeBuilder from '@axe-core/playwright';
15+
import { gotoStory } from '../../../../2nd-gen/test/a11y-helpers.js';
16+
17+
/**
18+
* Accessibility tests for Status Light component
19+
*
20+
* Tests both ARIA snapshot structure and aXe WCAG compliance
21+
*/
22+
23+
test.describe('Status Light - ARIA Snapshots', () => {
24+
test('should have correct accessibility tree structure', async ({
25+
page,
26+
}) => {
27+
const statusLight = await gotoStory(
28+
page,
29+
'statuslight--m',
30+
'sp-status-light'
31+
);
32+
33+
const snapshot = await statusLight.ariaSnapshot();
34+
expect(snapshot).toBeTruthy();
35+
await expect(statusLight).toMatchAriaSnapshot();
36+
});
37+
38+
test('should reflect different sizes', async ({ page }) => {
39+
const sizes = ['s', 'm', 'l'];
40+
41+
for (const size of sizes) {
42+
const statusLight = await gotoStory(
43+
page,
44+
`statuslight--${size}`,
45+
'sp-status-light'
46+
);
47+
48+
const snapshot = await statusLight.ariaSnapshot();
49+
expect(snapshot).toBeTruthy();
50+
}
51+
});
52+
53+
test('should handle disabled state', async ({ page }) => {
54+
const statusLight = await gotoStory(
55+
page,
56+
'statuslight--disabled-true',
57+
'sp-status-light'
58+
);
59+
60+
const snapshot = await statusLight.ariaSnapshot();
61+
expect(snapshot).toBeTruthy();
62+
});
63+
});
64+
65+
test.describe('Status Light - aXe Validation', () => {
66+
test('should not have accessibility violations - medium size', async ({
67+
page,
68+
}) => {
69+
await gotoStory(page, 'statuslight--m', 'sp-status-light');
70+
71+
const results = await new AxeBuilder({ page })
72+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
73+
.analyze();
74+
75+
expect(results.violations).toEqual([]);
76+
});
77+
78+
test('should not have violations - different sizes', async ({ page }) => {
79+
const sizes = ['s', 'l'];
80+
81+
for (const size of sizes) {
82+
await gotoStory(page, `statuslight--${size}`, 'sp-status-light');
83+
84+
const results = await new AxeBuilder({ page })
85+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
86+
.analyze();
87+
88+
expect(results.violations).toEqual([]);
89+
}
90+
});
91+
92+
test('should not have violations - disabled state', async ({ page }) => {
93+
await gotoStory(page, 'statuslight--disabled-true', 'sp-status-light');
94+
95+
const results = await new AxeBuilder({ page })
96+
.withTags(['wcag2a', 'wcag2aa', 'wcag21a', 'wcag21aa'])
97+
.analyze();
98+
99+
expect(results.violations).toEqual([]);
100+
});
101+
102+
test('should verify color contrast', async ({ page }) => {
103+
await gotoStory(page, 'statuslight--m', 'sp-status-light');
104+
105+
const results = await new AxeBuilder({ page })
106+
.withRules(['color-contrast'])
107+
.analyze();
108+
109+
expect(results.violations).toEqual([]);
110+
});
111+
});

‎1st-gen/packages/status-light/test/status-light.a11y.spec.ts-snapshots/Status-Light---ARIA-Snapshots-should-have-correct-accessibility-tree-structure-1.aria.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
- text: accent

‎1st-gen/playwright.config.ts‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎1st-gen/tsconfig-all.json‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,22 @@
11
{
22
"extends": "./tsconfig.json",
33
"compilerOptions": {
4-
"rootDir": "./",
4+
"emitDeclarationOnly": false,
55
"noEmit": true,
6-
"emitDeclarationOnly": false
6+
"rootDir": "./"
77
},
8+
"exclude": [
9+
"packages/*/node_modules/**/*.ts",
10+
"tools/*/node_modules/**/*.ts",
11+
"packages/**/test/*.a11y.spec.ts",
12+
"test/**/*.ts"
13+
],
814
"include": [
915
"packages/**/*.ts",
1016
"tools/**/*.ts",
1117
"projects/story-decorator/**/*.ts",
1218
"projects/vrt-compare/**/*.ts"
1319
],
14-
"exclude": [
15-
"packages/*/node_modules/**/*.ts",
16-
"tools/*/node_modules/**/*.ts"
17-
],
1820
"references": [
1921
{
2022
"path": "packages/accordion"

‎2nd-gen/package.json‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,12 +24,19 @@
2424
"start": "run-p dev:core dev:analyze storybook",
2525
"storybook": "yarn workspace @adobe/swc storybook",
2626
"storybook:build": "yarn workspace @adobe/swc storybook:build",
27-
"test": "yarn workspace @adobe/swc test"
27+
"test": "yarn workspace @adobe/swc test",
28+
"test:a11y": "playwright test --config=playwright.a11y.config.ts",
29+
"test:a11y:1st": "playwright test --config=playwright.a11y.config.ts --project=1st-gen",
30+
"test:a11y:2nd": "playwright test --config=playwright.a11y.config.ts --project=2nd-gen",
31+
"test:a11y:report": "playwright show-report test/playwright-a11y/report",
32+
"test:a11y:ui": "playwright test --config=playwright.a11y.config.ts --ui"
2833
},
2934
"workspaces": [
3035
"packages/*"
3136
],
3237
"devDependencies": {
38+
"@axe-core/playwright": "^4.11.0",
39+
"@playwright/test": "1.53.1",
3340
"eslint": "8.57.1",
3441
"eslint-plugin-simple-import-sort": "12.1.1",
3542
"npm-run-all2": "8.0.4"

0 commit comments

Comments
 (0)