Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,11 @@

Collects metadata for CG (Community Group) specifications, from various sources including GitHub, Mozilla, WebKit, Chromium, web-features, WPT, the W3C API, and the W3C Repo Manager.

`specs.json` lists the specifications to track. See [input metadata](metadata.md#input-metadata-specsjson).

The process performs the following operations on each spec in `specs.json`:

1. resolves the spec's URL and repository from browser-specs
1. fetches metadata from multiple sources
1. applies any [manual overrides](overrides/README.md)
1. writes all data collected to `data.json`
Expand Down
3 changes: 3 additions & 0 deletions collectors/contributions.js
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ const API = "https://labs.w3.org/repo-manager/api/repos";
const cache = new Map();

async function fetchContributors(repo) {
if (!repo) return null;
if (!cache.has(repo)) {
const url = `${API}/${repo}/contributors`;
const response = await fetch(url);
Expand Down Expand Up @@ -60,6 +61,7 @@ function activeLastYear(group) {
export async function collectContributions(spec) {
try {
const data = await fetchContributors(spec.repo);
if (!data) return null;
return {
substantive: tally(data.substantiveContributors),
nonSubstantive: tally(data.nonSubstantiveContributors),
Expand All @@ -75,6 +77,7 @@ export async function collectContributions(spec) {
export async function collectRecentSubstantiveContributions(spec) {
try {
const data = await fetchContributors(spec.repo);
if (!data) return null;
return activeLastYear(data.substantiveContributors);
} catch (err) {
logger.error(`[contributions] Error fetching contributors for ${spec.repo}: ${err.message}`);
Expand Down
6 changes: 3 additions & 3 deletions collectors/wpt.js
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,9 @@ const WPT_FYI_URL = "https://wpt.fyi/api/search";

export async function collectWPTFyi(spec) {
try {
const query = spec.feature
? spec.shortname + (spec.feature ? `/${spec.feature}` : "")
: spec.webFeaturesId;
// The WPT directory, which follows neither our shortname nor the
// web-features id reliably: `wptPath` states it when they diverge.
const query = spec.wptPath ?? spec.webFeaturesId;
const url = `${WPT_FYI_URL}?label=master&q=${encodeURIComponent(query)}`;

const res = await fetch(url);
Expand Down
48 changes: 36 additions & 12 deletions data.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@
"shortname": "file-system-access",
"specUrl": "https://wicg.github.io/file-system-access/",
"repo": "WICG/file-system-access",
"collectedAt": "2026-10-05T00:30:27.347Z",
"collectedAt": "2026-10-06T14:50:43.738Z",
"github": {
"stars": 800,
"forks": 70,
"openIssues": 88,
"openPRs": 7,
"closedPRs": 134,
"closedPRs": 135,
"lastCommitDate": "2025-10-10T14:22:22Z"
},
"mozilla": {
Expand Down Expand Up @@ -250,6 +250,14 @@
"wpt": {
"url": "https://wpt.fyi/results?q=feature:file-system-access"
},
"web-specs": [
{
"shortname": "file-system-access"
},
{
"shortname": "fs"
}
],
"state-of-surveys": [
{
"name": "State of JS 2025",
Expand Down Expand Up @@ -428,7 +436,7 @@
}
],
"chrome-use-counters": {
"percentageOfPageLoad": 0.00004491,
"percentageOfPageLoad": 0.00003873,
"url": "https://chromestatus.com/metrics/webfeature/timeline/popularity/258"
},
"chrome-status": [
Expand Down Expand Up @@ -575,11 +583,11 @@
}
},
{
"shortname": "speculation-rules",
"shortname": "prefetch",
"specUrl": "https://wicg.github.io/nav-speculation/prefetch.html",
"repo": "WICG/nav-speculation",
"feature": "prefetch",
"collectedAt": "2026-10-05T00:30:29.339Z",
"wptPath": "speculation-rules/prefetch",
"collectedAt": "2026-10-06T14:50:46.325Z",
"github": {
"stars": 200,
"forks": 44,
Expand Down Expand Up @@ -910,6 +918,17 @@
"wpt": {
"url": "https://wpt.fyi/results?q=feature:speculation-rules"
},
"web-specs": [
{
"shortname": "fenced-frame"
},
{
"shortname": "html"
},
{
"shortname": "prerendering-revamped"
}
],
"use-cases": [
{
"description": "My company has a web app in which users can create, edit, and save documents. The app's landing page is the documents list, from which users can open their documents in a proprietary editor in a new tab.\n\nThe editor has many features, so it loads and evaluates several megabytes of assets. Even with request & bytecode caching, loading and evaluating assets is pretty slow. \n\nWith the Speculation Rules API (specifically prerendering), I can start asset load and evaluation before a user opens their document, and then use the Broadcast Channel API to tell the prerendered page what document to open.",
Expand Down Expand Up @@ -981,10 +1000,10 @@
],
"developer-signals": {
"url": "https://github.com/web-platform-dx/developer-signals/issues/186",
"votes": 28
"votes": 30
},
"chrome-use-counters": {
"percentageOfPageLoad": 0.12091068,
"percentageOfPageLoad": 0.13209978,
"url": "https://chromestatus.com/metrics/webfeature/timeline/popularity/317"
},
"chrome-status": [
Expand Down Expand Up @@ -1286,9 +1305,9 @@
"shortname": "scheduling-apis",
"specUrl": "https://wicg.github.io/scheduling-apis/",
"repo": "WICG/scheduling-apis",
"collectedAt": "2026-10-05T00:30:31.390Z",
"collectedAt": "2026-10-06T14:50:49.004Z",
"github": {
"stars": 926,
"stars": 925,
"forks": 47,
"openIssues": 28,
"openPRs": 1,
Expand Down Expand Up @@ -1551,6 +1570,11 @@
"wpt": {
"url": "https://wpt.fyi/results?q=feature:scheduler"
},
"web-specs": [
{
"shortname": "scheduling-apis"
}
],
"state-of-surveys": [
{
"name": "State of JS 2025",
Expand Down Expand Up @@ -1578,10 +1602,10 @@
],
"developer-signals": {
"url": "https://github.com/web-platform-dx/developer-signals/issues/254",
"votes": 21
"votes": 23
},
"chrome-use-counters": {
"percentageOfPageLoad": 0.12550187,
"percentageOfPageLoad": 0.13588759,
"url": "https://chromestatus.com/metrics/webfeature/timeline/popularity/118"
},
"chrome-status": [
Expand Down
31 changes: 26 additions & 5 deletions index.js
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import { collectLastEdited } from "./collectors/last-edited.js";
import { collectW3CGroup } from "./collectors/w3c-group.js";
import { logger } from "./logger.js";
import { project } from "./projection.js";
import { resolveSpec } from "./specs.js";
import { loadOverrides, mergeResultsWithOverride } from "./utils.js";

import data from "./data.json" with { type: "json" };
Expand All @@ -32,6 +33,15 @@ const collectors = [
{ key: "w3cGroup", fn: collectW3CGroup },
];

// Find the URLs and their repositories
const resolvedSpecs = [];
const unlistedSpecs = [];
for (const spec of specs) {
const resolved = resolveSpec(spec);
if (resolved) resolvedSpecs.push(resolved);
else unlistedSpecs.push(spec.shortname);
}

const args = process.argv;

if (args.includes('--help') || args.includes('-h')) {
Expand All @@ -51,7 +61,7 @@ const SPECS_DIR = "specs";
// of specs that are no longer in specs.json.
async function updateSpecFiles(finalData) {
const written = new Set();
const inputs = new Map(specs.map(spec => [spec.shortname, spec]));
const inputs = new Map(resolvedSpecs.map(spec => [spec.shortname, spec]));

await mkdir(SPECS_DIR, { recursive: true });

Expand Down Expand Up @@ -90,9 +100,13 @@ async function updateDataFile(results) {
dataMap.set(result.shortname, result);
});

const merged = specs

Copy link
Copy Markdown
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there a reason to filter this way, vs. referencing resolvedSpecs?

Copy link
Copy Markdown
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

resolvedSpecs is a subset of specs (with additional properties). So if for some reasons, a spec listed in specs.json is dropped out from browser-specs, it will no longer be in resolvedSpecs and hence, will not be processed.
Filtering specs will ensure we keep the existing data available until specs.json is cleaned up. index.js will report an error (l.130) in such case and a manual cleanup is expected.

.map(spec => dataMap.get(spec.shortname))
.filter(entry => entry !== undefined);

// Apply the overrides to the merged results before writing to file
const overrides = await loadOverrides();
const finalData = mergeResultsWithOverride(Array.from(dataMap.values()), overrides);
const finalData = mergeResultsWithOverride(merged, overrides);

try {
await writeFile("data.json", JSON.stringify(finalData, null, 2), "utf8");
Expand All @@ -109,10 +123,17 @@ async function run() {
// Get arguments starting from the 3rd index (node index.js shortname1 shortname2)
const targetShortnames = args.slice(2);

// Collecting for a spec browser-specs does not list would mean guessing its
// URL and repository. Say which one, and leave whatever was collected for it
// last time in place rather than overwriting it with nulls.
for (const shortname of unlistedSpecs) {
logger.error(`[specs] "${shortname}" is unknown to browser-specs. Skipping it`);
}

// Filter specs: if no args provided, process all. Otherwise, filter by shortname.
const specsToProcess = targetShortnames.length > 0
? specs.filter(s => targetShortnames.includes(s.shortname))
: specs;
? resolvedSpecs.filter(s => targetShortnames.includes(s.shortname))
: resolvedSpecs;

if (targetShortnames.length > 0 && specsToProcess.length === 0) {
logger.error(`No specs found matching ${targetShortnames.join(', ')}`);
Expand All @@ -131,7 +152,7 @@ async function run() {
shortname: spec.shortname,
specUrl: spec.url,
repo: spec.repo,
feature: spec.feature,
wptPath: spec.wptPath,
collectedAt: new Date().toISOString(),
...Object.fromEntries(collectedEntries),
});
Expand Down
18 changes: 8 additions & 10 deletions metadata.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,9 @@ Every entry in `data.json` has the following top-level shape:
}
```

`specUrl`, `repo` and `collectedAt` are copied from `specs.json` (or stamped) when a spec is
collected, so an entry that has not been collected since they were introduced will not have them
yet. `collectedAt` is stamped per entry at collection time, not when the file is written, so a
`specUrl` and `repo` are resolved from [browser-specs](#where-the-url-and-the-repository-come-from)
and recorded when a spec is collected, so an entry that has not been collected since
they were introduced will not have them yet. `collectedAt` is stamped per entry at collection time, not when the file is written, so a
targeted run (`node index.js <shortname>`) leaves the timestamps of the other specs alone.

**Error convention:** when a collector fails (network error, HTTP error, unexpected payload), its key contains `{ "error": "<message>" }` instead of the normal object.
Expand All @@ -49,11 +49,9 @@ Each spec is declared with the following properties:

| Property | Type | Required | Description |
|---|---|---|---|
| `shortname` | string | yes | Unique identifier of the spec. Used as the merge key in `data.json`, as the name of its file in `overrides/`, as the CLI filter argument, and as the lookup key against Chrome Status and wpt.fyi. |
| `webFeaturesId` | string | yes | Feature identifier in the [web-features](https://github.com/web-platform-dx/web-features) dataset. Used by the `web_features`, `web_features_mapping` and (as fallback) `wpt` collectors. |
| `feature` | string | no | Sub-feature name for specs that cover several features (e.g. `prefetch` for `speculation-rules`). When present, the wpt.fyi query becomes `shortname/feature`. |
| `repo` | string | yes | GitHub repository in `owner/name` form (e.g. `WICG/file-system-access`). Used by the GitHub and substantive-contributions collectors. |
| `url` | string | yes | Canonical URL of the spec. Used to match entries in the Mozilla and WebKit standards-positions datasets. |
| `shortname` | string | yes | Unique identifier of the spec, and the shortname it has in [browser-specs](https://github.com/w3c/browser-specs). Used as the merge key in `data.json`, as the name of its file in `specs/` and `overrides/`, as the CLI filter argument, and as the lookup key against Chrome Status. |
| `webFeaturesId` | string | yes | Feature identifier in the [web-features](https://github.com/web-platform-dx/web-features) dataset. Used by the `web_features`, `web_features_mapping` and (as the default wpt.fyi query) `wpt` collectors. |
| `wptPath` | string | no | The web-platform-tests directory, when it is not the `webFeaturesId`. |
| `group` | number \| string | no | W3C group, as the numeric id or the shortname (`wicg`). Only needed when the repository has no `w3c.json` to read it from; see [`w3cGroup`](#w3cgroup--the-community-group). |

---
Expand Down Expand Up @@ -210,7 +208,7 @@ The `mappings` object links the feature to external resources. Possible keys (ea

### `wpt` — web-platform-tests coverage

- **Source:** [wpt.fyi search API](https://wpt.fyi/api/search) on the `master` label; the query is `shortname/feature` when `feature` is set, otherwise `webFeaturesId`, and results are restricted to test paths under `/<query>/`
- **Source:** [wpt.fyi search API](https://wpt.fyi/api/search) on the `master` label; the query is `wptPath` when set, otherwise `webFeaturesId`, and results are restricted to test paths under `/<query>/`
- **Collector:** `collectors/wpt.js`

| Property | Type | Description |
Expand Down Expand Up @@ -298,7 +296,7 @@ one Community Group cost one API request.

### `lastEdited` — when the document itself was last edited

- **Source:** the HTTP `Last-Modified` of `specs.json`'s `url`
- **Source:** the HTTP `Last-Modified` of the spec's `nightly.url` in browser-specs
- **Collector:** `collectors/last-edited.js`

```json
Expand Down
9 changes: 8 additions & 1 deletion package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion package.json
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,8 @@
"test": "node --test test/*.test.js"
},
"dependencies": {
"web-features": "*"
"web-features": "*",
"web-specs": "^4.17.0"
},
"engines": {
"node": ">=22",
Expand Down
54 changes: 54 additions & 0 deletions specs.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
/**
* Resolves the entries of specs.json against browser-specs.
*/
import webSpecs from "web-specs" with { type: "json" };
import { logger } from "./logger.js";

// Track former shortname so that an old shortname used in specs.json can still be resolved
const byShortname = new Map();
const byFormerName = new Map();
for (const spec of webSpecs) {
byShortname.set(spec.shortname, spec);
for (const former of spec.formerNames ?? []) {
byFormerName.set(former, spec);
}
}

/**
* The `owner/name` from the full GitHub repository URL
*
* @param {string | undefined} repository
* @returns {string | null}
*/
export function githubRepo(repository) {
if (!repository) return null;
let url;
try {
url = new URL(repository);
} catch {
return null;
}
// TODO: Add support for repositories hosted on platforms other than GitHub
if (url.hostname !== "github.com") return null;
const path = url.pathname.replace(/^\//, "").replace(/\.git$/, "");
return /^[^/]+\/[^/]+$/.test(path) ? path : null;
}

/**
* Adds the URL and the repository from browser-specs to an entry from specs.json
*
* @param {object} entry
* @returns {object | null} null when browser-specs doesn't know the shortname
*/
export function resolveSpec(entry) {
const found = byShortname.get(entry.shortname) ?? byFormerName.get(entry.shortname);
if (!found) return null;
if (found.shortname !== entry.shortname) {
logger.warn(`[specs] "${entry.shortname}" is a former shortname of "${found.shortname}" in browser-specs. Please rename it in specs.json`);
}
return {
...entry,
url: found.nightly?.url ?? null,
repo: githubRepo(found.nightly?.repository),
};
}
Loading
Loading