diff --git a/README.md b/README.md index 8aa78af..f930496 100644 --- a/README.md +++ b/README.md @@ -177,7 +177,9 @@ Current language support: ## Benchmarks -The repo ships with a **pinned, recreatable benchmark set** comparing known AI-generated repos against older solid OSS repos. +The repo ships with a **pinned, recreatable benchmark set** comparing known AI-generated repos against well-regarded OSS repos, with the mature-OSS cohort pinned to the latest default-branch commit on or before **2025-01-01**. + +_Why before Jan 1, 2025?_ Because this cutoff aims to catch mature OSS before AI coding had materially changed mainstream repository shape and review norms. **Blended score** = geometric mean of the six normalized-metric ratios versus the mature OSS cohort medians, then rescaled so the mature OSS cohort median is **1.00**. Higher means a repo is consistently noisier across the benchmark dimensions. @@ -185,13 +187,13 @@ The repo ships with a **pinned, recreatable benchmark set** comparing known AI-g | Metric | AI median | Mature OSS median | Ratio | | ------------------- | --------: | ----------------: | --------: | -| Blended score | **3.48** | **1.00** | **3.48x** | -| Score / file | **0.99** | **0.19** | **5.17x** | -| Score / KLOC | **9.51** | **4.42** | **2.15x** | -| Score / function | **0.23** | **0.09** | **2.49x** | -| Findings / file | **0.31** | **0.07** | **4.44x** | -| Findings / KLOC | **2.96** | **1.40** | **2.12x** | -| Findings / function | **0.08** | **0.03** | **2.99x** | +| Blended score | **3.02** | **1.00** | **3.02x** | +| Score / file | **0.99** | **0.24** | **4.11x** | +| Score / KLOC | **9.51** | **4.04** | **2.35x** | +| Score / function | **0.22** | **0.10** | **2.28x** | +| Findings / file | **0.31** | **0.08** | **3.74x** | +| Findings / KLOC | **2.96** | **1.38** | **2.14x** | +| Findings / function | **0.08** | **0.04** | **2.21x** | ### Pinned benchmark snapshot @@ -199,23 +201,24 @@ Ordered by blended score. | Repository | Cohort | Ref | Blended | Score/file | Score/KLOC | Findings/file | Findings/KLOC | | --------------------------------------------------------------------- | ---------- | --------- | -------: | ---------: | ---------: | ------------: | ------------: | -| [`garrytan/gstack`](https://github.com/garrytan/gstack) | ai | `6cc094c` | **5.94** | 2.34 | 21.71 | 0.52 | 4.85 | -| [`redwoodjs/agent-ci`](https://github.com/redwoodjs/agent-ci) | ai | `4de00d6` | **3.98** | 0.99 | 10.95 | 0.31 | 3.42 | -| [`jiayun/DevWorkbench`](https://github.com/jiayun/DevWorkbench) | ai | `ea50862` | **3.77** | 1.00 | 10.76 | 0.44 | 4.69 | -| [`openclaw/openclaw`](https://github.com/openclaw/openclaw) | ai | `44cf747` | **3.50** | 1.08 | 10.93 | 0.32 | 3.29 | -| [`robinebers/openusage`](https://github.com/robinebers/openusage) | ai | `857f537` | **3.48** | 1.33 | 8.30 | 0.34 | 2.11 | -| [`emdash-cms/emdash`](https://github.com/emdash-cms/emdash) | ai | `dbaf8c6` | **2.47** | 0.75 | 6.67 | 0.23 | 2.02 | -| [`FullAgent/fulling`](https://github.com/FullAgent/fulling) | ai | `d95060f` | **2.40** | 0.53 | 9.51 | 0.16 | 2.96 | -| [`cloudflare/vinext`](https://github.com/cloudflare/vinext) | ai | `28980b0` | **2.21** | 0.48 | 9.20 | 0.15 | 2.76 | -| [`vitejs/vite`](https://github.com/vitejs/vite) | mature-oss | `bdc53ab` | **1.65** | 0.26 | 7.95 | 0.08 | 2.45 | -| [`withastro/astro`](https://github.com/withastro/astro) | mature-oss | `2c9bf5e` | **1.63** | 0.27 | 5.68 | 0.09 | 2.02 | -| [`modem-dev/hunk`](https://github.com/modem-dev/hunk) | ai | `b37663f` | **1.32** | 0.38 | 4.71 | 0.13 | 1.55 | -| [`egoist/tsup`](https://github.com/egoist/tsup) | mature-oss | `b906f86` | **1.03** | 0.21 | 3.61 | 0.08 | 1.42 | -| [`umami-software/umami`](https://github.com/umami-software/umami) | mature-oss | `0a83864` | **1.01** | 0.15 | 4.17 | 0.06 | 1.61 | -| [`sindresorhus/execa`](https://github.com/sindresorhus/execa) | mature-oss | `f3a2e84` | **0.99** | 0.17 | 4.85 | 0.05 | 1.37 | -| [`antfu-collective/ni`](https://github.com/antfu-collective/ni) | mature-oss | `6d96905` | **0.73** | 0.11 | 4.68 | 0.02 | 0.94 | -| [`mikaelbr/node-notifier`](https://github.com/mikaelbr/node-notifier) | mature-oss | `b36c237` | **0.46** | 0.08 | 0.90 | 0.04 | 0.47 | -| [`vercel/hyper`](https://github.com/vercel/hyper) | mature-oss | `2a7bb18` | **0.46** | 0.65 | 1.12 | 0.16 | 0.28 | +| [`garrytan/gstack`](https://github.com/garrytan/gstack) | ai | `6cc094c` | **5.33** | 2.34 | 21.71 | 0.52 | 4.85 | +| [`redwoodjs/agent-ci`](https://github.com/redwoodjs/agent-ci) | ai | `4de00d6` | **3.57** | 0.99 | 10.95 | 0.31 | 3.42 | +| [`jiayun/DevWorkbench`](https://github.com/jiayun/DevWorkbench) | ai | `ea50862` | **3.39** | 1.00 | 10.76 | 0.44 | 4.69 | +| [`openclaw/openclaw`](https://github.com/openclaw/openclaw) | ai | `44cf747` | **3.06** | 1.04 | 10.60 | 0.32 | 3.22 | +| [`robinebers/openusage`](https://github.com/robinebers/openusage) | ai | `857f537` | **3.02** | 1.27 | 7.92 | 0.33 | 2.07 | +| [`emdash-cms/emdash`](https://github.com/emdash-cms/emdash) | ai | `dbaf8c6` | **2.17** | 0.73 | 6.54 | 0.22 | 1.98 | +| [`FullAgent/fulling`](https://github.com/FullAgent/fulling) | ai | `d95060f` | **2.16** | 0.53 | 9.51 | 0.16 | 2.96 | +| [`cloudflare/vinext`](https://github.com/cloudflare/vinext) | ai | `28980b0` | **1.99** | 0.48 | 9.20 | 0.15 | 2.76 | +| [`withastro/astro`](https://github.com/withastro/astro) | mature-oss | `f706899` | **1.58** | 0.28 | 6.75 | 0.10 | 2.31 | +| [`payloadcms/payload`](https://github.com/payloadcms/payload) | mature-oss | `f3f36d8` | **1.47** | 0.24 | 4.04 | 0.08 | 1.38 | +| [`vitejs/vite`](https://github.com/vitejs/vite) | mature-oss | `a492253` | **1.47** | 0.25 | 8.19 | 0.08 | 2.52 | +| [`pmndrs/zustand`](https://github.com/pmndrs/zustand) | mature-oss | `2e6d881` | **1.45** | 0.47 | 3.20 | 0.19 | 1.27 | +| [`modem-dev/hunk`](https://github.com/modem-dev/hunk) | ai | `b37663f` | **1.18** | 0.38 | 4.71 | 0.13 | 1.55 | +| [`umami-software/umami`](https://github.com/umami-software/umami) | mature-oss | `227b255` | **1.00** | 0.17 | 4.36 | 0.07 | 1.66 | +| [`egoist/tsup`](https://github.com/egoist/tsup) | mature-oss | `cd03e1e` | **0.95** | 0.22 | 3.83 | 0.09 | 1.50 | +| [`sindresorhus/execa`](https://github.com/sindresorhus/execa) | mature-oss | `99d1741` | **0.89** | 0.17 | 4.86 | 0.05 | 1.37 | +| [`mikaelbr/node-notifier`](https://github.com/mikaelbr/node-notifier) | mature-oss | `b36c237` | **0.41** | 0.08 | 0.90 | 0.04 | 0.47 | +| [`vercel/hyper`](https://github.com/vercel/hyper) | mature-oss | `2a7bb18` | **0.41** | 0.65 | 1.12 | 0.16 | 0.28 | Full benchmark assets: diff --git a/benchmarks/results/known-ai-vs-solid-oss.json b/benchmarks/results/known-ai-vs-solid-oss.json index b47bd79..17b187a 100644 --- a/benchmarks/results/known-ai-vs-solid-oss.json +++ b/benchmarks/results/known-ai-vs-solid-oss.json @@ -2,8 +2,8 @@ "schemaVersion": 1, "benchmarkSetId": "known-ai-vs-solid-oss", "benchmarkSetName": "Known AI repos vs older solid OSS repos", - "generatedAt": "2026-04-09T00:24:29.081Z", - "analyzerVersion": "0.2.0", + "generatedAt": "2026-04-17T23:32:37.950Z", + "analyzerVersion": "0.3.0", "configMode": "default", "repos": [ { @@ -14,23 +14,23 @@ "summary": { "fileCount": 139, "directoryCount": 29, - "findingCount": 47, - "repoScore": 184.8111111111111, + "findingCount": 46, + "repoScore": 176.3111111111111, "physicalLineCount": 33794, "logicalLineCount": 22270, "functionCount": 491, "normalized": { - "scorePerFile": 1.3295763389288568, - "scorePerKloc": 8.298657885546076, - "scorePerFunction": 0.37639737497171305, - "findingsPerFile": 0.3381294964028777, - "findingsPerKloc": 2.110462505612932, - "findingsPerFunction": 0.09572301425661914 + "scorePerFile": 1.2684252597921661, + "scorePerKloc": 7.9169784962330985, + "scorePerFunction": 0.3590857660104096, + "findingsPerFile": 0.33093525179856115, + "findingsPerKloc": 2.0655590480466994, + "findingsPerFunction": 0.09368635437881874 } }, - "blendedScore": 3.476442610225084, + "blendedScore": 3.016806784725735, "ruleCounts": { - "tests.duplicate-mock-setup": 26, + "tests.duplicate-mock-setup": 25, "structure.duplicate-function-signatures": 5, "structure.pass-through-wrappers": 5, "defensive.empty-catch": 4, @@ -100,7 +100,7 @@ "findingsPerFunction": 0.09523809523809523 } }, - "blendedScore": 3.7700791509994427, + "blendedScore": 3.3857673360055234, "ruleCounts": { "defensive.error-swallowing": 10, "defensive.empty-catch": 1, @@ -165,7 +165,7 @@ "findingsPerFunction": 0.0627177700348432 } }, - "blendedScore": 2.4020591614138374, + "blendedScore": 2.157199656063399, "ruleCounts": { "defensive.error-swallowing": 11, "defensive.error-obscuring": 5, @@ -229,23 +229,23 @@ "summary": { "fileCount": 10465, "directoryCount": 423, - "findingCount": 3398, - "repoScore": 11272.93866868018, + "findingCount": 3322, + "repoScore": 10933.93866868018, "physicalLineCount": 1939647, "logicalLineCount": 1031409, "functionCount": 40348, "normalized": { - "scorePerFile": 1.0772038861615079, - "scorePerKloc": 10.929649313395734, - "scorePerFunction": 0.27939274979379847, - "findingsPerFile": 0.3247013855709508, - "findingsPerKloc": 3.2945223475847114, - "findingsPerFunction": 0.0842173094081491 + "scorePerFile": 1.0448101928982494, + "scorePerKloc": 10.600972716623744, + "scorePerFunction": 0.2709908463537271, + "findingsPerFile": 0.31743908265647397, + "findingsPerKloc": 3.2208367388688672, + "findingsPerFunction": 0.08233369683751363 } }, - "blendedScore": 3.5018198889368195, + "blendedScore": 3.0623746367298774, "ruleCounts": { - "tests.duplicate-mock-setup": 1003, + "tests.duplicate-mock-setup": 927, "structure.pass-through-wrappers": 618, "structure.barrel-density": 412, "defensive.error-obscuring": 370, @@ -319,28 +319,28 @@ "summary": { "fileCount": 1072, "directoryCount": 306, - "findingCount": 243, - "repoScore": 803.2525793650789, + "findingCount": 239, + "repoScore": 787.2525793650789, "physicalLineCount": 257938, "logicalLineCount": 120432, "functionCount": 3513, "normalized": { - "scorePerFile": 0.7493027792584691, - "scorePerKloc": 6.669760357422271, - "scorePerFunction": 0.2286514601096154, - "findingsPerFile": 0.22667910447761194, - "findingsPerKloc": 2.017736149860502, - "findingsPerFunction": 0.06917164816396243 + "scorePerFile": 0.7343774061241407, + "scorePerKloc": 6.536905302287423, + "scorePerFunction": 0.22409694829635038, + "findingsPerFile": 0.22294776119402984, + "findingsPerKloc": 1.9845223860767902, + "findingsPerFunction": 0.06803302021064617 } }, - "blendedScore": 2.466144516579107, + "blendedScore": 2.1744626927629422, "ruleCounts": { "structure.duplicate-function-signatures": 55, "defensive.empty-catch": 40, "structure.barrel-density": 34, "defensive.error-obscuring": 27, "structure.pass-through-wrappers": 27, - "tests.duplicate-mock-setup": 25, + "tests.duplicate-mock-setup": 21, "structure.directory-fanout-hotspot": 17, "defensive.error-swallowing": 13, "defensive.async-noise": 5 @@ -422,7 +422,7 @@ "findingsPerFunction": 0.11057692307692307 } }, - "blendedScore": 5.938129873491111, + "blendedScore": 5.332812749380866, "ruleCounts": { "defensive.empty-catch": 55, "defensive.error-obscuring": 19, @@ -488,7 +488,7 @@ "findingsPerFunction": 0.027925531914893616 } }, - "blendedScore": 1.317830861442671, + "blendedScore": 1.1834946976828806, "ruleCounts": { "defensive.error-obscuring": 6, "structure.pass-through-wrappers": 6, @@ -564,7 +564,7 @@ "findingsPerFunction": 0.05622214604045252 } }, - "blendedScore": 2.2121663068836055, + "blendedScore": 1.986663972737263, "ruleCounts": { "structure.duplicate-function-signatures": 50, "structure.pass-through-wrappers": 29, @@ -654,7 +654,7 @@ "findingsPerFunction": 0.1318181818181818 } }, - "blendedScore": 3.97630246756507, + "blendedScore": 3.5709687976155604, "ruleCounts": { "defensive.empty-catch": 18, "defensive.error-obscuring": 3, @@ -699,44 +699,163 @@ ] }, { - "id": "ni", - "repo": "antfu-collective/ni", + "id": "zustand", + "repo": "pmndrs/zustand", "cohort": "mature-oss", - "ref": "6d9690555dcc75c2ef5badd2e61e7cfe5d605ffd", + "ref": "2e6d8813095c6a79ca208bae4c2cf5edc12049a1", "summary": { - "fileCount": 87, - "directoryCount": 16, - "findingCount": 2, - "repoScore": 10, - "physicalLineCount": 3569, - "logicalLineCount": 2138, - "functionCount": 99, + "fileCount": 48, + "directoryCount": 15, + "findingCount": 9, + "repoScore": 22.7, + "physicalLineCount": 8814, + "logicalLineCount": 7096, + "functionCount": 161, "normalized": { - "scorePerFile": 0.11494252873563218, - "scorePerKloc": 4.677268475210477, - "scorePerFunction": 0.10101010101010101, - "findingsPerFile": 0.022988505747126436, - "findingsPerKloc": 0.9354536950420954, - "findingsPerFunction": 0.020202020202020204 + "scorePerFile": 0.47291666666666665, + "scorePerKloc": 3.198985343855693, + "scorePerFunction": 0.14099378881987576, + "findingsPerFile": 0.1875, + "findingsPerKloc": 1.2683201803833146, + "findingsPerFunction": 0.055900621118012424 } }, - "blendedScore": 0.7263097619002122, + "blendedScore": 1.449021575841299, "ruleCounts": { - "structure.barrel-density": 1, - "structure.over-fragmentation": 1 + "structure.barrel-density": 3, + "structure.directory-fanout-hotspot": 2, + "defensive.empty-catch": 1, + "defensive.error-obscuring": 1, + "defensive.error-swallowing": 1, + "structure.pass-through-wrappers": 1 }, "topFiles": [ { - "path": "src/index.ts", + "path": "src/middleware/devtools.ts", + "score": 4.7, + "findingCount": 2 + }, + { + "path": "src/middleware.ts", + "score": 3, + "findingCount": 1 + }, + { + "path": "src/middleware/persist.ts", "score": 3, "findingCount": 1 + }, + { + "path": "examples/demo/src/utils/copy-to-clipboard.js", + "score": 2, + "findingCount": 1 + }, + { + "path": "src/index.ts", + "score": 2, + "findingCount": 1 } ], "topDirectories": [ { - "path": "bin", + "path": "examples/demo/src/components", + "score": 3, + "findingCount": 1 + }, + { + "path": "src/middleware", + "score": 3, + "findingCount": 1 + } + ] + }, + { + "id": "payload", + "repo": "payloadcms/payload", + "cohort": "mature-oss", + "ref": "f3f36d801010f3c95ae74655ff22a09ea66ab1ac", + "summary": { + "fileCount": 4234, + "directoryCount": 1837, + "findingCount": 349, + "repoScore": 1018.4828107237586, + "physicalLineCount": 384327, + "logicalLineCount": 251992, + "functionCount": 3544, + "normalized": { + "scorePerFile": 0.24054860905143094, + "scorePerKloc": 4.04172676403917, + "scorePerFunction": 0.2873822829355978, + "findingsPerFile": 0.0824279641001417, + "findingsPerKloc": 1.3849646020508588, + "findingsPerFunction": 0.0984762979683973 + } + }, + "blendedScore": 1.4739407776598403, + "ruleCounts": { + "structure.duplicate-function-signatures": 131, + "structure.directory-fanout-hotspot": 77, + "structure.barrel-density": 33, + "defensive.error-swallowing": 29, + "defensive.error-obscuring": 28, + "defensive.empty-catch": 21, + "structure.pass-through-wrappers": 13, + "defensive.async-noise": 6, + "tests.duplicate-mock-setup": 6, + "structure.over-fragmentation": 5 + }, + "topFiles": [ + { + "path": "packages/ui/src/providers/ServerFunctions/index.tsx", + "score": 8, + "findingCount": 1 + }, + { + "path": "packages/ui/src/utilities/abortAndIgnore.ts", "score": 7, "findingCount": 1 + }, + { + "path": "packages/ui/src/providers/DocumentInfo/index.tsx", + "score": 6.5, + "findingCount": 2 + }, + { + "path": "examples/localization/src/app/(frontend)/[locale]/[slug]/page.client.tsx", + "score": 6, + "findingCount": 1 + }, + { + "path": "examples/localization/src/app/(frontend)/[locale]/posts/[slug]/page.client.tsx", + "score": 6, + "findingCount": 1 + } + ], + "topDirectories": [ + { + "path": "packages/payload/src/exports/i18n", + "score": 13, + "findingCount": 2 + }, + { + "path": "packages/richtext-lexical/src/lexical-proxy/@lexical-react", + "score": 13, + "findingCount": 2 + }, + { + "path": "packages/payload/src/errors", + "score": 12.084291187739463, + "findingCount": 2 + }, + { + "path": "packages/richtext-lexical/src/lexical-proxy", + "score": 10.428571428571429, + "findingCount": 2 + }, + { + "path": "packages/plugin-cloud-storage", + "score": 10.166666666666668, + "findingCount": 2 } ] }, @@ -762,7 +881,7 @@ "findingsPerFunction": 0.023809523809523808 } }, - "blendedScore": 0.45939108356274544, + "blendedScore": 0.41256198156120816, "ruleCounts": { "defensive.empty-catch": 1 }, @@ -779,25 +898,25 @@ "id": "tsup", "repo": "egoist/tsup", "cohort": "mature-oss", - "ref": "b906f86102c02f1ef66ebd4a3f7ff50bdc07af65", + "ref": "cd03e1e00ec2bd6676ae1837cbc7e618ab6a2362", "summary": { - "fileCount": 48, + "fileCount": 46, "directoryCount": 8, "findingCount": 4, - "repoScore": 10.142857142857142, - "physicalLineCount": 6686, - "logicalLineCount": 2813, + "repoScore": 10.23076923076923, + "physicalLineCount": 6456, + "logicalLineCount": 2668, "functionCount": 140, "normalized": { - "scorePerFile": 0.2113095238095238, - "scorePerKloc": 3.605708191559595, - "scorePerFunction": 0.07244897959183673, - "findingsPerFile": 0.08333333333333333, - "findingsPerKloc": 1.4219694276573054, + "scorePerFile": 0.22240802675585283, + "scorePerKloc": 3.8346211509629797, + "scorePerFunction": 0.07307692307692307, + "findingsPerFile": 0.08695652173913043, + "findingsPerKloc": 1.4992503748125936, "findingsPerFunction": 0.02857142857142857 } }, - "blendedScore": 1.025474222156771, + "blendedScore": 0.9548338745328414, "ruleCounts": { "structure.pass-through-wrappers": 2, "defensive.error-obscuring": 1, @@ -818,7 +937,7 @@ "topDirectories": [ { "path": "src", - "score": 3.142857142857143, + "score": 3.230769230769231, "findingCount": 1 } ] @@ -827,25 +946,25 @@ "id": "execa", "repo": "sindresorhus/execa", "cohort": "mature-oss", - "ref": "f3a2e8481a1e9138de3895827895c834078b9456", + "ref": "99d1741d2525eca71b986282148bbf2983356428", "summary": { - "fileCount": 581, + "fileCount": 580, "directoryCount": 46, "findingCount": 28, "repoScore": 99.01486212489861, - "physicalLineCount": 36103, - "logicalLineCount": 20432, - "functionCount": 1008, + "physicalLineCount": 35995, + "logicalLineCount": 20374, + "functionCount": 1007, "normalized": { - "scorePerFile": 0.17042144944044513, - "scorePerKloc": 4.846068036653222, - "scorePerFunction": 0.09822902988581211, - "findingsPerFile": 0.04819277108433735, - "findingsPerKloc": 1.370399373531715, - "findingsPerFunction": 0.027777777777777776 + "scorePerFile": 0.17071527952568727, + "scorePerKloc": 4.859863655879975, + "scorePerFunction": 0.09832657609225284, + "findingsPerFile": 0.04827586206896552, + "findingsPerKloc": 1.37430057916953, + "findingsPerFunction": 0.027805362462760674 } }, - "blendedScore": 0.9872814122923419, + "blendedScore": 0.8882847293047132, "ruleCounts": { "defensive.empty-catch": 11, "structure.directory-fanout-hotspot": 7, @@ -931,7 +1050,7 @@ "findingsPerFunction": 0.0033619723571161747 } }, - "blendedScore": 0.45729755371185776, + "blendedScore": 0.4106818605605101, "ruleCounts": { "defensive.error-swallowing": 8, "structure.pass-through-wrappers": 5, @@ -972,45 +1091,44 @@ "id": "umami", "repo": "umami-software/umami", "cohort": "mature-oss", - "ref": "0a838649b773122cc68cbd0c3df78d4251b981c5", + "ref": "227b2554b4a373e63ceb7f48decdc60c8d3e6eaf", "summary": { - "fileCount": 674, - "directoryCount": 155, - "findingCount": 40, - "repoScore": 103.5878787878788, - "physicalLineCount": 37986, - "logicalLineCount": 24859, - "functionCount": 1209, + "fileCount": 512, + "directoryCount": 87, + "findingCount": 34, + "repoScore": 89.34296536796535, + "physicalLineCount": 29677, + "logicalLineCount": 20508, + "functionCount": 911, "normalized": { - "scorePerFile": 0.15369121481881126, - "scorePerKloc": 4.1670171281177355, - "scorePerFunction": 0.08568062761611149, - "findingsPerFile": 0.05934718100890208, - "findingsPerKloc": 1.609075184037974, - "findingsPerFunction": 0.033085194375516956 + "scorePerFile": 0.17449797923430732, + "scorePerKloc": 4.356493337622652, + "scorePerFunction": 0.09807131214924845, + "findingsPerFile": 0.06640625, + "findingsPerKloc": 1.6578896040569535, + "findingsPerFunction": 0.03732162458836443 } }, - "blendedScore": 1.0127185877076579, + "blendedScore": 1, "ruleCounts": { - "structure.duplicate-function-signatures": 14, - "defensive.error-obscuring": 7, + "structure.duplicate-function-signatures": 12, "structure.directory-fanout-hotspot": 6, - "structure.barrel-density": 5, - "structure.pass-through-wrappers": 4, - "defensive.empty-catch": 2, - "defensive.async-noise": 1, - "defensive.error-swallowing": 1 + "defensive.empty-catch": 4, + "defensive.error-obscuring": 4, + "structure.barrel-density": 3, + "structure.pass-through-wrappers": 3, + "defensive.async-noise": 2 }, "topFiles": [ { - "path": "src/lib/jwt.ts", - "score": 8, - "findingCount": 1 + "path": "src/lib/auth.ts", + "score": 5.75, + "findingCount": 3 }, { - "path": "scripts/seed/utils.ts", - "score": 4, - "findingCount": 1 + "path": "scripts/check-db.js", + "score": 4.4, + "findingCount": 2 }, { "path": "src/lib/params.ts", @@ -1023,35 +1141,35 @@ "findingCount": 1 }, { - "path": "src/components/svg/index.ts", - "score": 3, + "path": "src/queries/analytics/reports/getUTM.ts", + "score": 3.5, "findingCount": 1 } ], "topDirectories": [ { - "path": "src/components/hooks/queries", - "score": 6, + "path": "src/lib", + "score": 5.125, "findingCount": 1 }, { - "path": "src/lib", - "score": 5, + "path": "src/components/hooks/queries", + "score": 4.133333333333333, "findingCount": 1 }, { - "path": "src/app/(main)/websites/[websiteId]", - "score": 3.3636363636363633, + "path": "src/pages/api/reports", + "score": 3.5, "findingCount": 1 }, { - "path": "src/app/(main)/teams/[teamId]", - "score": 3.090909090909091, + "path": "src/app/(main)/profile", + "score": 3.428571428571429, "findingCount": 1 }, { - "path": "scripts", - "score": 3.083333333333333, + "path": "src/app/(main)/reports/[reportId]", + "score": 3.333333333333333, "findingCount": 1 } ] @@ -1060,78 +1178,71 @@ "id": "vite", "repo": "vitejs/vite", "cohort": "mature-oss", - "ref": "bdc53ab1e67f7e2e000112eeed9c85413ddb0e9e", + "ref": "a4922537a8d705da7769d30626a0d846511fc124", "summary": { - "fileCount": 1432, - "directoryCount": 618, - "findingCount": 114, - "repoScore": 370.44093704675544, - "physicalLineCount": 95795, - "logicalLineCount": 46589, - "functionCount": 2299, + "fileCount": 1229, + "directoryCount": 525, + "findingCount": 94, + "repoScore": 304.9507936507937, + "physicalLineCount": 77629, + "logicalLineCount": 37251, + "functionCount": 1904, "normalized": { - "scorePerFile": 0.2586878052002482, - "scorePerKloc": 7.951253236745916, - "scorePerFunction": 0.1611313340786235, - "findingsPerFile": 0.07960893854748603, - "findingsPerKloc": 2.4469295327223164, - "findingsPerFunction": 0.049586776859504134 + "scorePerFile": 0.2481292055742829, + "scorePerKloc": 8.186378718713422, + "scorePerFunction": 0.16016323195945045, + "findingsPerFile": 0.07648494711147275, + "findingsPerKloc": 2.523422190008322, + "findingsPerFunction": 0.04936974789915966 } }, - "blendedScore": 1.6464595022309005, + "blendedScore": 1.4707114224260587, "ruleCounts": { - "structure.pass-through-wrappers": 27, - "structure.directory-fanout-hotspot": 20, - "defensive.empty-catch": 14, - "structure.barrel-density": 11, - "defensive.error-obscuring": 10, - "defensive.async-noise": 9, + "structure.pass-through-wrappers": 25, + "structure.directory-fanout-hotspot": 21, + "defensive.empty-catch": 13, + "defensive.async-noise": 8, + "structure.barrel-density": 8, "structure.over-fragmentation": 8, "defensive.error-swallowing": 6, - "tests.duplicate-mock-setup": 6, - "structure.duplicate-function-signatures": 3 + "defensive.error-obscuring": 5 }, "topFiles": [ { "path": "packages/vite/src/node/server/pluginContainer.ts", - "score": 12.55, + "score": 11.05, "findingCount": 3 }, { "path": "packages/vite/src/node/utils.ts", - "score": 9.9, + "score": 7.9, "findingCount": 2 }, { - "path": "packages/vite/src/node/config.ts", - "score": 8, - "findingCount": 3 + "path": "packages/vite/bin/vite.js", + "score": 7, + "findingCount": 1 }, { - "path": "packages/vite/src/node/plugins/worker.ts", - "score": 8, + "path": "playground/test-utils.ts", + "score": 6.5, "findingCount": 2 }, { - "path": "packages/vite/src/node/ssr/__tests__/ssrLoadModule.spec.ts", - "score": 8, + "path": "packages/vite/src/node/server/moduleGraph.ts", + "score": 5.75, "findingCount": 2 } ], "topDirectories": [ { - "path": "playground/worker", - "score": 12.181818181818182, + "path": "playground/legacy", + "score": 11.166666666666666, "findingCount": 2 }, { "path": "playground/js-sourcemap", - "score": 11.480392156862745, - "findingCount": 2 - }, - { - "path": "playground/legacy", - "score": 11.166666666666666, + "score": 11.1, "findingCount": 2 }, { @@ -1143,6 +1254,11 @@ "path": "playground/html/inline", "score": 10, "findingCount": 2 + }, + { + "path": "playground/ssr-html/src", + "score": 10, + "findingCount": 2 } ] }, @@ -1150,51 +1266,51 @@ "id": "astro", "repo": "withastro/astro", "cohort": "mature-oss", - "ref": "2c9bf5e278792615dfd48de42f459cc4ad8e0b29", + "ref": "f7068995aa451dced13853789b0d51433c2373b5", "summary": { - "fileCount": 2798, - "directoryCount": 1144, - "findingCount": 265, - "repoScore": 745.8540043290039, - "physicalLineCount": 226544, - "logicalLineCount": 131228, - "functionCount": 4359, + "fileCount": 1949, + "directoryCount": 852, + "findingCount": 187, + "repoScore": 546.0060606060604, + "physicalLineCount": 138854, + "logicalLineCount": 80948, + "functionCount": 3018, "normalized": { - "scorePerFile": 0.266566834999644, - "scorePerKloc": 5.683649863817203, - "scorePerFunction": 0.17110667683620187, - "findingsPerFile": 0.09471050750536097, - "findingsPerKloc": 2.0193861066235863, - "findingsPerFunction": 0.06079376003670567 + "scorePerFile": 0.2801467730149104, + "scorePerKloc": 6.745145780081786, + "scorePerFunction": 0.18091652107556672, + "findingsPerFile": 0.09594663930220626, + "findingsPerKloc": 2.3101250185304147, + "findingsPerFunction": 0.06196156394963552 } }, - "blendedScore": 1.6300105660915536, + "blendedScore": 1.576067128942098, "ruleCounts": { - "structure.pass-through-wrappers": 55, - "defensive.empty-catch": 46, - "structure.duplicate-function-signatures": 37, - "defensive.error-obscuring": 34, - "structure.directory-fanout-hotspot": 32, - "structure.barrel-density": 29, - "defensive.async-noise": 21, - "defensive.error-swallowing": 10, + "defensive.empty-catch": 39, + "structure.duplicate-function-signatures": 28, + "structure.barrel-density": 27, + "defensive.error-obscuring": 26, + "structure.directory-fanout-hotspot": 24, + "structure.pass-through-wrappers": 24, + "defensive.async-noise": 11, + "defensive.error-swallowing": 7, "structure.over-fragmentation": 1 }, "topFiles": [ { - "path": "packages/astro/src/core/errors/dev/utils.ts", - "score": 11, - "findingCount": 2 + "path": "packages/astro/src/core/session.ts", + "score": 14.5, + "findingCount": 3 }, { - "path": "packages/astro/src/core/render-context.ts", + "path": "packages/astro/src/content/utils.ts", "score": 10.5, - "findingCount": 3 + "findingCount": 2 }, { - "path": "packages/astro/src/core/session/runtime.ts", - "score": 10.5, - "findingCount": 3 + "path": "packages/astro/src/core/errors/dev/utils.ts", + "score": 10, + "findingCount": 2 }, { "path": "packages/astro/src/content/mutable-data-store.ts", @@ -1202,9 +1318,9 @@ "findingCount": 2 }, { - "path": "packages/astro/src/assets/fonts/infra/unstorage-fs-storage.ts", - "score": 9, - "findingCount": 2 + "path": "packages/astro/src/transitions/router.ts", + "score": 8, + "findingCount": 1 } ], "topDirectories": [ @@ -1214,23 +1330,23 @@ "findingCount": 1 }, { - "path": "packages/astro/src/content", - "score": 4.666666666666666, + "path": "packages/astro/src/vite-plugin-astro-server", + "score": 4.5, "findingCount": 1 }, { - "path": "packages/astro/src/vite-plugin-astro-server", + "path": "packages/astro/src/core", "score": 4.333333333333334, "findingCount": 1 }, { - "path": "packages/astro/src/cli/info/infra", - "score": 4.166666666666666, + "path": "packages/astro/src/core/build/plugins", + "score": 4.333333333333334, "findingCount": 1 }, { - "path": "packages/markdown/remark/src", - "score": 4.142857142857142, + "path": "packages/astro/src/content", + "score": 4.166666666666666, "findingCount": 1 } ] @@ -1242,24 +1358,24 @@ "medians": { "scorePerFile": 0.9875000000000002, "scorePerKloc": 9.510586363885691, - "scorePerFunction": 0.2286514601096154, + "scorePerFunction": 0.22409694829635038, "findingsPerFile": 0.30851063829787234, "findingsPerKloc": 2.96198782293895, - "findingsPerFunction": 0.0842173094081491 + "findingsPerFunction": 0.08233369683751363 }, - "blendedScoreMedian": 3.476442610225084 + "blendedScoreMedian": 3.016806784725735 }, "mature-oss": { - "repoCount": 8, + "repoCount": 9, "medians": { - "scorePerFile": 0.19086548662498448, - "scorePerKloc": 4.422142801664107, - "scorePerFunction": 0.09195482875096181, - "findingsPerFile": 0.06947805977819406, - "findingsPerKloc": 1.3961844005945103, - "findingsPerFunction": 0.02817460317460317 + "scorePerFile": 0.24054860905143094, + "scorePerKloc": 4.04172676403917, + "scorePerFunction": 0.09832657609225284, + "findingsPerFile": 0.0824279641001417, + "findingsPerKloc": 1.3849646020508588, + "findingsPerFunction": 0.03732162458836443 }, - "blendedScoreMedian": 0.9999999999999999 + "blendedScoreMedian": 1 } }, "pairings": [ @@ -1281,12 +1397,12 @@ "solidRepoId": "umami", "notes": "Full-stack analytics-adjacent SaaS-style app comparison and negative control.", "ratios": { - "scorePerFile": 8.650958615274876, - "scorePerKloc": 1.9915103851024067, - "scorePerFunction": 4.393027752529381, - "findingsPerFile": 5.6974820143884894, - "findingsPerKloc": 1.311599685675797, - "findingsPerFunction": 2.8932281059063136 + "scorePerFile": 7.268996840868781, + "scorePerKloc": 1.8172823605311448, + "scorePerFunction": 3.6614761048974236, + "findingsPerFile": 4.9834955564959795, + "findingsPerKloc": 1.2458966163924032, + "findingsPerFunction": 2.510243201150114 } }, { @@ -1294,12 +1410,12 @@ "solidRepoId": "vite", "notes": "AI-built Vite-based framework layer vs the mature Vite baseline.", "ratios": { - "scorePerFile": 1.8744623580899764, - "scorePerKloc": 1.1567146190988122, - "scorePerFunction": 1.164743216238166, - "findingsPerFile": 1.8246857178375522, - "findingsPerKloc": 1.1259978819922476, - "findingsPerFunction": 1.133813278482459 + "scorePerFile": 1.9542260340635804, + "scorePerKloc": 1.1234919828563965, + "scorePerFunction": 1.1717834736189492, + "findingsPerFile": 1.8992141416806436, + "findingsPerKloc": 1.0918654366039802, + "findingsPerFunction": 1.1387975112874638 } } ] diff --git a/benchmarks/sets/known-ai-vs-solid-oss.json b/benchmarks/sets/known-ai-vs-solid-oss.json index f7631dd..27e9d4c 100644 --- a/benchmarks/sets/known-ai-vs-solid-oss.json +++ b/benchmarks/sets/known-ai-vs-solid-oss.json @@ -2,7 +2,7 @@ "schemaVersion": 1, "id": "known-ai-vs-solid-oss", "name": "Known AI repos vs older solid OSS repos", - "description": "Compare a cohort of known AI-generated JavaScript/TypeScript repos against older, well-regarded OSS repos using pinned commit SHAs and normalized analyzer metrics.", + "description": "Compare a cohort of known AI-generated JavaScript/TypeScript repos against well-regarded OSS repos, with the mature-OSS cohort pinned to the latest default-branch commit on or before 2025-01-01, using exact commit SHAs and normalized analyzer metrics.", "artifacts": { "checkoutsDir": "benchmarks/.cache/checkouts/known-ai-vs-solid-oss", "snapshotPath": "benchmarks/results/known-ai-vs-solid-oss.json", @@ -109,15 +109,26 @@ "notes": "Local GitHub Actions orchestration layer for agent workflows; useful benchmark for command-heavy CI tooling with strong AI involvement." }, { - "id": "ni", - "repo": "antfu-collective/ni", - "url": "https://github.com/antfu-collective/ni.git", + "id": "zustand", + "repo": "pmndrs/zustand", + "url": "https://github.com/pmndrs/zustand.git", "cohort": "mature-oss", - "ref": "6d9690555dcc75c2ef5badd2e61e7cfe5d605ffd", - "createdAt": "2020-11-05T07:56:23Z", - "stars": 8146, - "provenance": "Mature TypeScript package-manager CLI by Anthony Fu.", - "notes": "Original manual baseline for universal-pm." + "ref": "2e6d8813095c6a79ca208bae4c2cf5edc12049a1", + "createdAt": "2019-04-09T09:10:06Z", + "stars": 57758, + "provenance": "Mature React state management library by pmndrs.", + "notes": "Pinned to the latest default-branch commit on or before 2025-01-01 to test a later mature-OSS baseline without using a package-manager CLI." + }, + { + "id": "payload", + "repo": "payloadcms/payload", + "url": "https://github.com/payloadcms/payload.git", + "cohort": "mature-oss", + "ref": "f3f36d801010f3c95ae74655ff22a09ea66ab1ac", + "createdAt": "2021-01-05T18:49:45Z", + "stars": 41856, + "provenance": "Mature open-source full-stack Next.js CMS/framework by Payload.", + "notes": "Pinned to the latest default-branch commit on or before 2025-01-01 to test a later mature-OSS full-stack app/framework baseline." }, { "id": "node-notifier", @@ -135,7 +146,7 @@ "repo": "egoist/tsup", "url": "https://github.com/egoist/tsup.git", "cohort": "mature-oss", - "ref": "b906f86102c02f1ef66ebd4a3f7ff50bdc07af65", + "ref": "cd03e1e00ec2bd6676ae1837cbc7e618ab6a2362", "createdAt": "2020-03-19T16:23:41Z", "stars": 11198, "provenance": "Mature TypeScript build tool widely used in the ecosystem.", @@ -146,7 +157,7 @@ "repo": "sindresorhus/execa", "url": "https://github.com/sindresorhus/execa.git", "cohort": "mature-oss", - "ref": "f3a2e8481a1e9138de3895827895c834078b9456", + "ref": "99d1741d2525eca71b986282148bbf2983356428", "createdAt": "2015-12-05T22:57:03Z", "stars": 7481, "provenance": "Long-lived process execution library for Node.js.", @@ -168,7 +179,7 @@ "repo": "umami-software/umami", "url": "https://github.com/umami-software/umami.git", "cohort": "mature-oss", - "ref": "0a838649b773122cc68cbd0c3df78d4251b981c5", + "ref": "227b2554b4a373e63ceb7f48decdc60c8d3e6eaf", "createdAt": "2020-07-17T07:59:00Z", "stars": 36012, "provenance": "Mature privacy-focused analytics platform with a large TypeScript codebase.", @@ -179,7 +190,7 @@ "repo": "vitejs/vite", "url": "https://github.com/vitejs/vite.git", "cohort": "mature-oss", - "ref": "bdc53ab1e67f7e2e000112eeed9c85413ddb0e9e", + "ref": "a4922537a8d705da7769d30626a0d846511fc124", "createdAt": "2020-04-21T05:03:57Z", "stars": 79637, "provenance": "Mature frontend tooling project with a large TypeScript codebase.", @@ -190,7 +201,7 @@ "repo": "withastro/astro", "url": "https://github.com/withastro/astro.git", "cohort": "mature-oss", - "ref": "2c9bf5e278792615dfd48de42f459cc4ad8e0b29", + "ref": "f7068995aa451dced13853789b0d51433c2373b5", "createdAt": "2021-03-15T17:19:47Z", "stars": 58212, "provenance": "Mature web framework with a very large TypeScript/JavaScript monorepo.", diff --git a/reports/known-ai-vs-solid-oss-benchmark.md b/reports/known-ai-vs-solid-oss-benchmark.md index 5a03b4b..7c8833a 100644 --- a/reports/known-ai-vs-solid-oss-benchmark.md +++ b/reports/known-ai-vs-solid-oss-benchmark.md @@ -1,12 +1,12 @@ # Pinned benchmark: Known AI repos vs older solid OSS repos -Date: 2026-04-09 -Analyzer version: 0.2.0 +Date: 2026-04-17 +Analyzer version: 0.3.0 Config mode: default ## Goal -Compare a cohort of known AI-generated JavaScript/TypeScript repos against older, well-regarded OSS repos using pinned commit SHAs and normalized analyzer metrics. +Compare a cohort of known AI-generated JavaScript/TypeScript repos against well-regarded OSS repos, with the mature-OSS cohort pinned to the latest default-branch commit on or before 2025-01-01, using exact commit SHAs and normalized analyzer metrics. ## Reproduction @@ -28,70 +28,72 @@ Blended score = geometric mean of the six normalized-metric ratios versus the ma | Repo | Ref | Age | Stars | Blended | Files | Logical LOC | Functions | Score/file | Score/KLOC | Score/function | Findings/file | Findings/KLOC | Findings/function | |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| [garrytan/gstack](https://github.com/garrytan/gstack) | `6cc094c` | 0.1y | 65613 | **5.94** | 176 | 18958 | 832 | 2.34 | 21.71 | 0.49 | 0.52 | 4.85 | 0.11 | -| [redwoodjs/agent-ci](https://github.com/redwoodjs/agent-ci) | `4de00d6` | 0.2y | 120 | **3.98** | 94 | 8474 | 220 | 0.99 | 10.95 | 0.42 | 0.31 | 3.42 | 0.13 | -| [jiayun/DevWorkbench](https://github.com/jiayun/DevWorkbench) | `ea50862` | 0.8y | 17 | **3.77** | 32 | 2986 | 147 | 1.00 | 10.76 | 0.22 | 0.44 | 4.69 | 0.10 | -| [openclaw/openclaw](https://github.com/openclaw/openclaw) | `44cf747` | 0.4y | 350232 | **3.50** | 10465 | 1031409 | 40348 | 1.08 | 10.93 | 0.28 | 0.32 | 3.29 | 0.08 | -| [robinebers/openusage](https://github.com/robinebers/openusage) | `857f537` | 0.2y | 1715 | **3.48** | 139 | 22270 | 491 | 1.33 | 8.30 | 0.38 | 0.34 | 2.11 | 0.10 | -| [emdash-cms/emdash](https://github.com/emdash-cms/emdash) | `dbaf8c6` | 0.0y | 7842 | **2.47** | 1072 | 120432 | 3513 | 0.75 | 6.67 | 0.23 | 0.23 | 2.02 | 0.07 | -| [FullAgent/fulling](https://github.com/FullAgent/fulling) | `d95060f` | 0.5y | 2413 | **2.40** | 219 | 12154 | 574 | 0.53 | 9.51 | 0.20 | 0.16 | 2.96 | 0.06 | -| [cloudflare/vinext](https://github.com/cloudflare/vinext) | `28980b0` | 0.1y | 7709 | **2.21** | 1129 | 59523 | 2917 | 0.48 | 9.20 | 0.19 | 0.15 | 2.76 | 0.06 | -| [modem-dev/hunk](https://github.com/modem-dev/hunk) | `b37663f` | 0.1y | 352 | **1.32** | 166 | 13564 | 752 | 0.38 | 4.71 | 0.08 | 0.13 | 1.55 | 0.03 | +| [garrytan/gstack](https://github.com/garrytan/gstack) | `6cc094c` | 0.1y | 65613 | **5.33** | 176 | 18958 | 832 | 2.34 | 21.71 | 0.49 | 0.52 | 4.85 | 0.11 | +| [redwoodjs/agent-ci](https://github.com/redwoodjs/agent-ci) | `4de00d6` | 0.2y | 120 | **3.57** | 94 | 8474 | 220 | 0.99 | 10.95 | 0.42 | 0.31 | 3.42 | 0.13 | +| [jiayun/DevWorkbench](https://github.com/jiayun/DevWorkbench) | `ea50862` | 0.8y | 17 | **3.39** | 32 | 2986 | 147 | 1.00 | 10.76 | 0.22 | 0.44 | 4.69 | 0.10 | +| [openclaw/openclaw](https://github.com/openclaw/openclaw) | `44cf747` | 0.4y | 350232 | **3.06** | 10465 | 1031409 | 40348 | 1.04 | 10.60 | 0.27 | 0.32 | 3.22 | 0.08 | +| [robinebers/openusage](https://github.com/robinebers/openusage) | `857f537` | 0.2y | 1715 | **3.02** | 139 | 22270 | 491 | 1.27 | 7.92 | 0.36 | 0.33 | 2.07 | 0.09 | +| [emdash-cms/emdash](https://github.com/emdash-cms/emdash) | `dbaf8c6` | 0.0y | 7842 | **2.17** | 1072 | 120432 | 3513 | 0.73 | 6.54 | 0.22 | 0.22 | 1.98 | 0.07 | +| [FullAgent/fulling](https://github.com/FullAgent/fulling) | `d95060f` | 0.5y | 2413 | **2.16** | 219 | 12154 | 574 | 0.53 | 9.51 | 0.20 | 0.16 | 2.96 | 0.06 | +| [cloudflare/vinext](https://github.com/cloudflare/vinext) | `28980b0` | 0.1y | 7709 | **1.99** | 1129 | 59523 | 2917 | 0.48 | 9.20 | 0.19 | 0.15 | 2.76 | 0.06 | +| [modem-dev/hunk](https://github.com/modem-dev/hunk) | `b37663f` | 0.1y | 352 | **1.18** | 166 | 13564 | 752 | 0.38 | 4.71 | 0.08 | 0.13 | 1.55 | 0.03 | ## Mature OSS cohort | Repo | Ref | Age | Stars | Blended | Files | Logical LOC | Functions | Score/file | Score/KLOC | Score/function | Findings/file | Findings/KLOC | Findings/function | |---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:| -| [vitejs/vite](https://github.com/vitejs/vite) | `bdc53ab` | 6.0y | 79637 | **1.65** | 1432 | 46589 | 2299 | 0.26 | 7.95 | 0.16 | 0.08 | 2.45 | 0.05 | -| [withastro/astro](https://github.com/withastro/astro) | `2c9bf5e` | 5.1y | 58212 | **1.63** | 2798 | 131228 | 4359 | 0.27 | 5.68 | 0.17 | 0.09 | 2.02 | 0.06 | -| [egoist/tsup](https://github.com/egoist/tsup) | `b906f86` | 6.1y | 11198 | **1.03** | 48 | 2813 | 140 | 0.21 | 3.61 | 0.07 | 0.08 | 1.42 | 0.03 | -| [umami-software/umami](https://github.com/umami-software/umami) | `0a83864` | 5.7y | 36012 | **1.01** | 674 | 24859 | 1209 | 0.15 | 4.17 | 0.09 | 0.06 | 1.61 | 0.03 | -| [sindresorhus/execa](https://github.com/sindresorhus/execa) | `f3a2e84` | 10.3y | 7481 | **0.99** | 581 | 20432 | 1008 | 0.17 | 4.85 | 0.10 | 0.05 | 1.37 | 0.03 | -| [antfu-collective/ni](https://github.com/antfu-collective/ni) | `6d96905` | 5.4y | 8146 | **0.73** | 87 | 2138 | 99 | 0.11 | 4.68 | 0.10 | 0.02 | 0.94 | 0.02 | -| [mikaelbr/node-notifier](https://github.com/mikaelbr/node-notifier) | `b36c237` | 13.3y | 5843 | **0.46** | 24 | 2114 | 42 | 0.08 | 0.90 | 0.05 | 0.04 | 0.47 | 0.02 | -| [vercel/hyper](https://github.com/vercel/hyper) | `2a7bb18` | 9.8y | 44687 | **0.46** | 113 | 65075 | 5354 | 0.65 | 1.12 | 0.01 | 0.16 | 0.28 | 0.00 | +| [withastro/astro](https://github.com/withastro/astro) | `f706899` | 5.1y | 58212 | **1.58** | 1949 | 80948 | 3018 | 0.28 | 6.75 | 0.18 | 0.10 | 2.31 | 0.06 | +| [payloadcms/payload](https://github.com/payloadcms/payload) | `f3f36d8` | 5.3y | 41856 | **1.47** | 4234 | 251992 | 3544 | 0.24 | 4.04 | 0.29 | 0.08 | 1.38 | 0.10 | +| [vitejs/vite](https://github.com/vitejs/vite) | `a492253` | 6.0y | 79637 | **1.47** | 1229 | 37251 | 1904 | 0.25 | 8.19 | 0.16 | 0.08 | 2.52 | 0.05 | +| [pmndrs/zustand](https://github.com/pmndrs/zustand) | `2e6d881` | 7.0y | 57758 | **1.45** | 48 | 7096 | 161 | 0.47 | 3.20 | 0.14 | 0.19 | 1.27 | 0.06 | +| [umami-software/umami](https://github.com/umami-software/umami) | `227b255` | 5.8y | 36012 | **1.00** | 512 | 20508 | 911 | 0.17 | 4.36 | 0.10 | 0.07 | 1.66 | 0.04 | +| [egoist/tsup](https://github.com/egoist/tsup) | `cd03e1e` | 6.1y | 11198 | **0.95** | 46 | 2668 | 140 | 0.22 | 3.83 | 0.07 | 0.09 | 1.50 | 0.03 | +| [sindresorhus/execa](https://github.com/sindresorhus/execa) | `99d1741` | 10.4y | 7481 | **0.89** | 580 | 20374 | 1007 | 0.17 | 4.86 | 0.10 | 0.05 | 1.37 | 0.03 | +| [mikaelbr/node-notifier](https://github.com/mikaelbr/node-notifier) | `b36c237` | 13.4y | 5843 | **0.41** | 24 | 2114 | 42 | 0.08 | 0.90 | 0.05 | 0.04 | 0.47 | 0.02 | +| [vercel/hyper](https://github.com/vercel/hyper) | `2a7bb18` | 9.8y | 44687 | **0.41** | 113 | 65075 | 5354 | 0.65 | 1.12 | 0.01 | 0.16 | 0.28 | 0.00 | ## Cohort medians | Metric | AI median | Solid median | Ratio | |---|---:|---:|---:| -| Blended score | **3.48** | **1.00** | **3.48x** | -| Score / file | **0.99** | **0.19** | **5.17x** | -| Score / KLOC | **9.51** | **4.42** | **2.15x** | -| Score / function | **0.23** | **0.09** | **2.49x** | -| Findings / file | **0.31** | **0.07** | **4.44x** | -| Findings / KLOC | **2.96** | **1.40** | **2.12x** | -| Findings / function | **0.08** | **0.03** | **2.99x** | +| Blended score | **3.02** | **1.00** | **3.02x** | +| Score / file | **0.99** | **0.24** | **4.11x** | +| Score / KLOC | **9.51** | **4.04** | **2.35x** | +| Score / function | **0.22** | **0.10** | **2.28x** | +| Findings / file | **0.31** | **0.08** | **3.74x** | +| Findings / KLOC | **2.96** | **1.38** | **2.14x** | +| Findings / function | **0.08** | **0.04** | **2.21x** | ## Spot-check pairings | AI repo | Solid repo | Score/file ratio | Score/KLOC ratio | Score/function ratio | Findings/file ratio | Findings/KLOC ratio | Findings/function ratio | |---|---|---:|---:|---:|---:|---:|---:| | `devworkbench` | `hyper` | 1.55x | 9.58x | 16.01x | 2.75x | 16.95x | 28.33x | -| `openusage` | `umami` | 8.65x | 1.99x | 4.39x | 5.70x | 1.31x | 2.89x | -| `vinext` | `vite` | 1.87x | 1.16x | 1.16x | 1.82x | 1.13x | 1.13x | +| `openusage` | `umami` | 7.27x | 1.82x | 3.66x | 4.98x | 1.25x | 2.51x | +| `vinext` | `vite` | 1.95x | 1.12x | 1.17x | 1.90x | 1.09x | 1.14x | ## Top rule families by cohort ### AI cohort -- `tests.duplicate-mock-setup` — 1078 (26.7%) -- `structure.pass-through-wrappers` — 697 (17.2%) -- `defensive.empty-catch` — 463 (11.4%) -- `defensive.error-obscuring` — 456 (11.3%) -- `structure.barrel-density` — 455 (11.3%) -- `structure.duplicate-function-signatures` — 359 (8.9%) +- `tests.duplicate-mock-setup` — 997 (25.2%) +- `structure.pass-through-wrappers` — 697 (17.6%) +- `defensive.empty-catch` — 463 (11.7%) +- `defensive.error-obscuring` — 456 (11.5%) +- `structure.barrel-density` — 455 (11.5%) +- `structure.duplicate-function-signatures` — 359 (9.1%) ### Mature OSS cohort -- `structure.pass-through-wrappers` — 93 (19.7%) -- `defensive.empty-catch` — 77 (16.3%) -- `structure.directory-fanout-hotspot` — 66 (14.0%) -- `structure.duplicate-function-signatures` — 60 (12.7%) -- `defensive.error-obscuring` — 54 (11.4%) -- `structure.barrel-density` — 47 (10.0%) +- `structure.duplicate-function-signatures` — 177 (24.4%) +- `structure.directory-fanout-hotspot` — 138 (19.1%) +- `defensive.empty-catch` — 93 (12.8%) +- `structure.barrel-density` — 75 (10.4%) +- `structure.pass-through-wrappers` — 73 (10.1%) +- `defensive.error-obscuring` — 67 (9.3%) ## Notes - This benchmark is intentionally pinned to exact commit SHAs so future reruns can reproduce the same cohort. +- Why before 2025-01-01? The intent is to use a mature-OSS cutoff from before AI coding had materially changed mainstream repository shape and review norms. - AI provenance in the set may come from README disclosures or user-provided provenance recorded in the manifest. - The benchmark scanner uses the analyzer's default config for every repo to keep results comparable. - The analyzer still only scans JS/TS-family files, so non-JS/TS portions of mixed-language repos are out of scope. diff --git a/src/benchmarks/report.ts b/src/benchmarks/report.ts index a84c23e..95c284b 100644 --- a/src/benchmarks/report.ts +++ b/src/benchmarks/report.ts @@ -177,6 +177,7 @@ export function renderBenchmarkReport(set: BenchmarkSet, snapshot: BenchmarkSnap "## Notes", "", "- This benchmark is intentionally pinned to exact commit SHAs so future reruns can reproduce the same cohort.", + "- Why before 2025-01-01? The intent is to use a mature-OSS cutoff from before AI coding had materially changed mainstream repository shape and review norms.", "- AI provenance in the set may come from README disclosures or user-provided provenance recorded in the manifest.", "- The benchmark scanner uses the analyzer's default config for every repo to keep results comparable.", "- The analyzer still only scans JS/TS-family files, so non-JS/TS portions of mixed-language repos are out of scope.", diff --git a/tests/benchmark.test.ts b/tests/benchmark.test.ts index 8e82dba..4242233 100644 --- a/tests/benchmark.test.ts +++ b/tests/benchmark.test.ts @@ -21,7 +21,12 @@ describe("benchmark support", () => { expect(set.repos.some((repo) => repo.id === "gstack" && repo.cohort === "explicit-ai")).toBe( true, ); - expect(set.repos.some((repo) => repo.id === "ni" && repo.cohort === "mature-oss")).toBe(true); + expect(set.repos.some((repo) => repo.id === "zustand" && repo.cohort === "mature-oss")).toBe( + true, + ); + expect(set.repos.some((repo) => repo.id === "payload" && repo.cohort === "mature-oss")).toBe( + true, + ); }); test("creates a benchmark snapshot and report from local fixture repos", async () => {