known-ai-vs-solid-oss.json

{
  "schemaVersion": 1,
  "id": "known-ai-vs-solid-oss",
  "name": "Known AI repos vs older solid OSS repos",
  "description": "Compare a cohort of known AI-generated JavaScript/TypeScript repos against well-regarded OSS repos, with the mature-OSS cohort pinned to the latest default-branch commit on or before 2025-01-01, using exact commit SHAs and normalized analyzer metrics.",
  "artifacts": {
    "checkoutsDir": "benchmarks/.cache/checkouts/known-ai-vs-solid-oss",
    "snapshotPath": "benchmarks/results/known-ai-vs-solid-oss.json",
    "reportPath": "reports/known-ai-vs-solid-oss-benchmark.md"
  },
  "repos": [
    {
      "id": "openusage",
      "repo": "robinebers/openusage",
      "url": "https://github.com/robinebers/openusage.git",
      "cohort": "explicit-ai",
      "ref": "857f537a243483acf98ccd9ea32e20b380c63823",
      "createdAt": "2026-02-01T07:37:03Z",
      "stars": 1715,
      "provenance": "README: \"Not a single line of code in this project was read or written by hand. 100% AI-generated, AI-reviewed, AI-shipped\"",
      "notes": "Useful negative control because it does not look uniformly slop-heavy on every normalized metric."
    },
    {
      "id": "devworkbench",
      "repo": "jiayun/DevWorkbench",
      "url": "https://github.com/jiayun/DevWorkbench.git",
      "cohort": "explicit-ai",
      "ref": "ea50862107c377b0cdd6e508073ad0330b367576",
      "createdAt": "2025-06-25T03:39:53Z",
      "stars": 17,
      "provenance": "README: \"100% AI-Generated Code - A Totally Vibed Application\"",
      "notes": "Desktop-style TypeScript application with explicit AI-generation disclosure."
    },
    {
      "id": "fulling",
      "repo": "FullAgent/fulling",
      "url": "https://github.com/FullAgent/fulling.git",
      "cohort": "explicit-ai",
      "ref": "d95060f8421b5888564fc8517c1fe99ded463985",
      "createdAt": "2025-10-11T02:31:07Z",
      "stars": 2413,
      "provenance": "README: \"100% AI-generated code.\"",
      "notes": "Large AI-built full-stack engineering agent project."
    },
    {
      "id": "openclaw",
      "repo": "openclaw/openclaw",
      "url": "https://github.com/openclaw/openclaw.git",
      "cohort": "explicit-ai",
      "ref": "44cf74717b4507245648a0263f3b0df85aa2a813",
      "createdAt": "2025-11-24T10:16:47Z",
      "stars": 350232,
      "provenance": "User-provided provenance: repo is AI-generated.",
      "notes": "Very large TypeScript-heavy AI-generated product repo used to test whether the analyzer keeps signal on big codebases."
    },
    {
      "id": "emdash",
      "repo": "emdash-cms/emdash",
      "url": "https://github.com/emdash-cms/emdash.git",
      "cohort": "explicit-ai",
      "ref": "dbaf8c6f857456b6a449e68d26c5000c1143fadb",
      "createdAt": "2026-04-01T08:45:02Z",
      "stars": 7842,
      "provenance": "User-provided provenance: repo is AI-generated.",
      "notes": "Large Astro-based CMS that helps separate polished AI-generated product code from obvious slop-heavy toy repos."
    },
    {
      "id": "gstack",
      "repo": "garrytan/gstack",
      "url": "https://github.com/garrytan/gstack.git",
      "cohort": "explicit-ai",
      "ref": "6cc094cd4190e3fac65b69fb29e0c3cae7e0f214",
      "createdAt": "2026-03-11T21:22:45Z",
      "stars": 65613,
      "provenance": "User-provided provenance: repo is AI-generated.",
      "notes": "Popular AI-coding workflow/tooling repo with a substantial TypeScript surface and very high try/catch density."
    },
    {
      "id": "hunk",
      "repo": "modem-dev/hunk",
      "url": "https://github.com/modem-dev/hunk.git",
      "cohort": "explicit-ai",
      "ref": "b37663f3c3055f860290587865199192e15d9f35",
      "createdAt": "2026-03-17T19:16:15Z",
      "stars": 352,
      "provenance": "User-provided provenance: repo is AI-generated.",
      "notes": "Review-first terminal diff viewer that serves as a lower-slop AI-generated benchmark example."
    },
    {
      "id": "vinext",
      "repo": "cloudflare/vinext",
      "url": "https://github.com/cloudflare/vinext.git",
      "cohort": "explicit-ai",
      "ref": "28980b01f1692b47756a6768fc5733712c290f84",
      "createdAt": "2026-02-24T05:07:41Z",
      "stars": 7709,
      "provenance": "README: \"The vast majority of the code, tests, and documentation were written by AI (Claude Code). Humans direct architecture, priorities, and design decisions, but have not reviewed most of the code line-by-line.\"",
      "notes": "Polished, company-backed Vite-based reimplementation of the Next.js API surface built as an AI-driven software-development experiment."
    },
    {
      "id": "agent-ci",
      "repo": "redwoodjs/agent-ci",
      "url": "https://github.com/redwoodjs/agent-ci.git",
      "cohort": "explicit-ai",
      "ref": "4de00d69487e275d5bc37e30f7818a9303b22352",
      "createdAt": "2026-02-09T20:24:25Z",
      "stars": 120,
      "provenance": "User-provided provenance: repo is almost entirely Claude-authored.",
      "notes": "Local GitHub Actions orchestration layer for agent workflows; useful benchmark for command-heavy CI tooling with strong AI involvement."
    },
    {
      "id": "zustand",
      "repo": "pmndrs/zustand",
      "url": "https://github.com/pmndrs/zustand.git",
      "cohort": "mature-oss",
      "ref": "2e6d8813095c6a79ca208bae4c2cf5edc12049a1",
      "createdAt": "2019-04-09T09:10:06Z",
      "stars": 57758,
      "provenance": "Mature React state management library by pmndrs.",
      "notes": "Pinned to the latest default-branch commit on or before 2025-01-01 to test a later mature-OSS baseline without using a package-manager CLI."
    },
    {
      "id": "payload",
      "repo": "payloadcms/payload",
      "url": "https://github.com/payloadcms/payload.git",
      "cohort": "mature-oss",
      "ref": "f3f36d801010f3c95ae74655ff22a09ea66ab1ac",
      "createdAt": "2021-01-05T18:49:45Z",
      "stars": 41856,
      "provenance": "Mature open-source full-stack Next.js CMS/framework by Payload.",
      "notes": "Pinned to the latest default-branch commit on or before 2025-01-01 to test a later mature-OSS full-stack app/framework baseline."
    },
    {
      "id": "node-notifier",
      "repo": "mikaelbr/node-notifier",
      "url": "https://github.com/mikaelbr/node-notifier.git",
      "cohort": "mature-oss",
      "ref": "b36c237f0d913f9df3a2bd45adc08b33ff717f6a",
      "createdAt": "2012-12-02T09:53:24Z",
      "stars": 5843,
      "provenance": "Long-running OSS notification library for Node.js.",
      "notes": "Useful older utility-style baseline for a notification-oriented AI repo."
    },
    {
      "id": "tsup",
      "repo": "egoist/tsup",
      "url": "https://github.com/egoist/tsup.git",
      "cohort": "mature-oss",
      "ref": "cd03e1e00ec2bd6676ae1837cbc7e618ab6a2362",
      "createdAt": "2020-03-19T16:23:41Z",
      "stars": 11198,
      "provenance": "Mature TypeScript build tool widely used in the ecosystem.",
      "notes": "Compact older TS tool with fairly low normalized slop scores."
    },
    {
      "id": "execa",
      "repo": "sindresorhus/execa",
      "url": "https://github.com/sindresorhus/execa.git",
      "cohort": "mature-oss",
      "ref": "99d1741d2525eca71b986282148bbf2983356428",
      "createdAt": "2015-12-05T22:57:03Z",
      "stars": 7481,
      "provenance": "Long-lived process execution library for Node.js.",
      "notes": "Helpful benchmark for command-heavy code that still avoids extreme normalized scores."
    },
    {
      "id": "hyper",
      "repo": "vercel/hyper",
      "url": "https://github.com/vercel/hyper.git",
      "cohort": "mature-oss",
      "ref": "2a7bb18259d975f27b30b502af1be7576f6f5656",
      "createdAt": "2016-07-01T06:01:21Z",
      "stars": 44687,
      "provenance": "Older high-visibility TypeScript/Electron terminal project.",
      "notes": "Large, mature TS app baseline for GUI-oriented codebases."
    },
    {
      "id": "umami",
      "repo": "umami-software/umami",
      "url": "https://github.com/umami-software/umami.git",
      "cohort": "mature-oss",
      "ref": "227b2554b4a373e63ceb7f48decdc60c8d3e6eaf",
      "createdAt": "2020-07-17T07:59:00Z",
      "stars": 36012,
      "provenance": "Mature privacy-focused analytics platform with a large TypeScript codebase.",
      "notes": "Good counterweight for explicit-AI full-stack app comparisons."
    },
    {
      "id": "vite",
      "repo": "vitejs/vite",
      "url": "https://github.com/vitejs/vite.git",
      "cohort": "mature-oss",
      "ref": "a4922537a8d705da7769d30626a0d846511fc124",
      "createdAt": "2020-04-21T05:03:57Z",
      "stars": 79637,
      "provenance": "Mature frontend tooling project with a large TypeScript codebase.",
      "notes": "Added as a stress-test baseline to check that the analyzer does not overfit against only smaller mature repos."
    },
    {
      "id": "astro",
      "repo": "withastro/astro",
      "url": "https://github.com/withastro/astro.git",
      "cohort": "mature-oss",
      "ref": "f7068995aa451dced13853789b0d51433c2373b5",
      "createdAt": "2021-03-15T17:19:47Z",
      "stars": 58212,
      "provenance": "Mature web framework with a very large TypeScript/JavaScript monorepo.",
      "notes": "Added as a stress-test baseline for a large, actively maintained modern framework codebase."
    }
  ],
  "pairings": [
    {
      "aiRepoId": "devworkbench",
      "solidRepoId": "hyper",
      "notes": "GUI-oriented TypeScript app vs older terminal/desktop-style TS app."
    },
    {
      "aiRepoId": "openusage",
      "solidRepoId": "umami",
      "notes": "Full-stack analytics-adjacent SaaS-style app comparison and negative control."
    },
    {
      "aiRepoId": "vinext",
      "solidRepoId": "vite",
      "notes": "AI-built Vite-based framework layer vs the mature Vite baseline."
    }
  ]
}