diff --git a/.env-sample b/.env-sample deleted file mode 100644 index aecf9b4..0000000 --- a/.env-sample +++ /dev/null @@ -1,7 +0,0 @@ -SUPABASE_URL= -SUPABASE_ANON_KEY= -HOST= -TWITCH_CLIENT_ID= -TWITCH_CLIENT_SECRET= -TWITCH_CHANNEL_ID= -TWITCH_ACCESS_TOKEN= diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..411c8f5 --- /dev/null +++ b/.env.example @@ -0,0 +1,54 @@ +# Copy to .env and fill in. Everything here is also set on Netlify, per environment. +# +# The site builds and runs with none of these. Supabase driven parts stay quiet rather +# than failing, so a fork or a first clone works before anybody has been given keys. +# See supabase/README.md for where each value comes from. + +# Safe to expose. Ships in the browser bundle by design. +PUBLIC_SUPABASE_URL= + +# The v2 project ref, for pnpm types. Not defaulted on purpose: the old ref still serves +# the live site and regenerating against it would quietly empty the generated types. +SUPABASE_PROJECT_REF= + +PUBLIC_SUPABASE_ANON_KEY= + +# Never expose. Bypasses row level security completely. Server only, and it is what +# every write on the site goes through. +SUPABASE_SERVICE_ROLE_KEY= + +# Rotatable secret used to hash a visitor's IP into a like dedupe token. Rotating it +# forgets who liked what without ever having stored an address. Any long random string. +LIKE_IP_SECRET= + +# Drafts a title, a one line summary and a slug for a submitted dev disaster. Without a +# key the drafter falls back to the story's own opening line and never calls out, which +# works but reads flatter. AI_API_URL and AI_MODEL default to OpenAI chat completions and +# gpt-4o-mini, so only the key is usually needed. +AI_API_KEY= +AI_API_URL= +AI_MODEL= + +# Used once, after a data load, if the badge matcher is to key on stable Twitch ids +# rather than on logins. Not needed at build time or at runtime. See docs/backfill.md. +TWITCH_CLIENT_ID= +TWITCH_CLIENT_SECRET= + +# PARKED FOR V1. The three below are listed for completeness and must stay unset. v1 sends +# no email of any kind, so there is no sender, no address and no drain on a timer. Setting +# any of them is step one of turning notifications on, which is a decision with a real +# ongoing cost attached, not a configuration gap to be filled in. docs/notifications.md is +# the procedure, including the copy on submit, privacy, terms and account that has to +# change back at the same time. docs/new-project.md deliberately omits all three. +# +# RESEND_API_KEY Mail provider key. src/lib/mail.ts is written but nothing calls it. +# MAIL_FROM From address. Would need a domain verified with the provider. +# NOTIFY_SECRET Bearer token for the drain at /api/notifications/. Unset, that route +# 404s every request before it touches the database, which is the only +# reason it is safe to leave in the route table. The table it would drain +# does not exist either: the enqueue trigger and email_outbox are both +# held in supabase/deferred/, so nothing queues and there is nothing to +# send. Do not read these as "configured off". They are absent. +RESEND_API_KEY= +MAIL_FROM= +NOTIFY_SECRET= diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..f41fae8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +# Generated artifacts are committed so CI can regenerate them and diff. That only works +# if the checked out bytes match what the generator writes, so these stay LF on every +# platform regardless of core.autocrlf. +src/styles/themes.css text eol=lf +src/lib/themes.generated.ts text eol=lf +src/lib/ec-themes.generated.mjs text eol=lf +src/config/taxonomy.json text eol=lf +public/_redirects text eol=lf diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..a3c1210 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,190 @@ +name: CI + +on: + push: + pull_request: + workflow_dispatch: + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: true + +jobs: + build: + name: Build and gates + runs-on: ubuntu-latest + timeout-minutes: 30 + + steps: + + - uses: actions/checkout@v7 + - name: Check out the content submodule + env: + CONTENT_DEPLOY_KEY: ${{ secrets.CONTENT_DEPLOY_KEY }} + run: | + if [ -z "$CONTENT_DEPLOY_KEY" ]; then + echo "::error::CI cannot read src/content, so no collection would have any entries." + echo "src/content is a git submodule pointing at michaeljolley/content, which is" + echo "private. The default GITHUB_TOKEN is scoped to this repository only and" + echo "cannot clone another one." + echo "" + echo "Fix, once, by hand:" + echo " 1. ssh-keygen -t ed25519 -C 'baldbeardedbuilder.com CI' -f content-ci -N ''" + echo " 2. On michaeljolley/content, Settings, Deploy keys, Add deploy key." + echo " Paste content-ci.pub. Leave write access UNCHECKED, a build only reads." + echo " 3. On this repository, save the private half, the whole content-ci file" + echo " including its BEGIN and END lines, as the secret CONTENT_DEPLOY_KEY." + echo " 4. Delete both local files. GitHub keeps the only copies that matter." + echo "" + echo "A deploy key rather than a token on purpose. A token expires and is tied to" + echo "a person, so it brings this same failure back later without warning." + echo "" + echo "This job fails rather than building without content, because an empty" + echo "collection set makes every gate below pass for the wrong reason." + exit 1 + fi + mkdir -p ~/.ssh + chmod 700 ~/.ssh + # printf rather than echo, because a key is worthless if its final newline is lost. + printf '%s\n' "$CONTENT_DEPLOY_KEY" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + ssh-keyscan -t rsa,ecdsa,ed25519 github.com >> ~/.ssh/known_hosts 2>/dev/null + git submodule update --init --depth 1 src/content + if [ ! -f src/content/content.config.ts ]; then + echo "::error::src/content was cloned but content.config.ts is not in it." + exit 1 + fi + echo "src/content present, $(find src/content -name '*.md' | wc -l) markdown files." + + - uses: pnpm/action-setup@v6 + + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + + - run: pnpm install --frozen-lockfile + + # Fails if a generated artifact was hand edited. themes.css, the font CSS, the + # taxonomy map and _redirects are all outputs, never inputs. + - name: Generated artifacts are in sync + run: pnpm gen:check + + - name: Unit and redirect tests + run: pnpm test + + # The baseline was trimmed to the two legacy tables v2 actually reads, so a + # migration leaning on something that went would only fail on a db push against + # a fresh project, which is the worst place to find out. + - name: Migration chain is self contained + run: pnpm check:migrations + + # Catches the class of mistake that a build will happily ship: a Supabase column + # that changed shape under a query, a nullable view column read as if it were not. + - name: Types + run: pnpm check + + - name: Build + run: pnpm build + + # The step above is the first thing in this job that produces dist, and four tests + # in redirects.build.test.mjs need it. They were only in the run above, before the + # build, so they skipped on every run and reported green while asserting nothing. + # Run again here, where dist exists. REQUIRE_DIST turns the skip into a failure, so + # this cannot quietly stop working again if the steps are ever reordered. + - name: Redirect tests against the real build + run: pnpm test + env: + REQUIRE_DIST: '1' + + # Reads the built output rather than the source, because the sitemap and the + # Pagefind index are generated and so are never reviewed by a person. Catches a + # parked route that still ships, and any page listed in the sitemap whose own + # markup says noindex. + - name: Shipped output + run: pnpm check:dist + + # In this job rather than the browser one because it needs no browser, and it reads + # source as well as dist. /report/ is prerender = false, so it writes no file and + # every gate that works from the built output is blind to it. a11y is the exception, + # since it starts a dev server for exactly that reason. + - name: Published addresses + run: pnpm check:emails + + # Both browser gates need dist, and a build is slow enough that handing it over + # beats building it three times. + - name: Upload dist + uses: actions/upload-artifact@v7 + with: + name: dist + path: dist + retention-days: 3 + + a11y: + name: Accessibility and layout + runs-on: ubuntu-latest + needs: build + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist + + - run: pnpm exec playwright install --with-deps chromium + - run: pnpm a11y + + # Rides along in this job because it is the only one that pays for a browser, and + # installing chromium twice to run a second thirty second check is not worth it. + # Different question from accessibility, same requirement: a laid out page. + - name: Layout geometry + run: pnpm check:layout + + # Counts intents rather than measuring anything, so it is its own step: a doubled + # count has no visible symptom, and a failure here should not read as a layout one. + - name: Share intents + run: pnpm check:share + + # Separate from the accessibility job on purpose. axe only reports a missing id when + # the element needed a name to be usable, so this catches a class that job is right + # to stay quiet about, and a failure here should not read as an axe one. + - name: Id references + run: pnpm check:aria + + # Rides along here because it needs a browser and a built dist. Sixteen themes times + # five heading levels is 96 computed colors, and the failure being guarded is a + # heading that reads as body text or as a link rather than one that looks broken. + - name: Prose heading color + run: pnpm check:headings + + perf: + name: Performance budget + runs-on: ubuntu-latest + needs: build + timeout-minutes: 30 + + steps: + - uses: actions/checkout@v7 + - uses: pnpm/action-setup@v6 + - uses: actions/setup-node@v7 + with: + node-version: 22 + cache: pnpm + - run: pnpm install --frozen-lockfile + + - uses: actions/download-artifact@v8 + with: + name: dist + path: dist + + - run: pnpm exec playwright install --with-deps chromium + - run: pnpm perf diff --git a/.gitignore b/.gitignore index 16d54bb..9c023a5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,8 +1,13 @@ # build output dist/ +.netlify/ # generated types .astro/ +# Copied out of Fontsource by scripts/gen-fonts.mjs on every build. Fontsource is the +# source of truth, so a committed copy could only ever drift away from it. +public/fonts/ + # dependencies node_modules/ @@ -14,11 +19,41 @@ pnpm-debug.log* # environment variables -.env -.env.production +# +# Ignore every .env variant and name the exceptions, rather than naming the variants. +# This rule used to be the two files that happened to exist the day it was written, which +# meant .env.local, .env.audit-bak or anything else a debugging session leaves behind was +# untracked AND unignored, so one git add -A stages real credentials into a public repo. +# Now an unknown .env file is ignored by default and only an explicitly allowed one is +# tracked. tests/secrets.test.mjs asserts nothing env shaped is ever left visible. +.env* +!.env-sample +!.env.example + +# The same thing without the leading dot. env.local and env.backup get created by mistake +# often enough to be worth a rule, and tests/secrets.test.mjs proved this case was still +# open after the rule above was widened. Anchored to the root with a leading slash so it +# cannot reach src/env.d.ts, which is tracked source and must stay visible. +/env* + +# Screenshot output from the theme and state passes. Large, binary, and superseded on +# every run, so it belongs in the session files rather than in history. +# +# The harnesses that produce them are deliberately NOT ignored. They have to sit in the +# repo root to resolve their imports, and they are meant to be deleted once the pass is +# done. Left visible in git status, a leftover harness nags until it is removed. Ignored, +# it would quietly accumulate. +shots-*/ # macOS-specific files .DS_Store # jetbrains setting folder .idea/ + +# Supabase CLI scratch, holds the linked project ref +supabase/.temp/ + +# Resumable transcript cache and Supabase import files. These are large backfill artifacts, +# not application source, and can be regenerated from the video catalogue. +/backfill/ diff --git a/README.md b/README.md index 7629c53..b7a42b2 100644 --- a/README.md +++ b/README.md @@ -1,130 +1,88 @@ # baldbeardedbuilder.com -This is the source for my personal site. It's where the blog posts live, where the -YouTube videos get indexed, and where folks sign up for The .NET Drip. If you're here -to fix a typo in a post, poke at the layout, or just see how the sausage gets made, -you're in the right place. +The personal site of Michael Jolley, the Bald Bearded Builder. Articles, videos, +dev disasters and the .NET Drip signup. Built with Astro, deployed on Netlify, +with Supabase behind the parts that need an account. -It's an [Astro](https://astro.build) site that builds to static files and deploys to -Netlify. Nothing fancy. That's on purpose. - -## Get it running - -You'll need Node 18 or newer and git. +## Running it ```sh -git clone --recurse-submodules git@github.com:baldbeardedbuilder/website.git -cd website -npm install -cp .env-sample .env -npm run dev +pnpm install +pnpm dev ``` -That'll put the site on `http://localhost:4321`. +That serves on `localhost:4321`. Use pnpm, not npm. + +Versions are pinned in two places on purpose and they have to agree: `packageManager` +in `package.json` (currently `pnpm@9.1.1`) and `PNPM_VERSION` in `netlify.toml` +(currently `9`). Bumping one without the other means local and production build +with different tooling, which is the kind of difference that shows up as a broken +deploy and nothing else. `netlify.toml` also pins `NODE_VERSION = "24"`, while +`engines` asks only for Node 22 or newer, so local can be older than production. -If you already cloned without `--recurse-submodules`, you'll get a very empty site and -some confusing errors. Fix it with: +## The content is a submodule + +`src/content` is a git submodule pointing at `michaeljolley/content`, which is a +private repository. Without it every collection is empty, the site builds to +almost nothing, and any check you run passes for the wrong reason. ```sh git submodule update --init --recursive ``` -### About that submodule - -The content isn't in this repo. Blog posts, video metadata, and the content collection -schema all live in a separate private repo mounted at `src/content`. That means the -site build needs it, but you can't pull it unless you have access. - -If you don't have access to the content repo, the build won't complete. Sorry about -that. It's a tradeoff I took on purpose so I can edit posts without touching the site -code, but it does make outside contributions harder. If you want to help with something -that needs content to render, open an issue and I'll figure out a way to unblock you. - -## Environment variables - -Copy `.env-sample` to `.env` and fill in what you need. - -| Variable | What it does | -| :--- | :--- | -| `TWITCH_CLIENT_ID` | App credentials for the Twitch API | -| `TWITCH_CLIENT_SECRET` | App credentials for the Twitch API | -| `TWITCH_CHANNEL_ID` | Reserved, not currently read by the site | -| `TWITCH_ACCESS_TOKEN` | Reserved, the code fetches its own token at build time | -| `SUPABASE_URL` | Reserved, not currently read by the site | -| `SUPABASE_ANON_KEY` | Reserved, not currently read by the site | -| `HOST` | Reserved, not currently read by the site | - -The Twitch credentials are the only ones the build actually uses right now. They power -the "Live on Twitch Now" state on the homepage. Without them, the homepage build will -fail when it tries to call the Twitch API. The rest are leftovers from earlier versions -that I've left in the sample so I remember what the deployed environment expects. - -## Commands - -| Command | What it does | -| :--- | :--- | -| `npm install` | Installs dependencies | -| `npm run dev` | Dev server on `localhost:4321` | -| `npm run build` | Builds the production site to `./dist/` | -| `npm run preview` | Serves the built site so you can check it before deploy | -| `npm run astro ...` | Runs Astro CLI commands like `astro check` | - -Both `package-lock.json` and `pnpm-lock.yaml` are checked in. npm is what CI uses, so -that's the safe choice. - -## How it's laid out - -```text -public/ static assets, redirects, netlify.toml -src/ - components/ the reusable pieces (cards, sections, header, footer) - content/ git submodule, all posts and video metadata - layouts/ Layout.astro wraps every page - pages/ file-based routing, each file is a route - scripts/ twitch.ts and publish.ts, the bits of real logic - styles/ global.css -``` +Treat it as read only from here. Anything that would mean editing frontmatter +across the submodule belongs in `src/config/` instead. -A few things worth knowing before you go editing: +CI needs its own credential to read it, since the default `GITHUB_TOKEN` is scoped +to this repository and cannot clone another one. That is the secret +`CONTENT_DEPLOY_KEY`, and it holds the private half of a read only deploy key on +`michaeljolley/content` rather than a personal access token. The CI step prints the +four steps to create one when the secret is missing. -**Posts publish themselves.** `src/scripts/publish.ts` decides whether a post is live -based on its `pubDate` and an 8 AM Central publish time. It does its own daylight saving -math instead of pulling in a date library. That's a small amount of code doing a job a -dependency could do, and I'd rather own the fifty lines than the dependency tree. +A key rather than a token on purpose. A fine grained token expires at twelve months +at the most and dies with the account that issued it, so it brings this same failure +back later and silently. Netlify solved the same problem on the same pair of +repositories in 2024 with a deploy key, so the precedent was already live. -**Future posts get one preview page.** `src/pages/blog/[slug].astro` generates pages for -every published post plus exactly one upcoming post, rendered as a teaser. Everything -further out doesn't exist yet as far as the site is concerned. +## Generated files, and why the build fights you about them -**Twitch is checked at build time, not in the browser.** The homepage calls the Twitch -API during the build. So the "live now" badge is only as fresh as the last deploy. Good -enough for what it does, and it keeps the page static. +`pnpm gen` runs automatically before every build and at the start of `pnpm dev`. +It writes: -**Old URLs are honored.** `public/_redirects` and `public/netlify.toml` map a decade of -old blog paths to their current homes. If you rename a post slug, add the redirect. -Somebody out there has that link bookmarked. +| Output | From | +| :-- | :-- | +| `src/styles/themes.css`, `src/lib/themes.generated.ts`, `src/lib/ec-themes.generated.mjs` | `scripts/gen-themes.mjs`, which resolves real VS Code themes through shiki | +| `src/styles/fonts.generated.css` | `scripts/gen-fonts.mjs` | +| `src/config/taxonomy.json` | `scripts/gen-taxonomy.mjs` | +| `public/_redirects` | `scripts/gen-redirects.mjs` | -## Contributing +**Never hand edit any of those.** Edit the generator. `pnpm gen:check` fails the +build if a generated file differs from what its generator produces, which is +what stops a hand edit surviving to production. -Found a typo in a post? The post itself lives in the content submodule, so open an issue -here and I'll get to it. Found something broken in the site (layout, accessibility, a -link that goes nowhere, a build that falls over)? Pull requests are welcome. +## Checks -Keep changes focused. One idea per PR is easier for both of us. - -## When this repo isn't what you want - -If you're looking for an Astro blog starter you can fork and make your own, this isn't -it. The content is private, the styling is very much mine, and there's site-specific -logic baked in all over. Start from `npm create astro@latest` instead. You'll have a -better time. +```sh +pnpm test # unit and redirect tests +pnpm check # astro check +pnpm check:migrations # the SQL chain is self contained +pnpm check:dist # parked routes, sitemap and Pagefind agree with the routes +pnpm check:layout # thumbnail crop and dead space, measured in a browser +pnpm a11y # axe, WCAG 2.2 AA +pnpm perf # Lighthouse budget +``` -But if you want to read how a real site handles scheduled publishing, build-time API -calls, or a decade of URL redirects, dig in. That's the useful part. +`pnpm check:dist`, `check:layout`, `a11y` and `perf` all read `dist`, so run +`pnpm build` first. -## License +`pnpm verify:deploy` runs against a deployed URL rather than a local build. It is +the only check that can catch a Netlify setting being wrong, because everything +else passes happily on a machine where those settings do not apply. -MIT. See [LICENSE](./LICENSE). +## Further reading -The code is MIT. The blog posts, videos, and branding are not. Please don't repost the -writing as your own. +- `docs/deploy.md` for Netlify build settings and the branch deploy. +- `docs/new-project.md` for standing up the Supabase project, including the + GitHub OAuth callback trap that repoints production sign in if you get it + wrong. +- `docs/backfill.md` for data imports and `docs/notifications.md` for email operations. diff --git a/astro.config.mjs b/astro.config.mjs index 0ee0d4d..eea8cf7 100644 --- a/astro.config.mjs +++ b/astro.config.mjs @@ -1,35 +1,123 @@ // @ts-check +import fs from 'node:fs'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; import { defineConfig } from 'astro/config'; import expressiveCode from 'astro-expressive-code'; import sitemap from '@astrojs/sitemap'; +import preact from '@astrojs/preact'; +import netlify from '@astrojs/netlify'; +import { rehypeDemoteHeadings } from './src/lib/rehype-demote-headings.mjs'; + +/* + Every route whose page asks not to be indexed. + + Two mechanisms, because there are two kinds of page and neither covers the other. + + serialize below is the real one. It runs after the build has written dist, so it can + read the robots meta each page actually emitted and drop anything that says noindex. + That is the only thing that catches dynamic routes, and dynamic routes were where the + damage was: draft posts, filtered topic views and every dev disaster sort permutation + were all correctly marked noindex in their own markup and all listed in the sitemap + anyway. Five unpublished drafts were being submitted to Google. + + noindexRoutes covers what serialize cannot see. On demand pages write no file, so there + is no markup to read, and they reach the sitemap purely as routes. That is how + /unsubscribe/ stayed listed after being parked, and how /account/ was listed at all. + + This file used to assert that on demand pages are invisible to the sitemap and that + report was therefore excluded. Both halves were false. Nothing surfaced it because a + sitemap is generated and never read by a person. +*/ +function noindexRoutes() { + const root = fileURLToPath(new URL('./src/pages/', import.meta.url)); + /** @type {string[]} */ + const out = []; + /** @param {string} dir @param {string} prefix */ + const walk = (dir, prefix) => { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + // Astro does not route a leading underscore, so nothing under one can be listed. + if (entry.name.startsWith('_')) continue; + const full = path.join(dir, entry.name); + if (entry.isDirectory()) { + walk(full, `${prefix}${entry.name}/`); + } else if (entry.name.endsWith('.astro') && !entry.name.includes('[')) { + /* + Only counts as noindex when it is an attribute on a component tag. A bare word + match would exclude any page that merely writes the word in its prose, and this + site has pages that explain their own indexing. + */ + const src = fs.readFileSync(full, 'utf8'); + if (!/<[A-Z][\w.]*[^>]*\bnoindex\b/s.test(src)) continue; + const base = entry.name.replace(/\.astro$/, ''); + out.push(base === 'index' ? prefix : `${prefix}${base}/`); + } + } + }; + walk(root, '/'); + return out; +} + +const NOINDEX = noindexRoutes(); + +/** + True when the page this url built to emitted a noindex robots meta. + + Fails open on a missing file, which is the right direction here: a sitemap entry that + should not be there is caught by scripts/check-dist.mjs immediately afterwards, whereas + silently dropping real pages from the sitemap would be invisible. Config fails open, the + gate fails closed. + + @param {string} pathname +*/ +function builtNoindex(pathname) { + const file = fileURLToPath(new URL(`./dist${pathname}index.html`, import.meta.url)); + if (!fs.existsSync(file)) return false; + return /]+name=["']robots["'][^>]*noindex/i.test(fs.readFileSync(file, 'utf8')); +} // https://astro.build/config export default defineConfig({ site: 'https://baldbeardedbuilder.com', trailingSlash: 'always', + output: 'static', + adapter: netlify(), devToolbar: { enabled: false }, + /* + See src/lib/rehype-demote-headings.mjs. A markdown body that opens with its own title + would otherwise put a second h1 on a page that already has one. + */ + markdown: { + rehypePlugins: [rehypeDemoteHeadings] + }, integrations: [ - expressiveCode({ - themes: ['laserwave'], - styleOverrides: { - frames: { - tooltipSuccessBackground: '#9333ea' - }, - uiFontFamily: 'inherit', - codeCopyButtonBackground: 'transparent', - codeCopyButtonBorder: 'none', - codeCopyButtonBorderColor: 'transparent', - codeCopyButtonHoverBackground: 'rgba(147, 51, 234, 0.2)', - codeCopyButtonActiveBackground: 'rgba(147, 51, 234, 0.3)', - codeCopyButtonHoverOrFocusBackground: 'rgba(147, 51, 234, 0.2)' + // Options live in ec.config.mjs. See the comment at the top of that file. + expressiveCode(), + preact({ compat: false }), + sitemap({ + filter: (page) => { + // Profiles are noindex by decision 14, so they stay out of the sitemap too. + if (page.includes('/builders/')) return false; + const route = new URL(page).pathname; + return !NOINDEX.includes(route); }, - frames: { - showCopyToClipboardButton: true, - copyButtonTooltipText: 'Copy this snippet' - } - }), - sitemap() + + /* + Runs at astro:build:done, so dist exists and every prerendered page can be asked + directly whether it wants to be indexed. Returning undefined drops the entry. + */ + serialize: (item) => (builtNoindex(new URL(item.url).pathname) ? undefined : item), + + /* + Submit is rendered on demand, because it has to know whether the reader is signed + in before it draws a form that needs a sign in. It is a page people should be able + to find, so it is named here as well. Astro dedupes, so listing it twice is + harmless, and naming it keeps it in the sitemap if it is ever prerendered behind a + different route shape. + */ + customPages: ['https://baldbeardedbuilder.com/submit/'] + }) ] }); diff --git a/docs/backfill.md b/docs/backfill.md new file mode 100644 index 0000000..e9cf433 --- /dev/null +++ b/docs/backfill.md @@ -0,0 +1,439 @@ +# Backfill spec + +Everything the site can render but cannot produce for itself. The code is written and the +tables are live. This file describes the shape each one expects so a load can be prepared +without reading migrations. + +Treat it the way you would treat a migration. A wrong column type found halfway through a +24,000 row load is not an edit, it is a migration plus a reload. + +**No import script ships with this repo, by decision.** Loads happen outside it. What ships +is the schema, the constraints, and this document. + +## The badge tiers assume the whole history lands + +Read this before deciding to load part of it. + +`20260710000900_badge_thresholds_real_history.sql` did not pick round numbers. It measured +the real distribution and picked thresholds that land people on the tiers: 1,854 people, 113 +distinct stream days, a maximum attendance of 68. Front Row became 5, 15, 30 and 50 because +125 people clear 5, 38 clear 15, 19 clear 30, 6 clear 50, and nobody at all clears 100, so a +top tier of 100 was a plaque that could never light up. + +**None of that data exists in the new project until it is loaded.** A partial load makes +every tier wrong, and wrong in the direction that is hardest to notice: the badges still +grant, the shelves still render, the progress bars still fill. They are just measuring +somebody against a scale built for a history that is not there. Nothing in the code can flag +this, because a smaller history is indistinguishable from a quieter channel. + +So there are two honest options. Load the full history, and the thresholds mean what they +were measured to mean. Or load part of it, and rerun the distribution query in the +"Verifying a load" section at the bottom of this file, then move the thresholds to match +what actually landed. What does not work is loading half and leaving the numbers alone. + +## Read this before anything else + +1. **Two identity columns will bite you.** `streamEvents.id` and `disasters.id` are + `generated by default as identity`. Supplying explicit ids is allowed and is usually what + you want when preserving a legacy key, but the sequence does not move when you do it. The + next insert then collides on the primary key. Resync immediately after any load that + supplied ids: + + ```sql + select setval(pg_get_serial_sequence('public."streamEvents"', 'id'), + coalesce((select max(id) from public."streamEvents"), 1)); + select setval(pg_get_serial_sequence('public.disasters', 'id'), + coalesce((select max(id) from public.disasters), 1)); + ``` + +2. **Camel case table and column names need double quotes.** `"streamEvents"`, + `"streamUsers"`, `"eventType"`, `"streamDate"`, `"lastUpdated"`. Unquoted they fold to + lower case and the statement fails on a missing relation. + +3. **RLS is on and denies by default.** Load as the service role or as `postgres` in the SQL + editor. An anon key will silently insert nothing. + +4. **Badges are never granted by a load.** Grants are computed. Load the source rows, then + run the sweep in step 5 of the load order. + +5. **Filling `streamUsers.twitch_user_id` disables the login fallback for those rows.** This + is the single most likely way to make a correct looking load produce empty badge shelves, + so it is worth reading twice. The matcher pairs a profile to history in two ways: an id + match, which wins outright, or a login match, which applies *only when the history row has + no id*. That second clause is what stops somebody who takes over an abandoned Twitch name + from inheriting the previous owner's badges. The consequence for a load is that the moment + a `streamUsers` row gains an id, any profile still matching it by login alone stops + matching. So either leave `twitch_user_id` null everywhere, or fill it on `streamUsers` + **and** on the matching `profiles` rows in the same pass. Filling one side is worse than + filling neither. + +6. **Every `streamDate` in the table is cast to `date`, for everybody.** The Day One window + runs `min("streamDate"::date)` across the whole table with no filter. One malformed string + anywhere, a `2025-13-01` or an empty value, raises on every call, which means every badge + shelf and every progress bar on the site, not only the row that is wrong. Validate the + column before running the sweep: + + ```sql + select "streamDate", count(*) from public."streamEvents" + where "streamDate" !~ '^\d{4}-\d{2}-\d{2}$' group by 1; + ``` + + That query should return nothing. If it returns rows, fix them before step 5. + +## Load order + +Later steps read earlier ones. Nothing else depends on order. + +| Step | What | Depends on | +| --- | --- | --- | +| 1 | `streamUsers` | nothing | +| 2 | `streamEvents` | 1, on `login` | +| 3 | `profiles.twitch_user_id` and `twitch_login` | 1. **Required, not optional, if step 1 filled any `twitch_user_id`.** See warning 5 | +| 4 | `video_transcripts` | nothing | +| 5 | `select public.backfill_badges();` | 1, 2, 3 | +| 6 | `disasters` | optional, `profiles` if attributed | + +Steps 1, 2, 4 and 6 are independent of each other and can run in any order or at once. +Steps 1 and 3 are one unit if ids are involved, and should land in the same transaction if +that is practical. + +There is no `streams` step. That table is not in this project. See the header of +`supabase/migrations/20260101000000_baseline.sql` for what stayed behind and why. + +--- + +## 1. `streamUsers` + +One row per person seen on stream. Supplies the display name and avatar on a badge shelf, +and carries the stable Twitch id that the badge matcher prefers over a login. + +| Column | Type | Null | Notes | +| --- | --- | --- | --- | +| `login` | `text` | no | Twitch login, lower case. Half of the unique key | +| `platform` | `text` | no | Defaults to `twitch`. Other half of the unique key | +| `avatar_url` | `text` | **no** | No default. A load that omits it fails | +| `display_name` | `text` | yes | Cased name as Twitch renders it | +| `lastUpdated` | `timestamp` | yes | **Without** time zone. Everything else in the schema is `timestamptz`. Pass a naive UTC value | +| `twitch_user_id` | `text` | yes | Stable numeric id as a string. Unique where not null | + +Keys and constraints: + +- `unique (login, platform)`. There is no separate primary key, so `on conflict (login, + platform)` is the upsert target. +- `unique (twitch_user_id) where twitch_user_id is not null` via + `streamusers_twitch_user_id_unique`. Partial, so any number of rows may leave it null, but + two rows may not claim the same id. +- No foreign keys either way. `streamEvents.login` is a soft reference with nothing enforcing + it, so an event may name a login that has no user row. The badge counter tolerates that. + +Example row: + +```sql +insert into public."streamUsers" (login, display_name, avatar_url, "lastUpdated", platform, twitch_user_id) +values ('baldbeardedbuilder', 'BaldBeardedBuilder', + 'https://static-cdn.jtvnw.net/jtv_user_pictures/abc123-profile_image-300x300.png', + '2026-05-18 14:22:10', 'twitch', '123456789') +on conflict (login, platform) do update + set display_name = excluded.display_name, + avatar_url = excluded.avatar_url, + "lastUpdated" = excluded."lastUpdated", + twitch_user_id = coalesce(excluded.twitch_user_id, public."streamUsers".twitch_user_id); +``` + +**`twitch_user_id` is worth filling even though it is nullable, but it is all or nothing.** +Logins change. A person who renames disappears from every login based match, which shows up as +an empty badge shelf for exactly the long standing community member the badges exist to +recognise. Ids come from the Twitch Helix `/users` endpoint, up to 100 logins per request. + +Filling it here without also filling it on the matching `profiles` rows breaks the login +fallback for those people. Warning 5 at the top of this file is the long version. The short +version: fill both sides or neither. + +## 2. `streamEvents` + +The event history. The badge counter reads this table and nothing else for presence, chat, +raids, subs, gifts and cheers. + +| Column | Type | Null | Notes | +| --- | --- | --- | --- | +| `id` | `bigint` | no | Identity, by default. See the sequence warning above | +| `created_at` | `timestamptz` | no | Defaults to `now()`. Set it explicitly on a historical load or every row looks like it happened on load day | +| `eventType` | `text` | no | See the vocabulary below. No check constraint, so a typo loads cleanly and silently counts nothing | +| `login` | `text` | no | Lower case Twitch login. Matched case insensitively | +| `streamDate` | `text` | **no** | A date stored as text, format `YYYY-MM-DD`. Not a `date` column | +| `message` | `text` | yes | Chat text where relevant | +| `quantity` | `bigint` | yes | Bits for a cheer, viewers for a raid, months for a sub. Null elsewhere | +| `platform` | `text` | no | Defaults to `twitch` | + +`eventType` vocabulary, exactly as the counter matches it: + +| Value | Counts toward | How | +| --- | --- | --- | +| any value at all | Front Row, and Day One | Distinct `streamDate`. See the note below | +| `onChatMessage` | Talker | Sum of `coalesce(quantity, 1)` | +| `twitch:raid` | Raider | Row count | +| `twitch:sub` | Subscriber | Row count | +| `twitch:giftsub` | Gifter | Sum of `coalesce(quantity, 1)`, so one row gifting five subs counts five | +| `twitch:cheer` | Cheerer | Row count, not bits | +| `onJoin`, `onPart`, `onCommand`, `twitch:follow` | presence only | No badge of their own | + +Three things that decide badge outcomes, so they matter more than they look: + +- **Presence is any event, not `onJoin`.** Front Row counts distinct `streamDate` values + across every row belonging to a person, whatever the type. Somebody whose only row on a day + is a chat message still attended that day. A load that omits `onJoin` therefore loses + nothing, and a load that emits one per reconnect costs nothing either, because the count is + distinct on the date. +- **`quantity` is load bearing for chat and gifts, and ignored for cheers.** Talker sums + `quantity` on `onChatMessage`, so a row that rolls up a day of chat into one record with + `quantity = 40` counts forty. If the load emits one row per message, leave `quantity` null + and it counts one. Gifter behaves the same way. Cheerer counts rows, so bits in `quantity` + do not inflate it, which is deliberate: the badge is for cheering, not for cheering a lot. +- **Day One is derived, not hard coded.** It is the first ninety days measured from the + earliest `streamDate` present in the table. Loading history that predates the current + earliest row moves the window and changes who qualifies. Load oldest first, or load + everything and then run the sweep once. + +Example row: + +```sql +insert into public."streamEvents" ("eventType", login, "streamDate", message, quantity, platform, created_at) +values ('twitch:cheer', 'someviewer', '2025-08-14', 'cheer100 nice save', 100, 'twitch', '2025-08-14T19:04:11Z'); +``` + +Volume note: the counter is indexed on `lower(login)` for this table +(`streamevents_login_lower_idx`). Loading with the indexes in place is fine at this size. If a +load runs to hundreds of thousands of rows, dropping and recreating that index around it is +faster, but the index must exist before step 5 or the sweep degrades to a sequential scan per +profile. + +## 3. `profiles`, Twitch identity only + +Do not load `profiles` rows. Every row is created by the sign in trigger against +`auth.users`, and `profiles.id` is a foreign key onto it. A hand inserted profile with no +matching auth user fails the constraint, and one with a fabricated id creates an account +nobody can sign into. + +What can be backfilled is the Twitch linkage on a profile that already exists, for the case +where somebody linked before ids were captured: + +| Column | Type | Null | Notes | +| --- | --- | --- | --- | +| `twitch_user_id` | `text` | yes | Unique. This is what the matcher prefers | +| `twitch_login` | `text` | yes | Lower case. The fallback match | +| `twitch_linked_at` | `timestamptz` | yes | Set it, an unset value reads as never linked | + +```sql +update public.profiles + set twitch_user_id = '123456789', + twitch_login = 'someviewer', + twitch_linked_at = now() + where handle = 'someviewer'; +``` + +Two constraints worth knowing before a bulk update: `handle` matches +`^[a-z0-9][a-z0-9-]{0,38}$`, and `twitch_user_id` is unique, so two profiles cannot claim one +Twitch account. That is intentional. It is the thing stopping a second account inheriting +somebody else's history. + +## 4. `video_transcripts` + +Read at build time. Per decision 22 a long form video with no clean transcript gets no page, +so the size of this table is the size of the video catalogue at launch. + +| Column | Type | Null | Notes | +| --- | --- | --- | --- | +| `video_id` | `text` | no | Primary key. The YouTube id, which is also the content collection id | +| `source` | `text` | yes | One of `youtube`, `whisper`, `manual`. Checked | +| `language` | `text` | no | Defaults to `en` | +| `segments` | `jsonb` | yes | Array of `{ start, end, speaker?, text }`, seconds as numbers | +| `body` | `text` | yes | Flat transcript. **This is the field that decides whether a page exists.** Null or empty means no page | +| `chapters` | `jsonb` | yes | Array of `{ start, title }`, seconds as numbers | +| `duration` | `integer` | yes | Seconds | +| `transcript_updated_at` | `timestamptz` | yes | | +| `chapters_updated_at` | `timestamptz` | yes | | + +Chapters and transcripts arrive by different routes and are deliberately separable. Chapters +come back from YouTube's public player data with no credentials, and `scripts/gen-video-meta.mjs` +already fetches them. Transcripts do not: the public caption endpoint now returns an empty +body without a proof of origin token. The route that works is the YouTube Data API +authenticated as the channel owner, downloading captions from his own videos. So a row may +carry chapters long before it carries a body, and only the body gates the page. + +When YouTube has no chapters, topics may be curated in +`src/config/video-transcript-topics.json`. Both backfill scripts use that outline as their +fallback. The metadata refresh only replaces an existing chapter field when YouTube or the +curated file supplies a nonempty list, so an empty player response cannot erase an authored +transcript outline. + +```sql +insert into public.video_transcripts + (video_id, source, language, body, chapters, duration, transcript_updated_at, chapters_updated_at) +values + ('dQw4w9WgXcQ', 'youtube', 'en', + 'Today we are going to look at why the build broke...', + '[{"start": 0, "title": "Cold open"}, {"start": 84, "title": "The actual bug"}]'::jsonb, + 1420, now(), now()) +on conflict (video_id) do update + set body = coalesce(excluded.body, public.video_transcripts.body), + chapters = coalesce(excluded.chapters, public.video_transcripts.chapters), + transcript_updated_at = greatest(excluded.transcript_updated_at, + public.video_transcripts.transcript_updated_at); +``` + +### Build the import CSV + +`pnpm transcripts:csv` reads every YAML file in `src/content/videos`, skips rows where +`short: true`, and uses `yt-dlp` to fetch the best English caption track. Manual captions +win when both manual and generated tracks exist. It writes: + +- `backfill/video-transcripts/video_transcripts.csv`, ready for a Supabase table import +- `backfill/video-transcripts/rows/