diff --git a/.github/workflows/generate-feed.yml b/.github/workflows/generate-feed.yml index 7a1fa065c..6f0ab81ef 100644 --- a/.github/workflows/generate-feed.yml +++ b/.github/workflows/generate-feed.yml @@ -1,9 +1,12 @@ name: Generate Feeds on: + push: + paths: + - "config/default-sources.json" schedule: - # All feeds (tweets, podcasts, blogs): once daily at 6:17am UTC - - cron: '17 6 * * *' + # All feeds: 07:17 Asia/Shanghai (23:17 UTC on the previous day). + - cron: '17 23 * * *' workflow_dispatch: inputs: mode: diff --git a/README.md b/README.md index 553608b06..5840fa582 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ A daily or weekly digest delivered to your preferred messaging app (Telegram, Di WhatsApp, etc.) with: - Summaries of new podcast episodes from top AI podcasts -- Key posts and insights from 26 curated AI builders on X/Twitter +- Key posts and insights from 36 curated AI builders on X/Twitter - Full articles from official AI company blogs (Anthropic Engineering, Claude Blog) - Links to all original content - Available in English, Chinese, or bilingual @@ -61,8 +61,18 @@ Edit the files in the `prompts/` folder: - `summarize-tweets.md` — how X/Twitter posts are summarized - `summarize-blogs.md` — how blog posts are summarized - `digest-intro.md` — the overall digest format and tone +- `digest-intro-expanded.md` — expanded digest rules that retain medium-density items - `translate.md` — how English content is translated to Chinese +To generate the expanded edition explicitly: + +```bash +node scripts/prepare-digest.js --expanded +node scripts/deliver.js --file /tmp/fb-digest-expanded.md --expanded +``` + +The expanded edition is exported as `data/YYYY-MM-DD/digest-expanded.md` by default. + These are plain English instructions, not code. Changes take effect on the next digest. ## Default Sources @@ -75,8 +85,8 @@ These are plain English instructions, not code. Changes take effect on the next - [The MAD Podcast with Matt Turck](https://www.youtube.com/@DataDrivenNYC) - [AI & I by Every](https://www.youtube.com/playlist?list=PLuMcoKK9mKgHtW_o9h5sGO2vXrffKHwJL) -### AI Builders on X (26) -[Andrej Karpathy](https://x.com/karpathy), [Swyx](https://x.com/swyx), [Josh Woodward](https://x.com/joshwoodward), [Boris Cherny](https://x.com/bcherny), [Thibault Sottiaux](https://x.com/thsottiaux), [Peter Yang](https://x.com/petergyang), [Nan Yu](https://x.com/thenanyu), [Madhu Guru](https://x.com/realmadhuguru), [Amanda Askell](https://x.com/AmandaAskell), [Cat Wu](https://x.com/_catwu), [Thariq](https://x.com/trq212), [Google Labs](https://x.com/GoogleLabs), [Amjad Masad](https://x.com/amasad), [Guillermo Rauch](https://x.com/rauchg), [Alex Albert](https://x.com/alexalbert__), [Aaron Levie](https://x.com/levie), [Ryo Lu](https://x.com/ryolu_), [Garry Tan](https://x.com/garrytan), [Matt Turck](https://x.com/mattturck), [Zara Zhang](https://x.com/zarazhangrui), [Nikunj Kothari](https://x.com/nikunj), [Peter Steinberger](https://x.com/steipete), [Dan Shipper](https://x.com/danshipper), [Aditya Agarwal](https://x.com/adityaag), [Sam Altman](https://x.com/sama), [Claude](https://x.com/claudeai) +### AI Builders on X (36) +[Andrej Karpathy](https://x.com/karpathy), [Swyx](https://x.com/swyx), [Josh Woodward](https://x.com/joshwoodward), [Boris Cherny](https://x.com/bcherny), [Thibault Sottiaux](https://x.com/thsottiaux), [Kevin Weil](https://x.com/kevinweil), [OpenAI Developers](https://x.com/OpenAIDevs), [Anthropic](https://x.com/AnthropicAI), [Peter Yang](https://x.com/petergyang), [Nan Yu](https://x.com/thenanyu), [Madhu Guru](https://x.com/realmadhuguru), [Amanda Askell](https://x.com/AmandaAskell), [Cat Wu](https://x.com/_catwu), [Thariq](https://x.com/trq212), [Google Labs](https://x.com/GoogleLabs), [Amjad Masad](https://x.com/amasad), [Guillermo Rauch](https://x.com/rauchg), [Alex Albert](https://x.com/alexalbert__), [Aaron Levie](https://x.com/levie), [Ryo Lu](https://x.com/ryolu_), [Garry Tan](https://x.com/garrytan), [Matt Turck](https://x.com/mattturck), [Zara Zhang](https://x.com/zarazhangrui), [Nikunj Kothari](https://x.com/nikunj), [Peter Steinberger](https://x.com/steipete), [Dan Shipper](https://x.com/danshipper), [Aditya Agarwal](https://x.com/adityaag), [Sam Altman](https://x.com/sama), [Claude](https://x.com/claudeai), [Matt Van Horn](https://x.com/mvanhorn), [Simon Willison](https://x.com/simonw), [Harrison Chase](https://x.com/hwchase17), [Cognition](https://x.com/cognition_labs), [Cursor](https://x.com/cursor_ai), [LangChain](https://x.com/LangChainAI), [LlamaIndex](https://x.com/llama_index) ### Official Blogs (2) - [Anthropic Engineering](https://www.anthropic.com/engineering) — technical deep-dives from the Anthropic team @@ -114,7 +124,19 @@ is fetched centrally and updated daily. (blog articles via web scraping, YouTube transcripts via Supadata, X/Twitter via official API) 2. Your agent fetches the feed — one HTTP request, no API keys 3. Your agent remixes the raw content into a digestible summary using your preferences -4. The digest is delivered to your messaging app (or shown in-chat) +4. The digest is saved as Markdown and delivered to your messaging app (or shown in-chat) + +Each delivered digest is exported by default to: + +```text +data/YYYY-MM-DD/digest.md +``` + +Power users can override the export destination when calling the delivery script: + +```bash +node scripts/deliver.js --file /tmp/fb-digest.md --output data/2026-05-12 +``` See [examples/sample-digest.md](examples/sample-digest.md) for what the output looks like. diff --git a/README.zh-CN.md b/README.zh-CN.md index c6e6b0b36..53bf6df59 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -11,7 +11,7 @@ 每日或每周推送到你常用的通讯工具（Telegram、Discord、WhatsApp 等），包含： - 顶级 AI 播客新节目的精华摘要 -- 26 位精选 AI 建造者在 X/Twitter 上的关键观点和洞察 +- 36 位精选 AI 建造者在 X/Twitter 上的关键观点和洞察 - AI 公司官方博客的完整文章（Anthropic Engineering、Claude Blog） - 所有原始内容的链接 - 支持英文、中文或双语版本 @@ -54,8 +54,18 @@ Skill 使用纯文本 prompt 文件来控制内容的摘要方式。你可以通 - `summarize-tweets.md` — X/Twitter 帖子的摘要方式 - `summarize-blogs.md` — 博客文章的摘要方式 - `digest-intro.md` — 整体摘要的格式和语气 +- `digest-intro-expanded.md` — 扩展版摘要规则，保留更多中等信息密度动态 - `translate.md` — 英文内容翻译为中文的方式 +显式生成扩展版： + +```bash +node scripts/prepare-digest.js --expanded +node scripts/deliver.js --file /tmp/fb-digest-expanded.md --expanded +``` + +扩展版默认保存为 `data/YYYY-MM-DD/digest-expanded.md`。 + 这些都是纯文本指令，不是代码。修改后下次推送即生效。 ## 默认信息源 @@ -68,8 +78,8 @@ Skill 使用纯文本 prompt 文件来控制内容的摘要方式。你可以通 - [The MAD Podcast with Matt Turck](https://www.youtube.com/@DataDrivenNYC) - [AI & I by Every](https://www.youtube.com/playlist?list=PLuMcoKK9mKgHtW_o9h5sGO2vXrffKHwJL) -### X 上的 AI 建造者（26位） -[Andrej Karpathy](https://x.com/karpathy), [Swyx](https://x.com/swyx), [Josh Woodward](https://x.com/joshwoodward), [Boris Cherny](https://x.com/bcherny), [Thibault Sottiaux](https://x.com/thsottiaux), [Peter Yang](https://x.com/petergyang), [Nan Yu](https://x.com/thenanyu), [Madhu Guru](https://x.com/realmadhuguru), [Amanda Askell](https://x.com/AmandaAskell), [Cat Wu](https://x.com/_catwu), [Thariq](https://x.com/trq212), [Google Labs](https://x.com/GoogleLabs), [Amjad Masad](https://x.com/amasad), [Guillermo Rauch](https://x.com/rauchg), [Alex Albert](https://x.com/alexalbert__), [Aaron Levie](https://x.com/levie), [Ryo Lu](https://x.com/ryolu_), [Garry Tan](https://x.com/garrytan), [Matt Turck](https://x.com/mattturck), [Zara Zhang](https://x.com/zarazhangrui), [Nikunj Kothari](https://x.com/nikunj), [Peter Steinberger](https://x.com/steipete), [Dan Shipper](https://x.com/danshipper), [Aditya Agarwal](https://x.com/adityaag), [Sam Altman](https://x.com/sama), [Claude](https://x.com/claudeai) +### X 上的 AI 建造者（36位） +[Andrej Karpathy](https://x.com/karpathy), [Swyx](https://x.com/swyx), [Josh Woodward](https://x.com/joshwoodward), [Boris Cherny](https://x.com/bcherny), [Thibault Sottiaux](https://x.com/thsottiaux), [Kevin Weil](https://x.com/kevinweil), [OpenAI Developers](https://x.com/OpenAIDevs), [Anthropic](https://x.com/AnthropicAI), [Peter Yang](https://x.com/petergyang), [Nan Yu](https://x.com/thenanyu), [Madhu Guru](https://x.com/realmadhuguru), [Amanda Askell](https://x.com/AmandaAskell), [Cat Wu](https://x.com/_catwu), [Thariq](https://x.com/trq212), [Google Labs](https://x.com/GoogleLabs), [Amjad Masad](https://x.com/amasad), [Guillermo Rauch](https://x.com/rauchg), [Alex Albert](https://x.com/alexalbert__), [Aaron Levie](https://x.com/levie), [Ryo Lu](https://x.com/ryolu_), [Garry Tan](https://x.com/garrytan), [Matt Turck](https://x.com/mattturck), [Zara Zhang](https://x.com/zarazhangrui), [Nikunj Kothari](https://x.com/nikunj), [Peter Steinberger](https://x.com/steipete), [Dan Shipper](https://x.com/danshipper), [Aditya Agarwal](https://x.com/adityaag), [Sam Altman](https://x.com/sama), [Claude](https://x.com/claudeai), [Matt Van Horn](https://x.com/mvanhorn), [Simon Willison](https://x.com/simonw), [Harrison Chase](https://x.com/hwchase17), [Cognition](https://x.com/cognition_labs), [Cursor](https://x.com/cursor_ai), [LangChain](https://x.com/LangChainAI), [LlamaIndex](https://x.com/llama_index) ### 官方博客（2个） - [Anthropic Engineering](https://www.anthropic.com/engineering) — Anthropic 团队的技术深度文章 @@ -105,7 +115,19 @@ cd ~/.claude/skills/follow-builders/scripts && npm install 1. 中心化 feed 每日更新，抓取所有信息源的最新内容（博客文章通过网页抓取，YouTube 字幕通过 Supadata，X/Twitter 通过官方 API） 2. 你的 agent 获取 feed——一次 HTTP 请求，不需要 API key 3. 你的 agent 根据你的偏好将原始内容重新混编为易消化的摘要 -4. 摘要推送到你的通讯工具（或直接在聊天中显示） +4. 摘要保存为 Markdown，并推送到你的通讯工具（或直接在聊天中显示） + +每次生成的摘要默认会导出到： + +```text +data/YYYY-MM-DD/digest.md +``` + +高级用户也可以在调用投递脚本时指定导出位置： + +```bash +node scripts/deliver.js --file /tmp/fb-digest.md --output data/2026-05-12 +``` 查看 [examples/sample-digest.md](examples/sample-digest.md) 了解输出示例。 diff --git a/SKILL.md b/SKILL.md index d7a72d8fb..6659a6e5c 100644 --- a/SKILL.md +++ b/SKILL.md @@ -15,6 +15,10 @@ Philosophy: follow builders with original opinions, not influencers who regurgit (X/Twitter posts and YouTube transcripts) is fetched centrally and served via a public feed. Users only need API keys if they choose Telegram or email delivery. +The central feed is generated from `config/default-sources.json`. GitHub Actions +refreshes it daily at 07:17 Asia/Shanghai and also refreshes it when the central +source registry changes. + ## Detecting Platform Before doing anything, detect which platform you're running on by running: @@ -173,6 +177,7 @@ cat > ~/.follow-builders/config.json << 'CFGEOF' "language": "", "timezone": "", "frequency": "", + "digestMode": "", "deliveryTime": "", "weeklyDay": "", "delivery": { @@ -321,6 +326,11 @@ You do NOT fetch anything yourself. cd ${CLAUDE_SKILL_DIR}/scripts && node prepare-digest.js 2>/dev/null ``` +Use `node prepare-digest.js --expanded` for the expanded edition, which keeps +more medium-density product, workflow, tool, and early-signal updates. The same +edition can be selected persistently with `"digestMode": "expanded"` in +`~/.follow-builders/config.json`; the default remains `standard`. + The script outputs a single JSON blob with everything you need: - `config` — user's language and delivery preferences - `podcasts` — podcast episodes with full transcripts @@ -359,6 +369,10 @@ Read the prompts from the `prompts` field in the JSON: Assemble the digest following `prompts.digest_intro`. +The prepare script has already selected `prompts.digest_intro` for the requested +edition. Follow `config.digestMode` and do not silently switch between standard +and expanded output. + **ABSOLUTE RULES:** - NEVER invent or fabricate content. Only use what's in the JSON. - Every piece of content MUST have its URL. No URL = do not include. @@ -397,15 +411,31 @@ Read `config.language` from the JSON: Read `config.delivery.method` from the JSON: +First write the final digest text to `/tmp/fb-digest.md`, then call +`deliver.js`. The delivery script always exports a Markdown copy before sending. +Default export path: + +```text +data/YYYY-MM-DD/digest.md +``` + +Expanded output defaults to `data/YYYY-MM-DD/digest-expanded.md`. + +Use `--output ` only when the user explicitly asks for a custom export +file or directory. If `` has no file extension, treat it as a directory and +save `digest.md` inside it. + **If "telegram" or "email":** ```bash -echo '' > /tmp/fb-digest.txt -cd ${CLAUDE_SKILL_DIR}/scripts && node deliver.js --file /tmp/fb-digest.txt 2>/dev/null +cd ${CLAUDE_SKILL_DIR}/scripts && node deliver.js --file /tmp/fb-digest.md 2>/dev/null ``` If delivery fails, show the digest in the terminal as fallback. **If "stdout" (default):** -Just output the digest directly. +```bash +cd ${CLAUDE_SKILL_DIR}/scripts && node deliver.js --file /tmp/fb-digest.md +``` +Then show the digest text to the user and mention the saved Markdown path. --- @@ -414,16 +444,26 @@ Just output the digest directly. When the user says something that sounds like a settings change, handle it: ### Source Changes -The source list is managed centrally and cannot be modified by users. -If a user asks to add or remove sources, tell them: "The source list is curated -centrally and updates automatically. If you'd like to suggest a source, you can -open an issue at https://github.com/zarazhangrui/follow-builders." +The central source registry is `config/default-sources.json`. When maintaining +this repository, add or remove sources there, validate the JSON, and commit and +push that file. The GitHub Action then regenerates the central feed; editing only +`~/.follow-builders/config.json` or generated `feed-*.json` does not publish a +source change. For a suggestion without repository access, open an issue at +https://github.com/zarazhangrui/follow-builders. + +The registry currently contains 36 X builders, 6 podcasts, and 4 official blogs. ### Schedule Changes - "Switch to weekly/daily" → Update `frequency` in config.json - "Change time to X" → Update `deliveryTime` in config.json - "Change timezone to X" → Update `timezone` in config.json, also update the cron job +### Digest Edition Changes +- "Show the expanded digest" → Run `node prepare-digest.js --expanded` and pass + `--expanded` to `deliver.js` when exporting it +- "Use expanded digest by default" → Set `digestMode` to `expanded` in config.json +- "Use the concise digest" → Set `digestMode` to `standard` or pass `--standard` + ### Language Changes - "Switch to Chinese/English/bilingual" → Update `language` in config.json diff --git a/config/config-schema.json b/config/config-schema.json index bb15c45f8..0ffda3674 100644 --- a/config/config-schema.json +++ b/config/config-schema.json @@ -25,6 +25,12 @@ "default": "daily", "description": "How often to deliver the digest" }, + "digestMode": { + "type": "string", + "enum": ["standard", "expanded"], + "default": "standard", + "description": "Digest breadth: standard concise edition or expanded edition with medium-density items" + }, "deliveryTime": { "type": "string", "default": "08:00", diff --git a/config/default-sources.json b/config/default-sources.json index d35d444db..b6ee574ac 100644 --- a/config/default-sources.json +++ b/config/default-sources.json @@ -3,32 +3,62 @@ { "name": "Latent Space", "rssUrl": "https://pod2txt.vercel.app/api/feed?url=https://api.substack.com/feed/podcast/1084089.rss", - "url": "https://www.youtube.com/@LatentSpacePod" + "url": "https://www.youtube.com/@LatentSpacePod", + "topics": ["ai-engineering", "agents", "developer-tools"], + "wechat": { + "priority": "medium", + "fit": "AI Engineer / agent 生态深访，适合提炼成方法论或趋势辅助稿。" + } }, { "name": "Training Data", "rssUrl": "https://feeds.megaphone.fm/trainingdata", - "url": "https://www.youtube.com/playlist?list=PLOhHNjZItNnMm5tdW61JpnyxeYH5NDDx8" + "url": "https://www.youtube.com/playlist?list=PLOhHNjZItNnMm5tdW61JpnyxeYH5NDDx8", + "topics": ["ai-product", "agent-workflow", "ai-coding"], + "wechat": { + "priority": "high", + "fit": "AI 产品、AI coding 和企业工作流访谈，适合拆成公众号案例稿。" + } }, { "name": "No Priors", "rssUrl": "https://feeds.megaphone.fm/nopriors", - "url": "https://www.youtube.com/@NoPriorsPodcast" + "url": "https://www.youtube.com/@NoPriorsPodcast", + "topics": ["ai-product", "founders", "frontier-ai"], + "wechat": { + "priority": "medium", + "fit": "大厂 CEO、AI founder 和产品判断，适合做人物判断或宏观辅助稿。" + } }, { "name": "Unsupervised Learning", "rssUrl": "https://feeds.simplecast.com/dOSE_bdP", - "url": "https://www.youtube.com/@RedpointAI" + "url": "https://www.youtube.com/@RedpointAI", + "topics": ["ai-research", "agents", "frontier-ai"], + "wechat": { + "priority": "medium", + "fit": "AI 研究和 agent scaffold 判断，适合作为技术趋势辅助来源。" + } }, { "name": "The MAD Podcast with Matt Turck", "rssUrl": "https://anchor.fm/s/f2ee4948/podcast/rss", - "url": "https://www.youtube.com/@DataDrivenNYC/videos" + "url": "https://www.youtube.com/@DataDrivenNYC/videos", + "topics": ["data-infra", "ai-infra", "enterprise-ai"], + "wechat": { + "priority": "low", + "fit": "数据/AI infra 长访谈，适合补充企业 AI 背景，不优先做推荐流主稿。" + } }, { "name": "AI & I by Every", "rssUrl": "https://anchor.fm/s/ed1f5584/podcast/rss", - "url": "https://www.youtube.com/playlist?list=PLuMcoKK9mKgHtW_o9h5sGO2vXrffKHwJL" + "url": "https://www.youtube.com/playlist?list=PLuMcoKK9mKgHtW_o9h5sGO2vXrffKHwJL", + "topics": ["ai-writing", "agent-workflow", "ai-native-apps"], + "wechat": { + "priority": "medium", + "fit": "Every 的 AI workflow 和 AI-native 应用案例，适合做实操/产品辅助稿。" + } } ], "blogs": [ @@ -37,42 +67,252 @@ "type": "scrape", "indexUrl": "https://www.anthropic.com/engineering", "articleBaseUrl": "https://www.anthropic.com/engineering/", - "fetchMethod": "http" + "fetchMethod": "http", + "topics": ["claude", "agent-infra", "ai-coding"], + "wechat": { + "priority": "high", + "fit": "官方工程复盘、架构细节和产品质量事故，适合拆成公众号深度稿。" + } }, { "name": "Claude Blog", "type": "scrape", "indexUrl": "https://claude.com/blog", "articleBaseUrl": "https://claude.com/blog/", - "fetchMethod": "http" + "fetchMethod": "http", + "topics": ["claude", "claude-code", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Claude / Claude Code 产品更新和工作流案例，适合公众号推荐流。" + } + }, + { + "name": "Claude Code Releases", + "type": "atom", + "feedUrl": "https://github.com/anthropics/claude-code/releases.atom", + "indexUrl": "https://github.com/anthropics/claude-code/releases", + "topics": ["claude-code", "ai-coding", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Claude Code 版本变化、能力边界和开发者痛点，适合做短拆解。" + } + }, + { + "name": "OpenAI Codex Releases", + "type": "atom", + "feedUrl": "https://github.com/openai/codex/releases.atom", + "indexUrl": "https://github.com/openai/codex/releases", + "topics": ["codex", "openai", "ai-coding"], + "wechat": { + "priority": "high", + "fit": "Codex CLI 和 coding agent 更新，适合和 Claude Code 对照写。" + } } ], "x_accounts": [ { "name": "Andrej Karpathy", "handle": "karpathy" }, - { "name": "Swyx", "handle": "swyx" }, + { + "name": "Swyx", + "handle": "swyx", + "topics": ["ai-engineering", "agents"], + "wechat": { + "priority": "medium", + "fit": "AI Engineer 视角，适合作为趋势和方法论辅助来源。" + } + }, { "name": "Josh Woodward", "handle": "joshwoodward" }, - { "name": "Boris Cherny", "handle": "bcherny" }, - { "name": "Thibault Sottiaux", "handle": "thsottiaux" }, - { "name": "Peter Yang", "handle": "petergyang" }, + { + "name": "Boris Cherny", + "handle": "bcherny", + "topics": ["claude", "claude-code", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Claude / Claude Code 一线产品信号，适合公众号主选题。" + } + }, + { + "name": "Thibault Sottiaux", + "handle": "thsottiaux", + "topics": ["codex", "openai", "ai-coding"], + "wechat": { + "priority": "high", + "fit": "Codex 一线产品信号，适合和 Claude Code 做对照。" + } + }, + { "name": "Kevin Weil", "handle": "kevinweil" }, + { + "name": "OpenAI Developers", + "handle": "OpenAIDevs", + "topics": ["openai", "codex", "developer-platform"], + "wechat": { + "priority": "high", + "fit": "OpenAI 开发者平台与 Codex 相关更新，适合做官方变化速读。" + } + }, + { + "name": "Anthropic", + "handle": "AnthropicAI", + "topics": ["anthropic", "claude", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Anthropic 官方发布和 Claude 生态更新，适合补强官方来源。" + } + }, + { + "name": "Peter Yang", + "handle": "petergyang", + "topics": ["agent-workflow", "ai-product"], + "wechat": { + "priority": "high", + "fit": "Agent 工作流和产品化经验，适合写实用方法稿。" + } + }, { "name": "Nan Yu", "handle": "thenanyu" }, { "name": "Madhu Guru", "handle": "realmadhuguru" }, { "name": "Amanda Askell", "handle": "AmandaAskell" }, - { "name": "Cat Wu", "handle": "_catwu" }, - { "name": "Thariq", "handle": "trq212" }, + { + "name": "Cat Wu", + "handle": "_catwu", + "topics": ["claude-code", "agent-evals", "ai-product"], + "wechat": { + "priority": "high", + "fit": "Claude Code PM / eval 相关信号，适合拆 agent 产品质量。" + } + }, + { + "name": "Thariq", + "handle": "trq212", + "topics": ["claude-code", "agent-workflow", "html-artifacts"], + "wechat": { + "priority": "high", + "fit": "Claude Code 使用方法和 artifact/workflow，适合公众号实操稿。" + } + }, { "name": "Google Labs", "handle": "GoogleLabs" }, { "name": "Amjad Masad", "handle": "amasad" }, - { "name": "Guillermo Rauch", "handle": "rauchg" }, - { "name": "Alex Albert", "handle": "alexalbert__" }, + { + "name": "Guillermo Rauch", + "handle": "rauchg", + "topics": ["agent-infra", "vercel", "ai-coding"], + "wechat": { + "priority": "medium", + "fit": "Agent infra、runtime 和开发平台判断，适合辅助 Claude/Codex 主线。" + } + }, + { + "name": "Alex Albert", + "handle": "alexalbert__", + "topics": ["claude", "developer-relations", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Claude 开发者生态和产品提示，适合做 Claude 生态观察。" + } + }, { "name": "Aaron Levie", "handle": "levie" }, - { "name": "Ryo Lu", "handle": "ryolu_" }, + { + "name": "Ryo Lu", + "handle": "ryolu_", + "topics": ["cursor", "ai-coding", "design-in-code"], + "wechat": { + "priority": "medium", + "fit": "Cursor 和 design-in-code 视角，适合做 AI coding 产品体验辅助来源。" + } + }, { "name": "Garry Tan", "handle": "garrytan" }, { "name": "Matt Turck", "handle": "mattturck" }, { "name": "Zara Zhang", "handle": "zarazhangrui" }, { "name": "Nikunj Kothari", "handle": "nikunj" }, - { "name": "Peter Steinberger", "handle": "steipete" }, - { "name": "Dan Shipper", "handle": "danshipper" }, + { + "name": "Peter Steinberger", + "handle": "steipete", + "topics": ["codex", "openclaw", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "真实 coding agent 长任务和工具链经验，适合写实操复盘。" + } + }, + { + "name": "Dan Shipper", + "handle": "danshipper", + "topics": ["codex", "ai-native-apps", "agent-workflow"], + "wechat": { + "priority": "medium", + "fit": "AI-native apps 和写作/产品工作流，适合做案例辅助来源。" + } + }, { "name": "Aditya Agarwal", "handle": "adityaag" }, { "name": "Sam Altman", "handle": "sama" }, - { "name": "Claude", "handle": "claudeai" } + { + "name": "Claude", + "handle": "claudeai", + "topics": ["claude", "claude-code", "agent-workflow"], + "wechat": { + "priority": "high", + "fit": "Claude 官方产品发布，适合作为主选题事实来源。" + } + }, + { + "name": "Matt Van Horn", + "handle": "mvanhorn", + "topics": ["agentic-engineering", "claude-code", "codex"], + "wechat": { + "priority": "high", + "fit": "Agentic engineering 实操经验，适合拆成工具流和方法论稿。" + } + }, + { + "name": "Simon Willison", + "handle": "simonw", + "topics": ["llm-tools", "agents", "ai-coding"], + "wechat": { + "priority": "medium", + "fit": "LLM 工具、agent 安全和开发者实践，适合补技术可信度。" + } + }, + { + "name": "Harrison Chase", + "handle": "hwchase17", + "topics": ["langchain", "langgraph", "agents"], + "wechat": { + "priority": "medium", + "fit": "LangGraph / agent workflow 生态，适合做应用层和工程层辅助来源。" + } + }, + { + "name": "Cognition", + "handle": "cognition_labs", + "topics": ["devin", "ai-coding", "agent-evals"], + "wechat": { + "priority": "high", + "fit": "Devin、真实工程任务和 agent eval，适合做推荐池强关键词选题。" + } + }, + { + "name": "Cursor", + "handle": "cursor_ai", + "topics": ["cursor", "ai-coding", "developer-tools"], + "wechat": { + "priority": "high", + "fit": "AI coding 主流产品更新，适合和 Claude Code / Codex 对照。" + } + }, + { + "name": "LangChain", + "handle": "LangChainAI", + "topics": ["langchain", "langgraph", "agents"], + "wechat": { + "priority": "medium", + "fit": "Agent 框架和企业落地案例，适合做技术生态辅助来源。" + } + }, + { + "name": "LlamaIndex", + "handle": "llama_index", + "topics": ["agents", "retrieval", "agent-workflow"], + "wechat": { + "priority": "medium", + "fit": "Agent retrieval / workflow 生态，适合补充应用层素材。" + } + } ] } diff --git a/prompts/digest-intro-expanded.md b/prompts/digest-intro-expanded.md new file mode 100644 index 000000000..43a62db60 --- /dev/null +++ b/prompts/digest-intro-expanded.md @@ -0,0 +1,68 @@ +# Expanded Digest Intro Prompt + +You are assembling the expanded edition of the digest from individual source summaries. + +## Purpose + +Keep the standard digest's factual and source-link requirements, but preserve more +medium-density information so the reader can inspect the day's broader signal. The +expanded edition is not a dump of every post: include concrete product updates, +workflow observations, named tools, specific numbers, useful disagreements, and +credible early signals. Skip pure jokes, personal logistics, empty link drops, +engagement bait, and claims that cannot be understood from the source. + +## Format + +Start with: + +AI Builders Digest — [Date]（扩展版） + +Use these headings: + +```markdown +## Brief + +### 今日关键信号 +... + +### 适合谁读 +... + +### 公众号候选 +... + +## 文章详情 + +### X 动态 +... + +### 官方博客 +... + +### 播客转录 +... + +## Sources & Metadata +... +``` + +## Editorial rules + +- Include 5-8 signal bullets in the brief when the feed supports them. +- Keep the detailed X section broad: normally retain 12-20 useful items or compact + source groups, including medium-density items that the standard edition omits. +- Use 1-2 sentences for a medium-density item and 2-4 sentences for a major item. +- Group related posts from the same person or product rather than repeating context. +- Use the source person's bio for their role; never invent a title. +- Preserve the exact original URL for every included item. +- Include all new blogs and podcast episodes that have direct source URLs. +- For podcasts, use the exact episode `title` and specific video `url`; never use a + channel or playlist URL. +- In the metadata, report the raw feed counts and the selected digest mode. +- Keep the language and bilingual behavior from the standard digest prompt. +- Do not paste full copyrighted articles or transcripts. Paraphrase and use short + quotes only when necessary. + +At the very end, add: + +Generated through the Follow Builders skill: https://github.com/zarazhangrui/follow-builders diff --git a/prompts/summarize-tweets.md b/prompts/summarize-tweets.md index a0b0fca73..42b11f5a7 100644 --- a/prompts/summarize-tweets.md +++ b/prompts/summarize-tweets.md @@ -19,3 +19,12 @@ to know what this person is thinking and building. - If they shared a tool, demo, or resource, mention it by name with the link - If there's nothing substantive to report, say "No notable posts" rather than padding with fluff + +## Expanded digest mode + +When `config.digestMode` is `"expanded"`, retain medium-density items that contain +a concrete product update, workflow observation, named resource, specific number, +or useful implementation detail even when they are not important enough for the +standard edition. Summarize those items in 1-2 sentences and keep their original +URL. Still skip pure jokes, personal logistics, empty link drops, engagement bait, +and unsupported claims. diff --git a/scripts/deliver.js b/scripts/deliver.js index c212d083a..343bd9da1 100755 --- a/scripts/deliver.js +++ b/scripts/deliver.js @@ -10,6 +10,8 @@ // echo "digest text" | node deliver.js // node deliver.js --message "digest text" // node deliver.js --file /path/to/digest.txt +// node deliver.js --file /path/to/digest.txt --output data/2026-05-12 +// node deliver.js --file /path/to/digest.txt --no-export // // The script reads delivery config from ~/.follow-builders/config.json // and API keys from ~/.follow-builders/.env @@ -20,35 +22,111 @@ // - "stdout" (default): just prints to terminal // ============================================================================ -import { readFile } from 'fs/promises'; +import { mkdir, readFile, writeFile } from 'fs/promises'; import { existsSync } from 'fs'; -import { join } from 'path'; +import { dirname, extname, isAbsolute, join, resolve } from 'path'; import { homedir } from 'os'; -import { config as loadEnv } from 'dotenv'; // -- Constants --------------------------------------------------------------- const USER_DIR = join(homedir(), '.follow-builders'); const CONFIG_PATH = join(USER_DIR, 'config.json'); const ENV_PATH = join(USER_DIR, '.env'); +const SCRIPT_DIR = decodeURIComponent(new URL('.', import.meta.url).pathname); +const SKILL_DIR = join(SCRIPT_DIR, '..'); +const DIGEST_FILENAME = 'digest.md'; + +// -- CLI helpers ------------------------------------------------------------- + +function getArgValue(name) { + const args = process.argv.slice(2); + const idx = args.indexOf(name); + if (idx !== -1 && args[idx + 1]) return args[idx + 1]; + return null; +} + +function hasArg(name) { + return process.argv.slice(2).includes(name); +} + +function expandPath(filePath) { + if (filePath === '~') return homedir(); + if (filePath.startsWith('~/')) return join(homedir(), filePath.slice(2)); + return filePath; +} + +function resolveOutputPath(outputArg, dateStamp, digestMode = 'standard') { + if (!outputArg) { + const filename = digestMode === 'expanded' ? 'digest-expanded.md' : DIGEST_FILENAME; + return join(SKILL_DIR, 'data', dateStamp, filename); + } + + const expanded = expandPath(outputArg); + const absolute = isAbsolute(expanded) ? expanded : resolve(process.cwd(), expanded); + + // Treat paths without a file extension as directories. + if (!extname(absolute)) { + const filename = digestMode === 'expanded' ? 'digest-expanded.md' : DIGEST_FILENAME; + return join(absolute, filename); + } + return absolute; +} + +function dateStampFor(timezone) { + if (!timezone) return new Date().toISOString().slice(0, 10); + + try { + const parts = new Intl.DateTimeFormat('en-CA', { + timeZone: timezone, + year: 'numeric', + month: '2-digit', + day: '2-digit' + }).formatToParts(new Date()); + + const values = Object.fromEntries(parts.map(part => [part.type, part.value])); + return `${values.year}-${values.month}-${values.day}`; + } catch { + return new Date().toISOString().slice(0, 10); + } +} + +async function loadEnvFile(envPath) { + if (!existsSync(envPath)) return; + + const text = await readFile(envPath, 'utf-8'); + for (const rawLine of text.split(/\r?\n/)) { + const line = rawLine.trim(); + if (!line || line.startsWith('#')) continue; + + const eq = line.indexOf('='); + if (eq === -1) continue; + + const key = line.slice(0, eq).trim(); + let value = line.slice(eq + 1).trim(); + if (!key || process.env[key] !== undefined) continue; + + if ( + (value.startsWith('"') && value.endsWith('"')) || + (value.startsWith("'") && value.endsWith("'")) + ) { + value = value.slice(1, -1); + } + + process.env[key] = value; + } +} // -- Read input -------------------------------------------------------------- // The digest text can come from stdin, --message flag, or --file flag async function getDigestText() { - const args = process.argv.slice(2); - // Check --message flag - const msgIdx = args.indexOf('--message'); - if (msgIdx !== -1 && args[msgIdx + 1]) { - return args[msgIdx + 1]; - } + const message = getArgValue('--message'); + if (message) return message; // Check --file flag - const fileIdx = args.indexOf('--file'); - if (fileIdx !== -1 && args[fileIdx + 1]) { - return await readFile(args[fileIdx + 1], 'utf-8'); - } + const file = getArgValue('--file'); + if (file) return await readFile(file, 'utf-8'); // Read from stdin const chunks = []; @@ -58,6 +136,24 @@ async function getDigestText() { return Buffer.concat(chunks).toString('utf-8'); } +// -- Markdown Export --------------------------------------------------------- + +async function exportDigest(text, config) { + if (hasArg('--no-export')) return null; + + const outputArg = getArgValue('--output'); + const timezone = config.timezone || process.env.TZ; + const digestMode = hasArg('--expanded') || config.digestMode === 'expanded' + ? 'expanded' + : 'standard'; + const outputPath = resolveOutputPath(outputArg, dateStampFor(timezone), digestMode); + + await mkdir(dirname(outputPath), { recursive: true }); + await writeFile(outputPath, text, 'utf-8'); + + return outputPath; +} + // -- Telegram Delivery ------------------------------------------------------- // Sends the digest via Telegram Bot API. @@ -153,7 +249,7 @@ async function sendEmail(text, apiKey, toEmail) { async function main() { // Load env and config - loadEnv({ path: ENV_PATH }); + await loadEnvFile(ENV_PATH); let config = {}; if (existsSync(CONFIG_PATH)) { @@ -169,6 +265,8 @@ async function main() { } try { + const exportPath = await exportDigest(digestText, config); + switch (delivery.method) { case 'telegram': { const botToken = process.env.TELEGRAM_BOT_TOKEN; @@ -179,7 +277,8 @@ async function main() { console.log(JSON.stringify({ status: 'ok', method: 'telegram', - message: 'Digest sent to Telegram' + message: 'Digest sent to Telegram', + exportPath })); break; } @@ -193,7 +292,8 @@ async function main() { console.log(JSON.stringify({ status: 'ok', method: 'email', - message: `Digest sent to ${toEmail}` + message: `Digest sent to ${toEmail}`, + exportPath })); break; } @@ -202,6 +302,7 @@ async function main() { default: // Just print to terminal — the agent or OpenClaw handles delivery console.log(digestText); + if (exportPath) console.error(`Saved digest to ${exportPath}`); break; } } catch (err) { diff --git a/scripts/generate-feed.js b/scripts/generate-feed.js index 09183cea0..aa2363ebb 100755 --- a/scripts/generate-feed.js +++ b/scripts/generate-feed.js @@ -14,12 +14,20 @@ // ============================================================================ import { readFile, writeFile } from "fs/promises"; +import { setDefaultResultOrder } from "dns"; import { existsSync } from "fs"; import { join } from "path"; +import { fileURLToPath } from "url"; +import { execFile } from "child_process"; +import { promisify } from "util"; + +setDefaultResultOrder("ipv4first"); +const execFileAsync = promisify(execFile); // -- Constants --------------------------------------------------------------- const POD2TXT_BASE = "https://pod2txt.vercel.app/api"; +const SUPADATA_BASE = "https://api.supadata.ai/v1"; const X_API_BASE = "https://api.x.com/2"; // Some RSS hosts (notably Substack) block non-browser user agents from cloud IPs. // Using a real Chrome UA avoids 403 errors in GitHub Actions. @@ -28,11 +36,8 @@ const RSS_USER_AGENT = const TWEET_LOOKBACK_HOURS = 24; const PODCAST_LOOKBACK_HOURS = 336; // 14 days — podcasts publish weekly/biweekly, not daily const BLOG_LOOKBACK_HOURS = 72; -const MAX_TWEETS_PER_USER = 3; +const MAX_TWEETS_PER_USER = 5; const MAX_ARTICLES_PER_BLOG = 3; -const X_USER_LOOKUP_BATCH_SIZE = 5; -const X_RETRY_STATUSES = new Set([500, 502, 503, 504]); -const X_RETRY_ATTEMPTS = 3; // State file lives in the repo root so it gets committed by GitHub Actions const SCRIPT_DIR = decodeURIComponent(new URL(".", import.meta.url).pathname); @@ -79,6 +84,33 @@ async function loadSources() { return JSON.parse(await readFile(sourcesPath, "utf-8")); } +async function fetchTextWithCurlFallback(url, fetchOptions = {}) { + try { + const res = await fetch(url, fetchOptions); + if (res.ok) return await res.text(); + } catch { + // Fall through to curl. Node's undici can be flaky with some YouTube hosts. + } + + const { stdout } = await execFileAsync("curl", [ + "-L", + "--max-time", + "20", + "-s", + url, + ], { + maxBuffer: 20 * 1024 * 1024, + }); + return stdout; +} + +export function sourceMetadata(source) { + const metadata = {}; + if (source.topics) metadata.topics = source.topics; + if (source.wechat) metadata.wechat = source.wechat; + return metadata; +} + // -- Podcast Fetching (RSS + pod2txt) ---------------------------------------- // Parses an RSS feed XML string and returns episode objects with @@ -146,15 +178,13 @@ async function getYouTubeFeedUrl(channelUrl) { // "channelId":"UC..." pattern in the JSON blob is the most reliable. if (channelUrl.match(/\/@[A-Za-z0-9_.-]+/)) { try { - const res = await fetch(channelUrl, { + const html = await fetchTextWithCurlFallback(channelUrl, { headers: { "User-Agent": RSS_USER_AGENT, "Accept-Language": "en-US,en;q=0.9", }, signal: AbortSignal.timeout(15000), }); - if (!res.ok) return null; - const html = await res.text(); const idMatch = html.match(/"channelId":"(UC[A-Za-z0-9_-]{20,})"/) || html.match( @@ -175,7 +205,7 @@ async function getYouTubeFeedUrl(channelUrl) { // Atom RSS endpoint is unavailable. YouTube's internal data shapes change // occasionally, so we defensively navigate both the rich-grid (channel page) // and playlist-video-list (playlist page) structures. -function parseYouTubePageData(html) { +export function parseYouTubePageData(html) { const videos = []; const m = html.match(/var\s+ytInitialData\s*=\s*({[\s\S]*?});\s*<\/script>/); if (!m) return videos; @@ -198,6 +228,7 @@ function parseYouTubePageData(html) { if (title) { videos.push({ title, + videoId: v.videoId, url: `https://www.youtube.com/watch?v=${v.videoId}`, }); } @@ -216,6 +247,7 @@ function parseYouTubePageData(html) { if (title) { videos.push({ title, + videoId: v.videoId, url: `https://www.youtube.com/watch?v=${v.videoId}`, }); } @@ -232,14 +264,12 @@ async function fetchYouTubeVideos(channelUrl) { const feedUrl = await getYouTubeFeedUrl(channelUrl); if (feedUrl) { try { - const res = await fetch(feedUrl, { + const xml = await fetchTextWithCurlFallback(feedUrl, { headers: { "User-Agent": RSS_USER_AGENT }, signal: AbortSignal.timeout(15000), }); - if (res.ok) { - const videos = parseYouTubeFeed(await res.text()); - if (videos.length > 0) return videos; - } + const videos = parseYouTubeFeed(xml); + if (videos.length > 0) return videos; } catch { // fall through to scraping } @@ -252,22 +282,21 @@ async function fetchYouTubeVideos(channelUrl) { ? channelUrl : channelUrl.replace(/\/$/, "") + "/videos"; try { - const res = await fetch(videosPageUrl, { + const html = await fetchTextWithCurlFallback(videosPageUrl, { headers: { "User-Agent": RSS_USER_AGENT, "Accept-Language": "en-US,en;q=0.9", }, signal: AbortSignal.timeout(15000), }); - if (!res.ok) return []; - return parseYouTubePageData(await res.text()); + return parseYouTubePageData(html); } catch { return []; } } -// Parses a YouTube Atom feed and returns { title, url } for each entry. -function parseYouTubeFeed(xml) { +// Parses a YouTube Atom feed and returns { title, videoId, url } for each entry. +export function parseYouTubeFeed(xml) { const videos = []; const entryRegex = /([\s\S]*?)<\/entry>/g; let entryMatch; @@ -278,6 +307,7 @@ function parseYouTubeFeed(xml) { if (titleMatch && videoIdMatch) { videos.push({ title: titleMatch[1].trim(), + videoId: videoIdMatch[1].trim(), url: `https://www.youtube.com/watch?v=${videoIdMatch[1].trim()}`, }); } @@ -298,20 +328,19 @@ function normalizeTitle(t) { // Finds the YouTube video whose title best matches the podcast episode title. // Uses substring match first, then token overlap (>=50% of episode's content // words must appear in the video title). Returns null if no confident match. -async function findYouTubeEpisodeUrl(channelUrl, episodeTitle) { - const videos = await fetchYouTubeVideos(channelUrl); +export function matchYouTubeEpisode(videos, episodeTitle) { if (videos.length === 0) return null; const needle = normalizeTitle(episodeTitle); const needleTokens = new Set(needle.split(" ").filter((w) => w.length > 2)); if (needleTokens.size === 0) return null; - let bestUrl = null; + let bestVideo = null; let bestScore = 0; for (const v of videos) { const hay = normalizeTitle(v.title); if (hay && (hay.includes(needle) || needle.includes(hay))) { - return v.url; + return { videoId: v.videoId, url: v.url }; } const hayTokens = new Set(hay.split(" ").filter((w) => w.length > 2)); let overlap = 0; @@ -319,46 +348,136 @@ async function findYouTubeEpisodeUrl(channelUrl, episodeTitle) { const score = overlap / needleTokens.size; if (score > bestScore) { bestScore = score; - bestUrl = v.url; + bestVideo = v; } } - return bestScore >= 0.5 ? bestUrl : null; + return bestScore >= 0.5 && bestVideo + ? { videoId: bestVideo.videoId, url: bestVideo.url } + : null; +} + +export async function findYouTubeEpisode(channelUrl, episodeTitle) { + const videos = await fetchYouTubeVideos(channelUrl); + return matchYouTubeEpisode(videos, episodeTitle); +} + +// Backward-compatible URL-only helper for callers that only need a link. +export async function findYouTubeEpisodeUrl(channelUrl, episodeTitle) { + const episode = await findYouTubeEpisode(channelUrl, episodeTitle); + return episode?.url || null; +} + +export function buildPodcastFeedEntry({ selected, youtubeEpisode, transcript }) { + return { + source: "podcast", + name: selected.podcast.name, + title: selected.title, + guid: selected.guid, + videoId: youtubeEpisode?.videoId || null, + url: youtubeEpisode?.url || null, + podcastUrl: selected.link || null, + publishedAt: selected.publishedAt, + transcript, + ...sourceMetadata(selected.podcast), + }; } // Fetches a transcript from pod2txt. The API is async: first request may // return "processing", so we poll until "ready" (up to 5 attempts, ~2.5 min). -async function fetchPod2txtTranscript(rssUrl, guid, apiKey) { +export async function fetchPod2txtTranscript( + rssUrl, + guid, + apiKey, + fetcher = fetch, +) { const maxAttempts = 5; const pollInterval = 30000; // 30 seconds between polls + try { + for (let attempt = 1; attempt <= maxAttempts; attempt++) { + const res = await fetcher(`${POD2TXT_BASE}/transcript`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ feedurl: rssUrl, guid, apikey: apiKey }), + }); + + if (!res.ok) { + const text = await res.text().catch(() => ""); + return { error: `HTTP ${res.status}: ${text}` }; + } + + const data = await res.json(); + + if (data.status === "ready" && data.url) { + // Transcript is ready — fetch the text from the provided URL + const txtRes = await fetcher(data.url); + if (!txtRes.ok) + return { + error: `Failed to fetch transcript text: HTTP ${txtRes.status}`, + }; + const transcript = await txtRes.text(); + return { transcript }; + } + + if (data.status === "processing") { + console.error( + ` pod2txt: processing (attempt ${attempt}/${maxAttempts}), waiting ${pollInterval / 1000}s...`, + ); + if (attempt < maxAttempts) { + await new Promise((r) => setTimeout(r, pollInterval)); + } + continue; + } + + // Unexpected status or error from the API + return { error: data.message || `Unexpected status: ${data.status}` }; + } + } catch (err) { + return { error: err.message }; + } + + return { error: "Timed out waiting for transcript processing" }; +} + +function normalizeTranscriptContent(content) { + if (typeof content === "string") return content; + if (Array.isArray(content)) { + return content + .map((chunk) => (typeof chunk === "string" ? chunk : chunk?.text || "")) + .filter(Boolean) + .join("\n"); + } + return ""; +} + +function redactSecret(text, secret) { + if (!text || !secret) return text || ""; + return text.split(secret).join("[REDACTED]"); +} + +async function fetchSupadataTranscriptJob(jobId, apiKey, fetcher) { + const maxAttempts = 5; + const pollInterval = 30000; + for (let attempt = 1; attempt <= maxAttempts; attempt++) { - const res = await fetch(`${POD2TXT_BASE}/transcript`, { - method: "POST", - headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ feedurl: rssUrl, guid, apikey: apiKey }), + const res = await fetcher(`${SUPADATA_BASE}/transcript/${jobId}`, { + headers: { "x-api-key": apiKey }, }); - if (!res.ok) { const text = await res.text().catch(() => ""); - return { error: `HTTP ${res.status}: ${text}` }; + return { + error: `HTTP ${res.status}: ${redactSecret(text, apiKey)}`, + authError: res.status === 401 || res.status === 403, + }; } const data = await res.json(); + const transcript = normalizeTranscriptContent(data.content); + if (transcript) return { transcript }; - if (data.status === "ready" && data.url) { - // Transcript is ready — fetch the text from the provided URL - const txtRes = await fetch(data.url); - if (!txtRes.ok) - return { - error: `Failed to fetch transcript text: HTTP ${txtRes.status}`, - }; - const transcript = await txtRes.text(); - return { transcript }; - } - - if (data.status === "processing") { + if (data.status === "processing" || data.status === "queued") { console.error( - ` pod2txt: processing (attempt ${attempt}/${maxAttempts}), waiting ${pollInterval / 1000}s...`, + ` Supadata: processing (attempt ${attempt}/${maxAttempts}), waiting ${pollInterval / 1000}s...`, ); if (attempt < maxAttempts) { await new Promise((r) => setTimeout(r, pollInterval)); @@ -366,17 +485,49 @@ async function fetchPod2txtTranscript(rssUrl, guid, apiKey) { continue; } - // Unexpected status or error from the API - return { error: data.message || `Unexpected status: ${data.status}` }; + return { error: data.message || `Unexpected job status: ${data.status}` }; } - return { error: "Timed out waiting for transcript processing" }; + return { error: "Timed out waiting for Supadata transcript processing" }; +} + +export async function fetchSupadataTranscript(videoUrl, apiKey, fetcher = fetch) { + try { + const params = new URLSearchParams({ + url: videoUrl, + lang: "en", + text: "true", + mode: "native", + }); + const res = await fetcher(`${SUPADATA_BASE}/transcript?${params}`, { + headers: { "x-api-key": apiKey }, + signal: AbortSignal.timeout(30000), + }); + if (!res.ok && res.status !== 202) { + const text = await res.text().catch(() => ""); + return { + error: `HTTP ${res.status}: ${redactSecret(text, apiKey)}`, + authError: res.status === 401 || res.status === 403, + }; + } + + const data = await res.json(); + const transcript = normalizeTranscriptContent(data.content); + if (transcript) return { transcript }; + if (data.jobId) { + return fetchSupadataTranscriptJob(data.jobId, apiKey, fetcher); + } + + return { error: data.message || "No transcript content returned" }; + } catch (err) { + return { error: err.message }; + } } // Main podcast fetching function. For each podcast: // 1. Fetches the RSS feed to discover episodes // 2. Filters by lookback window and dedup -// 3. Fetches transcript via pod2txt for the newest unseen episode +// 3. Fetches transcript for the newest unseen episode async function fetchPodcastContent(podcasts, apiKey, state, errors) { const cutoff = new Date(Date.now() - PODCAST_LOOKBACK_HOURS * 60 * 60 * 1000); const allCandidates = []; @@ -455,15 +606,50 @@ async function fetchPodcastContent(podcasts, apiKey, state, errors) { console.error(` - "${v.title}" published=${v.publishedAt || "unknown"}`); } - // Step 3: Try each candidate until we get a transcript from pod2txt + // Step 3: Try each candidate until we get a transcript. for (const selected of withinWindow) { console.error(` Fetching transcript for "${selected.title}"...`); - const result = await fetchPod2txtTranscript( - selected.podcast.rssUrl, - selected.guid, - apiKey, + // Try to resolve the exact YouTube video URL first. Supadata's native + // transcript API works from the video URL, while the older pod2txt gateway + // works from podcast RSS GUIDs. + const youtubeEpisode = await findYouTubeEpisode( + selected.podcast.url, + selected.title, ); + const transcriptUrl = youtubeEpisode?.url || selected.link; + if (youtubeEpisode?.url) { + console.error(` Matched YouTube episode URL: ${youtubeEpisode.url}`); + } else if (selected.link) { + console.error(` Using podcast episode URL for transcript lookup: ${selected.link}`); + } else { + console.error( + ` No YouTube episode match found — falling back to podcast RSS transcript lookup`, + ); + } + + let result; + if (apiKey?.startsWith("sd_") && transcriptUrl) { + result = await fetchSupadataTranscript(transcriptUrl, apiKey); + if (result.error && !result.authError) { + console.error( + ` Supadata transcript error: ${result.error} — trying pod2txt fallback`, + ); + result = await fetchPod2txtTranscript( + selected.podcast.rssUrl, + selected.guid, + apiKey, + ); + } else if (result.error) { + console.error(` Supadata transcript error: ${result.error}`); + } + } else { + result = await fetchPod2txtTranscript( + selected.podcast.rssUrl, + selected.guid, + apiKey, + ); + } // Mark as seen regardless so we don't retry failed episodes daily state.seenVideos[selected.guid] = Date.now(); @@ -489,32 +675,11 @@ async function fetchPodcastContent(podcasts, apiKey, state, errors) { ` Selected: "${selected.title}" (transcript: ${result.transcript.length} chars)`, ); - // Try to resolve the exact YouTube video URL for this episode. If the - // lookup fails (no YouTube channel configured, no title match, network - // error), fall back to the channel URL so the feed still works. - const youtubeUrl = await findYouTubeEpisodeUrl( - selected.podcast.url, - selected.title, - ); - if (youtubeUrl) { - console.error(` Matched YouTube episode URL: ${youtubeUrl}`); - } else { - console.error( - ` No YouTube episode match found — falling back to channel URL`, - ); - } - - return [ - { - source: "podcast", - name: selected.podcast.name, - title: selected.title, - guid: selected.guid, - url: youtubeUrl || selected.podcast.url, - publishedAt: selected.publishedAt, - transcript: result.transcript, - }, - ]; + return [buildPodcastFeedEntry({ + selected, + youtubeEpisode, + transcript: result.transcript, + })]; } console.error(` No candidates had transcripts available`); @@ -523,48 +688,24 @@ async function fetchPodcastContent(podcasts, apiKey, state, errors) { // -- X/Twitter Fetching (Official API v2) ------------------------------------ -function sleep(ms) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} - -async function fetchXWithRetry(url, options) { - let lastResponse; - for (let attempt = 1; attempt <= X_RETRY_ATTEMPTS; attempt++) { - try { - const res = await fetch(url, options); - lastResponse = res; - if (!X_RETRY_STATUSES.has(res.status) || attempt === X_RETRY_ATTEMPTS) { - return res; - } - } catch (err) { - if (attempt === X_RETRY_ATTEMPTS) throw err; - } - await sleep(1000 * attempt); - } - return lastResponse; -} - async function fetchXContent(xAccounts, bearerToken, state, errors) { const results = []; const cutoff = new Date(Date.now() - TWEET_LOOKBACK_HOURS * 60 * 60 * 1000); - // Batch lookup user IDs. Smaller batches make one flaky X response less likely - // to wipe out the whole feed. + // Batch lookup all user IDs (1 API call) const handles = xAccounts.map((a) => a.handle); let userMap = {}; - for (let i = 0; i < handles.length; i += X_USER_LOOKUP_BATCH_SIZE) { - const batch = handles.slice(i, i + X_USER_LOOKUP_BATCH_SIZE); + for (let i = 0; i < handles.length; i += 100) { + const batch = handles.slice(i, i + 100); try { - const res = await fetchXWithRetry( + const res = await fetch( `${X_API_BASE}/users/by?usernames=${batch.join(",")}&user.fields=name,description`, { headers: { Authorization: `Bearer ${bearerToken}` } }, ); if (!res.ok) { - errors.push( - `X API: User lookup failed for ${batch.join(",")}: HTTP ${res.status}`, - ); + errors.push(`X API: User lookup failed: HTTP ${res.status}`); continue; } @@ -592,7 +733,7 @@ async function fetchXContent(xAccounts, bearerToken, state, errors) { if (!userData) continue; try { - const res = await fetchXWithRetry( + const res = await fetch( `${X_API_BASE}/users/${userData.id}/tweets?` + `max_results=5` + // fetch 5, then filter to 3 new ones `&tweet.fields=created_at,public_metrics,referenced_tweets,note_tweet` + @@ -648,6 +789,7 @@ async function fetchXContent(xAccounts, bearerToken, state, errors) { handle: account.handle, bio: userData.description, tweets: newTweets, + ...sourceMetadata(account), }); await new Promise((r) => setTimeout(r, 200)); @@ -661,6 +803,81 @@ async function fetchXContent(xAccounts, bearerToken, state, errors) { // -- Blog Fetching (HTML scraping) ------------------------------------------- +function decodeHtml(text) { + return text + .replace(/&/g, "&") + .replace(/</g, "<") + .replace(/>/g, ">") + .replace(/"/g, '"') + .replace(/'/g, "'") + .replace(/ /g, " "); +} + +function extractXmlText(block, tagName) { + const escapedTag = tagName.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); + const match = block.match( + new RegExp(`<${escapedTag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${escapedTag}>`, "i"), + ); + if (!match) return null; + + return decodeHtml(match[1].replace(/^$/, "")) + .replace(//gi, " ") + .replace(//gi, " ") + .replace(/<[^>]+>/g, " ") + .replace(/\s+/g, " ") + .trim(); +} + +function extractFeedLink(block) { + const hrefMatch = block.match(/]*href="([^"]+)"[^>]*>/i); + if (hrefMatch) return decodeHtml(hrefMatch[1].trim()); + return extractXmlText(block, "link"); +} + +function toIsoDate(dateText) { + if (!dateText) return null; + const date = new Date(dateText); + if (Number.isNaN(date.getTime())) return null; + return date.toISOString(); +} + +export function parseSyndicationFeed(xml) { + const articles = []; + const entryRegex = /<(item|entry)\b[^>]*>([\s\S]*?)<\/\1>/gi; + let entryMatch; + + while ((entryMatch = entryRegex.exec(xml)) !== null) { + const block = entryMatch[2]; + const title = extractXmlText(block, "title") || "Untitled"; + const url = extractFeedLink(block); + const guid = + extractXmlText(block, "guid") || + extractXmlText(block, "id") || + url || + title; + const dateText = + extractXmlText(block, "pubDate") || + extractXmlText(block, "published") || + extractXmlText(block, "updated"); + const content = + extractXmlText(block, "content:encoded") || + extractXmlText(block, "content") || + extractXmlText(block, "summary") || + extractXmlText(block, "description") || + ""; + + articles.push({ + title, + url, + guid, + publishedAt: toIsoDate(dateText), + content, + }); + } + + return articles.filter((article) => article.url); +} + // Scrapes the Anthropic Engineering blog index page. // The page is a Next.js app that embeds article data as JSON in `; + + assert.deepEqual(parseYouTubePageData(html), [ + { + title: "Episode title", + videoId: "abc123XYZ00", + url: "https://www.youtube.com/watch?v=abc123XYZ00", + }, + ]); +}); + +test("matchYouTubeEpisode returns both videoId and URL only for a confident match", () => { + const videos = [ + { + title: "How Every's Team Used AI to Ship Its Biggest Launch Ever", + videoId: "episode12345", + url: "https://www.youtube.com/watch?v=episode12345", + }, + ]; + + assert.deepEqual( + matchYouTubeEpisode(videos, "How Every's Team Used AI to Ship Its Biggest Launch Ever"), + { + videoId: "episode12345", + url: "https://www.youtube.com/watch?v=episode12345", + }, + ); + assert.equal(matchYouTubeEpisode(videos, "Unrelated episode"), null); +}); + +test("buildPodcastFeedEntry never falls back to a playlist URL", () => { + const entry = buildPodcastFeedEntry({ + selected: { + podcast: { + name: "AI & I by Every", + url: "https://www.youtube.com/playlist?list=playlist123", + }, + title: "Episode without a video match", + guid: "episode-guid", + link: "https://every.to/podcast/episode", + publishedAt: "2026-07-22T15:09:01.000Z", + }, + youtubeEpisode: null, + transcript: "transcript", + }); + + assert.equal(entry.videoId, null); + assert.equal(entry.url, null); + assert.equal(entry.podcastUrl, "https://every.to/podcast/episode"); + assert.doesNotMatch(JSON.stringify(entry), /youtube\.com\/playlist/); +}); + +test("parseSyndicationFeed reads Atom entries with escaped HTML content", () => { + const feed = ` + + + Codex v1.2.3 + tag:github.com,2008:Repository/1/v1.2.3 + 2026-06-07T08:00:00Z + + <p>Fixed agent workflow bugs.</p> + + `; + + assert.deepEqual(parseSyndicationFeed(feed), [ + { + title: "Codex v1.2.3", + url: "https://github.com/openai/codex/releases/tag/v1.2.3", + guid: "tag:github.com,2008:Repository/1/v1.2.3", + publishedAt: "2026-06-07T08:00:00.000Z", + content: "Fixed agent workflow bugs.", + }, + ]); +}); + +test("sourceMetadata keeps only editorial metadata", () => { + assert.deepEqual( + sourceMetadata({ + name: "Claude", + handle: "claudeai", + topics: ["claude-code"], + wechat: { priority: "high" }, + }), + { + topics: ["claude-code"], + wechat: { priority: "high" }, + }, + ); +}); diff --git a/scripts/prepare-digest.js b/scripts/prepare-digest.js index fe54170b3..f128d6af8 100755 --- a/scripts/prepare-digest.js +++ b/scripts/prepare-digest.js @@ -16,15 +16,26 @@ // Output: JSON to stdout // ============================================================================ -import { readFile, mkdir } from 'fs/promises'; +import { readFile } from 'fs/promises'; +import { setDefaultResultOrder } from 'dns'; import { existsSync } from 'fs'; import { join } from 'path'; +import { fileURLToPath } from 'url'; import { homedir } from 'os'; +import { execFile } from 'child_process'; +import { promisify } from 'util'; + +setDefaultResultOrder('ipv4first'); // -- Constants --------------------------------------------------------------- const USER_DIR = join(homedir(), '.follow-builders'); const CONFIG_PATH = join(USER_DIR, 'config.json'); +const SCRIPT_DIR = decodeURIComponent(new URL('.', import.meta.url).pathname); +const REPO_ROOT = join(SCRIPT_DIR, '..'); +const FETCH_TIMEOUT_MS = Number(process.env.FOLLOW_BUILDERS_FETCH_TIMEOUT_MS || 15000); +const execFileAsync = promisify(execFile); +let originMainFetchPromise = null; const FEED_X_URL = 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-x.json'; const FEED_PODCASTS_URL = 'https://raw.githubusercontent.com/zarazhangrui/follow-builders/main/feed-podcasts.json'; @@ -36,27 +47,161 @@ const PROMPT_FILES = [ 'summarize-tweets.md', 'summarize-blogs.md', 'digest-intro.md', + 'digest-intro-expanded.md', 'translate.md' ]; // -- Fetch helpers ----------------------------------------------------------- -async function fetchJSON(url) { - const res = await fetch(url); +async function fetchJSON(url, fetcher = fetch) { + const res = await fetcher(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); if (!res.ok) return null; return res.json(); } -async function fetchText(url) { - const res = await fetch(url); +async function fetchText(url, fetcher = fetch) { + const res = await fetcher(url, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) }); if (!res.ok) return null; return res.text(); } +export async function fetchJSONWithFallback({ url, localPath, label, errors, fetcher = fetch }) { + try { + const remote = await fetchJSON(url, fetcher); + if (remote) return remote; + errors.push(`Remote ${label} returned an unusable response`); + } catch (err) { + errors.push(`Remote ${label} failed: ${err.message}`); + } + + if (!existsSync(localPath)) { + errors.push(`Local fallback for ${label} not found: ${localPath}`); + return null; + } + + try { + return JSON.parse(await readFile(localPath, 'utf-8')); + } catch (err) { + errors.push(`Local fallback for ${label} failed: ${err.message}`); + return null; + } +} + +async function readGitBlob(ref, filePath) { + const { stdout } = await execFileAsync('git', ['show', `${ref}:${filePath}`], { + cwd: REPO_ROOT, + maxBuffer: 20 * 1024 * 1024 + }); + return JSON.parse(stdout); +} + +async function readGitTextBlob(ref, filePath) { + const { stdout } = await execFileAsync('git', ['show', `${ref}:${filePath}`], { + cwd: REPO_ROOT, + maxBuffer: 20 * 1024 * 1024 + }); + return stdout; +} + +async function ensureOriginMainFetched() { + if (!originMainFetchPromise) { + originMainFetchPromise = execFileAsync('git', ['fetch', '--quiet', 'origin', 'main'], { + cwd: REPO_ROOT, + timeout: FETCH_TIMEOUT_MS + }); + } + return originMainFetchPromise; +} + +export async function fetchFreshJSON({ + url, + filePath, + label, + errors, + fetcher = fetch, + gitFetcher = ensureOriginMainFetched, + gitReader = readGitBlob +}) { + try { + await gitFetcher(); + const fromOrigin = await gitReader('origin/main', filePath); + return { data: fromOrigin, source: 'origin_main' }; + } catch (err) { + errors.push(`Git origin/main ${label} failed: ${err.message}`); + } + + try { + const remote = await fetchJSON(url, fetcher); + if (remote) { + return { data: remote, source: 'github_raw' }; + } + errors.push(`Remote ${label} returned an unusable response`); + } catch (err) { + errors.push(`Remote ${label} failed: ${err.message}`); + } + + return { data: null, source: null }; +} + +export async function fetchTextWithFallback({ url, localPath, label, errors, fetcher = fetch }) { + try { + const remote = await fetchText(url, fetcher); + if (remote) return remote; + errors.push(`Remote ${label} returned an unusable response`); + } catch (err) { + errors.push(`Remote ${label} failed: ${err.message}`); + } + + if (!existsSync(localPath)) { + errors.push(`Local fallback for ${label} not found: ${localPath}`); + return null; + } + + try { + return readFile(localPath, 'utf-8'); + } catch (err) { + errors.push(`Local fallback for ${label} failed: ${err.message}`); + return null; + } +} + +export async function fetchTextWithRemoteFallback({ + url, + filePath, + localPath, + label, + errors, + fetcher = fetch, + gitFetcher = ensureOriginMainFetched, + gitTextReader = readGitTextBlob +}) { + try { + await gitFetcher(); + return await gitTextReader('origin/main', filePath); + } catch (err) { + errors.push(`Git origin/main ${label} failed: ${err.message}`); + } + + return fetchTextWithFallback({ url, localPath, label, errors, fetcher }); +} + +export function resolveDigestMode(args = [], config = {}) { + if (args.includes('--expanded')) return 'expanded'; + if (args.includes('--standard')) return 'standard'; + return config.digestMode === 'expanded' ? 'expanded' : 'standard'; +} + +function promptKeyFor(filename) { + if (filename === 'digest-intro.md') return 'digest_intro_standard'; + if (filename === 'digest-intro-expanded.md') return 'digest_intro_expanded'; + return filename.replace('.md', '').replace(/-/g, '_'); +} + // -- Main -------------------------------------------------------------------- async function main() { const errors = []; + const args = process.argv.slice(2); // 1. Read user config let config = { @@ -72,30 +217,67 @@ async function main() { } } - // 2. Fetch all three feeds - const [feedX, feedPodcasts, feedBlogs] = await Promise.all([ - fetchJSON(FEED_X_URL), - fetchJSON(FEED_PODCASTS_URL), - fetchJSON(FEED_BLOGS_URL) + const digestMode = resolveDigestMode(args, config); + + // 2. Fetch all three feeds. + // + // Feeds must be fresh enough to be tied to the current remote source. A local + // repo fallback can be weeks old, which is worse than no digest for news. + const [feedXResult, feedPodcastsResult, feedBlogsResult] = await Promise.all([ + fetchFreshJSON({ + url: FEED_X_URL, + filePath: 'feed-x.json', + label: 'tweet feed', + errors + }), + fetchFreshJSON({ + url: FEED_PODCASTS_URL, + filePath: 'feed-podcasts.json', + label: 'podcast feed', + errors + }), + fetchFreshJSON({ + url: FEED_BLOGS_URL, + filePath: 'feed-blogs.json', + label: 'blog feed', + errors + }) ]); + const feedX = feedXResult.data; + const feedPodcasts = feedPodcastsResult.data; + const feedBlogs = feedBlogsResult.data; + if (!feedX) errors.push('Could not fetch tweet feed'); if (!feedPodcasts) errors.push('Could not fetch podcast feed'); if (!feedBlogs) errors.push('Could not fetch blog feed'); - if (feedX?.errors?.length) { - errors.push( - ...feedX.errors.map((error) => `Tweet feed problem: ${error}`) - ); - } - if (feedPodcasts?.errors?.length) { - errors.push( - ...feedPodcasts.errors.map((error) => `Podcast feed problem: ${error}`) - ); - } - if (feedBlogs?.errors?.length) { - errors.push( - ...feedBlogs.errors.map((error) => `Blog feed problem: ${error}`) - ); + + if (!feedX || !feedPodcasts || !feedBlogs) { + console.log(JSON.stringify({ + status: 'no_fresh_data', + generatedAt: new Date().toISOString(), + message: 'Could not verify the latest remote feeds. No digest should be generated from local fallback data.', + config: { + language: config.language || 'en', + frequency: config.frequency || 'daily', + digestMode, + delivery: config.delivery || { method: 'stdout' } + }, + stats: { + podcastEpisodes: 0, + xBuilders: 0, + totalTweets: 0, + blogPosts: 0, + feedGeneratedAt: null + }, + feedSources: { + x: feedXResult.source, + podcasts: feedPodcastsResult.source, + blogs: feedBlogsResult.source + }, + errors + }, null, 2)); + return; } // 3. Load prompts with priority: user custom > remote (GitHub) > local default @@ -105,12 +287,11 @@ async function main() { // Otherwise, fetch the latest from GitHub so they get central improvements. // If GitHub is unreachable, fall back to the local copy shipped with the skill. const prompts = {}; - const scriptDir = decodeURIComponent(new URL('.', import.meta.url).pathname); - const localPromptsDir = join(scriptDir, '..', 'prompts'); + const localPromptsDir = join(REPO_ROOT, 'prompts'); const userPromptsDir = join(USER_DIR, 'prompts'); for (const filename of PROMPT_FILES) { - const key = filename.replace('.md', '').replace(/-/g, '_'); + const key = promptKeyFor(filename); const userPath = join(userPromptsDir, filename); const localPath = join(localPromptsDir, filename); @@ -120,21 +301,26 @@ async function main() { continue; } - // Priority 2: latest from GitHub (central updates) - const remote = await fetchText(`${PROMPTS_BASE}/${filename}`); - if (remote) { - prompts[key] = remote; + // Priority 2: latest from GitHub (central updates), falling back to local copy. + const prompt = await fetchTextWithRemoteFallback({ + url: `${PROMPTS_BASE}/${filename}`, + filePath: `prompts/${filename}`, + localPath, + label: `prompt ${filename}`, + errors + }); + if (prompt) { + prompts[key] = prompt; continue; } - - // Priority 3: local copy shipped with the skill - if (existsSync(localPath)) { - prompts[key] = await readFile(localPath, 'utf-8'); - } else { - errors.push(`Could not load prompt: ${filename}`); - } } + // Keep the historical key stable while selecting the requested edition. + prompts.digest_intro = + prompts[`digest_intro_${digestMode}`] || + prompts.digest_intro_standard || + prompts.digest_intro_expanded; + // 4. Build the output — everything the LLM needs in one blob const output = { status: 'ok', @@ -144,6 +330,7 @@ async function main() { config: { language: config.language || 'en', frequency: config.frequency || 'daily', + digestMode, delivery: config.delivery || { method: 'stdout' } }, @@ -161,6 +348,12 @@ async function main() { feedGeneratedAt: feedX?.generatedAt || feedPodcasts?.generatedAt || feedBlogs?.generatedAt || null }, + feedSources: { + x: feedXResult.source, + podcasts: feedPodcastsResult.source, + blogs: feedBlogsResult.source + }, + // Prompts — the LLM reads these and follows the instructions prompts, @@ -171,10 +364,12 @@ async function main() { console.log(JSON.stringify(output, null, 2)); } -main().catch(err => { - console.error(JSON.stringify({ - status: 'error', - message: err.message - })); - process.exit(1); -}); +if (process.argv[1] === fileURLToPath(import.meta.url)) { + main().catch(err => { + console.error(JSON.stringify({ + status: 'error', + message: err.message + })); + process.exit(1); + }); +} diff --git a/scripts/prepare-digest.test.js b/scripts/prepare-digest.test.js new file mode 100644 index 000000000..99e8e4e39 --- /dev/null +++ b/scripts/prepare-digest.test.js @@ -0,0 +1,142 @@ +import { mkdtemp, writeFile } from "fs/promises"; +import { getDefaultResultOrder } from "dns"; +import { tmpdir } from "os"; +import { join } from "path"; +import test from "node:test"; +import assert from "node:assert/strict"; + +import { + fetchFreshJSON, + fetchJSONWithFallback, + fetchTextWithFallback, + fetchTextWithRemoteFallback, + resolveDigestMode, +} from "./prepare-digest.js"; + +test("prepare-digest prefers IPv4 DNS results for GitHub raw fetches", () => { + assert.equal(getDefaultResultOrder(), "ipv4first"); +}); + +test("resolveDigestMode supports explicit and configured expanded editions", () => { + assert.equal(resolveDigestMode([], {}), "standard"); + assert.equal(resolveDigestMode(["--expanded"], {}), "expanded"); + assert.equal(resolveDigestMode(["--standard"], { digestMode: "expanded" }), "standard"); + assert.equal(resolveDigestMode([], { digestMode: "expanded" }), "expanded"); +}); + +test("fetchJSONWithFallback reads local feed when remote fetch fails", async () => { + const dir = await mkdtemp(join(tmpdir(), "fb-feed-")); + const localPath = join(dir, "feed.json"); + await writeFile(localPath, JSON.stringify({ x: [{ name: "Local feed" }] })); + + const errors = []; + const result = await fetchJSONWithFallback({ + url: "https://example.invalid/feed.json", + localPath, + label: "tweet feed", + errors, + fetcher: async () => { + throw new Error("network down"); + }, + }); + + assert.deepEqual(result, { x: [{ name: "Local feed" }] }); + assert.match(errors[0], /remote tweet feed failed/i); +}); + +test("fetchFreshJSON uses origin/main feed data when available", async () => { + const errors = []; + const result = await fetchFreshJSON({ + url: "https://example.com/feed.json", + filePath: "feed.json", + label: "tweet feed", + errors, + gitFetcher: async () => {}, + gitReader: async () => ({ generatedAt: "2026-05-27T07:45:34.065Z" }), + }); + + assert.deepEqual(result, { + data: { generatedAt: "2026-05-27T07:45:34.065Z" }, + source: "origin_main", + }); + assert.deepEqual(errors, []); +}); + +test("fetchFreshJSON falls back to GitHub raw when git is unavailable", async () => { + const errors = []; + const result = await fetchFreshJSON({ + url: "https://example.com/feed.json", + filePath: "feed.json", + label: "tweet feed", + errors, + gitFetcher: async () => { + throw new Error("not a git repo"); + }, + fetcher: async () => ({ + ok: true, + json: async () => ({ generatedAt: "2026-05-27T07:45:34.065Z" }), + }), + }); + + assert.deepEqual(result, { + data: { generatedAt: "2026-05-27T07:45:34.065Z" }, + source: "github_raw", + }); + assert.match(errors.join("\n"), /git origin\/main tweet feed failed/i); +}); + +test("fetchFreshJSON does not use repo local fallback", async () => { + const errors = []; + const result = await fetchFreshJSON({ + url: "https://example.invalid/feed.json", + filePath: "missing-feed.json", + label: "tweet feed", + errors, + gitFetcher: async () => { + throw new Error("not a git repo"); + }, + fetcher: async () => { + throw new Error("network down"); + }, + }); + + assert.deepEqual(result, { data: null, source: null }); + assert.match(errors.join("\n"), /git origin\/main tweet feed failed/i); + assert.match(errors.join("\n"), /remote tweet feed failed/i); +}); + +test("fetchTextWithFallback reads local prompt when remote fetch fails", async () => { + const dir = await mkdtemp(join(tmpdir(), "fb-prompt-")); + const localPath = join(dir, "prompt.md"); + await writeFile(localPath, "local prompt"); + + const errors = []; + const result = await fetchTextWithFallback({ + url: "https://example.invalid/prompt.md", + localPath, + label: "digest prompt", + errors, + fetcher: async () => { + throw new Error("network down"); + }, + }); + + assert.equal(result, "local prompt"); + assert.match(errors[0], /remote digest prompt failed/i); +}); + +test("fetchTextWithRemoteFallback uses origin/main prompt text when available", async () => { + const errors = []; + const result = await fetchTextWithRemoteFallback({ + url: "https://example.com/prompt.md", + filePath: "prompts/prompt.md", + localPath: "prompt.md", + label: "digest prompt", + errors, + gitFetcher: async () => {}, + gitTextReader: async () => "remote prompt from git", + }); + + assert.equal(result, "remote prompt from git"); + assert.deepEqual(errors, []); +});