diff --git a/data/leaderboard.json b/data/leaderboard.json index d84cc12f..480604e7 100644 --- a/data/leaderboard.json +++ b/data/leaderboard.json @@ -1,134 +1,150 @@ [ { "login": "zsxh1990", - "score": 22.75 + "score": 24.6 + }, + { + "login": "uncledad96-glitch", + "score": 15.26 }, { "login": "2lll5", - "score": 6.15 + "score": 5.02 }, { "login": "solaris", - "score": 3.44 + "score": 3.05 + }, + { + "login": "lb1192176991-lab", + "score": 1.87 }, { "login": "zeroknowledge0x", - "score": 1.82 + "score": 1.55 }, { "login": "sparshgarg999", - "score": 1.42 + "score": 1.26 + }, + { + "login": "namdamdoi68-oss", + "score": 0.97 + }, + { + "login": "ninghuagui-debug", + "score": 0.94 }, { "login": "yh-liao-07", - "score": 1.0 + "score": 0.93 }, { "login": "abhiavi", - "score": 1.0 + "score": 0.93 }, { "login": "ringotokens-commits", - "score": 0.99 + "score": 0.93 }, { "login": "ivegotahunnitonit", - "score": 0.99 + "score": 0.93 }, { "login": "root", - "score": 0.99 + "score": 0.62 }, { "login": "lushan888", - "score": 0.95 + "score": 0.58 }, { "login": "chiranjeevi7777", - "score": 0.95 + "score": 0.58 }, { "login": "zhangjianming7051", - "score": 0.95 + "score": 0.58 }, { "login": "xkes101116-hub", - "score": 0.94 + "score": 0.58 }, { "login": "foxymantou", - "score": 0.94 + "score": 0.58 }, { "login": "votienduong2208", - "score": 0.65 + "score": 0.56 }, { "login": "syzygys", - "score": 0.59 + "score": 0.53 }, { "login": "jhon12091986", - "score": 0.59 + "score": 0.53 }, { "login": "lovewave02", - "score": 0.58 + "score": 0.52 }, { "login": "gothundercats", - "score": 0.56 + "score": 0.51 }, { "login": "zqleslie", - "score": 0.55 + "score": 0.49 }, { "login": "andrianbalanesq", - "score": 0.52 + "score": 0.48 }, { "login": "cyprerask", - "score": 0.51 - }, - { - "login": "rohitmulani63-ops", - "score": 0.47 - }, - { - "login": "skyjames777", - "score": 0.47 + "score": 0.46 }, { "login": "doview1", - "score": 0.33 + "score": 0.28 }, { "login": "suresh chouksey", - "score": 0.33 + "score": 0.28 }, { "login": "iccccccccccccc", - "score": 0.33 + "score": 0.28 }, { "login": "sagarmaurya64-ai", - "score": 0.32 + "score": 0.27 + }, + { + "login": "rohitmulani63-ops", + "score": 0.23 + }, + { + "login": "skyjames777", + "score": 0.23 }, { "login": "pian0", - "score": 0.2 + "score": 0.17 }, { "login": "qi574", - "score": 0.16 + "score": 0.14 }, { "login": "cuongwf1711", - "score": 0.16 + "score": 0.14 }, { "login": "lqkhanh295", - "score": 0.13 + "score": 0.11 } ] \ No newline at end of file diff --git a/data/lessons.json b/data/lessons.json index 0e969407..b2a09e25 100644 --- a/data/lessons.json +++ b/data/lessons.json @@ -1945,6 +1945,28 @@ "status": "active", "verified": false }, + { + "id": "glama-introspection-gap", + "title": "Glama Introspection Gap — Build Success ≠ Tools Registered", + "domain": "devops", + "tags": [ + "glama", + "mcp", + "introspection", + "tools", + "registry" + ], + "summary": "After successfully building an MCP server on Glama, the tools don't appear in the Glama API or dashboard. Build success ≠ tools registered — they are separate a…", + "preview": "## Problem\n\nAfter successfully building an MCP server on Glama, the tools don't appear in the Glama API or dashboard. Build success ≠ tools registered — they are separate async processes.\n\n## Root Cause\n\nGlama's pipeline has two distinct steps:\n1. **Build** — Docker image creation (synchronous, returns immediately)\n2. **Introspection** — Runs the MCP server and calls `tools/list` (async, may take minutes to hours)\n\nThe build step can succeed while introspection fails silently. Common causes:\n- MCP server starts but doesn't respond to `tools/list` request\n- Server crashes during introspection\n- glama.json format issues (tools not detected)\n- Network timeout during introspection\n\n## Detection\n\n```bash\n# Check if tools are registered\ncurl -s \"https://glama.ai/api/mcp/v1/servers/OWNER/REPO\" | python3 -c \"\nimport json, sys\ndata = json.load(sys.stdin)\nprint(f'Tools: {len(data.get(\\\"tools\\\", []))}')\n\"\n```\n\nIf `tools: 0` but build succeeded, introspection failed.\n\n## Fix Action\n\n1. **Wait** — introspection is async, may take hours\n2. **Sync Server** — trigger Glama to re-read the repo\n3. **Rebuild** — force fresh introspection\n4. **Check glama.json** — must be minimal format (`$schema` + `maintainers` only)\n5. **Check Dockerfile** — ensure MCP server starts and responds to `initialize` request\n\n## Key Insight\n\n**glama.json is NOT for tool definitions.** Glama discovers tools via MCP introspection (calling `tools/list`), not from glama.json. The glama.json should only contain:\n```json\n{\n \"$schema\": \"https://glama.ai/mcp/schemas/server.json\",\n \"maintainers\": [\"username\"]\n}\n```\n\nComplex tool definitions in glama.json are ignored by Glama's introspection system.", + "url": "lessons/contrib/glama-introspection-gap.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "gpt-sovits-hubert-16khz", "title": "gpt sovits hubert 16khz", @@ -2649,6 +2671,28 @@ "status": "active", "verified": true }, + { + "id": "merge-probability-calibration", + "title": "Merge Probability Calibration — Honest Estimation vs Overfitting", + "domain": "devops", + "tags": [ + "prediction", + "calibration", + "merge-rate", + "a-b-test", + "coach" + ], + "summary": "PR coaches that predict merge probability often overfit to training data or give misleadingly precise estimates. A 70% accuracy claim may not generalize to new …", + "preview": "## Problem\n\nPR coaches that predict merge probability often overfit to training data or give misleadingly precise estimates. A 70% accuracy claim may not generalize to new repos or PR types.\n\n## Root Cause\n\n1. **Base rate trap** — Most repos have low external merge rates (20-30%). Predicting \"medium risk\" for everything gives 70%+ accuracy but zero discrimination.\n\n2. **Signal confusion** — Signals like \"needs_preflight\" or \"large_repo\" are risk markers, not success predictors. A PR can have many negative signals and still merge if the maintainer wants it.\n\n3. **Content blindness** — Current coaches analyze metadata (title, body, files_changed) but not actual diff content. Two PRs with identical metadata can have completely different merge outcomes.\n\n## A/B Test Results (445 cases)\n\n| Metric | Value |\n|--------|-------|\n| Merged PRs mean probability | 0.32 |\n| Closed PRs mean probability | 0.30 |\n| Gap | +0.02 |\n| Discrimination | YES (but limited) |\n\n**Conclusion:** Merge probability can't be more accurate than the repo's base merge rate without understanding PR content quality.\n\n## Fix Action\n\n1. **Use repo merge rate as base** — the most honest starting point\n2. **Only adjust for discriminating signals** — merge_conflict (×0.3), duplicate (×0.1), maintainer_internal (×0.05)\n3. **Don't double-count** — signals already affect tier, don't also affect probability\n4. **Be transparent** — tell users \"this repo has 20% merge rate, your PR is slightly better than average\"\n\n## Prevention\n\n- Always A/B test predictions against actual outcomes\n- Use LORO (Leave-One-Repo-Out) validation to detect overfitting\n- Report confidence intervals, not point estimates\n- Accept that some uncertainty is irreducible (depends on maintainer mood, timing, etc.)", + "url": "lessons/contrib/merge-probability-calibration.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "misakanet-heal-engine-bootstrap-workflow", "title": "MisakaNet --heal Engine Bootstrap Workflow", @@ -3348,6 +3392,28 @@ "status": "published", "verified": true }, + { + "id": "restricted-interactions-repos", + "title": "Restricted Interactions Repos — External PRs Not Accepted", + "domain": "devops", + "tags": [ + "github", + "restricted", + "external-pr", + "contribution", + "gatekeeping" + ], + "summary": "Some large open-source repos have GitHub \"restricted interactions\" enabled, which prevents non-collaborators from creating PRs, commenting, or even viewing cert…", + "preview": "## Problem\n\nSome large open-source repos have GitHub \"restricted interactions\" enabled, which prevents non-collaborators from creating PRs, commenting, or even viewing certain features. Contributors waste time preparing PRs that can never be submitted.\n\n## Root Cause\n\nGitHub allows repo admins to restrict interactions to collaborators only. This is separate from the standard fork-and-PR workflow. When enabled:\n- Non-collaborators cannot create PRs\n- Non-collaborators cannot comment on issues/PRs\n- The error message is generic: \"Interactions on this repository have been restricted to collaborators only\"\n\n## Detection\n\nBefore preparing a PR, check:\n```bash\n# Check if you can create a PR\ngh pr create --repo org/repo --title \"test\" --body \"test\" 2>&1 | grep \"restricted\"\n\n# Check if you can comment\ngh issue comment 1 --repo org/repo --body \"test\" 2>&1 | grep \"restricted\"\n```\n\n## Known Repos with Restricted Interactions\n\n- `encode/httpx` — Python HTTP library, no external PRs accepted\n- Some Grafana Labs repos — require signed commits + restricted interactions\n\n## Fix Action\n\n1. Check repo interaction permissions before investing time\n2. If restricted, look for alternative contribution channels (discussions, docs)\n3. If no alternatives, move to a different repo\n\n## Prevention\n\nAlways test repo permissions before preparing a PR:\n```bash\n# Quick permission check\ngh api repos/org/repo --jq '.permissions'\n```\n\nIf `pull` is false or interactions are restricted, don't invest time in PR preparation.", + "url": "lessons/contrib/restricted-interactions-repos.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "sag-lite-data-quality-cleaning", "title": "sag-lite-data-quality-cleaning", diff --git a/docs/data/feed.json b/docs/data/feed.json index fdcbfa6c..27a9ce86 100644 --- a/docs/data/feed.json +++ b/docs/data/feed.json @@ -1,102 +1,148 @@ { - "generated_at": "2026-07-20T14:36:00.707057+00:00", + "generated_at": "2026-07-27T22:28:45.800458+00:00", "repo": "https://github.com/Ikalus1988/MisakaNet", "site": "https://misakanet.org", "item_count": 15, "items": [ { "type": "merged_pr", - "title": "Add journey report: Newcomer onboarding experience test", - "url": "https://github.com/Ikalus1988/MisakaNet/pull/525", - "timestamp": "2026-07-20T02:22:26Z", + "title": "[Journey Report] Complete MisakaNet Onboarding Path & Friction Analysis (#510)", + "url": "https://github.com/Ikalus1988/MisakaNet/pull/603", + "timestamp": "2026-07-27T15:54:46Z", "source": "github" }, { "type": "merged_pr", - "title": "Fix #459: [Benchmark] Implement real LessonReuseBench runner or manual evidence protocol", - "url": "https://github.com/Ikalus1988/MisakaNet/pull/526", - "timestamp": "2026-07-19T17:35:55Z", + "title": "feat(lessons): 3 fact-checked community lessons", + "url": "https://github.com/Ikalus1988/MisakaNet/pull/608", + "timestamp": "2026-07-27T15:54:42Z", + "source": "github" + }, + { + "type": "challenge", + "title": "[Search][Bounty] Add search result quality feedback loop to search_knowledge.py", + "url": "https://github.com/Ikalus1988/MisakaNet/issues/604", + "labels": [ + "ready", + "bounty", + "agent-friendly", + "status:competition", + "pool:deep", + "priority:next" + ], + "timestamp": "2026-07-27T13:49:01Z", "source": "github" }, { "type": "merged_pr", - "title": "docs: add 5-minute quickstart section to README", - "url": "https://github.com/Ikalus1988/MisakaNet/pull/524", - "timestamp": "2026-07-19T16:05:30Z", + "title": "docs: update Glama lesson — add glama.json + introspection pitfalls", + "url": "https://github.com/Ikalus1988/MisakaNet/pull/594", + "timestamp": "2026-07-27T01:57:04Z", + "source": "github" + }, + { + "type": "challenge", + "title": "Build demand board: aggregate unsolved failure families into public task-family counts", + "url": "https://github.com/Ikalus1988/MisakaNet/issues/591", + "labels": [ + "enhancement", + "ready", + "bounty", + "agent-friendly", + "status:competition", + "pool:deep", + "status:needs-design", + "priority:next" + ], + "timestamp": "2026-07-26T16:51:21Z", "source": "github" }, { "type": "merged_pr", - "title": "docs: add contributor troubleshooting FAQ", - "url": "https://github.com/Ikalus1988/MisakaNet/pull/520", - "timestamp": "2026-07-19T12:05:05Z", + "title": "docs: add Glama MCP server deployment lesson", + "url": "https://github.com/Ikalus1988/MisakaNet/pull/590", + "timestamp": "2026-07-26T13:57:12Z", "source": "github" }, { "type": "merged_pr", - "title": "ci: stop fatal-guard false failure when paths do not match", - "url": "https://github.com/Ikalus1988/MisakaNet/pull/518", - "timestamp": "2026-07-19T12:02:58Z", + "title": "chore(deps-dev): bump postcss from 8.5.15 to 8.5.23 in /web", + "url": "https://github.com/Ikalus1988/MisakaNet/pull/586", + "timestamp": "2026-07-25T16:02:51Z", "source": "github" }, { "type": "challenge", - "title": "[Journey][Bounty] Test the full MisakaNet onboarding path and report real friction", - "url": "https://github.com/Ikalus1988/MisakaNet/issues/510", + "title": "Add curl-first intake endpoint: POST /api/intake for MCP, agent, and sandbox feedback", + "url": "https://github.com/Ikalus1988/MisakaNet/issues/589", "labels": [ - "good first issue", - "needs-ac", + "enhancement", "ready", "bounty", - "docs", "agent-friendly", "status:competition", - "user-research", - "journey-report", - "pool:quick", + "pool:deep", + "status:needs-design", "priority:now" ], - "timestamp": "2026-07-19T03:14:27Z", + "timestamp": "2026-07-25T15:55:06Z", + "source": "github" + }, + { + "type": "challenge", + "title": "Feedback hub: unify search/email/journey/danmaku intake", + "url": "https://github.com/Ikalus1988/MisakaNet/issues/574", + "labels": [ + "enhancement", + "ready", + "bounty", + "agent-friendly", + "status:competition", + "pool:deep", + "status:needs-design", + "priority:now" + ], + "timestamp": "2026-07-23T10:27:25Z", "source": "github" }, { "type": "lesson", - "title": "Bounty Contributors Are Not Always Users", - "url": "lessons/contrib/bounty-contributors-are-not-always-users.md", - "timestamp": "2026-07-17", - "domain": "growth", + "title": "Glama Introspection Gap — Build Success ≠ Tools Registered", + "url": "lessons/contrib/glama-introspection-gap.md", + "timestamp": "2026-07-22", + "domain": "devops", "source": "lessons" }, { "type": "lesson", - "title": "MCP Registry Readiness Requires QA Before Promotion", - "url": "lessons/contrib/mcp-registry-readiness-requires-qa-before-promotion.md", - "timestamp": "2026-07-17", - "domain": "mcp", + "title": "Merge Probability Calibration — Honest Estimation vs Overfitting", + "url": "lessons/contrib/merge-probability-calibration.md", + "timestamp": "2026-07-22", + "domain": "devops", "source": "lessons" }, { "type": "lesson", - "title": "Repository Traffic Is Not Lesson Use", - "url": "lessons/contrib/repository-traffic-is-not-lesson-use.md", - "timestamp": "2026-07-17", - "domain": "growth", + "title": "Restricted Interactions Repos — External PRs Not Accepted", + "url": "lessons/contrib/restricted-interactions-repos.md", + "timestamp": "2026-07-22", + "domain": "devops", "source": "lessons" }, { "type": "lesson", - "title": "When Lessons Are Too Heavy, Use Rescue Cards", - "url": "lessons/contrib/rescue-cards-for-non-github-users.md", + "title": "Bounty Contributors Are Not Always Users", + "url": "lessons/contrib/bounty-contributors-are-not-always-users.md", "timestamp": "2026-07-17", - "domain": "ux", + "domain": "growth", "source": "lessons" }, { "type": "lesson", - "title": "Two Evidence Loops for Failure Lessons", - "url": "lessons/contrib/two-evidence-loops-for-failure-lessons.md", + "title": "MCP Registry Readiness Requires QA Before Promotion", + "url": "lessons/contrib/mcp-registry-readiness-requires-qa-before-promotion.md", "timestamp": "2026-07-17", - "domain": "growth", + "domain": "mcp", "source": "lessons" }, { @@ -118,57 +164,6 @@ ], "timestamp": "2026-07-09T07:03:41Z", "source": "github" - }, - { - "type": "challenge", - "title": "[Ecosystem] Build Cursor integration for MisakaNet lessons", - "url": "https://github.com/Ikalus1988/MisakaNet/issues/318", - "labels": [ - "enhancement", - "Ring-3", - "bounty", - "agent-friendly", - "status:competition", - "pool:deep", - "status:needs-design", - "priority:later" - ], - "timestamp": "2026-07-02T16:01:30Z", - "source": "github" - }, - { - "type": "challenge", - "title": "[Ecosystem] Build VS Code extension for MisakaNet lesson search", - "url": "https://github.com/Ikalus1988/MisakaNet/issues/317", - "labels": [ - "enhancement", - "Ring-3", - "bounty", - "agent-friendly", - "status:competition", - "pool:deep", - "status:needs-design", - "priority:later" - ], - "timestamp": "2026-07-02T16:01:28Z", - "source": "github" - }, - { - "type": "challenge", - "title": "[API] Add GraphQL API for flexible lesson queries", - "url": "https://github.com/Ikalus1988/MisakaNet/issues/316", - "labels": [ - "enhancement", - "Ring-3", - "ready", - "bounty", - "agent-friendly", - "status:competition", - "pool:deep", - "priority:later" - ], - "timestamp": "2026-07-02T16:01:25Z", - "source": "github" } ] } diff --git a/docs/data/lessons.json b/docs/data/lessons.json index 0e969407..b2a09e25 100644 --- a/docs/data/lessons.json +++ b/docs/data/lessons.json @@ -1945,6 +1945,28 @@ "status": "active", "verified": false }, + { + "id": "glama-introspection-gap", + "title": "Glama Introspection Gap — Build Success ≠ Tools Registered", + "domain": "devops", + "tags": [ + "glama", + "mcp", + "introspection", + "tools", + "registry" + ], + "summary": "After successfully building an MCP server on Glama, the tools don't appear in the Glama API or dashboard. Build success ≠ tools registered — they are separate a…", + "preview": "## Problem\n\nAfter successfully building an MCP server on Glama, the tools don't appear in the Glama API or dashboard. Build success ≠ tools registered — they are separate async processes.\n\n## Root Cause\n\nGlama's pipeline has two distinct steps:\n1. **Build** — Docker image creation (synchronous, returns immediately)\n2. **Introspection** — Runs the MCP server and calls `tools/list` (async, may take minutes to hours)\n\nThe build step can succeed while introspection fails silently. Common causes:\n- MCP server starts but doesn't respond to `tools/list` request\n- Server crashes during introspection\n- glama.json format issues (tools not detected)\n- Network timeout during introspection\n\n## Detection\n\n```bash\n# Check if tools are registered\ncurl -s \"https://glama.ai/api/mcp/v1/servers/OWNER/REPO\" | python3 -c \"\nimport json, sys\ndata = json.load(sys.stdin)\nprint(f'Tools: {len(data.get(\\\"tools\\\", []))}')\n\"\n```\n\nIf `tools: 0` but build succeeded, introspection failed.\n\n## Fix Action\n\n1. **Wait** — introspection is async, may take hours\n2. **Sync Server** — trigger Glama to re-read the repo\n3. **Rebuild** — force fresh introspection\n4. **Check glama.json** — must be minimal format (`$schema` + `maintainers` only)\n5. **Check Dockerfile** — ensure MCP server starts and responds to `initialize` request\n\n## Key Insight\n\n**glama.json is NOT for tool definitions.** Glama discovers tools via MCP introspection (calling `tools/list`), not from glama.json. The glama.json should only contain:\n```json\n{\n \"$schema\": \"https://glama.ai/mcp/schemas/server.json\",\n \"maintainers\": [\"username\"]\n}\n```\n\nComplex tool definitions in glama.json are ignored by Glama's introspection system.", + "url": "lessons/contrib/glama-introspection-gap.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "gpt-sovits-hubert-16khz", "title": "gpt sovits hubert 16khz", @@ -2649,6 +2671,28 @@ "status": "active", "verified": true }, + { + "id": "merge-probability-calibration", + "title": "Merge Probability Calibration — Honest Estimation vs Overfitting", + "domain": "devops", + "tags": [ + "prediction", + "calibration", + "merge-rate", + "a-b-test", + "coach" + ], + "summary": "PR coaches that predict merge probability often overfit to training data or give misleadingly precise estimates. A 70% accuracy claim may not generalize to new …", + "preview": "## Problem\n\nPR coaches that predict merge probability often overfit to training data or give misleadingly precise estimates. A 70% accuracy claim may not generalize to new repos or PR types.\n\n## Root Cause\n\n1. **Base rate trap** — Most repos have low external merge rates (20-30%). Predicting \"medium risk\" for everything gives 70%+ accuracy but zero discrimination.\n\n2. **Signal confusion** — Signals like \"needs_preflight\" or \"large_repo\" are risk markers, not success predictors. A PR can have many negative signals and still merge if the maintainer wants it.\n\n3. **Content blindness** — Current coaches analyze metadata (title, body, files_changed) but not actual diff content. Two PRs with identical metadata can have completely different merge outcomes.\n\n## A/B Test Results (445 cases)\n\n| Metric | Value |\n|--------|-------|\n| Merged PRs mean probability | 0.32 |\n| Closed PRs mean probability | 0.30 |\n| Gap | +0.02 |\n| Discrimination | YES (but limited) |\n\n**Conclusion:** Merge probability can't be more accurate than the repo's base merge rate without understanding PR content quality.\n\n## Fix Action\n\n1. **Use repo merge rate as base** — the most honest starting point\n2. **Only adjust for discriminating signals** — merge_conflict (×0.3), duplicate (×0.1), maintainer_internal (×0.05)\n3. **Don't double-count** — signals already affect tier, don't also affect probability\n4. **Be transparent** — tell users \"this repo has 20% merge rate, your PR is slightly better than average\"\n\n## Prevention\n\n- Always A/B test predictions against actual outcomes\n- Use LORO (Leave-One-Repo-Out) validation to detect overfitting\n- Report confidence intervals, not point estimates\n- Accept that some uncertainty is irreducible (depends on maintainer mood, timing, etc.)", + "url": "lessons/contrib/merge-probability-calibration.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "misakanet-heal-engine-bootstrap-workflow", "title": "MisakaNet --heal Engine Bootstrap Workflow", @@ -3348,6 +3392,28 @@ "status": "published", "verified": true }, + { + "id": "restricted-interactions-repos", + "title": "Restricted Interactions Repos — External PRs Not Accepted", + "domain": "devops", + "tags": [ + "github", + "restricted", + "external-pr", + "contribution", + "gatekeeping" + ], + "summary": "Some large open-source repos have GitHub \"restricted interactions\" enabled, which prevents non-collaborators from creating PRs, commenting, or even viewing cert…", + "preview": "## Problem\n\nSome large open-source repos have GitHub \"restricted interactions\" enabled, which prevents non-collaborators from creating PRs, commenting, or even viewing certain features. Contributors waste time preparing PRs that can never be submitted.\n\n## Root Cause\n\nGitHub allows repo admins to restrict interactions to collaborators only. This is separate from the standard fork-and-PR workflow. When enabled:\n- Non-collaborators cannot create PRs\n- Non-collaborators cannot comment on issues/PRs\n- The error message is generic: \"Interactions on this repository have been restricted to collaborators only\"\n\n## Detection\n\nBefore preparing a PR, check:\n```bash\n# Check if you can create a PR\ngh pr create --repo org/repo --title \"test\" --body \"test\" 2>&1 | grep \"restricted\"\n\n# Check if you can comment\ngh issue comment 1 --repo org/repo --body \"test\" 2>&1 | grep \"restricted\"\n```\n\n## Known Repos with Restricted Interactions\n\n- `encode/httpx` — Python HTTP library, no external PRs accepted\n- Some Grafana Labs repos — require signed commits + restricted interactions\n\n## Fix Action\n\n1. Check repo interaction permissions before investing time\n2. If restricted, look for alternative contribution channels (discussions, docs)\n3. If no alternatives, move to a different repo\n\n## Prevention\n\nAlways test repo permissions before preparing a PR:\n```bash\n# Quick permission check\ngh api repos/org/repo --jq '.permissions'\n```\n\nIf `pull` is false or interactions are restricted, don't invest time in PR preparation.", + "url": "lessons/contrib/restricted-interactions-repos.md", + "created": "2026-07-22", + "updated": "", + "validity_period_days": 365, + "environment_version": "", + "confidence": 0.5, + "status": "published", + "verified": false + }, { "id": "sag-lite-data-quality-cleaning", "title": "sag-lite-data-quality-cleaning", diff --git a/lessons/contrib/finding-and-fixing-ghostty-s-largest-memory-leak.md b/lessons/contrib/finding-and-fixing-ghostty-s-largest-memory-leak.md new file mode 100644 index 00000000..4c7c0b0a --- /dev/null +++ b/lessons/contrib/finding-and-fixing-ghostty-s-largest-memory-leak.md @@ -0,0 +1,43 @@ +--- +{"title": "Finding and Fixing Ghostty's Largest Memory Leak", "domain": "systems_programming", "tags": ["memory_management", "memory_leak", "debugging", "terminal_emulator", "mmap"], "language": "en", "status": "published", "source": "https://mitchellh.com/writing/ghostty-memory-leak-fix", "created": "2026-07-28", "confidence": "0.85"} +--- + +## Problem + +Ghostty users reported the terminal emulator consuming absurd amounts of memory, with one user reporting 37 GB after 10 days of uptime. The leak was present since at least Ghostty 1.0, but only became apparent at scale when popular CLI applications like Claude Code started producing the correct conditions to trigger it. Claude Code's CLI produces multi-codepoint grapheme outputs which force Ghostty to regularly use non-standard memory pages, combined with significant scrollback output on the primary screen. + +## Root Cause + +Ghostty uses a PageList data structure (doubly-linked list of memory pages) to store terminal content. Most pages are standard-sized and allocated from a memory pool using mmap. When lines have many emoji, styles, or hyperlinks, larger non-standard pages are allocated directly with mmap, bypassing the pool. + +During scrollback pruning optimization, when the scrollback limit is reached, Ghostty reuses the oldest page as the newest page by moving it from the front to the back of the list. However, the code always resized the page metadata back to standard size without resizing the underlying memory allocation itself. This caused a metadata/memory desync: the metadata indicated standard size (eligible for pool reuse) but the underlying mmap allocation remained the large non-standard size. When the page was eventually freed, the code saw standard size in metadata, assumed it was part of the pool, and never called munmap on the large non-standard allocation, causing a classic memory leak. + +## Solution + +1. Never reuse non-standard pages during scrollback pruning +2. If a non-standard page is encountered during scrollback pruning, destroy it properly by calling munmap +3. Allocate a fresh standard-sized page from the pool instead +4. The core fix checks if the first page's memory length exceeds standard size and destroys the node if true, then breaks from the prune operation + +The code implementing this check: + +```zig +if (first.data.memory.len > std_size) { + self.destroyNode(first); + break :prune; +} +``` + +## Verification + +not specified in source + +## Notes + +The bug remained hidden for years because non-standard pages are rare by design—the architecture optimizes for standard pages as the common case. Only specific scenarios produce non-standard pages in large quantities. The rise of Claude Code as a popular CLI tool changed this by exercising Ghostty in a way that exposed the long-standing bug. The fix is conceptually simple: refuse to optimize (reuse) non-standard pages, treating them instead as exceptions that should be properly freed rather than recycled. This aligns with the current architectural assumption that standard pages are the common case. + +Additionally, virtual memory tags were added on macOS using the Mach kernel to help identify and debug memory allocations during future debugging scenarios. + +## References + +https://mitchellh.com/writing/ghostty-memory-leak-fix \ No newline at end of file diff --git a/lessons/contrib/glama-introspection-gap.md b/lessons/contrib/glama-introspection-gap.md new file mode 100644 index 00000000..cd235adb --- /dev/null +++ b/lessons/contrib/glama-introspection-gap.md @@ -0,0 +1,60 @@ +--- +{ + "title": "Glama Introspection Gap — Build Success ≠ Tools Registered", + "domain": "devops", + "tags": ["glama", "mcp", "introspection", "tools", "registry"], + "status": "published", + "source": "agent_experience", + "created": "2026-07-22", + "confidence": "0.90" +} +--- + +## Problem + +After successfully building an MCP server on Glama, the tools don't appear in the Glama API or dashboard. Build success ≠ tools registered — they are separate async processes. + +## Root Cause + +Glama's pipeline has two distinct steps: +1. **Build** — Docker image creation (synchronous, returns immediately) +2. **Introspection** — Runs the MCP server and calls `tools/list` (async, may take minutes to hours) + +The build step can succeed while introspection fails silently. Common causes: +- MCP server starts but doesn't respond to `tools/list` request +- Server crashes during introspection +- glama.json format issues (tools not detected) +- Network timeout during introspection + +## Detection + +```bash +# Check if tools are registered +curl -s "https://glama.ai/api/mcp/v1/servers/OWNER/REPO" | python3 -c " +import json, sys +data = json.load(sys.stdin) +print(f'Tools: {len(data.get(\"tools\", []))}') +" +``` + +If `tools: 0` but build succeeded, introspection failed. + +## Fix Action + +1. **Wait** — introspection is async, may take hours +2. **Sync Server** — trigger Glama to re-read the repo +3. **Rebuild** — force fresh introspection +4. **Check glama.json** — must be minimal format (`$schema` + `maintainers` only) +5. **Check Dockerfile** — ensure MCP server starts and responds to `initialize` request + +## Key Insight + +**glama.json is NOT for tool definitions.** Glama discovers tools via MCP introspection (calling `tools/list`), not from glama.json. The glama.json should only contain: +```json +{ + "$schema": "https://glama.ai/mcp/schemas/server.json", + "maintainers": ["username"] +} +``` + +Complex tool definitions in glama.json are ignored by Glama's introspection system. diff --git a/lessons/contrib/merge-probability-calibration.md b/lessons/contrib/merge-probability-calibration.md new file mode 100644 index 00000000..ee70661e --- /dev/null +++ b/lessons/contrib/merge-probability-calibration.md @@ -0,0 +1,48 @@ +--- +{ + "title": "Merge Probability Calibration — Honest Estimation vs Overfitting", + "domain": "devops", + "tags": ["prediction", "calibration", "merge-rate", "a-b-test", "coach"], + "status": "published", + "source": "agent_experience", + "created": "2026-07-22", + "confidence": "0.90" +} +--- + +## Problem + +PR coaches that predict merge probability often overfit to training data or give misleadingly precise estimates. A 70% accuracy claim may not generalize to new repos or PR types. + +## Root Cause + +1. **Base rate trap** — Most repos have low external merge rates (20-30%). Predicting "medium risk" for everything gives 70%+ accuracy but zero discrimination. + +2. **Signal confusion** — Signals like "needs_preflight" or "large_repo" are risk markers, not success predictors. A PR can have many negative signals and still merge if the maintainer wants it. + +3. **Content blindness** — Current coaches analyze metadata (title, body, files_changed) but not actual diff content. Two PRs with identical metadata can have completely different merge outcomes. + +## A/B Test Results (445 cases) + +| Metric | Value | +|--------|-------| +| Merged PRs mean probability | 0.32 | +| Closed PRs mean probability | 0.30 | +| Gap | +0.02 | +| Discrimination | YES (but limited) | + +**Conclusion:** Merge probability can't be more accurate than the repo's base merge rate without understanding PR content quality. + +## Fix Action + +1. **Use repo merge rate as base** — the most honest starting point +2. **Only adjust for discriminating signals** — merge_conflict (×0.3), duplicate (×0.1), maintainer_internal (×0.05) +3. **Don't double-count** — signals already affect tier, don't also affect probability +4. **Be transparent** — tell users "this repo has 20% merge rate, your PR is slightly better than average" + +## Prevention + +- Always A/B test predictions against actual outcomes +- Use LORO (Leave-One-Repo-Out) validation to detect overfitting +- Report confidence intervals, not point estimates +- Accept that some uncertainty is irreducible (depends on maintainer mood, timing, etc.) diff --git a/lessons/contrib/restricted-interactions-repos.md b/lessons/contrib/restricted-interactions-repos.md new file mode 100644 index 00000000..d30688a1 --- /dev/null +++ b/lessons/contrib/restricted-interactions-repos.md @@ -0,0 +1,54 @@ +--- +{ + "title": "Restricted Interactions Repos — External PRs Not Accepted", + "domain": "devops", + "tags": ["github", "restricted", "external-pr", "contribution", "gatekeeping"], + "status": "published", + "source": "agent_experience", + "created": "2026-07-22", + "confidence": "0.95" +} +--- + +## Problem + +Some large open-source repos have GitHub "restricted interactions" enabled, which prevents non-collaborators from creating PRs, commenting, or even viewing certain features. Contributors waste time preparing PRs that can never be submitted. + +## Root Cause + +GitHub allows repo admins to restrict interactions to collaborators only. This is separate from the standard fork-and-PR workflow. When enabled: +- Non-collaborators cannot create PRs +- Non-collaborators cannot comment on issues/PRs +- The error message is generic: "Interactions on this repository have been restricted to collaborators only" + +## Detection + +Before preparing a PR, check: +```bash +# Check if you can create a PR +gh pr create --repo org/repo --title "test" --body "test" 2>&1 | grep "restricted" + +# Check if you can comment +gh issue comment 1 --repo org/repo --body "test" 2>&1 | grep "restricted" +``` + +## Known Repos with Restricted Interactions + +- `encode/httpx` — Python HTTP library, no external PRs accepted +- Some Grafana Labs repos — require signed commits + restricted interactions + +## Fix Action + +1. Check repo interaction permissions before investing time +2. If restricted, look for alternative contribution channels (discussions, docs) +3. If no alternatives, move to a different repo + +## Prevention + +Always test repo permissions before preparing a PR: +```bash +# Quick permission check +gh api repos/org/repo --jq '.permissions' +``` + +If `pull` is false or interactions are restricted, don't invest time in PR preparation. diff --git a/scripts/heartbeat_lesson_pipeline.py b/scripts/heartbeat_lesson_pipeline.py new file mode 100644 index 00000000..6f1175ea --- /dev/null +++ b/scripts/heartbeat_lesson_pipeline.py @@ -0,0 +1,679 @@ +#!/usr/bin/env python3 +"""Heartbeat Lesson Pipeline — fetch → extract → score → PR. + +Automated daily lesson extraction from HN/Dev.to high-quality posts. +Each lesson must pass quality gate (≥75) before PR creation. + +Usage: + python3 scripts/heartbeat_lesson_pipeline.py # full pipeline + python3 scripts/heartbeat_lesson_pipeline.py --dry-run # preview only + python3 scripts/heartbeat_lesson_pipeline.py --target 5 # target count + python3 scripts/heartbeat_lesson_pipeline.py --sources hn # HN only + python3 scripts/heartbeat_lesson_pipeline.py --threshold 80 # stricter gate +""" +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +import textwrap +import urllib.request +from datetime import datetime, timedelta +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +LESSONS_DIR = REPO / "lessons" / "contrib" +SCORER = REPO / "scripts" / "quality_scorer.py" +DOMAIN_KEYWORDS = { + "security": ["vulnerability", "injection", "exploit", "CVE", "breach", "leak", "attack", "auth"], + "mcp": ["MCP", "model context protocol", "tool server", "tool use"], + "agent": ["agent", "autonomous", "multi-agent", "agentic", "orchestration"], + "devops": ["CI/CD", "deploy", "kubernetes", "docker", "infrastructure", "SRE"], + "llm": ["LLM", "GPT", "Claude", "Gemini", "fine-tune", "RAG", "embedding", "token"], + "python": ["Python", "asyncio", "FastAPI", "Django", "pip", "virtualenv"], + "frontend": ["React", "Vue", "Next.js", "TypeScript", "CSS", "Tailwind"], +} + +# Content type filters — only these produce real lessons +LESSON_WORTHY_PATTERNS = [ + # Incident/postmortem (real event with timeline) + r"(?i)(incident|postmortem|outage|downtime|breach|leak(ed)?|security incident|survival guide)", + # Bug/fix (real code issue) + r"(?i)(bug|fix(ed|es)?|patch|regression|crash|error|exception|segfault|broke|broken|doesn.t work)", + # How-to/tutorial (teaches a procedure) + r"(?i)(how to|tutorial|guide|step[- ]by[- ]step|walkthrough|setup|explained|built .*(?:server|tool|app|system))", + # Lessons learned (explicit experience) + r"(?i)(lessons? learned|what I learned|mistakes?|pitfall|gotcha|heads?[- ]up|what I built|what broke|what.s fixed)", + # Performance/debugging (measurable problem) + r"(?i)(performance|latency|memory leak|OOM|timeout|slow|optimiz(e|ation)|faster|speed up)", + # Show HN with technical depth (not just a product launch) + r"(?i)Show HN:.*(?:built|made|created|open[- ]source|library|tool|framework|server|engine|compiler)", + # Configuration/deployment issue + r"(?i)(config|deploy|migration|upgrade|compat|breaking change|deprecat|didn.t fix|it didn.t)", + # Security vulnerability (concrete, not policy) + r"(?i)(vulnerability|CVE|exploit|injection|token.*leak|secret.*expos|admin.*token|SQL.*inject)", + # MCP/Agent technical content + r"(?i)(MCP server|agent.*stack|tool.*use|context.*protocol|prompt.*inject)", + # "I tried X" experience posts + r"(?i)(I (?:tried|rewrote|replaced|migrated|built|debugged|fixed)|here.s what I)", + # Database/infrastructure lessons + r"(?i)(postgres|redis|kubernetes|docker|nginx|sqlite|database.*guide|database.*lesson)", +] + +# Anti-patterns — these are NOT lessons (news/opinion/announcement) +NOT_LESSON_PATTERNS = [ + r"(?i)^(?:Announcing|Introducing|Launch|Release)", # product announcements + r"(?i)(opinion|editorial|think piece|perspective)", # opinion pieces + r"(?i)(has died|obituary|memorial)", # obituaries + r"(?i)(regulation|policy|government|FCC|EU|congress)", # policy news + r"(?i)(advertise|ad|sponsor|pricing)", # ads/pricing + r"(?i)(competitive with|so ta|state[- ]of[- ]the[- ]art|benchmark)", # benchmark comparisons + r"(?i)(strategy|winning|losing|market|revenue)", # business strategy +] + +# ─── Sources ─────────────────────────────────────────────────────────────── + +def fetch_hn_stories(min_points: int = 100, days: int = 7) -> list[dict]: + """Fetch high-point HN stories from Algolia API.""" + cutoff = int((datetime.utcnow() - timedelta(days=days)).timestamp()) + url = ( + f"https://hn.algolia.com/api/v1/search?" + f"tags=story&hitsPerPage=30" + f"&numericFilters=points>{min_points},created_at_i>{cutoff}" + ) + try: + req = urllib.request.Request(url, headers={"User-Agent": "MisakaNet-Heartbeat/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read()) + except Exception as e: + print(f"⚠️ HN API error: {e}", file=sys.stderr) + return [] + + stories = [] + for h in data.get("hits", []): + stories.append({ + "source": "hn", + "id": h["objectID"], + "title": h.get("title", ""), + "url": h.get("url", ""), + "points": h.get("points", 0), + "comments": h.get("num_comments", 0), + "author": h.get("author", ""), + "created_at": h.get("created_at", ""), + "hn_url": f"https://news.ycombinator.com/item?id={h['objectID']}", + }) + return stories + + +def fetch_hn_by_keyword(keyword: str, min_points: int = 30, limit: int = 5) -> list[dict]: + """Search HN by keyword for targeted technical content.""" + url = ( + f"https://hn.algolia.com/api/v1/search?" + f"query={urllib.request.quote(keyword)}&tags=story" + f"&hitsPerPage={limit}&numericFilters=points>{min_points}" + ) + try: + req = urllib.request.Request(url, headers={"User-Agent": "MisakaNet-Heartbeat/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read()) + except Exception as e: + print(f"⚠️ HN keyword search error for '{keyword}': {e}", file=sys.stderr) + return [] + + stories = [] + for h in data.get("hits", []): + stories.append({ + "source": "hn", + "id": h["objectID"], + "title": h.get("title", ""), + "url": h.get("url", ""), + "points": h.get("points", 0), + "comments": h.get("num_comments", 0), + "author": h.get("author", ""), + "created_at": h.get("created_at", ""), + "hn_url": f"https://news.ycombinator.com/item?id={h['objectID']}", + }) + return stories + + +TECH_KEYWORDS = [ + "postmortem incident", + "prompt injection", + "debugging lesson", + "performance optimization", + "memory leak", + "database migration", + "deploy rollback", + "security vulnerability", + "CI CD broken", + "kubernetes crash", + "MCP server", + "agent architecture", + "Redis cache", + "Postgres tuning", + "Docker networking", +] + + +def fetch_devto_articles(tag: str = "mcp", days: int = 7, top: int = 7) -> list[dict]: + """Fetch top Dev.to articles.""" + url = f"https://dev.to/api/articles?tag={tag}&top={top}&per_page=20" + try: + req = urllib.request.Request(url, headers={"User-Agent": "MisakaNet-Heartbeat/1.0"}) + with urllib.request.urlopen(req, timeout=15) as resp: + data = json.loads(resp.read()) + except Exception as e: + print(f"⚠️ Dev.to API error: {e}", file=sys.stderr) + return [] + + cutoff = datetime.utcnow() - timedelta(days=days) + articles = [] + for a in data: + pub = a.get("published_at", "") + if pub: + try: + pub_dt = datetime.fromisoformat(pub.replace("Z", "+00:00")) + if pub_dt.replace(tzinfo=None) < cutoff: + continue + except ValueError: + pass + articles.append({ + "source": "devto", + "id": str(a["id"]), + "title": a["title"], + "url": a["url"], + "points": a.get("public_reactions_count", 0), + "comments": a.get("comments_count", 0), + "author": a.get("user", {}).get("username", ""), + "tags": a.get("tag_list", []), + }) + return articles + + +# ─── Scoring & Ranking ──────────────────────────────────────────────────── + +def classify_domain(title: str, url: str = "") -> str: + """Auto-classify domain from title/URL.""" + text = f"{title} {url}".lower() + for domain, keywords in DOMAIN_KEYWORDS.items(): + if any(kw.lower() in text for kw in keywords): + return domain + return "engineering" + + +def is_lesson_worthy(item: dict) -> bool: + """Coarse filter: reject obvious non-lesson content (news/ads/obituaries). + Let borderline cases through — LLM will do the real filtering.""" + title = item.get("title", "") + url = item.get("url", "") + + # Hard reject: these are NEVER lessons + for pattern in NOT_LESSON_PATTERNS: + if re.search(pattern, title): + return False + + # Accept: explicit lesson-worthy patterns + for pattern in LESSON_WORTHY_PATTERNS: + if re.search(pattern, f"{title} {url}"): + return True + + # Accept Dev.to with technical tags + if item.get("source") == "devto": + tech_tags = {"mcp", "agent", "devops", "python", "typescript", "security", "debugging"} + if set(item.get("tags", [])) & tech_tags: + return True + + # Accept HN posts with decent engagement (let LLM decide if it's a lesson) + if item.get("source") == "hn" and item.get("points", 0) >= 200: + return True + + return False + + + +def llm_is_lesson_worthy(candidate: dict, content: str) -> bool: + """Fine filter: ask LLM if article contains a reusable lesson.""" + prompt = ( + "You are a technical content evaluator. Read this article and decide:\n" + "Does it contain a REUSABLE technical lesson that an engineer could apply?\n\n" + "A lesson must have:\n" + "- A specific technical problem (not generic advice)\n" + "- A concrete cause (not vague 'it's hard')\n" + "- An actionable solution or mitigation (not just commentary)\n\n" + "NOT lessons: product announcements, opinion pieces, news, benchmarks, strategy analysis.\n\n" + 'Reply with ONLY a JSON object like: {"is_lesson": true, "reason": "one sentence why"}\n\n' + f"ARTICLE TITLE: {candidate['title']}\n" + f"ARTICLE CONTENT:\n{content[:3000]}" + ) + result = call_llm(prompt, max_tokens=100) + if not result: + return False # conservative: skip if LLM fails + try: + # Extract JSON from response + m = re.search(r"\{[^}]+\}", result) + if m: + data = json.loads(m.group()) + worthy = data.get("is_lesson", False) + reason = data.get("reason", "") + if not worthy: + print(f" 🚫 LLM skip: {reason}") + return worthy + except (json.JSONDecodeError, KeyError): + pass + return False + + +def rank_candidate(item: dict) -> float: + """Weighted score for prioritization.""" + score = 0.0 + score += min(item.get("points", 0), 500) * 0.1 # cap at 50 pts + score += min(item.get("comments", 0), 200) * 0.05 # cap at 10 pts + # Bonus for security/MCP topics + title_lower = item.get("title", "").lower() + if any(kw in title_lower for kw in ["security", "vulnerability", "injection"]): + score += 20 + if "mcp" in title_lower: + score += 15 + if "agent" in title_lower: + score += 10 + # Strong bonus for incident/postmortem patterns + if any(kw in title_lower for kw in ["incident", "postmortem", "outage", "breach", "leak"]): + score += 30 + if any(kw in title_lower for kw in ["how to", "tutorial", "lesson", "pitfall"]): + score += 25 + return score + + +def deduplicate(candidates: list[dict], existing_titles: set[str]) -> list[dict]: + """Remove duplicates by title similarity and already-covered topics.""" + seen = set() + unique = [] + for c in candidates: + # Normalize title for dedup + norm = re.sub(r"[^a-z0-9]", "", c["title"].lower())[:40] + if norm in seen: + continue + # Check against existing lessons + if any(norm[:20] in t for t in existing_titles): + continue + seen.add(norm) + unique.append(c) + return unique + + +# ─── Lesson Generation ──────────────────────────────────────────────────── + +def fetch_article_content(url: str) -> str | None: + """Fetch article text content.""" + try: + req = urllib.request.Request(url, headers={ + "User-Agent": "Mozilla/5.0 (compatible; MisakaNet/1.0)" + }) + with urllib.request.urlopen(req, timeout=15) as resp: + html = resp.read().decode("utf-8", errors="replace") + # Strip HTML + text = re.sub(r"]*>.*?", "", html, flags=re.DOTALL) + text = re.sub(r"]*>.*?", "", text, flags=re.DOTALL) + text = re.sub(r"<[^>]+>", "\n", text) + text = re.sub(r"\n\s*\n", "\n\n", text) + text = re.sub(r" +", " ", text).strip() + return text[:8000] # cap for LLM context + except Exception as e: + print(f"⚠️ Fetch failed for {url}: {e}", file=sys.stderr) + return None + + +def call_llm(prompt: str, max_tokens: int = 4000) -> str | None: + """Call LLM via Anthropic-compatible gateway (uses ANTHROPIC_* env vars).""" + base_url = os.environ.get("ANTHROPIC_BASE_URL") + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not base_url or not api_key: + print("❌ ANTHROPIC_BASE_URL and ANTHROPIC_API_KEY required", file=sys.stderr) + return None + + url = f"{base_url}/v1/messages" + body = json.dumps({ + "model": "ppio/pa/claude-haiku-4-5", + "max_tokens": max_tokens, + "messages": [{"role": "user", "content": prompt}], + }).encode() + req = urllib.request.Request(url, data=body, headers={ + "x-api-key": api_key, + "anthropic-version": "2023-06-01", + "Content-Type": "application/json", + }) + try: + with urllib.request.urlopen(req, timeout=60) as resp: + data = json.loads(resp.read()) + return data.get("content", [{}])[0].get("text", "") + except Exception as e: + print(f"⚠️ LLM call failed: {e}", file=sys.stderr) + return None + + +FACT_CHECK_PROMPT = """You are a fact-checker. Your ONLY job is to output a JSON object. + +Compare the LESSON below against the ARTICLE. Find any fabricated CONTENT claims. + +IGNORE these metadata fields (they are required by the schema and will always be "fabricated"): +- created date (always set to today) +- confidence value (always set by the system) +- status field (always "published") +- source URL (always set from the candidate) + +ONLY check these content claims: +- Problem description: must match the article +- Root Cause: must match the article (or "not specified in source") +- Solution steps: must be from the article (or "not specified in source") +- Verification: must be from the article (or "not specified in source") +- Code examples: must be from the article (or "not provided in source") +- Numbers/metrics: must match the article +- Any specific technical details: must be from the article + +Output ONLY this JSON, nothing else: +{{"pass": true, "issues": []}} +or +{{"pass": false, "issues": ["fabricated content claim 1", "fabricated content claim 2"]}} + +ARTICLE: +{article} + +LESSON: +{lesson}""" + + +def fact_check_lesson(lesson_text: str, article_content: str) -> tuple[bool, list[str]]: + """Verify lesson claims against original article. Returns (pass, issues).""" + prompt = FACT_CHECK_PROMPT.format( + article=article_content[:10000], + lesson=lesson_text[:3000], + ) + result = call_llm(prompt, max_tokens=300) + if not result: + return False, ["LLM fact-check failed"] + try: + m = re.search(r"\{[^}]+\}", result, re.DOTALL) + if m: + # Handle multi-line JSON + json_str = m.group() + # Fix common JSON issues + json_str = re.sub(r'\n', ' ', json_str) + data = json.loads(json_str) + passed = data.get("pass", False) + issues = data.get("issues", []) + return passed, issues + except (json.JSONDecodeError, KeyError): + pass + return False, ["Fact-check response unparseable"] + + +def generate_lesson_prompt(candidate: dict, content: str) -> str: + """Generate LLM prompt for lesson extraction.""" + return textwrap.dedent(f"""\ + Extract a MisakaNet lesson from this article. Output ONLY the lesson markdown file, nothing else. + + CRITICAL RULE — DO NOT FABRICATE: + - Every claim MUST come from the original article + - If the article doesn't provide code examples, write "not provided in source" + - If the article doesn't provide verification steps, write "not specified in source" + - If the article doesn't provide specific numbers, write "not specified in source" + - NEVER invent code, metrics, or steps that aren't in the article + + REQUIREMENTS (must follow exactly or quality gate fails): + 1. First line: JSON frontmatter between --- delimiters with these fields: + {{"title": "...", "domain": "...", "tags": [...], "language": "en", "status": "published", + "source": "article_url", "created": "{datetime.now().strftime('%Y-%m-%d')}", "confidence": "0.85"}} + 2. Required sections in this exact order: + - ## Problem (specific scenario from the article) + - ## Root Cause (technical detail from the article) + - ## Solution (steps from the article, or "not specified in source" if missing) + - ## Verification (from the article, or "not specified in source" if missing) + - ## Notes (generalization from the article) + - ## References (source URL) + 3. Code blocks MUST have language tags (```python, ```sql, ```bash, etc.) + 4. Problem section must describe a CONCRETE scenario from the article + 5. Solution must have numbered steps — use article's own words, don't invent + 6. Verification: if the article doesn't provide this, write "not specified in source" + + SOURCE: {candidate['url']} + TITLE: {candidate['title']} + POINTS: {candidate.get('points', 0)} + + ARTICLE CONTENT: + {content[:12000]} + """) + + +# ─── Quality Gate ────────────────────────────────────────────────────────── + +def run_quality_scorer(lesson_path: Path, threshold: int = 75) -> dict: + """Run quality scorer on a single lesson file.""" + result = subprocess.run( + [sys.executable, str(SCORER), str(lesson_path), "--json"], + capture_output=True, text=True, cwd=str(REPO), + ) + try: + data = json.loads(result.stdout) + lesson = data["lessons"][0] + return { + "score": lesson["score"], + "grade": lesson["grade"], + "pass": lesson["score"] >= threshold, + "breakdown": lesson["breakdown"], + } + except (json.JSONDecodeError, KeyError, IndexError) as e: + return {"score": 0, "grade": "F", "pass": False, "error": str(e)} + + +def save_and_score_lesson(content: str, slug: str, threshold: int = 75) -> dict | None: + """Save lesson to disk and run quality gate. Returns None if fails.""" + path = LESSONS_DIR / f"{slug}.md" + path.write_text(content, encoding="utf-8") + + result = run_quality_scorer(path, threshold) + if result["pass"]: + print(f" ✅ {slug}: {result['score']}/100 ({result['grade']})") + return result + else: + print(f" ❌ {slug}: {result['score']}/100 ({result['grade']}) — below {threshold}") + # Clean up failed lesson + path.unlink(missing_ok=True) + return None + + +# ─── Git Operations ──────────────────────────────────────────────────────── + +def git_operations(lesson_files: list[Path], branch_name: str) -> bool: + """Create branch on fork, commit, push, and create PR to upstream.""" + try: + # Create branch from origin/main (fork) + subprocess.run(["git", "fetch", "origin", "main"], cwd=REPO, check=True, capture_output=True) + subprocess.run(["git", "checkout", "-b", branch_name, "origin/main"], cwd=REPO, check=True, capture_output=True) + + # Add files + for f in lesson_files: + subprocess.run(["git", "add", str(f.relative_to(REPO))], cwd=REPO, check=True, capture_output=True) + + # Commit + count = len(lesson_files) + msg = f"feat(lessons): {count} high-quality lessons from heartbeat pipeline\n\n" + msg += "Lessons extracted from HN/Dev.to high-point posts.\n" + msg += f"All passed quality gate (≥75/100).\n\n" + msg += "Signed-off-by: Eric Jia <445655361@qq.com>" + subprocess.run(["git", "commit", "-m", msg], cwd=REPO, check=True, capture_output=True) + + # Push to fork + subprocess.run(["git", "push", "origin", branch_name, "--force"], cwd=REPO, check=True, capture_output=True) + + # Create PR to upstream + pr_body = f"## Heartbeat Lesson Batch\n\n" + pr_body += f"**{count} lessons** extracted from high-point HN/Dev.to posts.\n\n" + pr_body += "### Quality Scores\n\n" + pr_body += "| Lesson | Score | Source |\n|--------|-------|--------|\n" + for f in lesson_files: + pr_body += f"| `{f.stem}` | ✅ ≥75 | auto-extracted |\n" + pr_body += "\n---\n🤖 Auto-generated by heartbeat lesson pipeline" + + title_count = min(count, 10) + pr_title = f"feat(lessons): {title_count} community lessons (heartbeat batch)" + + result = subprocess.run( + ["gh", "pr", "create", "--repo", "Ikalus1988/MisakaNet", + "--head", f"zsxh1990:{branch_name}", "--base", "main", + "--title", pr_title, "--body", pr_body], + capture_output=True, text=True, cwd=REPO, + ) + if result.returncode == 0: + pr_url = result.stdout.strip() + print(f"\n🎉 PR created: {pr_url}") + return True + else: + # PR might already exist — try updating + print(f"⚠️ PR create returned: {result.stderr.strip()}") + return False + + except subprocess.CalledProcessError as e: + print(f"❌ Git operation failed: {e}") + return False + + +# ─── Main Pipeline ───────────────────────────────────────────────────────── + +def main(): + parser = argparse.ArgumentParser(description="Heartbeat Lesson Pipeline") + parser.add_argument("--dry-run", action="store_true", help="Preview only, don't create PR") + parser.add_argument("--target", type=int, default=10, help="Target lesson count") + parser.add_argument("--threshold", type=int, default=75, help="Quality gate threshold") + parser.add_argument("--sources", default="hn,devto", help="Comma-separated sources") + parser.add_argument("--min-points", type=int, default=100, help="Min HN points") + parser.add_argument("--days", type=int, default=7, help="Lookback days") + args = parser.parse_args() + + print(f"=== Heartbeat Lesson Pipeline ===") + print(f"Target: {args.target} lessons | Threshold: {args.threshold} | Sources: {args.sources}") + print() + + # 1. Fetch candidates + candidates = [] + if "hn" in args.sources: + print("📡 Fetching HN stories (popularity)...") + candidates.extend(fetch_hn_stories(args.min_points, args.days)) + print("📡 Fetching HN stories (keyword search)...") + for kw in TECH_KEYWORDS: + candidates.extend(fetch_hn_by_keyword(kw, min_points=30, limit=3)) + if "devto" in args.sources: + print("📡 Fetching Dev.to articles...") + for tag in ["mcp", "agent", "devops", "python"]: + candidates.extend(fetch_devto_articles(tag, args.days)) + + print(f"📊 Raw candidates: {len(candidates)}") + + # 2. Deduplicate & rank + existing = set() + for f in LESSONS_DIR.glob("*.md"): + title_match = re.search(r'"title":\s*"([^"]+)"', f.read_text(errors="replace")) + if title_match: + norm = re.sub(r"[^a-z0-9]", "", title_match.group(1).lower())[:40] + existing.add(norm) + + candidates = deduplicate(candidates, existing) + # Filter for lesson-worthy content (not news/opinion/announcement) + before_filter = len(candidates) + candidates = [c for c in candidates if is_lesson_worthy(c)] + print(f"📊 After lesson-worthiness filter: {len(candidates)} (dropped {before_filter - len(candidates)} news/opinion)") + candidates.sort(key=rank_candidate, reverse=True) + candidates = candidates[:args.target * 5] # fetch 5x to account for quality failures and fact-check rejections + + print(f"📊 After dedup & rank: {len(candidates)}") + print() + + if args.dry_run: + print("=== DRY RUN — Top candidates ===") + for i, c in enumerate(candidates[:args.target], 1): + domain = classify_domain(c["title"], c.get("url", "")) + print(f" {i}. [{domain}] ⭐{c.get('points',0)} | {c['title'][:60]}") + print(f" {c.get('url','')}") + return + + # 3. Extract & score via LLM + print(f"📝 Extracting lessons via LLM (target={args.target})...") + passed = [] + failed = 0 + for i, candidate in enumerate(candidates[:args.target + 5]): # extra buffer for failures + if len(passed) >= args.target: + break + domain = classify_domain(candidate["title"], candidate.get("url", "")) + url = candidate.get("url", "") + if not url: + print(f" ⏭️ [{domain}] No URL: {candidate['title'][:50]}") + continue + + print(f"\n [{i+1}/{args.target+5}] [{domain}] {candidate['title'][:60]}") + print(f" Fetching {url[:80]}...") + + content = fetch_article_content(url) + if not content or len(content) < 200: + print(f" ⏭️ Content too short or fetch failed") + continue + + # LLM gate: check if article is actually lesson-worthy + if not llm_is_lesson_worthy(candidate, content): + print(f" ⏭️ Not a reusable lesson (news/opinion/announcement)") + continue + + # Generate slug from title + slug = re.sub(r"[^a-z0-9]+", "-", candidate["title"].lower())[:60].strip("-") + prompt = generate_lesson_prompt(candidate, content) + + print(f" Calling LLM...") + lesson_text = call_llm(prompt) + if not lesson_text: + print(f" ❌ LLM returned nothing") + failed += 1 + continue + + # Strip markdown fences if LLM wrapped them + lesson_text = re.sub(r"^```(?:markdown)?\s*\n", "", lesson_text) + lesson_text = re.sub(r"\n```\s*$", "", lesson_text) + + # Fact-check: verify lesson claims against original article + print(f" Fact-checking against source...") + passed_check, issues = fact_check_lesson(lesson_text, content) + if not passed_check: + print(f" 🚫 Fact-check FAILED: {'; '.join(issues[:3])}") + failed += 1 + continue + + # Save and score + result = save_and_score_lesson(lesson_text, slug, args.threshold) + if result: + passed.append(LESSONS_DIR / f"{slug}.md") + else: + failed += 1 + + # Rate limit — 1 req/sec + if i < args.target + 4: + import time + time.sleep(1) + + # 4. Create PR if lessons passed + print(f"\n{'='*50}") + print(f"Results: {len(passed)} passed, {failed} failed, target was {args.target}") + + if passed and not args.dry_run: + branch = f"feat/heartbeat-lessons-{datetime.now().strftime('%Y%m%d')}" + # Always push to fork (origin) + PR to upstream + git_operations(passed, branch) + elif passed and args.dry_run: + print("\nDRY RUN — would create PR with:") + for f in passed: + print(f" ✅ {f.name}") + else: + print("\n❌ No lessons passed quality gate — no PR created") + + +if __name__ == "__main__": + main()