Feature/dynamic sitemap

.github/workflows/generate-sitemap.yml

@@ -0,0 +1,45 @@
+name: Generate Sitemap
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'fern/docs.yml'
+      - 'fern/pages/**'
+  workflow_dispatch:
+
+jobs:
+  generate-sitemap:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '18'
+
+      - name: Install dependencies
+        run: npm install js-yaml
+
+      - name: Generate sitemap
+        run: node generate-sitemap.js
+
+      - name: Check for changes
+        id: git-check
+        run: |
+          git diff --exit-code sitemap.xml || echo "changed=true" >> $GITHUB_OUTPUT
+
+      - name: Commit and push if changed
+        if: steps.git-check.outputs.changed == 'true'
+        run: |
+          git config --local user.email "github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add sitemap.xml
+          git commit -m "chore: auto-generate sitemap [skip ci]"
+          git push

.gitignore

@@ -1,3 +1,5 @@
 .env
 **/.preview/
 .DS_Store
+node_modules/
+package-lock.json

README.md

@@ -0,0 +1,117 @@
+# AgentMail Documentation
+
+
+
+## 📚 Documentation Site
+
+Visit [docs.agentmail.to](https://docs.agentmail.to) to view the live documentation. 
+Official documentation for [AgentMail](https://agentmail.to)
+
+## 🛠️ Development
+
+This documentation is built using [Fern](https://buildwithfern.com/).
+
+### Local Development Setup
+
+#### Prerequisites
+
+- Node.js (v18 or higher)
+- npm or yarn
+
+#### Installation
+
+1. Clone the repository:
+```bash
+git clone https://github.com/agentmail-to/agentmail-docs.git
+cd agentmail-docs
+```
+
+2. Install dependencies:
+```bash
+npm install
+```
+
+3. Install Fern CLI globally:
+```bash
+npm install -g fern-api
+```
+
+#### Running Locally
+
+To preview the documentation locally with hot-reload:
+
+```bash
+fern docs dev
+```
+
+This will start a local development server (usually at `http://localhost:3000`) where you can see your changes in real-time.
+
+#### Building Documentation
+
+To build the documentation:
+
+```bash
+fern generate --docs
+```
+
+#### Common Commands
+
+| Command | Description |
+|---------|-------------|
+| `fern docs dev` | Start local development server with hot-reload |
+| `fern generate --docs` | Build the documentation |
+| `fern check` | Validate your API definition and docs configuration |
+| `npm run generate-sitemap` | Generate sitemap.xml from docs.yml |
+
+#### Project Structure
+
+```
+agentmail-docs/
+├── fern/
+│   ├── docs.yml              # Main documentation configuration
+│   ├── pages/                # Documentation pages (MDX files)
+│   │   ├── get-started/
+│   │   ├── core-concepts/
+│   │   ├── guides/
+│   │   └── ...
+│   ├── definition/           # API definitions
+│   └── assets/              # Images and static files
+├── generate-sitemap.js       # Dynamic sitemap generator
+├── robots.txt               # SEO configuration
+└── sitemap.xml              # Auto-generated sitemap
+```
+
+### Generating Sitemap
+
+The sitemap is automatically generated from the `fern/docs.yml` file. You can generate it manually using:
+
+```bash
+npm install
+npm run generate-sitemap
+```
+
+### Automated Sitemap Updates
+
+The sitemap is automatically regenerated via GitHub Actions whenever:
+- Changes are pushed to `fern/docs.yml`
+- New pages are added in `fern/pages/`
+- Manually triggered via workflow dispatch
+
+See `.github/workflows/generate-sitemap.yml` for details.
+
+## 📝 Adding New Pages
+
+1. Create your `.mdx` file in the appropriate `fern/pages/` subdirectory
+2. Add the page to the navigation in `fern/docs.yml`
+3. Push your changes
+4. The sitemap will automatically update via GitHub Actions
+
+## 🔗 Links
+
+- [AgentMail Website](https://agentmail.to)
+- [Discord Community](https://discord.gg/hTYatWYWBc)
+- [Contact](mailto:contact@agentmail.cc)
+
+## 📄 License
+
+Copyright © 2024 AgentMail

generate-sitemap.js

@@ -0,0 +1,172 @@
+#!/usr/bin/env node
+/**
+ * Dynamic sitemap generator for AgentMail Documentation
+ * Automatically parses docs.yml and generates sitemap.xml
+ */
+
+const fs = require('fs');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const BASE_URL = 'https://docs.agentmail.to';
+
+/**
+ * Recursively extract all page paths from the navigation structure
+ */
+function extractPagesFromNavigation(navItem, pagesList) {
+  if (typeof navItem === 'object' && navItem !== null) {
+    // Handle page with path
+    if (navItem.page && navItem.path) {
+      let pagePath = navItem.path;
+
+      // Convert path to URL (remove pages/ prefix and .mdx extension)
+      if (pagePath.startsWith('pages/')) {
+        pagePath = pagePath.substring(6);
+      }
+      if (pagePath.endsWith('.mdx')) {
+        pagePath = pagePath.substring(0, pagePath.length - 4);
+      }
+
+      // Convert path to URL slug (remove directory structure, keep filename)
+      const urlSlug = pagePath.split('/').pop();
+      pagesList.push(urlSlug);
+    }
+
+    // Handle sections with contents
+    if (navItem.section && navItem.contents) {
+      navItem.contents.forEach(content => extractPagesFromNavigation(content, pagesList));
+    }
+
+    // Handle API reference
+    if (navItem.api) {
+      pagesList.push('api-reference');
+    }
+
+    // Recursively process object properties
+    Object.keys(navItem).forEach(key => {
+      if (!['page', 'path', 'icon', 'section', 'api', 'summary', 'snippets'].includes(key)) {
+        const value = navItem[key];
+        if (Array.isArray(value) || typeof value === 'object') {
+          extractPagesFromNavigation(value, pagesList);
+        }
+      }
+    });
+  } else if (Array.isArray(navItem)) {
+    navItem.forEach(item => extractPagesFromNavigation(item, pagesList));
+  }
+}
+
+/**
+ * Determine priority based on page importance
+ */
+function determinePriority(pageName) {
+  const highPriorityPages = ['welcome', 'introduction', 'quickstart', 'api-reference'];
+  const coreConceptPages = ['inboxes', 'messages', 'threads', 'drafts', 'labels', 'attachments', 'pods'];
+  const importantPages = [
+    'webhooks-overview', 'webhooks-events', 'webhook-setup', 'websockets',
+    'sending-receiving-email', 'custom-domains', 'email-deliverability',
+    'faq', 'support'
+  ];
+
+  if (highPriorityPages.includes(pageName)) return '0.9';
+  if (coreConceptPages.includes(pageName)) return '0.8';
+  if (importantPages.includes(pageName)) return '0.8';
+  return '0.7';
+}
+
+/**
+ * Determine change frequency based on page type
+ */
+function determineChangefreq(pageName) {
+  const frequentPages = ['welcome', 'quickstart', 'faq', 'community'];
+  const weeklyPages = [
+    'introduction', 'inboxes', 'messages', 'threads', 'webhooks-overview',
+    'api-reference', 'support', 'sending-receiving-email'
+  ];
+
+  if (frequentPages.includes(pageName)) return 'weekly';
+  if (weeklyPages.includes(pageName)) return 'weekly';
+  return 'monthly';
+}
+
+/**
+ * Generate sitemap.xml from docs.yml
+ */
+function generateSitemap(docsYmlPath, outputPath, baseUrl) {
+  // Read docs.yml
+  const docsYmlContent = fs.readFileSync(docsYmlPath, 'utf8');
+  const docsConfig = yaml.load(docsYmlContent);
+
+  // Extract pages from navigation
+  const pages = [];
+  if (docsConfig.navigation) {
+    extractPagesFromNavigation(docsConfig.navigation, pages);
+  }
+
+  // Remove duplicates while preserving order
+  const uniquePages = [...new Set(pages)];
+
+  // Build XML
+  let xml = '<?xml version="1.0" encoding="UTF-8"?>\n';
+  xml += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"\n';
+  xml += '        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"\n';
+  xml += '        xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9\n';
+  xml += '        http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd">\n\n';
+
+  // Add homepage
+  xml += '  <!-- Homepage -->\n';
+  xml += '  <url>\n';
+  xml += `    <loc>${baseUrl}/</loc>\n`;
+  xml += '    <priority>1.0</priority>\n';
+  xml += '    <changefreq>daily</changefreq>\n';
+  xml += '  </url>\n\n';
+
+  // Add all pages
+  uniquePages.forEach(page => {
+    const pageUrl = `${baseUrl}/${page}`;
+    const priority = determinePriority(page);
+    const changefreq = determineChangefreq(page);
+
+    xml += '  <url>\n';
+    xml += `    <loc>${pageUrl}</loc>\n`;
+    xml += `    <priority>${priority}</priority>\n`;
+    xml += `    <changefreq>${changefreq}</changefreq>\n`;
+    xml += '  </url>\n';
+  });
+
+  xml += '\n</urlset>\n';
+
+  // Write to file
+  fs.writeFileSync(outputPath, xml, 'utf8');
+
+  console.log(`✅ Sitemap generated successfully at ${outputPath}`);
+  console.log(`📄 Total URLs: ${uniquePages.length + 1}`); // +1 for homepage
+}
+
+/**
+ * Main function
+ */
+function main() {
+  const scriptDir = __dirname;
+  const docsYmlPath = path.join(scriptDir, 'fern', 'docs.yml');
+  const outputPath = path.join(scriptDir, 'sitemap.xml');
+
+  if (!fs.existsSync(docsYmlPath)) {
+    console.error(`❌ Error: docs.yml not found at ${docsYmlPath}`);
+    process.exit(1);
+  }
+
+  try {
+    generateSitemap(docsYmlPath, outputPath, BASE_URL);
+  } catch (error) {
+    console.error('❌ Error generating sitemap:', error.message);
+    process.exit(1);
+  }
+}
+
+// Run if called directly
+if (require.main === module) {
+  main();
+}
+
+module.exports = { generateSitemap };

package.json

@@ -0,0 +1,16 @@
+{
+  "name": "agentmail-docs",
+  "version": "1.0.0",
+  "description": "AgentMail Documentation",
+  "scripts": {
+    "generate-sitemap": "node generate-sitemap.js",
+    "prebuild": "npm run generate-sitemap"
+  },
+  "dependencies": {
+    "js-yaml": "^4.1.1"
+  },
+  "keywords": [
+    "agentmail",
+    "documentation"
+  ]
+}

robots.txt

@@ -0,0 +1,6 @@
+# robots.txt for AgentMail Documentation
+User-agent: *
+Allow: /
+
+# Sitemap
+Sitemap: https://docs.agentmail.to/sitemap.xml