Merge pull request #8 from mathiasesn/7-extract-website-as-markdown

Extract website as markdown

Author: mathiasesn

pyproject.toml

@@ -1,6 +1,10 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
 [project]
 name = "repo-context"
-version = "0.3.0"
+version = "0.4.0"
 description = "Convert Git repositories into LLM-friendly context format"
 authors = [{ name = "Mathias Nielsen", email = "[email protected]" }]
 maintainers = [{ name = "Mathias Nielsen", email = "[email protected]" }]
@@ -9,7 +13,9 @@ license = { file = "LICENSE" }
 requires-python = ">=3.10"
 dependencies = [
     "gitpython>=3.1.43",
+    "markdownify>=0.14.1",
     "python-dotenv>=1.0.1",
+    "requests>=2.32.3",
     "rich>=13.9.4",
     "tqdm>=4.67.1",
 ]
@@ -23,7 +29,3 @@ dev-dependencies = [
 
 [project.scripts]
 repo-context = "repo_context.cli:main"
-
-[build-system]
-requires = ["hatchling"]
-build-backend = "hatchling.build"

repo_context/cli.py

@@ -42,6 +42,11 @@ def parse_args() -> Namespace:
         default=None,
         help="Maximum number of lines in context files",
     )
+    parser.add_argument(
+        "--web",
+        action="store_true",
+        help="Convert a webpage instead of a repository",
+    )
     args = parser.parse_args()
     return args
 
@@ -50,6 +55,23 @@ def main():
     # Parse arguments
     args = parse_args()
 
+    if args.web:
+        from repo_context.webpage import Webpage
+
+        # Create the webpage converter and get markdown
+        webpage = Webpage()
+        context = webpage.get_markdown(args.source)
+
+        # Get the filename from the URL
+        fname = urlparse(args.source).path.strip("/").replace("/", "-")
+
+        # Write context to file
+        output_path = Path(f"{args.output}/{fname}.md")
+        output_path.write_text(context)
+
+        logger.info(f"Context written to {output_path}")
+        return
+
     # Concat ignore patterns
     ignore_patterns = args.ignore.copy() if args.ignore else []
     if args.ignore_file:

repo_context/webpage.py

@@ -0,0 +1,79 @@
+import logging
+import re
+from functools import lru_cache
+from urllib.parse import urlparse
+
+import requests
+from markdownify import markdownify
+from requests.exceptions import RequestException, Timeout
+
+logger = logging.getLogger("repo_context.webpage")
+
+
+class Webpage:
+    """A class for fetching and converting webpages to markdown format."""
+
+    def __init__(
+        self,
+        timeout: int = 20,
+        allowed_schemes: tuple[str] = ("http", "https"),
+        max_retries: int = 3,
+    ) -> None:
+        self.timeout = timeout
+        self.allowed_schemes = allowed_schemes
+        self.max_retries = max_retries
+
+        self.user_agent: str = "Mozilla/5.0 (compatible; WebpageFetcher/1.0)"
+
+        self.session = requests.Session()
+        self.session.headers.update({"User-Agent": self.user_agent})
+
+    def _validate_url(self, url: str) -> None:
+        """Validates URL scheme and format."""
+        parsed = urlparse(url)
+        if parsed.scheme not in self.allowed_schemes:
+            raise ValueError(f"Invalid URL scheme. Allowed: {self.allowed_schemes}")
+
+    def _fetch_content(self, url: str) -> str:
+        """Fetches webpage content with retries."""
+        for attempt in range(self.max_retries):
+            try:
+                response = self.session.get(url, timeout=self.timeout)
+                response.raise_for_status()
+                return response.text
+            except Timeout:
+                if attempt == self.max_retries - 1:
+                    raise RuntimeError("Request timed out after retries")
+            except RequestException as e:
+                raise RuntimeError(f"Failed to fetch webpage: {e}")
+
+    def _convert_to_markdown(self, html: str) -> str:
+        """Converts HTML to clean markdown format."""
+        try:
+            markdown = markdownify(html).strip()
+            return re.sub(r"\n{3,}", "\n\n", markdown)
+        except Exception as e:
+            raise RuntimeError(f"Failed to convert HTML to markdown: {e}")
+
+    @lru_cache(maxsize=100)
+    def get_markdown(self, url: str) -> str:
+        """
+        Fetches webpage and converts to markdown format with caching.
+
+        Args:
+            url: Webpage URL to fetch
+
+        Returns:
+            Converted markdown content
+
+        Raises:
+            WebpageError: If fetching or conversion fails
+            ValueError: If URL is invalid
+        """
+        try:
+            self._validate_url(url)
+            content = self._fetch_content(url)
+            return self._convert_to_markdown(content)
+        except Exception as e:
+            logger.error(f"Failed to process {url}: {e}")
+            raise

tests/test_webpage.py

@@ -0,0 +1,110 @@
+from unittest.mock import Mock, patch
+
+import pytest
+from requests.exceptions import RequestException, Timeout
+from repo_context.webpage import Webpage
+
+
+@pytest.fixture
+def webpage():
+    """Base webpage instance with default settings."""
+    return Webpage()
+
+
+@pytest.fixture
+def mock_response():
+    """Mock successful response fixture."""
+    response = Mock()
+    response.text = "<h1>Test</h1><p>Content</p>"
+    response.raise_for_status.return_value = None
+    return response
+
+
+class TestWebpage:
+    def test_init_default_values(self):
+        """Test initialization with default values."""
+        webpage = Webpage()
+        assert webpage.timeout == 20
+        assert webpage.allowed_schemes == ("http", "https")
+        assert webpage.max_retries == 3
+        assert "Mozilla" in webpage.user_agent
+
+    @pytest.mark.parametrize(
+        "url,valid",
+        [
+            ("https://example.com", True),
+            ("http://test.com", True),
+            ("ftp://invalid.com", False),
+            ("invalid-url", False),
+        ],
+    )
+    def test_validate_url(self, webpage, url, valid):
+        """Test URL validation with various inputs."""
+        if valid:
+            webpage._validate_url(url)
+        else:
+            with pytest.raises(ValueError):
+                webpage._validate_url(url)
+
+    @patch("requests.Session.get")
+    def test_fetch_content_success(self, mock_get, webpage, mock_response):
+        """Test successful content fetching."""
+        mock_get.return_value = mock_response
+        content = webpage._fetch_content("https://example.com")
+        assert content == "<h1>Test</h1><p>Content</p>"
+        mock_get.assert_called_once()
+
+    @patch("requests.Session.get")
+    def test_fetch_content_timeout_retry(self, mock_get, webpage):
+        """Test timeout handling with retries."""
+        mock_get.side_effect = Timeout()
+        with pytest.raises(RuntimeError, match="timed out"):
+            webpage._fetch_content("https://example.com")
+        assert mock_get.call_count == webpage.max_retries
+
+    @patch("requests.Session.get")
+    def test_fetch_content_request_error(self, mock_get, webpage):
+        """Test request exception handling."""
+        mock_get.side_effect = RequestException("Network error")
+        with pytest.raises(RuntimeError, match="Failed to fetch"):
+            webpage._fetch_content("https://example.com")
+
+    def test_convert_to_markdown(self, webpage):
+        """Test HTML to markdown conversion."""
+        html = "<h1>Test</h1><p>Content</p>\n\n\n<p>More</p>"
+        markdown = webpage._convert_to_markdown(html)
+        assert "Test\n====" in markdown
+        assert "\n\n\n" not in markdown
+
+    @patch("requests.Session.get")
+    def test_get_markdown_integration(self, mock_get, webpage, mock_response):
+        """Test complete markdown conversion flow."""
+        mock_get.return_value = mock_response
+        result = webpage.get_markdown("https://example.com")
+        assert "Test\n====" in result
+        assert "Content" in result
+
+    @patch("requests.Session.get")
+    def test_get_markdown_caching(self, mock_get, webpage, mock_response):
+        """Test LRU caching functionality."""
+        mock_get.return_value = mock_response
+        url = "https://example.com"
+
+        # First call
+        webpage.get_markdown(url)
+        # Second call (should use cache)
+        webpage.get_markdown(url)
+
+        mock_get.assert_called_once()
+
+    def test_custom_timeout(self):
+        """Test custom timeout configuration."""
+        webpage = Webpage(timeout=30)
+        assert webpage.timeout == 30
+
+    def test_custom_schemes(self):
+        """Test custom allowed schemes."""
+        webpage = Webpage(allowed_schemes=("https",))
+        assert webpage.allowed_schemes == ("https",)
+        with pytest.raises(ValueError):
+            webpage._validate_url("http://example.com")