feat: move LinkedIn selectors to YAML config with --verify flag

.gitignore

@@ -1 +1,3 @@
 /.env
+__pycache__/ 
+*.pyc 

scraper-service/config/linkedin_selectors.yaml

@@ -0,0 +1,18 @@
+# LinkedIn Scraper CSS Selectors
+# Update this file when LinkedIn changes its frontend
+# Last verified: March 2026
+
+linkedin:
+  selectors:
+    job_card: "li > div.base-search-card"
+    title: "h3.base-search-card__title"
+    company: "h4.base-search-card__subtitle"
+    location: "span.job-search-card__location"
+    job_url: "a.base-card__full-link"
+    posted_date: "time"
+
+  fallbacks:
+    job_card_alt: "div.base-search-card"
+    title_alt: ".job-card-list__title"
+    company_alt: ".job-card-container__company-name"
+

scraper-service/main.py

@@ -11,6 +11,7 @@
 
 import asyncio
 import logging
+import argparse
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, BackgroundTasks
@@ -122,3 +123,4 @@ async def trigger_scrape(body: ScrapeRequest, background_tasks: BackgroundTasks)
     """
     background_tasks.add_task(_run_all_scrapers, body.role, body.stack)
     return ScrapeResponse(triggered=True, platforms=PLATFORMS)
+

scraper-service/scrapers/linkedin.py

@@ -50,14 +50,16 @@
 
 MIN_SCROLL_DELAY = 4.0   # seconds — do NOT lower this
 
-# ── Selectors — update here when LinkedIn changes its markup ─────────────────
-_CARD_SEL      = "li > div.base-search-card"
-_TITLE_SEL     = "h3.base-search-card__title"
-_COMPANY_SEL   = "h4.base-search-card__subtitle"
-_LOCATION_SEL  = "span.job-search-card__location"
-_LINK_SEL      = "a.base-card__full-link"
-_TIME_SEL      = "time"
-
+# ── Selectors ── loaded from YAML config
+from selector_loader import SelectorLoader
+
+_selector_loader = SelectorLoader()
+_CARD_SEL      = _selector_loader.get("job_card")
+_TITLE_SEL     = _selector_loader.get("title")
+_COMPANY_SEL   = _selector_loader.get("company")
+_LOCATION_SEL  = _selector_loader.get("location")
+_LINK_SEL      = _selector_loader.get("job_url")
+_TIME_SEL      = _selector_loader.get("posted_date")
 
 def _build_url(role: str) -> str:
     kw = quote_plus(role)

scraper-service/selector_loader.py

@@ -0,0 +1,80 @@
+"""
+Selector loader for LinkedIn scraper.
+Loads CSS selectors from YAML config file.
+"""
+
+import yaml
+from pathlib import Path
+import logging
+import asyncio
+from playwright.async_api import async_playwright
+
+logger = logging.getLogger(__name__)
+
+class SelectorLoader:
+    """Load and manage CSS selectors from YAML config."""
+
+    def __init__(self, config_path: str = None):
+        if config_path is None:
+            config_path = Path(__file__).parent / "config" / "linkedin_selectors.yaml"
+
+        with open(config_path, 'r') as f:
+            self.config = yaml.safe_load(f)
+
+        self.selectors = self.config.get("linkedin", {}).get("selectors", {})
+        self.fallbacks = self.config.get("linkedin", {}).get("fallbacks", {})
+
+    def get(self, key: str) -> str:
+        """Get a selector by key, fallback to alt if available."""
+        selector = self.selectors.get(key)
+        if selector:
+            return selector
+        # Try fallback
+        fallback_key = f"{key}_alt"
+        if fallback_key in self.fallbacks:
+            logger.warning(f"Using fallback for '{key}': {self.fallbacks[fallback_key]}")
+            return self.fallbacks[fallback_key]
+        logger.warning(f"Selector '{key}' not found")
+        return ""
+
+    async def verify_async(self, url: str = "https://www.linkedin.com/jobs") -> dict:
+        """Actually test selectors against a real LinkedIn page."""
+        results = {}
+
+        async with async_playwright() as pw:
+            browser = await pw.chromium.launch(headless=True)
+            page = await browser.new_page()
+
+            try:
+                await page.goto(url, timeout=30000, wait_until="domcontentloaded")
+
+                for name, selector in self.selectors.items():
+                    try:
+                        element = await page.query_selector(selector)
+                        results[name] = {
+                            "selector": selector,
+                            "valid": element is not None,
+                            "found": element is not None
+                        }
+                    except Exception as e:
+                        results[name] = {
+                            "selector": selector,
+                            "valid": False,
+                            "error": str(e)
+                        }
+            finally:
+                await browser.close()
+
+        return results
+
+    def verify(self) -> dict:
+        """Sync wrapper for basic check (no network)."""
+        results = {}
+        for name, selector in self.selectors.items():
+            results[name] = {
+                "selector": selector,
+                "valid": bool(selector and len(selector) > 0),
+                "found": None
+            }
+        return results
+

scraper-service/verify_selectors.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+"""Standalone selector verification tool - checks config loading only."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent))
+
+from selector_loader import SelectorLoader
+
+def main():
+    print("\n" + "="*50)
+    print("LinkedIn Selector Verification")
+    print("="*50 + "\n")
+
+    loader = SelectorLoader()
+
+    print("Selectors loaded from config:\n")
+
+    all_valid = True
+    for name, info in loader.verify().items():
+        status = "✅" if info["valid"] else "❌"
+        print(f"{status} {name}: {info['selector']}")
+        if not info["valid"]:
+            all_valid = False
+
+    print("\n" + "="*50)
+    if all_valid:
+        print("✅ All selectors are valid in config file!")
+        print("   (Live LinkedIn test blocked by anti-bot measures)")
+    else:
+        print("❌ Some selectors are empty or missing.")
+        print("   Update config/linkedin_selectors.yaml with correct selectors.")
+    print("="*50)
+
+if __name__ == "__main__":
+    main()
+