feat: move LinkedIn selectors to YAML config with --verify flag

scraper-service/config/linkedin_selectors.yaml

@@ -0,0 +1,18 @@
+# LinkedIn Scraper CSS Selectors
+# Update this file when LinkedIn changes its frontend
+# Last verified: March 2026
+
+linkedin:
+  selectors:
+    job_card: "li > div.base-search-card"
+    title: "h3.base-search-card__title"
+    company: "h4.base-search-card__subtitle"
+    location: "span.job-search-card__location"
+    job_url: "a.base-card__full-link"
+    posted_date: "time"
+
+  fallbacks:
+    job_card_alt: "div.base-search-card"
+    title_alt: ".job-card-list__title"
+    company_alt: ".job-card-container__company-name"
+

scraper-service/main.py

@@ -11,6 +11,7 @@
 
 import asyncio
 import logging
+import argparse
 from contextlib import asynccontextmanager
 
 from fastapi import FastAPI, BackgroundTasks
@@ -122,3 +123,55 @@ async def trigger_scrape(body: ScrapeRequest, background_tasks: BackgroundTasks)
     """
     background_tasks.add_task(_run_all_scrapers, body.role, body.stack)
     return ScrapeResponse(triggered=True, platforms=PLATFORMS)
+
+
+# ---------------------------------------------------------------------------
+# CLI entry point for --verify flag
+# ---------------------------------------------------------------------------
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="LinkedIn Scraper Selector Verification")
+    parser.add_argument(
+        '--verify', 
+        action='store_true', 
+        help='Verify LinkedIn selectors from config file'
+    )
+    args = parser.parse_args()
+
+    if args.verify:
+        print("\n" + "="*50)
+        print("LinkedIn Selector Verification")
+        print("="*50 + "\n")
+
+        try:
+            from selectors import SelectorLoader
+            loader = SelectorLoader()
+            results = loader.verify()
+
+            all_valid = True
+            for name, info in results.items():
+                status = "✅" if info["valid"] else "❌"
+                print(f"{status} {name}: {info['selector']}")
+                if not info["valid"]:
+                    all_valid = False
+
+            print("\n" + "="*50)
+            if all_valid:
+                print("✅ All selectors are valid!")
+            else:
+                print("❌ Some selectors are invalid or missing.")
+                print("   Update config/linkedin_selectors.yaml with correct selectors.")
+            print("="*50)
+
+        except FileNotFoundError:
+            print("❌ Config file not found: config/linkedin_selectors.yaml")
+            print("   Please create the YAML config file first.")
+        except Exception as e:
+            print(f"❌ Error during verification: {e}")
+
+        exit(0)
+
+    # Normal FastAPI startup
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)
+

scraper-service/scrapers/linkedin.py

@@ -50,14 +50,16 @@
 
 MIN_SCROLL_DELAY = 4.0   # seconds — do NOT lower this
 
-# ── Selectors — update here when LinkedIn changes its markup ─────────────────
-_CARD_SEL      = "li > div.base-search-card"
-_TITLE_SEL     = "h3.base-search-card__title"
-_COMPANY_SEL   = "h4.base-search-card__subtitle"
-_LOCATION_SEL  = "span.job-search-card__location"
-_LINK_SEL      = "a.base-card__full-link"
-_TIME_SEL      = "time"
-
+# ── Selectors ── loaded from YAML config
+from selectors import SelectorLoader
+
+_selector_loader = SelectorLoader()
+_CARD_SEL      = _selector_loader.get("job_card")
+_TITLE_SEL     = _selector_loader.get("title")
+_COMPANY_SEL   = _selector_loader.get("company")
+_LOCATION_SEL  = _selector_loader.get("location")
+_LINK_SEL      = _selector_loader.get("job_url")
+_TIME_SEL      = _selector_loader.get("posted_date")
 
 def _build_url(role: str) -> str:
     kw = quote_plus(role)

scraper-service/selectors.py

@@ -0,0 +1,39 @@
+"""
+Selector loader for LinkedIn scraper.
+Loads CSS selectors from YAML config file.
+"""
+
+import yaml
+from pathlib import Path
+import logging
+
+logger = logging.getLogger(__name__)
+
+class SelectorLoader:
+    """Load and manage CSS selectors from YAML config."""
+
+    def __init__(self, config_path: str = None):
+        if config_path is None:
+            config_path = Path(__file__).parent / "config" / "linkedin_selectors.yaml"
+
+        with open(config_path, 'r') as f:
+            self.config = yaml.safe_load(f)
+
+        self.selectors = self.config.get("linkedin", {}).get("selectors", {})
+        self.fallbacks = self.config.get("linkedin", {}).get("fallbacks", {})
+
+    def get(self, key: str) -> str:
+        """Get a selector by key."""
+        selector = self.selectors.get(key)
+        if selector is None:
+            logger.warning(f"Selector '{key}' not found in config, using fallback")
+            return self.fallbacks.get(f"{key}_alt", "")
+        return selector
+
+    def verify(self) -> dict:
+        """Verify all selectors (returns dict with status)."""
+        results = {}
+        for key, selector in self.selectors.items():
+            results[key] = {"selector": selector, "valid": bool(selector)}
+        return results
+