ENH Add Springer

.env.example

@@ -2,3 +2,9 @@ ELSEVIER_API_KEY=YOUR_API_KEY
 ELSEVIER_HTTP_PROXY=socks5://127.0.0.1:1080
 ELSEVIER_HTTPS_PROXY=socks5://127.0.0.1:1080
 ELSEVIER_USE_PROXY=false
+
+# Optional Springer Open Access + PubMed fallback settings
+SPRINGER_API_KEY=
+SPRINGER_BASE_URL=https://api.springernature.com
+PUBMED_BASE_URL=https://eutils.ncbi.nlm.nih.gov/entrez/eutils
+NCBI_API_KEY=

README.md

@@ -39,7 +39,18 @@ After installing the package, the `elsevier-extract` script becomes available vi
 - `--dois` for comma-separated DOIs or a text file containing one DOI per line
 - `--jsonl` for a JSON Lines file where each line is `{"doi": "...", "pmid": "..."}`
 
-Additional flags allow users to skip writing specific outputs (`--skip-xml`, `--skip-text`, `--skip-tables`, `--skip-coordinates`), continue past failures (`--continue-on-error`), disable caching (`--no-cache`), or adjust verbosity (`-v/--verbose`, `-q/--quiet`). `--output-dir` controls the base directory for results, and the CLI honors `ELSEVIER_EXTRACTION_WORKERS` when no `--max-workers` override is provided.
+Download order is fixed to `Elsevier -> Springer Open Access`:
+
+- Elsevier full-text is attempted first for DOI/PMID records.
+- On Elsevier miss, PMID-only records are resolved to DOI via NCBI ESummary and retried against Springer Open Access (`/openaccess/json` + `/openaccess/jats`).
+
+Additional flags allow users to skip writing specific outputs (`--skip-xml`, `--skip-text`, `--skip-tables`, `--skip-coordinates`), continue past failures by default (`--continue-on-error`), opt into fail-fast behavior (`--fail-fast`), disable caching (`--no-cache`), or adjust verbosity (`-v/--verbose`, `-q/--quiet`). `--output-dir` controls the base directory for results, and the CLI honors `ELSEVIER_EXTRACTION_WORKERS` when no `--max-workers` override is provided.
+
+### Environment Variables
+
+- Required: `ELSEVIER_API_KEY`
+- Optional Springer fallback: `SPRINGER_API_KEY`, `SPRINGER_BASE_URL`
+- Optional PubMed resolution: `PUBMED_BASE_URL`, `NCBI_API_KEY`
 
 ### Output layout
 
@@ -51,4 +62,12 @@ Each article is saved under `output-dir/{identifier}` where `{identifier}` is th
 - `coordinates.json` – NIMADS-style evaluation of extracted coordinates
 - `tables/*.csv` – extracted tables named after their labels/captions
 
-The CLI also appends every run to `manifest.jsonl` (with status, timing, and file list) and records failures in `errors.jsonl`, enabling audit and resumable processing.
+The CLI also appends every run to `manifest.jsonl` (with status, source provider, timing, file list, and a `reason` field for skipped entries) and records failures in `errors.jsonl` (including provider source), enabling audit and resumable processing. Springer/PubMed skip reasons include:
+
+- `springer_unconfigured`
+- `pmid_to_doi_unresolved`
+- `springer_not_found`
+- `springer_jats_unavailable`
+- `springer_rate_limited`
+
+When `springer_rate_limited` occurs, the runner stops querying Springer for the remainder of that run and marks subsequent Springer-fallback records as skipped with the same reason to avoid unnecessary quota pressure.

elsevier_coordinate_extraction/cli/main.py

@@ -77,10 +77,18 @@ def create_parser() -> argparse.ArgumentParser:
         action="store_true",
         help="Skip writing coordinates JSON",
     )
+    parser.set_defaults(continue_on_error=True)
     parser.add_argument(
         "--continue-on-error",
         action="store_true",
-        help="Keep going after failures",
+        dest="continue_on_error",
+        help="Keep going after failures (default)",
+    )
+    parser.add_argument(
+        "--fail-fast",
+        action="store_false",
+        dest="continue_on_error",
+        help="Stop on first failed article extraction",
     )
     parser.add_argument(
         "--max-workers",

elsevier_coordinate_extraction/cli/orchestrator.py

@@ -4,6 +4,7 @@
 from pathlib import Path
 from typing import Dict, List
 
+import httpx
 from tqdm import tqdm
 
 from elsevier_coordinate_extraction.cache import FileCache
@@ -34,6 +35,36 @@
 Record = Dict[str, str]
 
 
+def _normalize_source(value: object) -> str | None:
+    if isinstance(value, str):
+        lowered = value.strip().lower()
+        if lowered in {"elsevier", "springer"}:
+            return lowered
+    return None
+
+
+def _infer_source_from_error(
+    error: BaseException | None,
+    reason: str | None,
+) -> str | None:
+    if reason:
+        if reason == "pmid_to_doi_unresolved" or reason.startswith("springer_"):
+            return "springer"
+        if reason.startswith("elsevier_") or reason == "not_found_404":
+            return "elsevier"
+
+    if isinstance(error, httpx.HTTPStatusError):
+        request = error.request or getattr(error.response, "request", None)
+        if request is not None:
+            host = request.url.host or ""
+            lowered = host.lower()
+            if "springernature.com" in lowered:
+                return "springer"
+            if "elsevier.com" in lowered:
+                return "elsevier"
+    return None
+
+
 async def process_articles(
     records: List[Record],
     output_dir: Path,
@@ -43,27 +74,34 @@ async def process_articles(
     skip_text: bool = False,
     skip_tables: bool = False,
     skip_coordinates: bool = False,
-    continue_on_error: bool = False,
+    continue_on_error: bool = True,
     use_cache: bool = True,
     verbose: bool = False,
 ) -> Dict[str, int]:
     output_dir.mkdir(parents=True, exist_ok=True)
     cache = FileCache(output_dir / ".cache") if use_cache else None
 
-    downloaded_errors: List[Record] = []
-    download_exceptions: List[Exception] = []
-    skipped_records: List[Record] = []
+    downloaded_errors: List[tuple[Record, Exception, str | None]] = []
+    skipped_records: List[tuple[Record, str, str | None, str | None]] = []
 
     async def _progress_callback(
         record: Record,
         article: ArticleContent | None,
         error: BaseException | None,
     ) -> None:
+        download_bar.update(1)
         if error is not None:
-            downloaded_errors.append(record.copy())
-            download_exceptions.append(error)
+            reason = getattr(error, "skip_reason", None)
+            source = _infer_source_from_error(error, reason if isinstance(reason, str) else None)
+            if isinstance(reason, str) and reason:
+                skipped_records.append((record.copy(), reason, source, str(error)))
+            else:
+                if isinstance(error, Exception):
+                    downloaded_errors.append((record.copy(), error, source))
+                else:
+                    downloaded_errors.append((record.copy(), Exception(str(error)), source))
         elif article is None:
-            skipped_records.append(record.copy())
+            skipped_records.append((record.copy(), "not_found_404", "elsevier", None))
 
     stats = {"success": 0, "failed": 0, "skipped": 0}
 
@@ -73,6 +111,7 @@ async def _progress_callback(
             records,
             client=client,
             cache=cache,
+            settings=settings,
             progress_callback=_progress_callback,
         )
     download_bar.close()
@@ -83,6 +122,7 @@ async def _progress_callback(
             article.metadata.get("identifier_lookup")
             or {"doi": article.doi}
         )
+        source = _normalize_source(article.metadata.get("provider"))
         start = time.monotonic()
         files_written: List[Path] = []
         try:
@@ -114,8 +154,10 @@ async def _progress_callback(
                 output_dir,
                 record=record,
                 status="success",
+                source=source,
                 files=files_written,
                 error=None,
+                reason=None,
                 duration=duration,
             )
             stats["success"] += 1
@@ -128,37 +170,43 @@ async def _progress_callback(
                 output_dir,
                 record=record,
                 status="failed",
+                source=source,
                 files=files_written,
                 error=str(exc),
+                reason=None,
                 duration=duration,
             )
-            append_error_entry(output_dir, record=record, error=exc)
+            append_error_entry(output_dir, record=record, error=exc, source=source)
             stats["failed"] += 1
             extract_bar.write(f"Error processing {record}: {exc}")
             if not continue_on_error:
                 extract_bar.close()
                 raise
     extract_bar.close()
 
-    for record, error in zip(downloaded_errors, download_exceptions):
+    for record, error, source in downloaded_errors:
         append_manifest_entry(
             output_dir,
             record=record,
             status="failed",
+            source=source,
             files=[],
             error=str(error),
+            reason=None,
             duration=0.0,
         )
-        append_error_entry(output_dir, record=record, error=error)
+        append_error_entry(output_dir, record=record, error=error, source=source)
         stats["failed"] += 1
 
-    for record in skipped_records:
+    for record, reason, source, detail in skipped_records:
         append_manifest_entry(
             output_dir,
             record=record,
             status="skipped",
+            source=source,
             files=[],
-            error=None,
+            error=detail,
+            reason=reason,
             duration=0.0,
         )
         stats["skipped"] += 1

elsevier_coordinate_extraction/cli/outputs.py

@@ -94,8 +94,10 @@ def append_manifest_entry(
     *,
     record: Dict[str, str],
     status: str,
+    source: str | None,
     files: list[Path],
     error: str | None,
+    reason: str | None,
     duration: float,
 ) -> None:
     manifest_path = output_dir / "manifest.jsonl"
@@ -104,8 +106,10 @@ def append_manifest_entry(
         "timestamp": datetime.utcnow().isoformat() + "Z",
         "identifier": record,
         "status": status,
+        "source": source,
         "files": [str(path.relative_to(output_dir)) for path in files],
         "error": error,
+        "reason": reason,
         "duration_seconds": round(duration, 3),
     }
     with manifest_path.open("a", encoding="utf-8") as fh:
@@ -117,11 +121,13 @@ def append_error_entry(
     *,
     record: Dict[str, str],
     error: Exception,
+    source: str | None,
 ) -> None:
     errors_path = output_dir / "errors.jsonl"
     entry = {
         "timestamp": datetime.utcnow().isoformat() + "Z",
         "identifier": record,
+        "source": source,
         "error_type": type(error).__name__,
         "error_message": str(error),
     }

elsevier_coordinate_extraction/client.py

@@ -14,6 +14,14 @@
 __all__ = ["ScienceDirectClient"]
 
 
+def _http2_enabled() -> bool:
+    try:
+        import h2  # noqa: F401
+    except ImportError:
+        return False
+    return True
+
+
 class ScienceDirectClient:
     """Thin wrapper around httpx.AsyncClient with Elsevier defaults."""
 
@@ -96,7 +104,7 @@ async def _ensure_client(self) -> None:
             "timeout": timeout,
             "headers": headers,
             "transport": self._transport,
-            "http2": True,
+            "http2": _http2_enabled(),
         }
         if self._settings.use_proxy:
             proxy_value = self._settings.https_proxy or self._settings.http_proxy