generate_publist.py

#!/usr/bin/env python3
"""Generate the publications page for garrethmartin.github.io.

Fetches papers from an ADS library and writes _posts/2019-04-06-Publications.md.
Run from the repo root:  python generate_publist.py
"""

import json
import os
import re
import requests
from collections import defaultdict

ADS_TOKEN    = "7rbhCKGe8EGuSLM3XpV2RZ1v5RpjvnOqUgb13X7i"
LIBRARY_ID   = "nThU2Yw3SUytqSYjksZ8uA"
OUT_FILE     = "_posts/2019-04-06-Publications.md"
HEADERS      = {"Authorization": f"Bearer {ADS_TOKEN}"}
SCHOLAR_CACHE = os.path.join(os.path.expanduser("~/Code/CV_stuff"), "scholar_cache.json")


# ADS records still carry authors' former names; override the display form here
# without touching the underlying ADS query (author search strings stay as-is).
AUTHOR_NAME_OVERRIDES = {
    "butler, j": "Butler, M.",
    "butler, joseph": "Butler, Madeline",
}


def apply_author_override(name):
    key = name.strip().lower().rstrip(".")
    return AUTHOR_NAME_OVERRIDES.get(key, name)


def latex_to_utf8(text):
    replacements = {
        r"\'a": "á", r'\"a': "ä", r"\'e": "é", r'\"e': "ë",
        r"\'i": "í", r'\"i': "ï", r"\'o": "ó", r'\"o': "ö",
        r"\'u": "ú", r'\"u': "ü", r"\~n": "ñ", r"\c{c}": "ç",
        r"\'A": "Á", r'\"A': "Ä", r"\&": "&", r"``": "“", r"''": "”",
    }
    for latex, char in replacements.items():
        text = text.replace(latex, char)
    return text


def load_scholar_data(path):
    try:
        with open(path) as f:
            data = json.load(f)
    except (OSError, json.JSONDecodeError) as e:
        print(f"WARNING: could not load scholar cache ({e}); skipping Scholar stats")
        return {}
    return data


def is_catalog(p):
    doctype = (p.get("doctype") or "").lower()
    journal = ((p.get("bibstem") or [""])[0] if p.get("bibstem") else (p.get("pub") or "")).lower()
    title   = ((p.get("title") or [""])[0]).lower()
    return (
        doctype == "catalog"
        or journal.startswith("ycat")
        or "vizier online data catalog" in title
    )


def format_paper(p):
    raw_title = (p.get("title") or [""])[0]
    title     = f"*{latex_to_utf8(raw_title)}*"
    authors   = [apply_author_override(latex_to_utf8(a)) for a in (p.get("author") or [])]
    authors   = [f"**{a}**" if "Martin, G" in a else a for a in authors]
    author_str = ", ".join(authors)
    year    = p.get("year") or "n.d."
    date    = p.get("pubdate") or ""
    month   = date.split("-")[1] if date else ""
    bibstem = p.get("bibstem") or []
    journal = bibstem[0] if bibstem else (p.get("pub") or "")
    volume  = p.get("volume") or ""
    pages   = p.get("page") or []
    page    = pages[0] if pages else ""
    dois    = p.get("doi") or []
    doi     = dois[0] if dois else ""
    ref     = f"{journal} {volume}, {page}".strip()
    ref_str = f"[{ref}](https://doi.org/{doi})" if doi else ref
    return f"{title}  \n{author_str}  \n{month}/{year} | {ref_str}"


def main():
    scholar_data = load_scholar_data(SCHOLAR_CACHE)

    lib_url  = f"https://api.adsabs.harvard.edu/v1/biblib/libraries/{LIBRARY_ID}"
    lib_resp = requests.get(lib_url, headers=HEADERS, params={"start": 0, "rows": 500})
    lib_resp.raise_for_status()
    solr     = lib_resp.json()["solr"]["response"]
    bibcodes = [d["bibcode"] for d in solr["docs"]]
    print(f"Library contains {solr['numFound']} docs, fetching {len(bibcodes)}.")

    query       = " OR ".join(f"bibcode:{bc}" for bc in bibcodes)
    search_resp = requests.get(
        "https://api.adsabs.harvard.edu/v1/search/query",
        headers=HEADERS,
        params={
            "q":    query,
            "fl":   "title,author,pubdate,year,pub,bibstem,volume,page,doi,doctype",
            "rows": len(bibcodes),
        },
    )
    search_resp.raise_for_status()
    all_papers = search_resp.json()["response"]["docs"]

    first_author_papers = []
    all_by_year = defaultdict(list)

    for p in all_papers:
        if is_catalog(p):
            continue
        year  = p.get("year") or "0"
        date  = p.get("pubdate") or ""
        month = int(date.split("-")[1]) if date and "-" in date else 0
        formatted = format_paper(p)
        authors   = [latex_to_utf8(a) for a in (p.get("author") or [])]
        if authors and authors[0].startswith("Martin, G"):
            first_author_papers.append((year, month, formatted))
        all_by_year[year].append((month, formatted))

    first_author_papers.sort(key=lambda x: (x[0], x[1]), reverse=True)
    for year in all_by_year:
        all_by_year[year].sort(key=lambda x: x[0], reverse=True)

    n_papers = sum(len(v) for v in all_by_year.values())

    sep = "  ■  "
    if scholar_data:
        stats_parts = [
            f"{n_papers} peer-reviewed publications and preprints",
            f"{scholar_data['citedby']:,} citations (Scholar)",
            f"h-index {scholar_data['hindex']} (Scholar)",
            f"i10-index {scholar_data['i10index']} (Scholar)",
            __import__('datetime').date.today().strftime('%B %Y'),
        ]
    else:
        stats_parts = [
            f"{n_papers} peer-reviewed publications and preprints",
            "citations",
            "h-index",
            __import__('datetime').date.today().strftime('%B %Y'),
        ]
    stats_line = f"*{sep.join(stats_parts)}*"

    lines = [
        "---",
        "layout: post",
        "title: Publications",
        "cover: NHz.png",
        "date:   2013-12-09 12:00:00",
        "categories: posts",
        'icon: "📄"',
        "---",
        "",
        "[`ADS Library`](https://ui.adsabs.harvard.edu/public-libraries/nThU2Yw3SUytqSYjksZ8uA \"ADS library\")"
        f"{sep}[`Google Scholar`](https://scholar.google.com/citations?user=4O8TNrgAAAAJ \"Google Scholar\")"
        f"{sep}[`Publication list (PDF)`](/files/publication_list.pdf \"Publication list\")",
        "",
        stats_line,
        "",
        "## First-author publications",
        "",
    ]

    for _, _, entry in first_author_papers:
        lines.append(entry + "\n")

    lines += ["", "## All peer-reviewed publications and preprints", ""]

    for year in sorted(all_by_year.keys(), reverse=True):
        lines.append(f"### {year}\n")
        for _, entry in all_by_year[year]:
            indented = "\n".join(">" + line for line in entry.split("\n"))
            lines.append(indented + "\n")

    with open(OUT_FILE, "w") as f:
        f.write("\n".join(lines))

    print(f"Written to {OUT_FILE} ({n_papers} papers after filtering)")


if __name__ == "__main__":
    main()