From 9a227759277f8f3d06fce08d725e1c2fcbe43dec Mon Sep 17 00:00:00 2001 From: Marine Denolle Date: Wed, 22 Jul 2026 12:26:38 +0200 Subject: [PATCH] Add publication-pipeline guide: prompt -> code -> paper -> journal A validated, end-to-end workflow for version-controlling a paper the same way code is version-controlled: a Quarto Markdown source in the same repo as the analysis code, one build script that renders it to a journal's LaTeX class, and a GitHub Action that syncs the built manuscript to a paper-only repo Overleaf's GitHub Sync reads from -- so co-authors keep working in Overleaf with no new tooling, and their feedback comes back to the code repo as GitHub issues instead of evaporating in email/Slack. Extracted from a real submission draft (Denolle-Lab/codameter / Denolle-Lab/codameter-paper), including a gotchas appendix of the specific LaTeX/pandoc/class conflicts that cost real debugging time (a natbib length-register collision, a hardcoded bibliography-heading case Quarto's own template does not expose a metadata knob for, a class that silently drops the LaTeX float-placement argument, and a Pandoc raw-table parsing edge case), plus a copyable template (build.py + sync workflow) genericized from the same two files. Linked from research/README.md alongside the existing Gaia entry. --- research/README.md | 9 + research/publication-pipeline/README.md | 167 ++++++++++++++++++ .../.github/workflows/sync-to-overleaf.yml | 112 ++++++++++++ .../publication-pipeline/template/README.md | 16 ++ .../publication-pipeline/template/build.py | 162 +++++++++++++++++ 5 files changed, 466 insertions(+) create mode 100644 research/publication-pipeline/README.md create mode 100644 research/publication-pipeline/template/.github/workflows/sync-to-overleaf.yml create mode 100644 research/publication-pipeline/template/README.md create mode 100644 research/publication-pipeline/template/build.py diff --git a/research/README.md b/research/README.md index 3c9ba48..d29eddc 100644 --- a/research/README.md +++ b/research/README.md @@ -43,5 +43,14 @@ three-plane split — **content** (the agents), **harness** (FrugalMind), **gold ## Specs in this folder +- [`publication-pipeline/`](publication-pipeline/) — **prompt → code → paper → journal**, + version-controlled end to end. Not an agent persona but the infrastructure a + Scientist's paper-writing work runs on: a Quarto Markdown source, one Python build + script that renders it to a journal's LaTeX class, and a GitHub Action that syncs the + built manuscript to a paper-only repo Overleaf reads from — so co-authors keep working + in Overleaf with zero new tooling, and their comments come back to the code repo as + GitHub issues instead of evaporating in a chat thread. Validated on a real submission + draft (`Denolle-Lab/codameter`); includes a copyable template. + The research role's agents are maintained in the repositories above. To propose a *new role* in this collection, [open an issue](../.github/ISSUE_TEMPLATE/new-role.yml). diff --git a/research/publication-pipeline/README.md b/research/publication-pipeline/README.md new file mode 100644 index 0000000..f4045b8 --- /dev/null +++ b/research/publication-pipeline/README.md @@ -0,0 +1,167 @@ +# Publication pipeline: prompt → code → paper → journal, without losing the thread + +**Version:** v0.1 · **Status:** active (validated on a real paper, see [Worked example](#worked-example)) · **Tag:** `publication-pipeline/v0.1` + +> One-sentence purpose statement: version-control a paper the same way you version-control code, so that "what changed and why" is answerable at every stage from first prompt to journal PDF, and a colleague's comment in Overleaf can always be traced back to a line of code. + +## The problem this solves + +A typical academic paper's provenance looks like this: exploratory code in a notebook nobody kept, a chat log with an AI assistant that produced a paragraph nobody remembers the reasoning for, a `.docx` emailed between three co-authors with `_v2_final_ACTUALFINAL.docx` in the filename, and a LaTeX file hand-typeset from that prose the week before submission. None of it is in the same version-control system. A reviewer's comment three months later ("can you check this with a different frequency band?") requires reconstructing which script produced which number, from memory. + +This guide describes a pipeline where **the code, the figures, and the paper text are versioned together in one git history**, the LaTeX a journal wants is *generated*, not hand-typeset, and the one genuinely manual step — turning a colleague's Overleaf comment into a code change — is captured as a GitHub issue instead of a Slack message that evaporates. + +It is not a framework to install. It is five small, standard tools wired together in a specific order. Every piece is something you already have a account/license for as an academic: Python, git, GitHub, Quarto, Overleaf. + +## The five stages + +``` + (1) PROMPT/CODE (2) MARKDOWN REPORT (3) VERSION CONTROL + Claude / Claude Code → .qmd or .ipynb narrating → git commit, GitHub repo + in your code repo methods + results (code AND prose, same repo) + │ + ▼ + (5) COLLABORATIVE REVIEW ← (4) AUTOMATED TYPESET + Overleaf (comments, quarto render .qmd → .tex + track changes, co-authors) pushed to a paper-only GitHub repo + │ Overleaf's GitHub Sync pulls it + ▼ + GitHub issue in the CODE repo ──────────────────────────────┘ + ("Fig. 3b needs bigger fonts") → code edit → back to stage 4 +``` + +Two repos, not one: a **code repo** (this project's actual analysis code, tests, and the paper's Markdown source) and a **paper repo** (nothing but the compiled `.tex`, bibliography, figures, and the journal's class files — the thing Overleaf actually reads). Splitting them matters for one reason: **Overleaf syncs a whole repo**, and you do not want your Python source, test suite, and CI history inside a project a co-author opens just to fix a typo in the abstract. + +## Worked example + +This pipeline is not hypothetical — it is running today on: +- **Code repo**: [`Denolle-Lab/codameter`](https://github.com/Denolle-Lab/codameter) — `paper/manuscript_marine.qmd` is the source, `paper/build.py` is the generator, `src/codameter/` is the analysis code the figures come from. +- **Paper repo**: [`Denolle-Lab/codameter-paper`](https://github.com/Denolle-Lab/codameter-paper) — synced automatically from the code repo, linked to an Overleaf project via Overleaf's GitHub Sync. +- **Sync automation**: `.github/workflows/sync-paper-to-overleaf.yml` in the code repo. + +Every claim below is grounded in that project, not a hypothetical. The [`template/`](template/) folder in this guide is a genericized copy of the same two files, ready to adapt. + +--- + +## Stage 1 — Prompt → code + +The AI assistant (Claude Code, or any agent) works **inside the code repo**, on the actual analysis modules and tests — not in a disposable chat window. The reason is not process for its own sake: an assistant that edits `src/yourpackage/figures.py` produces a diff you can review, test, and revert; an assistant that pastes a finished paragraph into a chat produces nothing you can check. + +**Minimal-tooling choices:** +- Interaction happens as normal Python development — edit a module, run the test suite, look at the diff. No bespoke "agent workflow" tooling. +- Notebooks are for *exploration only* and are not the artifact of record — anything a paper depends on should end up as a tested function in a module, not a cell in a `.ipynb` that can be re-run out of order. If a notebook exists, treat it as disposable scratch work, same as you would a REPL session. +- Standing instructions for the assistant (tone, what to preserve, what never to guess) live in a checked-in `CLAUDE.md`/`AGENTS.md` at the repo root, not in a chat history that disappears. This is the "three-layer model" this repository (`academic-practice-agents`) already documents for agent specs generally — apply the same discipline here. + +## Stage 2 — Code → Markdown report + +The paper's prose lives in one **Quarto Markdown** file (`.qmd`). Markdown with inline citations (`[@key]`) and executable code blocks, not LaTeX, is the thing a human *and* an AI assistant edit directly. Figures are referenced by path, not pasted as images — regenerating a figure means re-running a script, not re-drawing something in Illustrator. + +``` +paper/manuscript.qmd # the single editable source — never hand-edit the generated .tex +src/yourpackage/figures.py # the actual figure-generating code the paper cites +literature/figs/*.png # generated output, committed so the build is reproducible offline +``` + +Why Quarto over raw Pandoc or a bespoke script: it is one well-maintained binary, handles citation processing (`natbib`/`citeproc`) and LaTeX passthrough for tables/equations without extra plumbing, and — critically — it lets raw LaTeX blocks sit directly in the Markdown for anything Markdown cannot express (a `tabularx` table, a `keywords` environment your target class defines). You do not need to learn a new templating language; you need Markdown plus the ability to drop in a fenced ` ```{=latex} ` block when Markdown genuinely cannot say what you need. + +## Stage 3 — GitHub for version control (code *and* prose, together) + +Both the analysis code and the paper's `.qmd` are committed to the **same** git repository, on the same branches, reviewed through the same pull requests. This is the single highest-leverage decision in the whole pipeline: it means a commit that changes a coefficient in the model and a commit that updates the sentence describing that coefficient can be the *same* commit, and `git log -p -- paper/manuscript.qmd src/yourpackage/model.py` answers "did the paper text and the code actually change together?" — a question that is unanswerable once the paper lives in a `.docx` on someone's laptop. + +Practical conventions that cost nothing and pay for themselves: +- **pre-commit hooks** (`ruff`, `black`, `mypy`, trailing-whitespace) on the code, so review time goes to substance, not formatting nits. +- **One feature branch per logical change**, PR review even if you are the sole author for now — it is the mechanism that makes `git bisect` and `git blame` useful later, and costs nothing more than typing `git checkout -b`. +- The generated `.tex` **is** committed (unusually, for a generated artifact) — because it is the thing the paper repo (stage 4) needs to sync, and because a reviewer wants to see the exact LaTeX diff, not just the Markdown diff, when a citation or equation changes. + +## Stage 4 — Automated Markdown → journal LaTeX + +One script, `paper/build.py`, does the whole render: regenerate any figures/tables that come from code, run `quarto render` (which runs the full `lualatex` + `bibtex` toolchain and emits both `.tex` and `.pdf`), and — the one non-obvious step — **patch the two or three things Quarto's own template hardcodes that your target journal's class disagrees with**, then recompile. + +This last step deserves honesty: **Quarto's LaTeX template is opinionated, and it will not always agree with an old or unusual `.cls` file.** In the worked example, three separate small conflicts surfaced against a real 1998-vintage journal class (`gji.cls` for *Geophysical Journal International*): + +1. Quarto's `\usepackage{natbib}` and the class's own `\newlength{\bibhang}` collide (`natbib` loads *after* the class in Quarto's template, and its own `\newlength` call errors if the class already claimed that register). **Fix pattern**: let whichever package loads first win; provide the value, not the definition, in your own preamble. +2. Quarto hardcodes the bibliography heading as the literal string `"References"`, overriding the class's own house style (which might want `"REFERENCES"`). **There is no YAML metadata key that fixes this under `cite-method: natbib`** — tested, confirmed absent. **Fix pattern**: a small, targeted string patch on the rendered `.tex`, applied by `build.py` itself (not by hand), followed by a recompile with `latexmk` (which re-runs `lualatex`/`bibtex` until stable, so you never have to hand-count how many passes a class needs). +3. Some classes' `\figure`/`\table` don't accept the `[htbp]` placement argument Markdown-generated LaTeX defaults to — it prints literally instead of being consumed. **Fix pattern**: know your class's own float defaults, and don't pass an argument it doesn't expect. + +None of this is specific to GJI — it is the generic shape of "an old but real journal class meets a modern automated pipeline." The [gotchas appendix](#appendix-hard-won-latex--pandoc-gotchas) below catalogs the specific failures with their fixes, because the failure *messages* are searchable and the fixes are not obvious from the message alone. + +**Minimal-tooling choice**: `build.py` is one plain Python script with no framework — `argparse`, `subprocess`, `pathlib`. It is the one piece of this pipeline worth reading end-to-end before you trust it, because it is the one piece doing something non-standard (patching a generated file). Keep it that way: resist the urge to grow it into a build system. + +## Stage 5 — The paper repo, synced to Overleaf, and the return path + +A **GitHub Action**, triggered on push to the code repo's `paper/**` (and the figures/class-files it depends on — see the [gotchas appendix](#appendix-hard-won-latex--pandoc-gotchas) for why the path list matters), copies the built `.tex`, bibliography, figures, and the journal's class/style files into a **separate, paper-only GitHub repo**. That repo is the thing Overleaf's **GitHub Sync** feature reads from — either automatically, or via a "Pull from GitHub" click in Overleaf's menu. + +```yaml +# .github/workflows/sync-paper-to-overleaf.yml — the shape of it (full version in template/) +on: + push: + branches: [main] + paths: ["paper/manuscript.tex", "paper/*.bib", "paper/*.cls", "literature/figs/**"] +jobs: + sync: + steps: + - checkout the code repo (sparse: just paper/ and the figures) + - checkout the paper repo, authenticated with a deploy key + - copy the built files across + - commit and push, only if something actually changed +``` + +Co-authors and reviewers then work in Overleaf exactly as they always have — comments, track changes, a shared editor — with **zero new tooling asked of them**. This is deliberate: the automation exists so that *you* don't hand-copy files, not so that your colleagues learn a new workflow. + +### Deploy keys: the one credential this needs + +The sync workflow needs write access to the paper repo. Use a **deploy key scoped to that one repo** (an SSH key pair, the public half registered on the paper repo with write access, the private half stored as a GitHub Actions secret on the code repo) — not a personal access token with broader account access. Generate it **outside any git working tree** (`~/.ssh/`, never a bare relative filename run from inside a checkout — that leaves the private key sitting untracked in a working directory, one `git add -A` away from a real leak). `gh repo deploy-key add` and `gh secret set` do the GitHub-side wiring from the command line; nothing about this needs a browser. + +### The return path: comments back to code + +This is the one stage that stays deliberately manual, and the reason is worth stating plainly: **the generated `.tex` is a one-way artifact.** If a colleague edits prose directly in Overleaf and that edit is not ported back into the `.qmd`, the *next* automated sync from the code repo will silently overwrite it — because the workflow always regenerates the paper repo's files from the code repo's build output. + +Given that constraint, the practice that actually works: +1. Ask colleagues to use Overleaf's **comments**, not direct text edits, during review. (Overleaf's comments are Overleaf-side metadata — they do not travel through git at all, in either direction. This is a real limitation, not a bug to route around: automating comment-scraping would need a scraper against an UI Overleaf does not expose an API for, which is the opposite of minimal tooling.) +2. When you read a comment worth acting on, **open a GitHub issue in the code repo** describing it in one sentence — "Fig. 3b legend overlaps the data," "reviewer wants the $Q_c$ definition moved earlier." This is the paper trail. It costs one sentence and gives you `git log` provenance for why a figure changed. +3. Resolve the issue with a normal code change — edit the `.qmd` or the figure-generating function, rerun `build.py`, let the existing forward sync push the correction back to Overleaf. Close the issue. + +This turns "a comment in a tool with no git history" into "an issue with a linked commit," which is the entire point of the exercise. + +--- + +## Minimal-dependency principles + +Every tool choice above was made against one test: *would an academic with no DevOps background be able to set this up and keep it running without a dedicated engineer?* Concretely: + +| Choice | Not this | Why | +|---|---|---| +| Quarto | Bespoke Pandoc scripting, or a hand-rolled `.tex` template | One binary, handles citations and raw-LaTeX passthrough already | +| `paper/build.py`, plain Python | A build system (Make, a task runner) | One file you can read in five minutes; no new syntax to learn | +| GitHub Actions | A self-hosted CI server | Already included with the GitHub account you have | +| A scoped deploy key | A broad personal access token | Least privilege; one repo, one purpose, easy to revoke | +| Overleaf's native GitHub Sync | A custom Overleaf API integration | Zero new tooling for co-authors; they keep using Overleaf exactly as before | +| GitHub issues for feedback | A bespoke comment-import script | The friction (one sentence, one click) is the *feature* — it forces triage instead of blind automation | +| `git`/GitHub for the paper text | Google Docs / `.docx` with tracked changes | One history for code and prose; `git blame` works on both | + +## Setup checklist + +1. **Code repo**: add `paper/manuscript.qmd`, a `paper/build.py` (adapt from [`template/build.py`](template/build.py)), your journal's `.cls`/`.bst` if it has one, and a `_preamble.tex` for anything Quarto's YAML front matter cannot express. +2. **Build locally once**, end to end, before automating anything: `python paper/build.py`. Fix whatever your specific class disagrees with (see the [gotchas](#appendix-hard-won-latex--pandoc-gotchas)) *before* wiring up CI — debugging a LaTeX class conflict through a CI log is much slower than doing it in your own terminal. +3. **Create the paper repo** (empty, or seeded from your journal's official Overleaf template export). +4. **Generate a deploy key** in `~/.ssh/`, register the public half on the paper repo with write access, store the private half as a secret on the code repo (see [`template/`](template/) for the exact `gh` commands). +5. **Add the sync workflow** (adapt from [`template/.github/workflows/sync-to-overleaf.yml`](template/.github/workflows/sync-to-overleaf.yml)), listing every file your build touches in its trigger `paths:` (see the gotcha below). +6. **Link the paper repo to an Overleaf project** via Overleaf's GitHub Sync (Overleaf → New Project → Import from GitHub, or link an existing project via its menu). +7. Push a change to `paper/manuscript.qmd` in the code repo and confirm it appears in Overleaf within a few minutes. + +## Appendix: hard-won LaTeX / pandoc gotchas + +Each of these cost real debugging time on the worked example. They are recorded here because the *error message* is the thing you will search for, and the fix is rarely obvious from the message alone. + +- **`Command \bibhang already defined`** — your journal's class and `natbib` both try to define the same length register. Let whichever loads first win; `\providecommand`/re-`\setlength` the value in your own preamble rather than `\newlength`-ing it again. +- **The bibliography heading won't match your class's house style** — Quarto hardcodes it and blanks the class's own heading mechanism to avoid a double heading. There is no `reference-section-title` effect under `cite-method: natbib` (tested). Patch the rendered `.tex` directly in your build script, and patch **both** the visible `\section*{...}` and the matching `\addcontentsline{toc}{section}{...}` together — the second one feeds the PDF bookmark panel via `hyperref` even when there is no printed table of contents, and leaving it inconsistent with the heading is its own small bug. +- **A stray `[htbp]` prints literally above every figure/table** — some classes redefine `\figure`/`\table` to take *no* placement argument at all (unlike standard LaTeX). Check your class's own `\fps@figure`/`\fps@table` defaults before passing one. +- **A closing `$` immediately followed by a digit doesn't parse as math** — e.g. `$\sim$0.02` silently escapes to garbled literal text instead of erroring, because Pandoc's dollar-math rule specifically disallows a digit right after the closing `$` (to avoid misreading `$20,000` as math). Merge into one span: `$\sim\!0.02\,\%$`. +- **A raw `\begin{table}...\end{table}` block leaks stray literal text right after it, silently corrupting any citations inside** — a real, reproducible Pandoc parsing edge case with no single isolated trigger found (tried caption length, LaTeX-style double-backtick quotes, custom macros, math-dollar adjacency — no single feature explained it in isolation). The reliable fix: wrap the whole table in an explicit raw-LaTeX fence (` ```{=latex} ... ``` `), which forces Pandoc to treat it as fully opaque. This means any `[@key]` citations *inside* the table must be converted to raw `\citep{key}` first, since a raw fence bypasses Pandoc's citation processing entirely. +- **Literal Unicode math symbols in YAML metadata (e.g. an `abstract:` field) render as missing glyphs** under an unusual class's font setup, even though the same symbols are fine in the body text once escaped through a macro. If your body prose already uses `\dvv\ ` (a macro, with an escaped trailing space to stop the macro eating it) instead of a literal `δv/v`, do the same in YAML metadata fields — don't assume they get the same LaTeX processing pass as body Markdown. +- **The sync trigger must cover every file the build touches.** If your class/style files carry local patches (like the `\bibhang` fix above), the paper repo needs *your* patched copies, not whatever it already has — and the workflow's `paths:` filter needs to include those class files, not just the manuscript and bibliography, or a class fix you make will never propagate. + +## Version history + +| Version | Date | Change | Reason | +|---|---|---|---| +| v0.1 | 2026-07-22 | Initial version, extracted from the `codameter`/`codameter-paper` pipeline | First validated end-to-end run: build, class-conflict fixes, deploy-key sync, all working on a real GJI submission draft | diff --git a/research/publication-pipeline/template/.github/workflows/sync-to-overleaf.yml b/research/publication-pipeline/template/.github/workflows/sync-to-overleaf.yml new file mode 100644 index 0000000..79ceb51 --- /dev/null +++ b/research/publication-pipeline/template/.github/workflows/sync-to-overleaf.yml @@ -0,0 +1,112 @@ +name: sync-paper-to-overleaf + +# Adapted from the validated pipeline in Denolle-Lab/codameter (see the parent +# README.md for the full write-up). Copy this into your code repo's +# .github/workflows/, then edit the four CONFIG blocks marked below. +# +# Mirrors the already-built manuscript (the rendered .tex, its bibliography, +# any journal class/style files it depends on, and the figures it references) +# into a separate, paper-only GitHub repo -- the one Overleaf's GitHub Sync +# reads from. Overleaf itself then picks up the change (automatically, if +# GitHub Sync is enabled on the project, or via "Menu -> GitHub -> Pull" +# otherwise). +# +# Your code repo's paper/ directory is the single source of truth for every +# file this workflow copies -- including the journal's class/style files, if +# you carry any local patches to them (see the parent README's gotchas +# appendix). An unpatched copy sitting only in the paper repo will make +# Overleaf hit exactly the errors those patches exist to fix. +# +# Deliberately does NOT re-run the quarto/lualatex build in CI: the manuscript +# is built and verified locally (`python paper/build.py`), where the full +# TeX Live toolchain your manuscript needs is already known to work. This +# workflow only syncs the already-committed, already-verified output. Trying +# to reproduce a from-scratch LaTeX toolchain in a fresh CI image adds a +# second place version drift can happen, for no benefit once the build is +# already verified locally. +# +# One-time setup this workflow depends on (not done by this workflow): +# 1. Generate a deploy key OUTSIDE any git working tree, e.g. in ~/.ssh/ -- +# NOT a bare relative filename run from inside a repo checkout, which +# leaves the private key sitting untracked (and easy to `git add -A` +# into a commit) right in the working directory: +# ssh-keygen -t ed25519 -C "paper-sync" -f ~/.ssh/paper-deploy -N "" +# 2. Add ~/.ssh/paper-deploy.pub to your paper repo -> Settings -> +# Deploy keys -> Add deploy key -> check "Allow write access". +# (or: `gh repo deploy-key add ~/.ssh/paper-deploy.pub +# --repo / --title "paper-sync" --allow-write`) +# 3. Add the contents of ~/.ssh/paper-deploy (the private half) as a +# repository secret named PAPER_DEPLOY_KEY on your code repo +# (Settings -> Secrets and variables -> Actions -> New repository secret; +# or `gh secret set PAPER_DEPLOY_KEY --repo / +# < ~/.ssh/paper-deploy`). +# 4. Keep the key pair in ~/.ssh/ as your local record of what the secret +# contains; there is nothing to delete since it was never inside a repo. + +on: + push: + branches: [main] # CONFIG 1 of 4 -- your default branch name (main/master) + paths: + # CONFIG 2 of 4 -- every file paper/build.py produces or reads that + # Overleaf needs. Missing an entry here means a change to that file + # never triggers a sync. + - "paper/manuscript.tex" + - "paper/references.bib" + - "paper/*.cls" # your journal's class file, if any + - "paper/*.bst" # your journal's bibliography style, if any + - "figures/**" # wherever your figures live + - ".github/workflows/sync-to-overleaf.yml" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: sync-paper-to-overleaf + cancel-in-progress: true + +jobs: + sync: + runs-on: ubuntu-latest + steps: + - name: Checkout code repo (paper source) + uses: actions/checkout@v4 + with: + sparse-checkout: | + paper + figures + sparse-checkout-cone-mode: false + + - name: Checkout paper repo + uses: actions/checkout@v4 + with: + repository: OWNER/PAPER-REPO # CONFIG 3 of 4 -- your paper repo + ssh-key: ${{ secrets.PAPER_DEPLOY_KEY }} + path: paper-repo + + - name: Sync manuscript, bibliography, class files, and figures + run: | + mkdir -p paper-repo/figures + cp paper/manuscript.tex paper-repo/ + cp paper/references.bib paper-repo/ + cp paper/*.cls paper-repo/ 2>/dev/null || true + cp paper/*.bst paper-repo/ 2>/dev/null || true + rsync -a --delete figures/ paper-repo/figures/ + + - name: Commit and push if changed + working-directory: paper-repo + run: | + git config user.name "paper-sync[bot]" + git config user.email "actions@github.com" + # CONFIG 4 of 4 -- list exact filenames, matching the copy step + # above, rather than `git add -A`: this is a shared repo your + # co-authors' Overleaf project reads, so only stage what this + # workflow is actually meant to manage. + git add manuscript.tex references.bib figures/ + git add *.cls *.bst 2>/dev/null || true + if git diff --cached --quiet; then + echo "No changes to sync." + exit 0 + fi + git commit -m "Sync from $(basename "$GITHUB_REPOSITORY")@${GITHUB_SHA::7}" + git push diff --git a/research/publication-pipeline/template/README.md b/research/publication-pipeline/template/README.md new file mode 100644 index 0000000..9aac724 --- /dev/null +++ b/research/publication-pipeline/template/README.md @@ -0,0 +1,16 @@ +# Template: paper → Overleaf sync + +Two files, adapted from a working pipeline (see the parent [README.md](../README.md) for the full write-up and the reasoning behind each choice). Copy both into your own project and edit the marked `CONFIG` blocks. + +## Files + +- **`build.py`** → copy to `paper/build.py`. Edit the three `CONFIG` blocks at the top: your manuscript's filename(s), any figure-regeneration commands, and (only if your journal's class needs it) the literal `.tex` string patches described in the parent README's gotchas appendix. +- **`.github/workflows/sync-to-overleaf.yml`** → copy to `.github/workflows/sync-to-overleaf.yml` in your code repo. Edit the four `CONFIG` blocks: branch name, trigger paths, paper repo name, and the exact filenames staged in the commit step. + +## Order of operations + +1. Get `build.py` working **locally** first — `python paper/build.py` should produce a clean `.pdf` before you touch the GitHub Action at all. Any LaTeX class conflict is far faster to debug in your own terminal than through a CI log. +2. Create the paper-only GitHub repo (empty, or seeded from your journal's Overleaf template export). +3. Generate a deploy key in `~/.ssh/` (commands in the workflow file's header comment) and wire it up with `gh repo deploy-key add` + `gh secret set`. +4. Push the edited workflow file. It will fail until the secret exists (expected) — once the secret is set, trigger it manually once via the Actions tab (`workflow_dispatch`) to test before relying on the automatic push trigger. +5. Link the paper repo to an Overleaf project via Overleaf's GitHub Sync. diff --git a/research/publication-pipeline/template/build.py b/research/publication-pipeline/template/build.py new file mode 100644 index 0000000..cc507f3 --- /dev/null +++ b/research/publication-pipeline/template/build.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Build the manuscript: Quarto Markdown -> LaTeX -> PDF, one command. + +Adapted from the validated pipeline in Denolle-Lab/codameter (see the parent +README.md for the full write-up). Copy this into your own paper/ directory +and edit the three CONFIG blocks below for your project; the rest should not +need changes. + +The single editable source is a Quarto Markdown (.qmd) file. This script: + + 1. (optionally) regenerates any figures/tables your analysis code produces; + 2. runs `quarto render .qmd --to pdf`, which (with `keep-tex: true` + set in the .qmd's YAML front matter) emits both `.tex` and + `.pdf` via the full quarto->pandoc->lualatex->bibtex pipeline; + 3. if your journal's LaTeX class disagrees with something Quarto's own + template hardcodes (see the README's gotchas appendix -- a wrong + bibliography-heading case is the most common one), applies the targeted + string patches you list in TEX_PATCHES and recompiles with `latexmk` + (which re-runs lualatex/bibtex until stable, so you never have to + hand-count passes). + +So you edit Markdown; you get TeX and PDF. + +Usage:: + + python paper/build.py # render PDF (+ TeX) + python paper/build.py --figures # also regenerate figures/tables first + python paper/build.py --qmd manuscript.qmd # pin the source explicitly +""" +from __future__ import annotations + +import argparse +import shutil +import subprocess +import sys +from pathlib import Path + +HERE = Path(__file__).resolve().parent +ROOT = HERE.parent + +# --------------------------------------------------------------------------- +# CONFIG 1 of 3 -- your manuscript's source filename(s). +# +# List candidates in preference order (not just one name): a manuscript source +# does get renamed mid-project, and a script that fails with a clear "no +# source found, looked for X/Y/Z" is much easier to debug than one that +# silently picks up nothing. +# --------------------------------------------------------------------------- +SOURCE_CANDIDATES = ["manuscript.qmd"] + +# --------------------------------------------------------------------------- +# CONFIG 2 of 3 -- optional command(s) that regenerate figures/tables from +# your analysis code, run when --figures is passed. Each is +# [sys.executable, "path/to/script.py"] or similar, relative to ROOT. +# --------------------------------------------------------------------------- +FIGURE_COMMANDS: list[list[str]] = [ + # [sys.executable, "scripts/make_figures.py"], +] + +# --------------------------------------------------------------------------- +# CONFIG 3 of 3 -- literal string patches to apply to the rendered .tex +# before the final compile, for anything your journal's class wants that +# Quarto's own LaTeX template does not expose a YAML-metadata knob for. +# Leave empty if your class has no such conflict. See the README's gotchas +# appendix for the two most common real examples (bibliography heading case, +# and its matching \addcontentsline entry so the PDF bookmark panel agrees). +# +# Example (uncomment and adapt): +# TEX_PATCHES = [ +# (r"\section*{References}\label{references}", +# r"\section*{REFERENCES}\label{references}"), +# (r"\addcontentsline{toc}{section}{References}", +# r"\addcontentsline{toc}{section}{REFERENCES}"), +# ] +# --------------------------------------------------------------------------- +TEX_PATCHES: list[tuple[str, str]] = [] + + +def run(cmd: list[str], cwd: Path) -> None: + print(f"$ {' '.join(cmd)} (in {cwd})") + subprocess.run(cmd, cwd=cwd, check=True) + + +def find_source(explicit: str | None) -> Path: + if explicit: + p = HERE / explicit + if not p.exists(): + sys.exit(f"error: --qmd {explicit!r} not found in {HERE}") + return p + for name in SOURCE_CANDIDATES: + p = HERE / name + if p.exists(): + return p + found = sorted(q.name for q in HERE.glob("*.qmd")) + sys.exit( + "error: no manuscript source found (looked for " + f"{', '.join(SOURCE_CANDIDATES)} in {HERE}).\n" + f" .qmd files present: {found or '(none)'}\n" + " pass --qmd to pin one explicitly, or edit " + "SOURCE_CANDIDATES at the top of this script." + ) + + +def patch_and_recompile(tex: Path) -> None: + if not TEX_PATCHES: + return + text = tex.read_text(encoding="utf-8") + changed = False + for old, new in TEX_PATCHES: + if old in text: + text = text.replace(old, new) + changed = True + if not changed: + return + tex.write_text(text, encoding="utf-8") + print(f"patched {tex.name} (applied TEX_PATCHES)") + if not shutil.which("latexmk"): + print( + "warning: 'latexmk' not found -- .tex was patched but the .pdf " + "still reflects the pre-patch text. Install latexmk (part of any " + "TeX Live install) to recompile automatically." + ) + return + run(["latexmk", "-pdflua", "-interaction=nonstopmode", tex.name], tex.parent) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument( + "--figures", action="store_true", help="regenerate figures/tables first" + ) + ap.add_argument( + "--qmd", default=None, help="manuscript source filename (default: autodetect)" + ) + args = ap.parse_args() + + if shutil.which("quarto") is None: + sys.exit("error: 'quarto' not found on PATH (install from quarto.org).") + + source = find_source(args.qmd) + print(f"manuscript source: {source.relative_to(ROOT)}") + + if args.figures: + for cmd in FIGURE_COMMANDS: + run(cmd, ROOT) + + # Quarto reads/writes relative to the .qmd directory. + run(["quarto", "render", source.name, "--to", "pdf"], HERE) + + pdf = source.with_suffix(".pdf") + tex = source.with_suffix(".tex") + if tex.exists(): + patch_and_recompile(tex) + + print("\nBuild complete:") + for p in (tex, pdf): + print(f" {'ok ' if p.exists() else 'MISSING '}{p.relative_to(ROOT)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())