Repository navigation
sync-manual-links correlation=sefaria:33860162971:1:2026-09-04_12-50-33860162971-1:55560ab01678fdc30eed26f7da38ac70ad6d0e7a6de36ce17963c90aa9e66827 #39
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Sync manual links | |
| run-name: ${{ inputs.mode == 'migrate' && format('sync-manual-links migrate correlation={0}', inputs.correlation_id) || format('sync-manual-links correlation={0}', inputs.correlation_id) }} | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| sefaria_tag: | |
| required: true | |
| type: string | |
| sefaria_release_metadata_sha256: | |
| required: true | |
| type: string | |
| sefaria_run_id: | |
| required: true | |
| type: string | |
| sefaria_run_attempt: | |
| required: true | |
| type: string | |
| correlation_id: | |
| required: true | |
| type: string | |
| mode: | |
| description: refresh is the weekly saga; migrate re-issues lineage after a config or tool change | |
| required: false | |
| default: refresh | |
| type: choice | |
| options: | |
| - refresh | |
| - migrate | |
| expected_old_config_sha256: | |
| description: migrate only - the config_sha256 the operator expects to find in the committed lineage | |
| required: false | |
| default: '' | |
| type: string | |
| expected_old_tool_commit: | |
| description: migrate only - the seforim_tool_commit the operator expects to find in the committed lineage | |
| required: false | |
| default: '' | |
| type: string | |
| permissions: | |
| contents: write | |
| actions: read | |
| concurrency: | |
| group: manual-links-sync | |
| cancel-in-progress: false | |
| # queue:max lifts GitHub's single-pending slot to a durable queue, so a later weekly | |
| # dispatch can't supersede/cancel an already-pending saga. (Shipped 2026-05-07; confirm | |
| # at canary — no actionlint gate runs here.) | |
| queue: max | |
| jobs: | |
| sync-and-commit: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 180 | |
| concurrency: | |
| group: otzaria-main-writer | |
| cancel-in-progress: false | |
| # Shared writer group (also used by update-library): each queued run carries a | |
| # DIFFERENT correlation/intent, so a late dispatch must never supersede a | |
| # pending one — queue every intent durably. | |
| queue: max | |
| outputs: | |
| expected_links_commit: ${{ steps.apply.outputs.expected_links_commit }} | |
| seforim_tool_commit: ${{ steps.tool.outputs.sha }} | |
| sefaria_archive_sha256: ${{ steps.chain.outputs.archive_sha256 }} | |
| env: | |
| GH_TOKEN: ${{ secrets.PIPELINE_TOKEN }} | |
| SEFARIA_TAG: ${{ inputs.sefaria_tag }} | |
| METADATA_SHA: ${{ inputs.sefaria_release_metadata_sha256 }} | |
| CORRELATION_ID: ${{ inputs.correlation_id }} | |
| MODE: ${{ inputs.mode }} | |
| EXPECTED_OLD_CONFIG_SHA: ${{ inputs.expected_old_config_sha256 }} | |
| EXPECTED_OLD_TOOL_COMMIT: ${{ inputs.expected_old_tool_commit }} | |
| steps: | |
| - name: Free disk space for the verified Sefaria export | |
| uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be | |
| with: | |
| tool-cache: true | |
| android: true | |
| dotnet: true | |
| haskell: true | |
| large-packages: true | |
| docker-images: true | |
| swap-storage: false | |
| - name: Validate dispatch identity | |
| env: | |
| RUN_ID: ${{ inputs.sefaria_run_id }} | |
| RUN_ATTEMPT: ${{ inputs.sefaria_run_attempt }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| [[ "$RUN_ID" =~ ^[1-9][0-9]*$ ]] | |
| [[ "$RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] | |
| [[ "$SEFARIA_TAG" =~ ^[A-Za-z0-9._-]+$ ]] | |
| [[ "$METADATA_SHA" =~ ^[0-9a-f]{64}$ ]] | |
| [[ "$CORRELATION_ID" == "sefaria:${RUN_ID}:${RUN_ATTEMPT}:${SEFARIA_TAG}:${METADATA_SHA}" ]] | |
| [[ "$MODE" == refresh || "$MODE" == migrate ]] | |
| # The migrate gates are operator-supplied on purpose; refresh must never carry them. | |
| if [[ "$MODE" == migrate ]]; then | |
| [[ "$EXPECTED_OLD_CONFIG_SHA" =~ ^[0-9a-f]{64}$ ]] | |
| [[ "$EXPECTED_OLD_TOOL_COMMIT" =~ ^[0-9a-f]{40}$ ]] | |
| else | |
| [[ -z "$EXPECTED_OLD_CONFIG_SHA" && -z "$EXPECTED_OLD_TOOL_COMMIT" ]] | |
| fi | |
| - name: Preflight PIPELINE_TOKEN can push to Otzaria main | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Every repo here except LinkerToOtzaria is public, so reads succeed even with | |
| # an empty/underscoped token — the failure only surfaces at the atomic git push. | |
| # Fail loud and specific instead of a cryptic "could not read Username". | |
| [ -n "${GH_TOKEN:-}" ] || { echo "::error::PIPELINE_TOKEN secret is empty on otzaria-library — set it: gh secret set PIPELINE_TOKEN -R Otzaria/otzaria-library"; exit 1; } | |
| push="$(gh api repos/Otzaria/otzaria-library --jq '.permissions.push // false' 2>/dev/null || echo error)" | |
| [ "$push" = true ] || { echo "::error::PIPELINE_TOKEN cannot push to otzaria-library (permissions.push=$push) — the classic PAT needs the 'repo' scope (public_repo is not enough for cross-repo automation) and must belong to a user with write access"; exit 1; } | |
| - name: Checkout current Otzaria main | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| lfs: true | |
| persist-credentials: false | |
| - uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.12.10' | |
| - name: Validate committed synchronization contract | |
| run: | | |
| python3 -m unittest -v test_manual_links_packaging.py test_sefaria_release_chain.py | |
| test -f manual_links_lineage.json | |
| # Source roots may legitimately differ from the previous lineage after | |
| # prepare_only. Validate config/state/collisions now; require the new | |
| # exact lineage only after the updater has processed that source tree. | |
| python3 manual_links_packaging.py check --workspace . | |
| - name: Prove the migrate gates against the committed lineage | |
| if: inputs.mode == 'migrate' | |
| env: | |
| RUN_ID: ${{ inputs.sefaria_run_id }} | |
| RUN_ATTEMPT: ${{ inputs.sefaria_run_attempt }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Verified against the lineage, never derived from it: a wrong expectation stops here. | |
| [[ "$(jq -r '.config_sha256' manual_links_lineage.json)" == "$EXPECTED_OLD_CONFIG_SHA" ]] | |
| [[ "$(jq -r '.seforim_tool_commit' manual_links_lineage.json)" == "$EXPECTED_OLD_TOOL_COMMIT" ]] | |
| # validateMigrationGates forbids moving the Sefaria target, so the dispatch must | |
| # name the lineage's own release - fail here instead of after a 500MB download. | |
| [[ "$(jq -r '.sefaria.tag' manual_links_lineage.json)" == "$SEFARIA_TAG" ]] | |
| [[ "$(jq -r '.sefaria.release_metadata_sha256' manual_links_lineage.json)" == "$METADATA_SHA" ]] | |
| [[ "$(jq -r '.sefaria.run_id' manual_links_lineage.json)" == "$RUN_ID" ]] | |
| [[ "$(jq -r '.sefaria.run_attempt' manual_links_lineage.json)" == "$RUN_ATTEMPT" ]] | |
| - name: Resolve one authorized Seforim tool commit | |
| id: tool | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| lineage_tag="$(jq -r '.sefaria.tag' manual_links_lineage.json)" | |
| lineage_digest="$(jq -r '.sefaria.release_metadata_sha256' manual_links_lineage.json)" | |
| # migrate exists to adopt a new config/tool at the pinned Sefaria target, so it | |
| # always resolves the authorized branch head instead of the lineage's old pin. | |
| if [[ "$MODE" == refresh && "$lineage_tag" == "$SEFARIA_TAG" && "$lineage_digest" == "$METADATA_SHA" ]]; then | |
| sha="$(jq -r '.seforim_tool_commit' manual_links_lineage.json)" | |
| else | |
| ref="$(jq -r '.seforim_tool_ref' manual_links_sync.json)" | |
| sha="$(git ls-remote https://github.com/Otzaria/SeforimLibrary.git "$ref" | awk 'NR==1 {print $1}')" | |
| fi | |
| [[ "$sha" =~ ^[0-9a-f]{40}$ ]] | |
| echo "sha=$sha" >> "$GITHUB_OUTPUT" | |
| - name: Checkout the exact updater tool | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: Otzaria/SeforimLibrary | |
| ref: ${{ steps.tool.outputs.sha }} | |
| path: seforim-tool | |
| fetch-depth: 0 | |
| persist-credentials: false | |
| - name: Verify tool authorization | |
| working-directory: seforim-tool | |
| env: | |
| TOOL_SHA: ${{ steps.tool.outputs.sha }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| [[ "$(git rev-parse HEAD)" == "$TOOL_SHA" ]] | |
| ref="$(jq -r '.seforim_tool_ref' ../manual_links_sync.json)" | |
| branch="${ref#refs/heads/}" | |
| git fetch origin "$branch" | |
| git merge-base --is-ancestor "$TOOL_SHA" "origin/$branch" | |
| - name: Preflight resources before downloading the export | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| available_kib="$(awk '/MemAvailable/ {print $2}' /proc/meminfo)" | |
| (( available_kib >= 6 * 1024 * 1024 )) | |
| available_blocks="$(df -Pk . | awk 'NR==2 {print $4}')" | |
| (( available_blocks >= 12 * 1024 * 1024 )) | |
| - name: Fetch and verify Sefaria metadata chain and archive | |
| id: chain | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| rm -rf sefaria-chain | |
| python3 sefaria_release_chain.py \ | |
| --repo Otzaria/SefariaExport \ | |
| --target-tag "$SEFARIA_TAG" \ | |
| --target-metadata-sha256 "$METADATA_SHA" \ | |
| --lineage manual_links_lineage.json \ | |
| --output sefaria-chain | |
| archive_sha="$(jq -r '.archive.sha256' sefaria-chain/chain-result.json)" | |
| echo "archive_sha256=$archive_sha" >> "$GITHUB_OUTPUT" | |
| mapfile -t parts < <(jq -r '.archive.parts[].name' sefaria-chain/chain-result.json) | |
| : > sefaria-chain/combined.tar.zst | |
| for part in "${parts[@]}"; do | |
| cat "sefaria-chain/target-assets/$part" >> sefaria-chain/combined.tar.zst | |
| done | |
| [[ "$(sha256sum sefaria-chain/combined.tar.zst | cut -d' ' -f1)" == "$archive_sha" ]] | |
| mkdir sefaria-chain/export | |
| tar --zstd -xf sefaria-chain/combined.tar.zst -C sefaria-chain/export | |
| # The export root is the single directory holding both json/ and schemas/ | |
| # (the Docker-Compose archives no longer ship a database_export folder). | |
| mapfile -t export_roots < <(find sefaria-chain/export -type d -name schemas -print | sort | while read -r s; do d="$(dirname "$s")"; [[ -d "$d/json" ]] && printf '%s\n' "$d"; done) | |
| [[ ${#export_roots[@]} -eq 1 ]] | |
| printf '%s\n' "${export_roots[0]}" > sefaria-chain/export-root.txt | |
| - name: Refresh, apply, commit and push atomically | |
| id: apply | |
| env: | |
| TOOL_SHA: ${{ steps.tool.outputs.sha }} | |
| PIPELINE_TOKEN: ${{ secrets.PIPELINE_TOKEN }} | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| export_root="$(realpath "$(cat sefaria-chain/export-root.txt)")" | |
| auth="$(printf 'x-access-token:%s' "$PIPELINE_TOKEN" | base64 | tr -d '\n')" | |
| success=false | |
| migrate_args=() | |
| if [[ "$MODE" == migrate ]]; then | |
| migrate_args=(-PexpectedOldConfigSha256="$EXPECTED_OLD_CONFIG_SHA" | |
| -PexpectedOldToolCommit="$EXPECTED_OLD_TOOL_COMMIT") | |
| fi | |
| for attempt in 1 2 3; do | |
| git fetch origin main | |
| worktree="$RUNNER_TEMP/otzaria-attempt-$attempt" | |
| output="$RUNNER_TEMP/manual-links-output-$attempt" | |
| rm -rf "$worktree" "$output" | |
| git worktree add --detach "$worktree" origin/main | |
| mkdir "$output" | |
| ./seforim-tool/gradlew -p seforim-tool :sefariasqlite:refreshManualLinks \ | |
| -PmanualLinksMode="$MODE" \ | |
| "${migrate_args[@]}" \ | |
| -PmanualLinksRepo="$worktree" \ | |
| -PmanualLinksConfig="$worktree/manual_links_sync.json" \ | |
| -PmanualLinksLineage="$worktree/manual_links_lineage.json" \ | |
| -PsefariaExport="$export_root" \ | |
| -PsefariaReleaseMetadata="$GITHUB_WORKSPACE/sefaria-chain/release_metadata.json" \ | |
| -PsefariaReleaseMetadataSha256="$METADATA_SHA" \ | |
| -PsefariaChangelogDir="$GITHUB_WORKSPACE/sefaria-chain/changelogs" \ | |
| -PseforimToolCommit="$TOOL_SHA" \ | |
| -PmanualLinksOutput="$output" \ | |
| -Dorg.gradle.jvmargs=-Xmx4g | |
| test -f "$output/.manual-links-refresh-complete" | |
| test -f "$output/manual_links_refresh_report.json" | |
| test -f "$output/manual_links_lineage.json" | |
| status="$(jq -r '.status' "$output/manual_links_refresh_report.json")" | |
| [[ "$status" == ok || "$status" == no_op ]] | |
| python3 - "$output" <<'PY' | |
| import hashlib | |
| import json | |
| import sys | |
| from pathlib import Path | |
| root = Path(sys.argv[1]) | |
| def load_strict(path): | |
| def pairs(items): | |
| value = {} | |
| for key, item in items: | |
| if key in value: | |
| raise SystemExit(f"duplicate JSON key {key!r} in {path}") | |
| value[key] = item | |
| return value | |
| return json.loads(path.read_text(encoding="utf-8"), object_pairs_hook=pairs) | |
| marker_path = root / ".manual-links-refresh-complete" | |
| marker = load_strict(marker_path) | |
| if set(marker) != {"schema_version", "report_sha256", "lineage_sha256", "status"} or type(marker["schema_version"]) is not int or marker["schema_version"] != 1: | |
| raise SystemExit("completion marker does not match schema version 1") | |
| canonical_marker = json.dumps(marker, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" | |
| if marker_path.read_bytes() != canonical_marker: | |
| raise SystemExit("completion marker is not canonical JSON with one trailing LF") | |
| report = root / "manual_links_refresh_report.json" | |
| lineage = root / "manual_links_lineage.json" | |
| for path in (report, lineage): | |
| value = load_strict(path) | |
| canonical = json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")).encode("utf-8") + b"\n" | |
| if path.read_bytes() != canonical: | |
| raise SystemExit(f"{path.name} is not canonical JSON with one trailing LF") | |
| file_hash = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() | |
| if marker["report_sha256"] != file_hash(report) or marker["lineage_sha256"] != file_hash(lineage): | |
| raise SystemExit("completion marker hashes do not match output files") | |
| report_status = load_strict(report).get("status") | |
| if marker["status"] not in {"ok", "no_op"} or marker["status"] != report_status: | |
| raise SystemExit("completion marker status differs from report") | |
| PY | |
| if [[ "$status" == ok ]]; then | |
| while IFS= read -r root; do | |
| test -d "$output/$root" | |
| mkdir -p "$worktree/$root" | |
| rsync --archive --delete "$output/$root/" "$worktree/$root/" | |
| done < <(jq -r '.links_roots[] | select(.expected_state == "present") | .path' "$worktree/manual_links_sync.json") | |
| cp "$output/manual_links_lineage.json" "$worktree/manual_links_lineage.json" | |
| fi | |
| python3 "$worktree/manual_links_packaging.py" check --workspace "$worktree" --require-lineage >/dev/null | |
| python3 - "$worktree" <<'PY' | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| root = Path(sys.argv[1]) | |
| config = json.loads((root / "manual_links_sync.json").read_text(encoding="utf-8")) | |
| allowed_roots = tuple(entry["path"] + "/" for entry in config["links_roots"]) | |
| changed = set() | |
| for command in (["git", "diff", "--name-only", "-z"], ["git", "ls-files", "--others", "--exclude-standard", "-z"]): | |
| raw = subprocess.check_output(command, cwd=root) | |
| changed.update(item.decode("utf-8") for item in raw.split(b"\0") if item) | |
| forbidden = sorted(path for path in changed if path != "manual_links_lineage.json" and not path.startswith(allowed_roots)) | |
| if forbidden: | |
| raise SystemExit(f"updater changed paths outside the strict allowlist: {forbidden}") | |
| PY | |
| mapfile -t roots < <(jq -r '.links_roots[] | select(.expected_state == "present") | .path' "$worktree/manual_links_sync.json") | |
| git -C "$worktree" config user.name github-actions[bot] | |
| git -C "$worktree" config user.email github-actions[bot]@users.noreply.github.com | |
| git -C "$worktree" add -- manual_links_lineage.json "${roots[@]}" | |
| python3 - "$worktree" <<'PY' | |
| import json | |
| import subprocess | |
| import sys | |
| from pathlib import Path | |
| root = Path(sys.argv[1]) | |
| config = json.loads((root / "manual_links_sync.json").read_text(encoding="utf-8")) | |
| prefixes = tuple(entry["path"] + "/" for entry in config["links_roots"]) | |
| raw = subprocess.check_output(["git", "diff", "--cached", "--name-only", "-z"], cwd=root) | |
| paths = [item.decode("utf-8") for item in raw.split(b"\0") if item] | |
| forbidden = [path for path in paths if path != "manual_links_lineage.json" and not path.startswith(prefixes)] | |
| if forbidden: | |
| raise SystemExit(f"staged paths outside strict allowlist: {forbidden}") | |
| PY | |
| message="chore(manual-links): sync Sefaria $SEFARIA_TAG" | |
| [[ "$MODE" == refresh ]] || message="chore(manual-links): migrate lineage at Sefaria $SEFARIA_TAG" | |
| if ! git -C "$worktree" diff --cached --quiet; then | |
| git -C "$worktree" commit -m "$message" | |
| fi | |
| expected="$(git -C "$worktree" rev-parse HEAD)" | |
| if git -C "$worktree" -c "http.https://github.com/.extraheader=AUTHORIZATION: basic $auth" push origin HEAD:main; then | |
| mkdir -p manual-links-refresh-artifact | |
| cp "$output/manual_links_refresh_report.json" manual-links-refresh-artifact/ | |
| cp "$output/manual_links_lineage.json" manual-links-refresh-artifact/ | |
| cp "$output/.manual-links-refresh-complete" \ | |
| manual-links-refresh-artifact/manual-links-refresh-complete | |
| echo "expected_links_commit=$expected" >> "$GITHUB_OUTPUT" | |
| success=true | |
| git worktree remove --force "$worktree" | |
| break | |
| fi | |
| git worktree remove --force "$worktree" | |
| echo "Push conflict on attempt $attempt; discarding output and recomputing from origin/main." | |
| done | |
| [[ "$success" == true ]] | |
| - name: Publish immutable manual-links refresh report release | |
| run: >- | |
| bash .github/scripts/publish_release_handoff.sh | |
| "manual-links-refresh-report-run-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" | |
| "Immutable manual-links refresh report ${GITHUB_RUN_ID}:${GITHUB_RUN_ATTEMPT}" | |
| "$GITHUB_SHA" manual-links-refresh-artifact/manual_links_refresh_report.json | |
| manual-links-refresh-artifact/manual_links_lineage.json | |
| manual-links-refresh-artifact/manual-links-refresh-complete | |
| start-saga: | |
| name: Persist saga state and dispatch Otzaria publisher | |
| needs: sync-and-commit | |
| # A migrate is operator maintenance, not a saga root: it publishes no saga state and | |
| # dispatches no publisher; its links reach the DB through the next weekly refresh. | |
| if: inputs.mode == 'refresh' | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| permissions: | |
| contents: read | |
| actions: read | |
| env: | |
| GH_TOKEN: ${{ secrets.PIPELINE_TOKEN }} | |
| EXPECTED_COMMIT: ${{ needs.sync-and-commit.outputs.expected_links_commit }} | |
| TOOL_SHA: ${{ needs.sync-and-commit.outputs.seforim_tool_commit }} | |
| SEFARIA_ARCHIVE_SHA: ${{ needs.sync-and-commit.outputs.sefaria_archive_sha256 }} | |
| CORRELATION_ID: ${{ inputs.correlation_id }} | |
| SEFARIA_TAG: ${{ inputs.sefaria_tag }} | |
| SEFARIA_METADATA_SHA: ${{ inputs.sefaria_release_metadata_sha256 }} | |
| SEFARIA_RUN_ID: ${{ inputs.sefaria_run_id }} | |
| SEFARIA_RUN_ATTEMPT: ${{ inputs.sefaria_run_attempt }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ needs.sync-and-commit.outputs.expected_links_commit }} | |
| sparse-checkout: | | |
| .github/scripts | |
| fordb_latest_pointer.json | |
| persist-credentials: false | |
| - name: Write canonical immutable saga state | |
| id: state | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| [[ "$EXPECTED_COMMIT" =~ ^[0-9a-f]{40}$ ]] | |
| [[ "$TOOL_SHA" =~ ^[0-9a-f]{40}$ ]] | |
| [[ "$SEFARIA_ARCHIVE_SHA" =~ ^[0-9a-f]{64}$ ]] | |
| [[ "$SEFARIA_METADATA_SHA" =~ ^[0-9a-f]{64}$ ]] | |
| [[ "$SEFARIA_TAG" =~ ^[A-Za-z0-9._-]{1,100}$ ]] | |
| [[ "$SEFARIA_RUN_ID" =~ ^[1-9][0-9]*$ ]] | |
| [[ "$SEFARIA_RUN_ATTEMPT" =~ ^[1-9][0-9]*$ ]] | |
| [[ "$CORRELATION_ID" == "sefaria:${SEFARIA_RUN_ID}:${SEFARIA_RUN_ATTEMPT}:${SEFARIA_TAG}:${SEFARIA_METADATA_SHA}" ]] | |
| python3 - <<'PY' | |
| import json, re | |
| from pathlib import Path | |
| path=Path("fordb_latest_pointer.json") | |
| def pairs(items): | |
| out={} | |
| for key,value in items: | |
| if key in out: raise SystemExit(f"duplicate ForDB pointer key {key}") | |
| out[key]=value | |
| return out | |
| value=json.loads(path.read_text(encoding="utf-8"),object_pairs_hook=pairs) | |
| keys={"schema_version","tag","asset","sha256","provenance_asset","provenance_sha256"} | |
| if set(value)!=keys or type(value["schema_version"]) is not int or value["schema_version"]!=1: | |
| raise SystemExit("invalid ForDB pointer schema") | |
| for key in keys-{"schema_version"}: | |
| if type(value[key]) is not str: raise SystemExit(f"ForDB pointer {key} must be a string") | |
| if value["asset"]!="fordb_latest.zip" or value["provenance_asset"]!="fordb_provenance.json": | |
| raise SystemExit("unexpected ForDB pointer asset names") | |
| if not re.fullmatch(r"[0-9a-f]{64}",value["sha256"]) or not re.fullmatch(r"[0-9a-f]{64}",value["provenance_sha256"]): | |
| raise SystemExit("invalid ForDB pointer digest") | |
| if value["tag"]!="fordb-sha256-"+value["sha256"]: | |
| raise SystemExit("ForDB pointer is not content-addressed") | |
| PY | |
| FORDB_TAG="$(jq -r .tag fordb_latest_pointer.json)" | |
| FORDB_ARCHIVE_SHA="$(jq -r .sha256 fordb_latest_pointer.json)" | |
| FORDB_PROVENANCE_SHA="$(jq -r .provenance_sha256 fordb_latest_pointer.json)" | |
| correlation_sha="$(printf '%s' "$CORRELATION_ID" | sha256sum | cut -d' ' -f1)" | |
| mkdir saga-state | |
| jq -cS -n \ | |
| --arg correlation_id "$CORRELATION_ID" --arg correlation_sha256 "$correlation_sha" \ | |
| --argjson saga_run_id "$GITHUB_RUN_ID" --argjson saga_run_attempt "$GITHUB_RUN_ATTEMPT" \ | |
| --arg expected_links_commit "$EXPECTED_COMMIT" --arg seforim_tool_commit "$TOOL_SHA" \ | |
| --arg sefaria_tag "$SEFARIA_TAG" --arg sefaria_release_metadata_sha256 "$SEFARIA_METADATA_SHA" \ | |
| --arg sefaria_archive_sha256 "$SEFARIA_ARCHIVE_SHA" \ | |
| --arg fordb_tag "$FORDB_TAG" --arg fordb_archive_sha256 "$FORDB_ARCHIVE_SHA" \ | |
| --arg fordb_provenance_sha256 "$FORDB_PROVENANCE_SHA" \ | |
| '{schema_version:1,correlation_id:$correlation_id,correlation_sha256:$correlation_sha256, | |
| saga_run_id:$saga_run_id,saga_run_attempt:$saga_run_attempt, | |
| expected_links_commit:$expected_links_commit,seforim_tool_commit:$seforim_tool_commit, | |
| sefaria_tag:$sefaria_tag,sefaria_release_metadata_sha256:$sefaria_release_metadata_sha256, | |
| sefaria_archive_sha256:$sefaria_archive_sha256,fordb_tag:$fordb_tag, | |
| fordb_archive_sha256:$fordb_archive_sha256,fordb_provenance_sha256:$fordb_provenance_sha256}' > saga-state/saga-state.json | |
| sha256sum saga-state/saga-state.json | cut -d' ' -f1 > saga-state/saga-state.sha256 | |
| echo "correlation_sha=$correlation_sha" >> "$GITHUB_OUTPUT" | |
| - name: Publish immutable saga state release | |
| run: >- | |
| bash .github/scripts/publish_release_handoff.sh | |
| "saga-state-${{ steps.state.outputs.correlation_sha }}-attempt-${GITHUB_RUN_ATTEMPT}" | |
| "Immutable weekly saga state ${{ steps.state.outputs.correlation_sha }}:${GITHUB_RUN_ATTEMPT}" | |
| "$GITHUB_SHA" saga-state/saga-state.json saga-state/saga-state.sha256 | |
| - name: Dispatch exact Otzaria publisher and return immediately | |
| shell: bash | |
| run: | | |
| set -euo pipefail | |
| # Submit at most once. A non-zero response may still have delivered; | |
| # the state release is already durable and reconcile-sagas will first | |
| # search the exact title, then dispatch only if it proves there is none. | |
| if gh workflow run update-library.yml -R Otzaria/otzaria-library \ | |
| -f mode=links_sync_mode -f expected_links_commit="$EXPECTED_COMMIT" \ | |
| -f correlation_id="$CORRELATION_ID" -f saga_run_id="$GITHUB_RUN_ID" \ | |
| -f saga_run_attempt="$GITHUB_RUN_ATTEMPT"; then | |
| echo "Saga persisted; Otzaria child submission accepted. Continuation is callback/reconciler driven." | |
| else | |
| echo "::warning::Saga persisted but child dispatch response was ambiguous; reconciler will observe before retrying." | |
| fi |