feat(dh): store the printed dibbur alongside its key (line_dh.dhDisplay)

.github/scripts/test_manual_release_workflow.py

@@ -273,6 +273,10 @@ def test_patch_fan_allows_only_the_supported_schema_transitions(self):
             '[ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 3 ]',
             '[ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 4 ]',
             '[ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 4 ]',
+            '[ "$PREV_SCHEMA" = 1 ] && [ "$THIS_SCHEMA" = 5 ]',
+            '[ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 5 ]',
+            '[ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 5 ]',
+            '[ "$PREV_SCHEMA" = 4 ] && [ "$THIS_SCHEMA" = 5 ]',
         )
         for transition in supported:
             self.assertIn(transition, patch_fan)

.github/workflows/contract.yml

@@ -18,10 +18,38 @@ jobs:
     steps:
       - uses: actions/checkout@v4
 
+      # A coordinated contract change spans two repositories. On a PR, prefer
+      # a linked open companion PR by the same author; on push (or when none is
+      # linked) keep validating against the companion's default branch.
+      - name: Resolve linked updater PR
+        id: updater
+        env:
+          GH_TOKEN: ${{ github.token }}
+        run: |
+          set -euo pipefail
+          repo=Otzaria/otzaria_library_updater
+          if [ "$GITHUB_EVENT_NAME" = pull_request ]; then
+            marker="$GITHUB_REPOSITORY#${{ github.event.pull_request.number }}"
+            author='${{ github.event.pull_request.user.login }}'
+            number=$(gh api -X GET search/issues \
+              -f q="repo:$repo is:pr is:open author:$author \"$marker\"" \
+              --jq '.items[0].number // empty')
+            if [ -n "$number" ]; then
+              companion_repo=$(gh api "repos/$repo/pulls/$number" --jq .head.repo.full_name)
+              companion_ref=$(gh api "repos/$repo/pulls/$number" --jq .head.ref)
+              echo "repository=$companion_repo" >> "$GITHUB_OUTPUT"
+              echo "ref=$companion_ref" >> "$GITHUB_OUTPUT"
+              exit 0
+            fi
+          fi
+          echo "repository=$repo" >> "$GITHUB_OUTPUT"
+          echo "ref=main" >> "$GITHUB_OUTPUT"
+
       - name: Checkout Otzaria/otzaria_library_updater (צד ה-Dart)
         uses: actions/checkout@v4
         with:
-          repository: Otzaria/otzaria_library_updater
+          repository: ${{ steps.updater.outputs.repository }}
+          ref: ${{ steps.updater.outputs.ref }}
           path: seforim-library-updater
 
       - name: Checkout Otzaria/SefariaExport visibility contract

.github/workflows/manual-generate-release.yml

@@ -1495,7 +1495,11 @@ jobs:
               if { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 3 ]; } || \
                  { [ "$PREV_SCHEMA" = 1 ] && [ "$THIS_SCHEMA" = 4 ]; } || \
                  { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 4 ]; } || \
-                 { [ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 4 ]; }; then
+                 { [ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 4 ]; } || \
+                 { [ "$PREV_SCHEMA" = 1 ] && [ "$THIS_SCHEMA" = 5 ]; } || \
+                 { [ "$PREV_SCHEMA" = 2 ] && [ "$THIS_SCHEMA" = 5 ]; } || \
+                 { [ "$PREV_SCHEMA" = 3 ] && [ "$THIS_SCHEMA" = 5 ]; } || \
+                 { [ "$PREV_SCHEMA" = 4 ] && [ "$THIS_SCHEMA" = 5 ]; }; then
                 echo "schema $PREV_SCHEMA → $THIS_SCHEMA — producing the supported cross-schema delta"
               else
                 echo "schema $PREV_SCHEMA → $THIS_SCHEMA is unsupported — skip anchor"

dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/Database.sq

@@ -306,14 +306,17 @@ CREATE TABLE IF NOT EXISTS line_ref (
 -- Dibbur-hamatchil index: the normalised opening words of each commentary
 -- comment (the bold or dash-separated prefix of the line's content), keyed
 -- as text so the client can prefix-match while the user types, and render
--- them as virtual sub-headings under the book's TOC. Populated by the
+-- them as virtual sub-headings under the book's TOC. dhDisplay is the same
+-- dibbur as printed (points and quote marks kept) for those sub-headings —
+-- the normalised key is unfit to show. Populated by the
 -- buildLineDhIndex stage. Existing client DBs receive the table and its
 -- contents through the schema-4 delta migration registered by the patch
 -- pipeline; fresh DBs include it directly.
 CREATE TABLE IF NOT EXISTS line_dh (
     bookId INTEGER NOT NULL,
     dhText TEXT NOT NULL,
     lineIndex INTEGER NOT NULL,
+    dhDisplay TEXT NOT NULL,
     PRIMARY KEY (bookId, dhText, lineIndex)
 ) WITHOUT ROWID;
 

dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/LineDhQueries.sq

@@ -1,20 +1,20 @@
 -- Queries for the dibbur-hamatchil index
 
 selectByBook:
-SELECT dhText, lineIndex FROM line_dh WHERE bookId = ? ORDER BY lineIndex;
+SELECT dhText, lineIndex, dhDisplay FROM line_dh WHERE bookId = ? ORDER BY lineIndex;
 
 -- Prefix match over the PK btree: the caller passes the normalised prefix
 -- and the same prefix with its last character incremented (exclusive bound),
 -- which keeps the scan index-only regardless of LIKE collation rules.
 selectByBookAndPrefixRange:
-SELECT dhText, lineIndex FROM line_dh
+SELECT dhText, lineIndex, dhDisplay FROM line_dh
 WHERE bookId = ? AND dhText >= :prefix AND dhText < :prefixEnd
 ORDER BY dhText, lineIndex
 LIMIT :limit;
 
 insert:
-INSERT OR IGNORE INTO line_dh (bookId, dhText, lineIndex)
-VALUES (?, ?, ?);
+INSERT OR IGNORE INTO line_dh (bookId, dhText, lineIndex, dhDisplay)
+VALUES (?, ?, ?, ?);
 
 deleteByBookId:
 DELETE FROM line_dh WHERE bookId = ?;

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/dh/BuildLineDhIndexCli.kt

@@ -10,7 +10,7 @@ import java.sql.DriverManager
 /**
  * Fills `line_dh` — the (bookId, dhText) -> lineIndex dibbur-hamatchil index
  * the Otzaria client uses to search commentaries by their opening words and
- * to render them as virtual sub-headings.
+ * to render them as virtual sub-headings (dhDisplay carries the printed form).
  *
  * Runs as a late DB-writing stage (`:generator-common:buildLineDhIndex`),
  * after every book-writing stage, and is idempotent: the table is rebuilt
@@ -60,6 +60,7 @@ internal fun rebuildLineDhIndex(conn: Connection, logger: Logger): LineDhIndexRe
                     bookId INTEGER NOT NULL,
                     dhText TEXT NOT NULL,
                     lineIndex INTEGER NOT NULL,
+                    dhDisplay TEXT NOT NULL,
                     PRIMARY KEY (bookId, dhText, lineIndex)
                 ) WITHOUT ROWID
                 """.trimIndent(),
@@ -108,15 +109,15 @@ internal fun indexAllBooks(conn: Connection, logger: Logger): LineDhIndexReport
     }
 
     conn.prepareStatement(
-        "INSERT OR IGNORE INTO line_dh (bookId, dhText, lineIndex) VALUES (?, ?, ?)",
+        "INSERT OR IGNORE INTO line_dh (bookId, dhText, lineIndex, dhDisplay) VALUES (?, ?, ?, ?)",
     ).use { insert ->
         conn.prepareStatement(
             "SELECT lineIndex, content FROM line WHERE bookId = ? ORDER BY lineIndex",
         ).use { selectLines ->
             for (bookId in bookIds) {
                 var contentLines = 0
-                val bold = ArrayList<Pair<Long, String>>()
-                val dash = ArrayList<Pair<Long, String>>()
+                val bold = ArrayList<Pair<Long, DhExtractor.Dh>>()
+                val dash = ArrayList<Pair<Long, DhExtractor.Dh>>()
 
                 selectLines.setLong(1, bookId)
                 selectLines.executeQuery().use { rs ->
@@ -141,10 +142,11 @@ internal fun indexAllBooks(conn: Connection, logger: Logger): LineDhIndexReport
                     continue
                 }
 
-                for ((lineIndex, dhText) in winner) {
+                for ((lineIndex, dh) in winner) {
                     insert.setLong(1, bookId)
-                    insert.setString(2, dhText)
+                    insert.setString(2, dh.key)
                     insert.setLong(3, lineIndex)
+                    insert.setString(4, dh.display)
                     insert.addBatch()
                 }
                 insert.executeBatch()

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/dh/DhExtractor.kt

@@ -24,6 +24,14 @@ object DhExtractor {
 
     enum class Format { BOLD, DASH }
 
+    /**
+     * One extracted dibbur: [key] is the [DhKey] form the index is searched
+     * by; [display] is the dibbur as printed — points and quote marks kept,
+     * whitespace collapsed, edge punctuation trimmed — for showing it as a
+     * sub-heading.
+     */
+    data class Dh(val key: String, val display: String)
+
     /** Longest raw dibbur accepted, in characters (real ones median ~11). */
     private const val MAX_DH_LENGTH = 100
 
@@ -66,25 +74,29 @@ object DhExtractor {
     )
 
     private val POINTS = Regex("[֑-ׇ]")
+    private val WHITESPACE = Regex("""\s+""")
+
+    /** Same edge trim as DhKey, so key and display agree on where the dibbur ends. */
+    private const val EDGE_PUNCTUATION = ".,:;?!()[]"
 
-    /** Extracts and normalises the dibbur of [line] in [format], or `null`. */
-    fun extract(line: String, format: Format): String? = when (format) {
+    /** Extracts the dibbur of [line] in [format], or `null`. */
+    fun extract(line: String, format: Format): Dh? = when (format) {
         Format.BOLD -> extractBold(line)
         Format.DASH -> extractDash(line)
     }
 
     /** `true` when [line] is a `<h1>`–`<h6>` heading (never carries a dibbur). */
     fun isHeadingLine(line: String): Boolean = HEADING_LINE.containsMatchIn(line)
 
-    private fun extractBold(line: String): String? {
+    private fun extractBold(line: String): Dh? {
         if (isHeadingLine(line)) return null
         val m = BOLD_PREFIX.find(line) ?: return null
         val rest = TAG.replace(m.groupValues[2], "").trim()
         if (rest.isEmpty()) return null // whole-line bold: a heading, not a dibbur
         return accept(m.groupValues[1])
     }
 
-    private fun extractDash(line: String): String? {
+    private fun extractDash(line: String): Dh? {
         if (isHeadingLine(line)) return null
         val m = SPACED_DASH.find(line) ?: return null
         var dh = line.substring(0, m.range.first)
@@ -99,7 +111,7 @@ object DhExtractor {
         return accept(dh)
     }
 
-    private fun accept(rawDh: String): String? {
+    private fun accept(rawDh: String): Dh? {
         // Match DhKey's edge trimming while deliberately preserving quote
         // marks: תוד"ה is a locator, while תודה is a genuine Hebrew word.
         val marker = POINTS.replace(rawDh, "")
@@ -111,8 +123,10 @@ object DhExtractor {
             .replace('‘', '\'')
             .replace("''", "\"")
             .trim()
-            .trim { it in ".,:;?!()[]" || it == ' ' }
+            .trim { it in EDGE_PUNCTUATION || it == ' ' }
         if (marker in STOP_MARKERS) return null
-        return DhKey.normalize(rawDh)
+        val key = DhKey.normalize(rawDh) ?: return null
+        val display = WHITESPACE.replace(rawDh, " ").trim { it in EDGE_PUNCTUATION || it == ' ' }
+        return Dh(key, display)
     }
 }

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt

@@ -126,14 +126,18 @@ class LogicalContentHasher(
             add(indexOf("line_ref") + 1, "line_dh")
         }
 
+        /** Schema 5 changes line_dh columns, not the set or order of tables. */
+        val TABLES_SCHEMA_5: List<String> = TABLES_SCHEMA_4
+
         /** Current-schema default for build-time diagnostics and current DB tests. */
-        val DEFAULT_TABLES: List<String> = TABLES_SCHEMA_4
+        val DEFAULT_TABLES: List<String> = TABLES_SCHEMA_5
 
         fun tablesForSchemaVersion(schemaVersion: Int): List<String> = when (schemaVersion) {
             1 -> TABLES_SCHEMA_1
             2 -> TABLES_SCHEMA_2
             3 -> TABLES_SCHEMA_3
             4 -> TABLES_SCHEMA_4
+            5 -> TABLES_SCHEMA_5
             else -> error("Unsupported logical-hash schema version $schemaVersion")
         }
 

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchDbProducer.kt

@@ -58,8 +58,8 @@ class PatchDbProducer(
         fromVersion: Int,
         toVersion: Int,
         migrations: List<Pair<Int, String>> = emptyList(),
-        fromSchemaVersion: Int = PatchDbSchema.CURRENT_VERSION,
-        toSchemaVersion: Int = PatchDbSchema.CURRENT_VERSION,
+        fromSchemaVersion: Int = CURRENT_DB_SCHEMA_VERSION,
+        toSchemaVersion: Int = CURRENT_DB_SCHEMA_VERSION,
     ): Output {
         require(fromSchemaVersion <= toSchemaVersion) {
             "Schema downgrade $fromSchemaVersion -> $toSchemaVersion is not supported"

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchTables.kt

@@ -18,6 +18,9 @@ internal data class PatchTable(
     val updatable: Boolean,
 )
 
+/** Current `seforim.db` schema produced by this revision. */
+internal const val CURRENT_DB_SCHEMA_VERSION: Int = 5
+
 /**
  * Canonical table order — parents (referenced) come before children
  * (referencing) for upserts. The applier runs upserts in this order and
@@ -62,8 +65,8 @@ internal val PATCH_TABLES_IN_FK_ORDER: List<PatchTable> = listOf(
     // Schema 4. Canonical line-reference index — pure key table (PK == all
     // columns), so there is nothing to update on conflict.
     PatchTable("line_ref",           listOf("bookId", "refKeyHash", "lineIndex"), updatable = false),
-    // Schema 4. Dibbur-hamatchil index — pure key table, same shape.
-    PatchTable("line_dh",            listOf("bookId", "dhText", "lineIndex"), updatable = false),
+    // Schema 5. Dibbur-hamatchil index — dhDisplay rides along the key.
+    PatchTable("line_dh",            listOf("bookId", "dhText", "lineIndex"), updatable = true),
 
     // Links.
     PatchTable("link",               listOf("id"),       updatable = true),
@@ -91,9 +94,15 @@ internal val PATCH_TABLES_IN_FK_ORDER: List<PatchTable> = listOf(
     PatchTable("schema_meta",        listOf("key"),      updatable = true),
 )
 
+/** Schema-4 contract shipped in v26, before line_dh gained dhDisplay. */
+internal val PATCH_TABLES_SCHEMA_4: List<PatchTable> =
+    PATCH_TABLES_IN_FK_ORDER.map { table ->
+        if (table.name == "line_dh") table.copy(updatable = false) else table
+    }
+
 /** Schema-3 contract retained for updater compatibility tests. */
 internal val PATCH_TABLES_SCHEMA_3: List<PatchTable> =
-    PATCH_TABLES_IN_FK_ORDER.filterNot { it.name in setOf("line_ref", "line_dh") }
+    PATCH_TABLES_SCHEMA_4.filterNot { it.name in setOf("line_ref", "line_dh") }
 
 /** Schema-2 contract retained for updater compatibility tests. */
 internal val PATCH_TABLES_SCHEMA_2: List<PatchTable> =
@@ -108,6 +117,7 @@ internal fun patchTablesForSchemaVersion(schemaVersion: Int): List<PatchTable> =
     1 -> PATCH_TABLES_SCHEMA_1
     2 -> PATCH_TABLES_SCHEMA_2
     3 -> PATCH_TABLES_SCHEMA_3
-    4 -> PATCH_TABLES_IN_FK_ORDER
+    4 -> PATCH_TABLES_SCHEMA_4
+    5 -> PATCH_TABLES_IN_FK_ORDER
     else -> error("Unsupported patch-table schema version $schemaVersion")
 }

generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/StampSchemaVersionCli.kt

@@ -18,7 +18,7 @@ import java.sql.DriverManager
  *   - `dbVersion`      integer release version (matches release_meta.json
  *                      `latestVersion` and `deltas[].toVersion`)
  *   - `dbSchemaVersion` integer SQLDelight schema version (optional, defaults
- *                      to [PatchDbSchema.CURRENT_VERSION]); matches the
+ *                      to [CURRENT_DB_SCHEMA_VERSION]); matches the
  *                      manifest's `toSchemaVersion`
  *
  * Idempotent: re-running with the same values is a no-op
@@ -32,7 +32,7 @@ fun main() {
     val dbVersion = System.getProperty("dbVersion")?.toIntOrNull()
         ?: error("-PdbVersion= missing or not an integer")
     val dbSchemaVersion = System.getProperty("dbSchemaVersion")?.toIntOrNull()
-        ?: PatchDbSchema.CURRENT_VERSION
+        ?: CURRENT_DB_SCHEMA_VERSION
 
     val path = Paths.get(dbPath)
     require(Files.isRegularFile(path)) { "Database file not found: $dbPath" }
@@ -48,13 +48,13 @@ fun main() {
 /** Writes release/schema metadata only after the DB satisfies that schema's table contract. */
 internal fun stampSchemaVersion(conn: Connection, dbVersion: Int, dbSchemaVersion: Int) {
     require(dbVersion >= 1) { "dbVersion=$dbVersion must be positive" }
-    require(dbSchemaVersion in 1..PatchDbSchema.CURRENT_VERSION) {
-        "dbSchemaVersion=$dbSchemaVersion is outside the supported range 1..${PatchDbSchema.CURRENT_VERSION}"
+    require(dbSchemaVersion in 1..CURRENT_DB_SCHEMA_VERSION) {
+        "dbSchemaVersion=$dbSchemaVersion is outside the supported range 1..$CURRENT_DB_SCHEMA_VERSION"
     }
     check(conn.autoCommit) { "stampSchemaVersion requires an unowned JDBC connection" }
 
     val requiredTables = when (dbSchemaVersion) {
-        4 -> setOf("line_ref", "line_dh")
+        4, 5 -> setOf("line_ref", "line_dh")
         else -> emptySet()
     }
     val existingTables = if (requiredTables.isEmpty()) {
@@ -74,6 +74,19 @@ internal fun stampSchemaVersion(conn: Connection, dbVersion: Int, dbSchemaVersio
             missingTables.sorted().joinToString()
     }
 
+    if (dbSchemaVersion >= 5) {
+        val dhDisplay = PatchDbSchema.readTableInfo(conn, "main", "line_dh")
+            .find { it.name == "dhDisplay" }
+        require(
+            dhDisplay != null &&
+                dhDisplay.type.equals("TEXT", ignoreCase = true) &&
+                dhDisplay.notNull,
+        ) {
+            "Cannot stamp DB as schema $dbSchemaVersion; " +
+                "line_dh.dhDisplay must exist as TEXT NOT NULL"
+        }
+    }
+
     conn.autoCommit = false
     try {
         conn.prepareStatement(