From 602d9502a547cfe1e5d41744f0b01f0eb066b8c3 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Thu, 8 Oct 2026 13:57:43 +0300 Subject: [PATCH] fix(sefariasqlite): number and index Midrash Rabbah paragraphs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Midrash Rabbah's paragraphs (אותיות) are Sefaria "Integer" segments, which the importer leaves unlabelled and keeps out of the TOC, so the books could only be navigated by parasha. For the ten Midrash Rabbah books, prefix each paragraph with its printed number ("(א) ") and give it a TOC entry ("אות א") under its parasha. The entry points at the paragraph line itself, so no heading line is added. Line ids are unaffected: Sefaria lines are keyed on their heRef, which is unchanged. Fixes kdroidFilter/Zayit#509 Signed-off-by: Elie Gambache --- .../sefariasqlite/SefariaBookPayloadReader.kt | 15 ++- .../sefariasqlite/SefariaImportText.kt | 18 ++++ .../SefariaNumberedParagraphsTest.kt | 96 +++++++++++++++++++ 3 files changed, 127 insertions(+), 2 deletions(-) create mode 100644 generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaNumberedParagraphsTest.kt diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index c6139ffa..e5c78b45 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -542,9 +542,13 @@ internal class SefariaBookPayloadReader( val sectionIndex = sectionNames.size - depth val isReferenceable = referenceableSections.getOrNull(sectionIndex) ?: true + val isNumberedParagraph = depth == 1 && hasNumberedParagraphs(bookEnTitle) val nextLinePrefix = if ( - depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1 && - !hasSelfLabeledSegments(bookEnTitle) + isNumberedParagraph || + ( + depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1 && + !hasSelfLabeledSegments(bookEnTitle) + ) ) { "($letter) " } else { @@ -586,6 +590,7 @@ internal class SefariaBookPayloadReader( append(", ") } val nextRefIndexOffset = childRefOffsets?.getOrNull(idx) ?: 0 + val lineCountBefore = output.size recursiveSections( sectionNames = sectionNames, @@ -604,6 +609,12 @@ internal class SefariaBookPayloadReader( referenceableSections = referenceableSections, refIndexOffset = nextRefIndexOffset ) + + // The paragraph's TOC entry points at the paragraph line itself, so the + // text gets no extra heading line. + if (isNumberedParagraph && output.size > lineCountBefore) { + headings += Heading(title = "אות $letter", level = level, lineIndex = lineCountBefore) + } } } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 87a544f6..fff464f5 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -35,6 +35,24 @@ private val SELF_LABELED_SEGMENT_BOOKS = setOf("Mishnah Berurah") internal fun hasSelfLabeledSegments(bookEnTitle: String): Boolean = bookEnTitle in SELF_LABELED_SEGMENT_BOOKS +// Books whose unnamed ("Integer") segments are the printed numbered paragraphs (אותיות) +// readers navigate by: each segment gets its "(letter) " label and its own TOC entry +// (kdroidFilter/Zayit#509). +private val NUMBERED_PARAGRAPH_BOOKS = setOf( + "Bereshit Rabbah", + "Shemot Rabbah", + "Vayikra Rabbah", + "Bamidbar Rabbah", + "Devarim Rabbah", + "Ruth Rabbah", + "Esther Rabbah", + "Eikhah Rabbah", + "Kohelet Rabbah", + "Shir HaShirim Rabbah", +) + +internal fun hasNumberedParagraphs(bookEnTitle: String): Boolean = bookEnTitle in NUMBERED_PARAGRAPH_BOOKS + // Some simanim of the Mishnah Berurah (494-529) mark the seif katan as "{א}" // instead of the printed "(א)". private val BRACED_SEGMENT_MARKER_REGEX = Regex("""^\{([א-ת]{1,4})\}""") diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaNumberedParagraphsTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaNumberedParagraphsTest.kt new file mode 100644 index 00000000..10bb1b29 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaNumberedParagraphsTest.kt @@ -0,0 +1,96 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import co.touchlab.kermit.Logger +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Regression for kdroidFilter/Zayit#509: the Midrash Rabbah's paragraphs (אותיות) + * are unnamed "Integer" segments, so they had neither a number nor a TOC entry. + */ +class SefariaNumberedParagraphsTest { + @Test + fun midrashRabbahParagraphsGetLabelAndTocEntry() { + val payload = readBook("Bereshit Rabbah", "בראשית רבה") + + assertEquals( + listOf( + "

בראשית רבה

", + "

פרק א

", + "(א) רבי הושעיה רבה פתח", + "(ב) רבי יהושע דסכנין", + "

פרק ב

", + "(א) והארץ היתה תהו", + ), + payload.lines, + ) + assertEquals( + listOf( + Heading("בראשית רבה", 0, 0), + Heading("פרק א", 1, 1), + Heading("אות א", 2, 2), + Heading("אות ב", 2, 3), + Heading("פרק ב", 1, 4), + Heading("אות א", 2, 5), + ), + payload.headings, + ) + // heRefs (the line id natural keys) are unchanged + assertEquals( + listOf("בראשית רבה א, א", "בראשית רבה א, ב", "בראשית רבה ב, א"), + payload.refEntries.map { it.heRef.replace(Regex(" +"), " ") }, + ) + } + + @Test + fun otherBooksKeepUnlabeledIntegerSegments() { + val payload = readBook("FakeBook", "ספר") + + assertEquals("רבי הושעיה רבה פתח", payload.lines[2]) + assertEquals(listOf(0, 1, 1), payload.headings.map { it.level }) + } + + private fun readBook(title: String, heTitle: String) = runBlocking { + val tempDir = Files.createTempDirectory("seforim-numbered-paragraphs") + val schemaDir = Files.createDirectories(tempDir.resolve("schemas")) + val bookDir = Files.createDirectories(tempDir.resolve("json").resolve(title)) + Files.writeString(schemaDir.resolve("${title.replace(' ', '_')}.json"), schemaJson(title, heTitle)) + Files.writeString(bookDir.resolve("merged.json"), mergedJson(title, heTitle)) + + val reader = SefariaBookPayloadReader( + Json { ignoreUnknownKeys = true; coerceInputValues = true }, + Logger.withTag("SefariaNumberedParagraphsTest"), + ) + val schemaLookup = reader.buildSchemaLookup(schemaDir) + reader.readBooksInParallel(tempDir.resolve("json"), schemaDir, schemaLookup).single() + } + + private fun schemaJson(title: String, heTitle: String) = """ + { + "title": "$title", + "heTitle": "$heTitle", + "schema": { + "nodeType": "JaggedArrayNode", + "depth": 2, + "addressTypes": ["Perek", "Integer"], + "sectionNames": ["Chapter", "Paragraph"], + "title": "$title", + "heTitle": "$heTitle" + } + } + """.trimIndent() + + private fun mergedJson(title: String, heTitle: String) = """ + { + "title": "$title", + "heTitle": "$heTitle", + "text": [ + ["רבי הושעיה רבה פתח", "רבי יהושע דסכנין"], + ["והארץ היתה תהו"] + ] + } + """.trimIndent() +}