From 641666f41540d3f089ef21003a692452177270c1 Mon Sep 17 00:00:00 2001 From: Elie Gambache Date: Thu, 8 Oct 2026 13:46:02 +0300 Subject: [PATCH] fix(sefariasqlite): stop labelling Mishnah Berurah segments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sefaria's Mishnah Berurah segments already open with their printed seif katan marker, so the generated "(letter) " label doubled it ("(יד) (יד) נהגו לתייג"), and gave fake numbers to the kuntresim that have none (Mishnat Soferim in siman 36: "(טז) משנת סופרים", "(יז) תפילין..."). Simanim 494-529 mark the seif katan as "{א}", which read "(א) {א}". Skip the generated label for this book and normalize a leading "{א}" to "(א)". Line ids are unaffected: Sefaria lines are keyed on their heRef, which is unchanged. Checked against the 2026-10-07 export: same 18,119 lines, no heRef difference, no doubled marker left. Fixes #74 Fixes #73 Signed-off-by: Elie Gambache --- .../sefariasqlite/SefariaBookPayloadReader.kt | 6 +- .../sefariasqlite/SefariaImportText.kt | 14 +++ .../SefariaSelfLabeledSegmentsTest.kt | 90 +++++++++++++++++++ 3 files changed, 109 insertions(+), 1 deletion(-) create mode 100644 generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSelfLabeledSegmentsTest.kt diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index fa6e89de..c6139ffa 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -502,6 +502,7 @@ internal class SefariaBookPayloadReader( val content = leafPrimitive?.takeIf { it.isString }?.content if (!content.isNullOrEmpty()) { val cleaned = cleanSefariaLine(content, stripFootnotes = hasTranslatorFootnotes(refPrefix)) + .let { if (hasSelfLabeledSegments(bookEnTitle)) normalizeSegmentMarker(it) else it } if (cleaned.isNotEmpty()) { output += linePrefix + cleaned val cleanRef = trimTrailingSeparators(refPrefix) @@ -541,7 +542,10 @@ internal class SefariaBookPayloadReader( val sectionIndex = sectionNames.size - depth val isReferenceable = referenceableSections.getOrNull(sectionIndex) ?: true - val nextLinePrefix = if (depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1) { + val nextLinePrefix = if ( + depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1 && + !hasSelfLabeledSegments(bookEnTitle) + ) { "($letter) " } else { "" diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt index 6020d4d1..87a544f6 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt @@ -27,6 +27,20 @@ private val TRANSLATOR_FOOTNOTE_REF_PREFIXES = listOf("Noda BiYehudah I, Orach C internal fun hasTranslatorFootnotes(ref: String): Boolean = TRANSLATOR_FOOTNOTE_REF_PREFIXES.any { ref.startsWith(it) } +// Books whose segments already open with their printed marker ("(יד) נהגו לתייג"), +// so the generated "(letter) " segment label doubles it. The Mishnah Berurah's +// kuntresim (e.g. Mishnat Soferim in siman 36) have no seif katan at all, so a +// generated label there is wrong rather than redundant (kdroidFilter/SeforimLibrary#73, #74). +private val SELF_LABELED_SEGMENT_BOOKS = setOf("Mishnah Berurah") + +internal fun hasSelfLabeledSegments(bookEnTitle: String): Boolean = bookEnTitle in SELF_LABELED_SEGMENT_BOOKS + +// Some simanim of the Mishnah Berurah (494-529) mark the seif katan as "{א}" +// instead of the printed "(א)". +private val BRACED_SEGMENT_MARKER_REGEX = Regex("""^\{([א-ת]{1,4})\}""") + +internal fun normalizeSegmentMarker(line: String): String = BRACED_SEGMENT_MARKER_REGEX.replace(line, "($1)") + private val FOOTNOTE_MARKER_REGEX = Regex(""".*?""") private const val FOOTNOTE_OPEN = """""" private val ITALIC_TAG_REGEX = Regex("""]*>|""", RegexOption.IGNORE_CASE) diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSelfLabeledSegmentsTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSelfLabeledSegmentsTest.kt new file mode 100644 index 00000000..2627242f --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSelfLabeledSegmentsTest.kt @@ -0,0 +1,90 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import co.touchlab.kermit.Logger +import kotlinx.coroutines.runBlocking +import kotlinx.serialization.json.Json +import java.nio.file.Files +import kotlin.test.Test +import kotlin.test.assertEquals + +/** + * Regression for kdroidFilter/SeforimLibrary#73 and #74: the Mishnah Berurah's + * segments carry their own seif katan marker, so the generated "(letter) " label + * doubled it, and labelled the kuntresim (Mishnat Soferim) that have none. + */ +class SefariaSelfLabeledSegmentsTest { + @Test + fun selfLabeledBookKeepsOnlyItsOwnMarkers() { + val payload = readBook("Mishnah Berurah", "משנה ברורה") + + assertEquals( + listOf( + "

משנה ברורה

", + "

סימן א

", + "(א) נהגו לתייג", + "משנת סופרים", + "תפילין ומזוזות", + "

סימן ב

", + "(א) אין מראין", + "(ב) לראות", + ), + payload.lines, + ) + // heRefs (the line id natural keys) are unchanged + assertEquals( + listOf("משנה ברורה א, א", "משנה ברורה א, ב", "משנה ברורה א, ג", "משנה ברורה ב, א", "משנה ברורה ב, ב"), + payload.refEntries.map { it.heRef.replace(Regex(" +"), " ") }, + ) + } + + @Test + fun otherBooksStillGetGeneratedLabels() { + val payload = readBook("FakeBook", "ספר") + + assertEquals("(א) (א) נהגו לתייג", payload.lines[2]) + assertEquals("(ב) משנת סופרים", payload.lines[3]) + assertEquals("(א) {א} אין מראין", payload.lines[6]) + } + + private fun readBook(title: String, heTitle: String) = runBlocking { + val tempDir = Files.createTempDirectory("seforim-self-labeled") + val schemaDir = Files.createDirectories(tempDir.resolve("schemas")) + val bookDir = Files.createDirectories(tempDir.resolve("json").resolve(title)) + Files.writeString(schemaDir.resolve("${title.replace(' ', '_')}.json"), schemaJson(title, heTitle)) + Files.writeString(bookDir.resolve("merged.json"), mergedJson(title, heTitle)) + + val reader = SefariaBookPayloadReader( + Json { ignoreUnknownKeys = true; coerceInputValues = true }, + Logger.withTag("SefariaSelfLabeledSegmentsTest"), + ) + val schemaLookup = reader.buildSchemaLookup(schemaDir) + reader.readBooksInParallel(tempDir.resolve("json"), schemaDir, schemaLookup).single() + } + + private fun schemaJson(title: String, heTitle: String) = """ + { + "title": "$title", + "heTitle": "$heTitle", + "schema": { + "nodeType": "JaggedArrayNode", + "depth": 2, + "addressTypes": ["Siman", "Seif Katan"], + "sectionNames": ["Siman", "Seif Katan"], + "heSectionNames": ["סימן", "סעיף קטן"], + "title": "$title", + "heTitle": "$heTitle" + } + } + """.trimIndent() + + private fun mergedJson(title: String, heTitle: String) = """ + { + "title": "$title", + "heTitle": "$heTitle", + "text": [ + ["(א) נהגו לתייג", "משנת סופרים", "תפילין ומזוזות"], + ["{א} אין מראין", "{ב} לראות"] + ] + } + """.trimIndent() +}