fix(sefariasqlite): stop labelling Mishnah Berurah segments

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt

@@ -502,6 +502,7 @@ internal class SefariaBookPayloadReader(
             val content = leafPrimitive?.takeIf { it.isString }?.content
             if (!content.isNullOrEmpty()) {
                 val cleaned = cleanSefariaLine(content, stripFootnotes = hasTranslatorFootnotes(refPrefix))
+                    .let { if (hasSelfLabeledSegments(bookEnTitle)) normalizeSegmentMarker(it) else it }
                 if (cleaned.isNotEmpty()) {
                     output += linePrefix + cleaned
                     val cleanRef = trimTrailingSeparators(refPrefix)
@@ -541,7 +542,10 @@ internal class SefariaBookPayloadReader(
 
             val sectionIndex = sectionNames.size - depth
             val isReferenceable = referenceableSections.getOrNull(sectionIndex) ?: true
-            val nextLinePrefix = if (depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1) {
+            val nextLinePrefix = if (
+                depth == 1 && isReferenceable && currentAddressType != "Integer" && nonEmptyCount > 1 &&
+                !hasSelfLabeledSegments(bookEnTitle)
+            ) {
                 "($letter) "
             } else {
                 ""

generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportText.kt

@@ -27,6 +27,20 @@ private val TRANSLATOR_FOOTNOTE_REF_PREFIXES = listOf("Noda BiYehudah I, Orach C
 internal fun hasTranslatorFootnotes(ref: String): Boolean =
     TRANSLATOR_FOOTNOTE_REF_PREFIXES.any { ref.startsWith(it) }
 
+// Books whose segments already open with their printed marker ("(יד) נהגו לתייג"),
+// so the generated "(letter) " segment label doubles it. The Mishnah Berurah's
+// kuntresim (e.g. Mishnat Soferim in siman 36) have no seif katan at all, so a
+// generated label there is wrong rather than redundant (kdroidFilter/SeforimLibrary#73, #74).
+private val SELF_LABELED_SEGMENT_BOOKS = setOf("Mishnah Berurah")
+
+internal fun hasSelfLabeledSegments(bookEnTitle: String): Boolean = bookEnTitle in SELF_LABELED_SEGMENT_BOOKS
+
+// Some simanim of the Mishnah Berurah (494-529) mark the seif katan as "{א}"
+// instead of the printed "(א)".
+private val BRACED_SEGMENT_MARKER_REGEX = Regex("""^\{([א-ת]{1,4})\}""")
+
+internal fun normalizeSegmentMarker(line: String): String = BRACED_SEGMENT_MARKER_REGEX.replace(line, "($1)")
+
 private val FOOTNOTE_MARKER_REGEX = Regex("""<sup class="footnote-marker">.*?</sup>""")
 private const val FOOTNOTE_OPEN = """<i class="footnote">"""
 private val ITALIC_TAG_REGEX = Regex("""<i\b[^>]*>|</i>""", RegexOption.IGNORE_CASE)

generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaSelfLabeledSegmentsTest.kt

@@ -0,0 +1,90 @@
+package io.github.kdroidfilter.seforimlibrary.sefariasqlite
+
+import co.touchlab.kermit.Logger
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.json.Json
+import java.nio.file.Files
+import kotlin.test.Test
+import kotlin.test.assertEquals
+
+/**
+ * Regression for kdroidFilter/SeforimLibrary#73 and #74: the Mishnah Berurah's
+ * segments carry their own seif katan marker, so the generated "(letter) " label
+ * doubled it, and labelled the kuntresim (Mishnat Soferim) that have none.
+ */
+class SefariaSelfLabeledSegmentsTest {
+    @Test
+    fun selfLabeledBookKeepsOnlyItsOwnMarkers() {
+        val payload = readBook("Mishnah Berurah", "משנה ברורה")
+
+        assertEquals(
+            listOf(
+                "<h1>משנה ברורה</h1>",
+                "<h2>סימן א</h2>",
+                "(א) נהגו לתייג",
+                "<b>משנת סופרים</b>",
+                "תפילין ומזוזות",
+                "<h2>סימן ב</h2>",
+                "(א) אין מראין",
+                "(ב) לראות",
+            ),
+            payload.lines,
+        )
+        // heRefs (the line id natural keys) are unchanged
+        assertEquals(
+            listOf("משנה ברורה א, א", "משנה ברורה א, ב", "משנה ברורה א, ג", "משנה ברורה ב, א", "משנה ברורה ב, ב"),
+            payload.refEntries.map { it.heRef.replace(Regex(" +"), " ") },
+        )
+    }
+
+    @Test
+    fun otherBooksStillGetGeneratedLabels() {
+        val payload = readBook("FakeBook", "ספר")
+
+        assertEquals("(א) (א) נהגו לתייג", payload.lines[2])
+        assertEquals("(ב) <b>משנת סופרים</b>", payload.lines[3])
+        assertEquals("(א) {א} אין מראין", payload.lines[6])
+    }
+
+    private fun readBook(title: String, heTitle: String) = runBlocking {
+        val tempDir = Files.createTempDirectory("seforim-self-labeled")
+        val schemaDir = Files.createDirectories(tempDir.resolve("schemas"))
+        val bookDir = Files.createDirectories(tempDir.resolve("json").resolve(title))
+        Files.writeString(schemaDir.resolve("${title.replace(' ', '_')}.json"), schemaJson(title, heTitle))
+        Files.writeString(bookDir.resolve("merged.json"), mergedJson(title, heTitle))
+
+        val reader = SefariaBookPayloadReader(
+            Json { ignoreUnknownKeys = true; coerceInputValues = true },
+            Logger.withTag("SefariaSelfLabeledSegmentsTest"),
+        )
+        val schemaLookup = reader.buildSchemaLookup(schemaDir)
+        reader.readBooksInParallel(tempDir.resolve("json"), schemaDir, schemaLookup).single()
+    }
+
+    private fun schemaJson(title: String, heTitle: String) = """
+        {
+          "title": "$title",
+          "heTitle": "$heTitle",
+          "schema": {
+            "nodeType": "JaggedArrayNode",
+            "depth": 2,
+            "addressTypes": ["Siman", "Seif Katan"],
+            "sectionNames": ["Siman", "Seif Katan"],
+            "heSectionNames": ["סימן", "סעיף קטן"],
+            "title": "$title",
+            "heTitle": "$heTitle"
+          }
+        }
+    """.trimIndent()
+
+    private fun mergedJson(title: String, heTitle: String) = """
+        {
+          "title": "$title",
+          "heTitle": "$heTitle",
+          "text": [
+            ["(א) נהגו לתייג", "<b>משנת סופרים</b>", "תפילין ומזוזות"],
+            ["{א} אין מראין", "{ב} לראות"]
+          ]
+        }
+    """.trimIndent()
+}