diff --git a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt index ae94ca89..2989896f 100644 --- a/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt +++ b/dao/src/commonMain/kotlin/io/github/kdroidfilter/seforimlibrary/dao/repository/SeforimRepository.kt @@ -2708,6 +2708,34 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L database.authorQueriesQueries.insertWithId(id, name) } + suspend fun insertLicenseWithId( + id: Long, + code: String, + attribution: Boolean?, + shareAlike: Boolean?, + commercial: Boolean?, + derivatives: Boolean?, + ) = withContext(Dispatchers.IO) { + database.licenseQueriesQueries.insertLicenseWithId( + id, + code, + attribution?.let { if (it) 1L else 0L }, + shareAlike?.let { if (it) 1L else 0L }, + commercial?.let { if (it) 1L else 0L }, + derivatives?.let { if (it) 1L else 0L }, + ) + } + + suspend fun insertEditionWithId(id: Long, title: String, heTitle: String?, sourceUrl: String?) = + withContext(Dispatchers.IO) { + database.licenseQueriesQueries.insertEditionWithId(id, title, heTitle, sourceUrl) + } + + suspend fun linkBookEdition(bookId: Long, editionId: Long, licenseId: Long, licenseInferred: Boolean) = + withContext(Dispatchers.IO) { + database.licenseQueriesQueries.linkBookEdition(bookId, editionId, licenseId, if (licenseInferred) 1L else 0L) + } + suspend fun insertTopicWithId(id: Long, name: String) = withContext(Dispatchers.IO) { database.topicQueriesQueries.insertWithId(id, name) } diff --git a/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/Database.sq b/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/Database.sq index dc83bedf..e77b2cb2 100644 --- a/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/Database.sq +++ b/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/Database.sq @@ -55,6 +55,10 @@ -- default_commentator.commentatorBookId -> book(id) ON DELETE CASCADE -- default_targum.bookId -> book(id) ON DELETE CASCADE -- default_targum.targumBookId -> book(id) ON DELETE CASCADE +-- edition.licenseId -> license(id) ON DELETE RESTRICT +-- book_edition.bookId -> book(id) ON DELETE CASCADE +-- book_edition.editionId -> edition(id) ON DELETE CASCADE +-- book_edition.licenseId -> license(id) ON DELETE RESTRICT -- ───────────────────────────────────────────────────────────────────────────── -- Schema-level metadata (db_version, db_schema_version, content_hash, ...) @@ -207,6 +211,58 @@ CREATE TABLE IF NOT EXISTS book_author ( CREATE INDEX IF NOT EXISTS idx_book_author_book ON book_author(bookId); CREATE INDEX IF NOT EXISTS idx_book_author_author ON book_author(authorId); +-- Licenses, one row per distinct license met while importing (get-or-create on `code`). +-- The permission flags are NULL for a license the generator does not recognize. +CREATE TABLE IF NOT EXISTS license ( + id INTEGER PRIMARY KEY NOT NULL, + code TEXT NOT NULL UNIQUE, + attribution INTEGER, + shareAlike INTEGER, + commercial INTEGER, + derivatives INTEGER +); + +-- Upstream editions a book is built from (e.g. a Sefaria version), declared once. +CREATE TABLE IF NOT EXISTS edition ( + id INTEGER PRIMARY KEY NOT NULL, + title TEXT NOT NULL, + heTitle TEXT, + sourceUrl TEXT, + UNIQUE (title, sourceUrl) +); + +-- Book ↔ edition, with the license that applies to that book's copy of the edition. +-- `licenseInferred` = 1 when the upstream license was unknown and was inherited from the +-- other books of the same edition, all sharing a single license. +CREATE TABLE IF NOT EXISTS book_edition ( + bookId INTEGER NOT NULL, + editionId INTEGER NOT NULL, + licenseId INTEGER NOT NULL, + licenseInferred INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (bookId, editionId), + FOREIGN KEY (bookId) REFERENCES book(id) ON DELETE CASCADE, + FOREIGN KEY (editionId) REFERENCES edition(id) ON DELETE CASCADE, + FOREIGN KEY (licenseId) REFERENCES license(id) ON DELETE RESTRICT +); + +CREATE INDEX IF NOT EXISTS idx_book_edition_book ON book_edition(bookId); +CREATE INDEX IF NOT EXISTS idx_book_edition_license ON book_edition(licenseId); + +-- The single place that resolves a book's licenses: one row per (book, license). +-- Books without any edition (non-Sefaria sources for now) have no row = unknown. +CREATE VIEW IF NOT EXISTS book_license_v AS +SELECT be.bookId AS bookId, + l.id AS licenseId, + l.code AS code, + l.attribution AS attribution, + l.shareAlike AS shareAlike, + l.commercial AS commercial, + l.derivatives AS derivatives, + MAX(be.licenseInferred) AS inferred +FROM book_edition be +JOIN license l ON l.id = be.licenseId +GROUP BY be.bookId, l.id; + -- Lines table CREATE TABLE IF NOT EXISTS line ( id INTEGER PRIMARY KEY NOT NULL, diff --git a/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/LicenseQueries.sq b/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/LicenseQueries.sq new file mode 100644 index 00000000..8a00c3df --- /dev/null +++ b/dao/src/commonMain/sqldelight/io/github/kdroidfilter/seforimlibrary/db/LicenseQueries.sq @@ -0,0 +1,27 @@ +-- Queries for licenses, editions and the book_edition junction + +insertLicenseWithId: +INSERT OR IGNORE INTO license (id, code, attribution, shareAlike, commercial, derivatives) +VALUES (?, ?, ?, ?, ?, ?); + +selectAllLicenses: +SELECT * FROM license ORDER BY code; + +insertEditionWithId: +INSERT OR IGNORE INTO edition (id, title, heTitle, sourceUrl) VALUES (?, ?, ?, ?); + +linkBookEdition: +INSERT INTO book_edition (bookId, editionId, licenseId, licenseInferred) +VALUES (?, ?, ?, ?) +ON CONFLICT (bookId, editionId) DO NOTHING; + +selectEditionsByBookId: +SELECT e.*, l.code AS licenseCode, be.licenseInferred +FROM book_edition be +JOIN edition e ON e.id = be.editionId +JOIN license l ON l.id = be.licenseId +WHERE be.bookId = ? +ORDER BY e.title; + +selectLicensesByBookId: +SELECT * FROM book_license_v WHERE bookId = ? ORDER BY code; diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/buildstate/NaturalKey.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/buildstate/NaturalKey.kt index a98a576b..15b27257 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/buildstate/NaturalKey.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/buildstate/NaturalKey.kt @@ -17,6 +17,8 @@ enum class IdTable(val tableName: String, val lookupKind: String?) { CONNECTION_TYPE("connection_type", "connection_type"), CATEGORY("category", "category"), TOC_TEXT("tocText", "toc_text"), + LICENSE("license", "license"), + EDITION("edition", "edition"), // Composite-keyed tables — dedicated storage. BOOK("book", null), diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocator.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocator.kt index 2c538e83..e045cef4 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocator.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocator.kt @@ -31,6 +31,8 @@ interface IdAllocator { fun connectionTypeId(name: String): Long fun categoryId(canonicalPath: String): Long fun tocTextId(text: String): Long + fun licenseId(code: String): Long + fun editionId(naturalKey: String): Long // ─── Composite-keyed tables ──────────────────────────────────────────────── fun bookId(sourceName: String, canonicalHeTitle: String): Long diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocatorBindings.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocatorBindings.kt index e5bc7dac..f672e236 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocatorBindings.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/IdAllocatorBindings.kt @@ -1,5 +1,6 @@ package io.github.kdroidfilter.seforimlibrary.common.ids +import io.github.kdroidfilter.seforimlibrary.common.licenses.Licenses import io.github.kdroidfilter.seforimlibrary.core.models.Book import io.github.kdroidfilter.seforimlibrary.core.models.Line import io.github.kdroidfilter.seforimlibrary.core.models.TocEntry @@ -45,6 +46,8 @@ class IdAllocatorBindings( private val connectionTypesInserted = java.util.concurrent.ConcurrentHashMap.newKeySet() private val categoriesInserted = java.util.concurrent.ConcurrentHashMap.newKeySet() private val tocTextsInserted = java.util.concurrent.ConcurrentHashMap.newKeySet() + private val licensesInserted = java.util.concurrent.ConcurrentHashMap.newKeySet() + private val editionsInserted = java.util.concurrent.ConcurrentHashMap.newKeySet() // ─── Lookup-table helpers ────────────────────────────────────────────────── @@ -104,6 +107,32 @@ class IdAllocatorBindings( return id } + /** Get-or-create the license for an upstream code (see [Licenses.canonicalCode]). */ + suspend fun upsertLicense(rawCode: String?): Long { + val code = Licenses.canonicalCode(rawCode) + val id = allocator.licenseId(code) + if (licensesInserted.add(code)) { + val permissions = Licenses.permissionsOf(code) + repo.insertLicenseWithId( + id = id, + code = code, + attribution = permissions?.attribution, + shareAlike = permissions?.shareAlike, + commercial = permissions?.commercial, + derivatives = permissions?.derivatives, + ) + } + return id + } + + /** Get-or-create an upstream edition, identified by its title and source URL. */ + suspend fun upsertEdition(title: String, heTitle: String?, sourceUrl: String?): Long { + val key = "$title\u001F${sourceUrl.orEmpty()}" + val id = allocator.editionId(key) + if (editionsInserted.add(key)) repo.insertEditionWithId(id, title, heTitle, sourceUrl) + return id + } + // ─── Composite-keyed entities ────────────────────────────────────────────── /** diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/InMemoryIdAllocator.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/InMemoryIdAllocator.kt index 3723db9f..888bbdb0 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/InMemoryIdAllocator.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/ids/InMemoryIdAllocator.kt @@ -110,6 +110,8 @@ class InMemoryIdAllocator private constructor( override fun connectionTypeId(name: String): Long = allocateLookup(IdTable.CONNECTION_TYPE, name) override fun categoryId(canonicalPath: String): Long = allocateLookup(IdTable.CATEGORY, canonicalPath) override fun tocTextId(text: String): Long = allocateLookup(IdTable.TOC_TEXT, text) + override fun licenseId(code: String): Long = allocateLookup(IdTable.LICENSE, code) + override fun editionId(naturalKey: String): Long = allocateLookup(IdTable.EDITION, naturalKey) // ─── Composite-keyed accessors ───────────────────────────────────────────── diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/Licenses.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/Licenses.kt new file mode 100644 index 00000000..7dd251ce --- /dev/null +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/Licenses.kt @@ -0,0 +1,47 @@ +package io.github.kdroidfilter.seforimlibrary.common.licenses + +/** What a license allows. Only known for the licenses listed in [Licenses]. */ +data class LicensePermissions( + val attribution: Boolean, + val shareAlike: Boolean, + val commercial: Boolean, + val derivatives: Boolean, +) + +/** + * Single source of truth for license codes, shared by every importer. + * + * Licenses are get-or-create: any code met upstream is stored as-is (after [canonicalCode] + * folds spelling variants of the same license), and only the codes below carry permission + * flags. A new upstream code is stored with unknown permissions until it is added here. + */ +object Licenses { + const val UNKNOWN = "unknown" + const val PUBLIC_DOMAIN = "Public Domain" + + private val ALIASES = mapOf( + "pd" to PUBLIC_DOMAIN, + "public domain" to PUBLIC_DOMAIN, + "unknown" to UNKNOWN, + ) + + private val PERMISSIONS = mapOf( + PUBLIC_DOMAIN to LicensePermissions(attribution = false, shareAlike = false, commercial = true, derivatives = true), + "CC0" to LicensePermissions(attribution = false, shareAlike = false, commercial = true, derivatives = true), + "CC-BY" to LicensePermissions(attribution = true, shareAlike = false, commercial = true, derivatives = true), + "CC-BY-SA" to LicensePermissions(attribution = true, shareAlike = true, commercial = true, derivatives = true), + "CC-BY-NC" to LicensePermissions(attribution = true, shareAlike = false, commercial = false, derivatives = true), + "CC-BY-NC-SA" to LicensePermissions(attribution = true, shareAlike = true, commercial = false, derivatives = true), + "CC-BY-ND" to LicensePermissions(attribution = true, shareAlike = false, commercial = true, derivatives = false), + "CC-BY-NC-ND" to LicensePermissions(attribution = true, shareAlike = false, commercial = false, derivatives = false), + ) + + /** Folds upstream spellings of the same license into one code; null/blank → [UNKNOWN]. */ + fun canonicalCode(raw: String?): String { + val trimmed = raw?.trim().orEmpty() + if (trimmed.isEmpty()) return UNKNOWN + return ALIASES[trimmed.lowercase()] ?: trimmed + } + + fun permissionsOf(code: String): LicensePermissions? = PERMISSIONS[code] +} diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt index 32eabf76..8f8114dc 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/LogicalContentHasher.kt @@ -82,11 +82,14 @@ class LogicalContentHasher( "category", "category_closure", "tocText", + "license", + "edition", "book", "book_topic", "book_author", "book_pub_place", "book_pub_date", + "book_edition", "tocEntry", "line", "line_toc", diff --git a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchTables.kt b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchTables.kt index 38568714..f04521c3 100644 --- a/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchTables.kt +++ b/generator/common/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/common/patch/PatchTables.kt @@ -36,6 +36,8 @@ internal val PATCH_TABLES_IN_FK_ORDER: List = listOf( PatchTable("pub_date", listOf("id"), updatable = true), PatchTable("connection_type", listOf("id"), updatable = true), PatchTable("tocText", listOf("id"), updatable = true), + PatchTable("license", listOf("id"), updatable = true), + PatchTable("edition", listOf("id"), updatable = true), // Self-ref tree — categories. parentId FK is self → same table, OK. PatchTable("category", listOf("id"), updatable = true), @@ -50,6 +52,7 @@ internal val PATCH_TABLES_IN_FK_ORDER: List = listOf( PatchTable("book_pub_place", listOf("bookId", "pubPlaceId"), updatable = false), PatchTable("book_pub_date", listOf("bookId", "pubDateId"), updatable = false), PatchTable("book_acronym", listOf("bookId", "term"), updatable = false), + PatchTable("book_edition", listOf("bookId", "editionId"), updatable = true), // TOC. tocEntry FK to line (lineId) and line FK to tocEntry (tocEntryId) // form a cycle, broken at apply time with PRAGMA defer_foreign_keys = ON. diff --git a/generator/common/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/LicensesTest.kt b/generator/common/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/LicensesTest.kt new file mode 100644 index 00000000..bc7709be --- /dev/null +++ b/generator/common/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/common/licenses/LicensesTest.kt @@ -0,0 +1,35 @@ +package io.github.kdroidfilter.seforimlibrary.common.licenses + +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNotNull +import kotlin.test.assertNull + +class LicensesTest { + @Test + fun foldsPublicDomainSpellings() { + assertEquals(Licenses.PUBLIC_DOMAIN, Licenses.canonicalCode("PD")) + assertEquals(Licenses.PUBLIC_DOMAIN, Licenses.canonicalCode(" Public Domain ")) + } + + @Test + fun missingOrBlankIsUnknown() { + assertEquals(Licenses.UNKNOWN, Licenses.canonicalCode(null)) + assertEquals(Licenses.UNKNOWN, Licenses.canonicalCode("")) + assertEquals(Licenses.UNKNOWN, Licenses.canonicalCode("Unknown")) + } + + @Test + fun keepsUnrecognizedCodesWithoutPermissions() { + assertEquals("Copyright: Koren", Licenses.canonicalCode("Copyright: Koren")) + assertNull(Licenses.permissionsOf("Copyright: Koren")) + assertNull(Licenses.permissionsOf(Licenses.UNKNOWN)) + } + + @Test + fun nonCommercialForbidsCommercialUse() { + val nc = assertNotNull(Licenses.permissionsOf("CC-BY-NC")) + assertEquals(false, nc.commercial) + assertEquals(true, nc.attribution) + } +} diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt index e5c78b45..416c3f33 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaBookPayloadReader.kt @@ -134,6 +134,11 @@ internal class SefariaBookPayloadReader( val titleAliasKeys = extractTitleAliasKeys(schemaJson, schemaObj) val isReferenceWork = schemaJson["categories"]?.jsonArray ?.firstOrNull()?.jsonPrimitive?.contentOrNull == "Reference" + val versions = (textJson["versions"] as? JsonArray).orEmpty().mapNotNull { entry -> + val pair = entry as? JsonArray ?: return@mapNotNull null + val versionTitle = pair.getOrNull(0).stringOrNull() ?: return@mapNotNull null + SefariaVersionRef(versionTitle, pair.getOrNull(1).stringOrNull()) + } BookPayload( heTitle = hebrewTitle, @@ -151,6 +156,8 @@ internal class SefariaBookPayloadReader( collectiveTitleEn = collectiveTitleEn, titleAliasKeys = titleAliasKeys, isReferenceWork = isReferenceWork, + sefariaTitle = fileTitle ?: englishTitle, + versions = versions, ) }.onFailure { e -> logger.w(e) { "Failed to prepare book from $textPath" } diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt index b1d11b29..41d27048 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaDirectImporter.kt @@ -84,6 +84,16 @@ class SefariaDirectImporter( val bookPayloads = bookPayloadReader.readBooksInParallel(jsonDir, schemaDir, schemaLookup) logger.i { "Parsed ${bookPayloads.size} books" } + // Resolve every merged version's license up front: a version missing from + // versions.json means the export is inconsistent, so fail before inserting anything. + val versionLicenses = SefariaVersionLicenses.load(dbRoot, json) + val licensesByTitle = bookPayloads.associate { payload -> + payload.sefariaTitle to payload.versions.map { version -> + versionLicenses.resolve(payload.sefariaTitle, version.versionTitle) + ?: error("Version '${version.versionTitle}' of '${payload.sefariaTitle}' missing from versions.json") + } + } + val classLoader = javaClass.classLoader val blacklists = loadSefariaBlacklists(classLoader, logger) if (!blacklists.isEmpty()) { @@ -255,6 +265,11 @@ class SefariaDirectImporter( hasNekudot = hasNekudot ) repository.insertBook(book) + licensesByTitle[payload.sefariaTitle].orEmpty().forEach { version -> + val editionId = bindings.upsertEdition(version.versionTitle, version.heVersionTitle, version.versionSource) + val licenseId = bindings.upsertLicense(version.licenseCode) + repository.linkBookEdition(bookId, editionId, licenseId, version.inferred) + } // Track normalized titles (Hebrew/English) for later default-commentator mapping. // Indexes the primary titles plus all Sefaria-known aliases (titleVariants / diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt index 26b03828..0bd0db3a 100644 --- a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaImportModels.kt @@ -71,6 +71,15 @@ internal data class BookPayload( val titleAliasKeys: List = emptyList(), // Top-level Sefaria category is `Reference` (dictionaries, lexicons, bibliographies). val isReferenceWork: Boolean = false, + // The Sefaria versions merged into this book (merged.json `title` + `versions`), + // joined against versions.json to resolve each version's license. + val sefariaTitle: String = enTitle, + val versions: List = emptyList(), +) + +internal data class SefariaVersionRef( + val versionTitle: String, + val versionSource: String?, ) internal data class RefEntry( diff --git a/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicenses.kt b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicenses.kt new file mode 100644 index 00000000..63f96cb3 --- /dev/null +++ b/generator/sefariasqlite/src/jvmMain/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicenses.kt @@ -0,0 +1,73 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.common.licenses.Licenses +import kotlinx.serialization.Serializable +import kotlinx.serialization.json.Json +import java.nio.file.Path +import kotlin.io.path.exists +import kotlin.io.path.readText + +/** One Hebrew version record of `versions.json`, as dumped raw by SefariaExport. */ +@Serializable +internal data class SefariaVersionRecord( + val title: String, + val versionTitle: String, + val versionTitleInHebrew: String? = null, + val versionSource: String? = null, + val license: String? = null, +) + +/** The license of one book's version, after canonicalization and edition inheritance. */ +internal data class ResolvedVersionLicense( + val versionTitle: String, + val heVersionTitle: String?, + val versionSource: String?, + val licenseCode: String, + val inferred: Boolean, +) + +/** + * Resolves the license of each Sefaria version from `versions.json`. + * + * Sefaria declares licenses per (book, version). When a version is `unknown` but every + * known version of the same edition (versionTitle + versionSource) shares a single license, + * that license is inherited and flagged as inferred. Anything else stays unknown. + */ +internal class SefariaVersionLicenses(records: List) { + private val byVersion = records.associateBy { it.title to it.versionTitle } + + private val editionLicense: Map, String> = records + .map { (it.versionTitle to it.versionSource) to Licenses.canonicalCode(it.license) } + .filter { (_, code) -> code != Licenses.UNKNOWN } + .groupBy({ it.first }, { it.second }) + .mapNotNull { (edition, codes) -> codes.toSet().singleOrNull()?.let { edition to it } } + .toMap() + + /** Returns null when the version is absent from `versions.json`. */ + fun resolve(title: String, versionTitle: String): ResolvedVersionLicense? { + val record = byVersion[title to versionTitle] ?: return null + val declared = Licenses.canonicalCode(record.license) + val inherited = editionLicense[record.versionTitle to record.versionSource] + .takeIf { declared == Licenses.UNKNOWN } + return ResolvedVersionLicense( + versionTitle = record.versionTitle, + heVersionTitle = record.versionTitleInHebrew?.takeIf { it.isNotBlank() }, + versionSource = record.versionSource?.takeIf { it.isNotBlank() }, + licenseCode = inherited ?: declared, + inferred = inherited != null, + ) + } + + companion object { + const val FILE_NAME = "versions.json" + + fun load(dbRoot: Path, json: Json): SefariaVersionLicenses { + val file = dbRoot.resolve(FILE_NAME) + require(file.exists()) { + "$FILE_NAME missing from the Sefaria export at $dbRoot " + + "(produced by SefariaExport since the 2026-10-08 release)" + } + return SefariaVersionLicenses(json.decodeFromString>(file.readText())) + } + } +} diff --git a/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicensesTest.kt b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicensesTest.kt new file mode 100644 index 00000000..d2cb2c78 --- /dev/null +++ b/generator/sefariasqlite/src/jvmTest/kotlin/io/github/kdroidfilter/seforimlibrary/sefariasqlite/SefariaVersionLicensesTest.kt @@ -0,0 +1,69 @@ +package io.github.kdroidfilter.seforimlibrary.sefariasqlite + +import io.github.kdroidfilter.seforimlibrary.common.licenses.Licenses +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertFalse +import kotlin.test.assertNotNull +import kotlin.test.assertNull +import kotlin.test.assertTrue + +class SefariaVersionLicensesTest { + private val davidson = "William Davidson Edition - Vocalized Aramaic" + private val koren = "https://korenpub.co.il" + + private fun record(title: String, versionTitle: String, source: String?, license: String?) = + SefariaVersionRecord(title = title, versionTitle = versionTitle, versionSource = source, license = license) + + @Test + fun declaredLicenseIsKeptAndCanonicalized() { + val licenses = SefariaVersionLicenses(listOf(record("Mishnah Chullin", "Kaufmann", "x", "PD"))) + val resolved = assertNotNull(licenses.resolve("Mishnah Chullin", "Kaufmann")) + assertEquals(Licenses.PUBLIC_DOMAIN, resolved.licenseCode) + assertFalse(resolved.inferred) + } + + @Test + fun unknownInheritsTheSingleLicenseOfItsEdition() { + val licenses = SefariaVersionLicenses( + listOf( + record("Berakhot", davidson, koren, "CC-BY-NC"), + record("Shabbat", davidson, koren, "CC-BY-NC"), + record("Chullin", davidson, koren, "unknown"), + ), + ) + val resolved = assertNotNull(licenses.resolve("Chullin", davidson)) + assertEquals("CC-BY-NC", resolved.licenseCode) + assertTrue(resolved.inferred) + } + + @Test + fun unknownStaysUnknownWhenTheEditionIsMixed() { + val licenses = SefariaVersionLicenses( + listOf( + record("A", "Torat Emet", "t", "Public Domain"), + record("B", "Torat Emet", "t", "CC-BY-NC"), + record("C", "Torat Emet", "t", null), + ), + ) + val resolved = assertNotNull(licenses.resolve("C", "Torat Emet")) + assertEquals(Licenses.UNKNOWN, resolved.licenseCode) + assertFalse(resolved.inferred) + } + + @Test + fun sameTitleFromAnotherSourceIsAnotherEdition() { + val licenses = SefariaVersionLicenses( + listOf( + record("A", "Vilna Edition", "nli", "Public Domain"), + record("B", "Vilna Edition", "other", "unknown"), + ), + ) + assertEquals(Licenses.UNKNOWN, licenses.resolve("B", "Vilna Edition")?.licenseCode) + } + + @Test + fun missingVersionIsNull() { + assertNull(SefariaVersionLicenses(emptyList()).resolve("Chullin", davidson)) + } +}