Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -2708,6 +2708,28 @@ class SeforimRepository(databasePath: String, private val driver: SqlDriver) : L
database.authorQueriesQueries.insertWithId(id, name)
}

suspend fun updateAuthorLifeData(
authorId: Long,
birthYear: Int?,
birthYearApprox: Boolean?,
deathYear: Int?,
deathYearApprox: Boolean?,
era: String?,
) = withContext(Dispatchers.IO) {
database.authorQueriesQueries.updateLifeData(
birthYear?.toLong(),
birthYearApprox?.let { if (it) 1L else 0L },
deathYear?.toLong(),
deathYearApprox?.let { if (it) 1L else 0L },
era,
authorId,
)
}

suspend fun insertAuthorAlias(authorId: Long, alias: String) = withContext(Dispatchers.IO) {
database.authorQueriesQueries.insertAlias(authorId, alias)
}

suspend fun insertLicenseWithId(
id: Long,
code: String,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,17 @@ VALUES (?);
insertWithId:
INSERT OR IGNORE INTO author (id, name) VALUES (?, ?);

updateLifeData:
UPDATE author
SET birthYear = ?, birthYearApprox = ?, deathYear = ?, deathYearApprox = ?, era = ?
WHERE id = ?;

insertAlias:
INSERT OR IGNORE INTO author_alias (authorId, alias) VALUES (?, ?);

selectAliasesByAuthorId:
SELECT alias FROM author_alias WHERE authorId = ? ORDER BY alias;

selectIdByName:
SELECT id FROM author WHERE name = ? LIMIT 1;

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
-- default_commentator.commentatorBookId -> book(id) ON DELETE CASCADE
-- default_targum.bookId -> book(id) ON DELETE CASCADE
-- default_targum.targumBookId -> book(id) ON DELETE CASCADE
-- author_alias.authorId -> author(id) ON DELETE CASCADE
-- edition.licenseId -> license(id) ON DELETE RESTRICT
-- book_edition.bookId -> book(id) ON DELETE CASCADE
-- book_edition.editionId -> edition(id) ON DELETE CASCADE
Expand Down Expand Up @@ -95,11 +96,29 @@ CREATE INDEX IF NOT EXISTS idx_category_closure_descendant ON category_closure(d
-- Authors table
CREATE TABLE IF NOT EXISTS author (
id INTEGER PRIMARY KEY NOT NULL,
name TEXT NOT NULL UNIQUE
-- Raw upstream name: the id's natural key, never rewritten.
name TEXT NOT NULL UNIQUE,
-- Life data from the Sefaria author topic, when known.
birthYear INTEGER,
birthYearApprox INTEGER,
deathYear INTEGER,
deathYearApprox INTEGER,
-- Sefaria era code (T, A, GN, RI, AH, CO...).
era TEXT
);

CREATE INDEX IF NOT EXISTS idx_author_name ON author(name);

-- Other names of an author (רעק"א, ראב"ע...), for author search.
CREATE TABLE IF NOT EXISTS author_alias (
authorId INTEGER NOT NULL,
alias TEXT NOT NULL,
PRIMARY KEY (authorId, alias),
FOREIGN KEY (authorId) REFERENCES author(id) ON DELETE CASCADE
);

CREATE INDEX IF NOT EXISTS idx_author_alias_alias ON author_alias(alias);

-- Table des topics
CREATE TABLE IF NOT EXISTS topic (
id INTEGER PRIMARY KEY NOT NULL,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ class LogicalContentHasher(
"book",
"book_topic",
"book_author",
"author_alias",
"book_pub_place",
"book_pub_date",
"book_edition",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ internal val PATCH_TABLES_IN_FK_ORDER: List<PatchTable> = listOf(

// Book-attribute junctions — depend on book + author/topic/pubPlace/pubDate.
PatchTable("book_author", listOf("bookId", "authorId"), updatable = false),
PatchTable("author_alias", listOf("authorId", "alias"), updatable = false),
PatchTable("book_topic", listOf("bookId", "topicId"), updatable = false),
PatchTable("book_pub_place", listOf("bookId", "pubPlaceId"), updatable = false),
PatchTable("book_pub_date", listOf("bookId", "pubDateId"), updatable = false),
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package io.github.kdroidfilter.seforimlibrary.sefariasqlite

import kotlinx.serialization.Serializable
import kotlinx.serialization.json.Json
import java.nio.file.Path
import kotlin.io.path.exists
import kotlin.io.path.readText

/** One author topic of `authors.json`, as dumped raw by SefariaExport. */
@Serializable
internal data class SefariaAuthorTopic(
val slug: String,
val heTitles: List<String> = emptyList(),
val birthYear: Int? = null,
val birthYearIsApprox: Boolean? = null,
val deathYear: Int? = null,
val deathYearIsApprox: Boolean? = null,
val era: String? = null,
) {
/** The Hebrew titles other than [name], trimmed and deduplicated. */
fun aliasesOf(name: String): List<String> =
heTitles.map { it.trim() }.filter { it.isNotEmpty() && it != name }.distinct()
}

/** Sefaria author topics keyed by slug, from `authors.json`. */
internal class SefariaAuthorTopics(topics: List<SefariaAuthorTopic>) {
private val bySlug = topics.associateBy { it.slug }

operator fun get(slug: String): SefariaAuthorTopic? = bySlug[slug]

companion object {
const val FILE_NAME = "authors.json"

fun load(dbRoot: Path, json: Json): SefariaAuthorTopics {
val file = dbRoot.resolve(FILE_NAME)
require(file.exists()) {
"$FILE_NAME missing from the Sefaria export at $dbRoot " +
"(produced by SefariaExport since PR #7)"
}
return SefariaAuthorTopics(json.decodeFromString<List<SefariaAuthorTopic>>(file.readText()))
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,13 @@ internal class SefariaBookPayloadReader(
?: textJson["categories"]?.jsonArray?.mapNotNull { it.jsonPrimitive.contentOrNull }
?: emptyList()

val authors = schemaJson["authors"]?.jsonArray?.mapNotNull(::resolveSefariaAuthorName).orEmpty()
val authorEntries = schemaJson["authors"]?.jsonArray.orEmpty()
val authors = authorEntries.mapNotNull(::resolveSefariaAuthorName)
val authorSlugs = authorEntries.mapNotNull { entry ->
val name = resolveSefariaAuthorName(entry) ?: return@mapNotNull null
val slug = (entry as? JsonObject)?.get("slug")?.stringOrNull() ?: return@mapNotNull null
name to slug
}.toMap()

val (lines, refs, headings) = buildBookContent(
schemaObj = schemaObj,
Expand Down Expand Up @@ -157,6 +163,7 @@ internal class SefariaBookPayloadReader(
titleAliasKeys = titleAliasKeys,
isReferenceWork = isReferenceWork,
sefariaTitle = fileTitle ?: englishTitle,
authorSlugs = authorSlugs,
versions = versions,
)
}.onFailure { e ->
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ class SefariaDirectImporter(
// Resolve every merged version's license up front: a version missing from
// versions.json means the export is inconsistent, so fail before inserting anything.
val versionLicenses = SefariaVersionLicenses.load(dbRoot, json)
val authorTopics = SefariaAuthorTopics.load(dbRoot, json)
val enrichedAuthorIds = HashSet<Long>()
val licensesByTitle = bookPayloads.associate { payload ->
payload.sefariaTitle to payload.versions.map { version ->
versionLicenses.resolve(payload.sefariaTitle, version.versionTitle)
Expand Down Expand Up @@ -237,7 +239,20 @@ class SefariaDirectImporter(
// would assign a fresh auto-increment id on every build and break the
// delta producer's secondary-UNIQUE collision pre-check).
val resolvedAuthors = payload.authors.map { name ->
Author(id = bindings.upsertAuthor(name), name = name)
val authorId = bindings.upsertAuthor(name)
val topic = payload.authorSlugs[name]?.let { authorTopics[it] }
if (topic != null && enrichedAuthorIds.add(authorId)) {
repository.updateAuthorLifeData(
authorId = authorId,
birthYear = topic.birthYear,
birthYearApprox = topic.birthYearIsApprox,
deathYear = topic.deathYear,
deathYearApprox = topic.deathYearIsApprox,
era = topic.era,
)
topic.aliasesOf(name).forEach { repository.insertAuthorAlias(authorId, it) }
}
Author(id = authorId, name = name)
}
val resolvedPubDates = payload.pubDates.map { pd ->
io.github.kdroidfilter.seforimlibrary.core.models.PubDate(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,8 @@ internal data class BookPayload(
// joined against versions.json to resolve each version's license.
val sefariaTitle: String = enTitle,
val versions: List<SefariaVersionRef> = emptyList(),
// Sefaria topic slug of each author, keyed by the resolved author name.
val authorSlugs: Map<String, String> = emptyMap(),
)

internal data class SefariaVersionRef(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
package io.github.kdroidfilter.seforimlibrary.sefariasqlite

import kotlinx.serialization.json.Json
import kotlin.test.Test
import kotlin.test.assertEquals
import kotlin.test.assertNull

class SefariaAuthorTopicsTest {
private val json = Json { ignoreUnknownKeys = true }

@Test
fun parsesExportRecords() {
val topics = SefariaAuthorTopics(
json.decodeFromString(
"""[{"slug":"rambam","heTitles":["רמב\"ם","משה בן מימון"],"birthYear":1137,
"birthYearIsApprox":null,"deathYear":1204,"deathYearIsApprox":false,"era":"RI"}]""",
),
)
val rambam = topics["rambam"]!!
assertEquals(1204, rambam.deathYear)
assertNull(rambam.birthYearIsApprox)
assertEquals("RI", rambam.era)
assertNull(topics["unknown-slug"])
}

@Test
fun aliasesExcludeTheAuthorNameAndBlanks() {
val topic = SefariaAuthorTopic(slug = "akiva-eiger", heTitles = listOf("עקיבא איגר", " רעק\"א ", "", "רעק\"א"))
assertEquals(listOf("רעק\"א"), topic.aliasesOf("עקיבא איגר"))
}
}
Loading