From 6e9bbfafc65657b4aa3dce1c09af810368a550d9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:44:43 +0200 Subject: [PATCH 001/434] Update build.gradle.kts --- build.gradle.kts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 81ed09e1..c21b32c1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } group = "org.usagi" -version = "1.0.1" +version = "1.0.2" tasks.test { useJUnitPlatform() @@ -34,7 +34,7 @@ tasks.withType().configureEach "-opt-in=kotlin.RequiresOptIn", "-opt-in=kotlin.contracts.ExperimentalContracts", "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", - "-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi", + "-opt-in=tsuki.InternalParsersApi", ) } } @@ -54,18 +54,14 @@ publishing { dependencies { implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) implementation(libs.okhttp) implementation(libs.okio) implementation(libs.json) implementation(libs.androidx.collection) - implementation("com.squareup.okhttp3:okhttp-brotli:5.4.0") - api(libs.core.parsers) + api(libs.tsuki) api(libs.jsoup) - compileOnly(libs.android.stubs) - ksp(project(":plugins-ksp")) testImplementation(libs.junit.api) @@ -77,14 +73,18 @@ dependencies { } tasks.register("buildJar") { + description = "Build all sources to a JAR file" dependsOn("dexJar") } tasks.register("dexJar") { + description = "Dex classes after build" dependsOn(tasks.jar) inputJar.set(tasks.jar.flatMap { it.archiveFile }) outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) classpath.from(configurations.runtimeClasspath) } -tasks.register("generateTestsReport") +tasks.register("generateTestsReport") { + description = "Generate a HTML file to get tests report" +} From 30747881b34fb5a04e2e63b143dea698f4a233dd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:44:59 +0200 Subject: [PATCH 002/434] Update library versions and dependencies Removed unused dependencies and added 'tsuki' library. --- gradle/libs.versions.toml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e1bb9889..d781b981 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,5 @@ [versions] collection = "1.6.0" -core = "9b51187e96" kotlin = "2.2.10" ksp = "2.2.10-2.0.2" coroutines = "1.10.2" @@ -13,9 +12,8 @@ json = "20251224" jsoup = "1.17.2" quickjs = "1.1.0" korte = "4.0.10" -serialization = "1.9.0" simplexml = "2.7.1" -stubs = "4.1.1.4" +tsuki = "1.0.1" [plugins] kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } @@ -23,12 +21,10 @@ kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", versi ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } [libraries] -android-stubs = { module = "com.google.android:android", version.ref = "stubs" } -core-parsers = { module = "com.github.UsagiApp:core-parsers", version.ref = "core"} +tsuki = { module = "com.github.UsagiApp:Tsuki", version.ref = "tsuki"} ksp-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp" } kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines" } -kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" } junit-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } @@ -40,4 +36,4 @@ androidx-collection = { module = "androidx.collection:collection", version.ref = jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } quickjs = { module = "io.webfolder:quickjs", version.ref = "quickjs" } korte = { module = "com.soywiz.korlibs.korte:korte-jvm", version.ref = "korte" } -simplexml = { module = "org.simpleframework:simple-xml", version.ref = "simplexml" } \ No newline at end of file +simplexml = { module = "org.simpleframework:simple-xml", version.ref = "simplexml" } From 0b5afb83083ab238e191bc864801d8fabce413a3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:45:14 +0200 Subject: [PATCH 003/434] Update ParserProcessor.kt --- .../kotatsu/parsers/ksp/ParserProcessor.kt | 128 ++---------------- 1 file changed, 14 insertions(+), 114 deletions(-) diff --git a/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt b/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt index bacd2079..bfb3674c 100644 --- a/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt +++ b/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt @@ -20,21 +20,17 @@ class ParserProcessor( private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}") override fun process(resolver: Resolver): List { - val kotatsuSymbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser") - val tachiyomiSymbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.TachiyomiSource") - + val kotatsuSymbols = resolver.getSymbolsWithAnnotation("tsuki.MangaSourceParser") val kotatsuRet = kotatsuSymbols.filterNot { it.validate() }.toList() - val tachiyomiRet = tachiyomiSymbols.filterNot { it.validate() }.toList() - - if (!kotatsuSymbols.iterator().hasNext() && !tachiyomiSymbols.iterator().hasNext()) { - return kotatsuRet + tachiyomiRet + if (!kotatsuSymbols.iterator().hasNext()) { + return kotatsuRet } val dependencies = Dependencies.ALL_FILES val factoryFile = try { codeGenerator.createNewFile( dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers", + packageName = "tsuki", fileName = "MangaParserFactory", ) } catch (e: FileAlreadyExistsException) { @@ -45,7 +41,7 @@ class ParserProcessor( try { codeGenerator.createNewFile( dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers.model", + packageName = "tsuki.model", fileName = "MangaSource", ) } catch (e: FileAlreadyExistsException) { @@ -54,29 +50,27 @@ class ParserProcessor( } val totalCount = sourcesFile?.writer().use { sourcesWriter -> factoryFile?.writer().use { factoryWriter -> - writeContent(sourcesWriter, factoryWriter, kotatsuSymbols, tachiyomiSymbols) + writeContent(sourcesWriter, factoryWriter, kotatsuSymbols) } } writeSummary(totalCount) - return kotatsuRet + tachiyomiRet + return kotatsuRet } private fun writeContent( sourcesWriter: Writer?, factoryWriter: Writer?, kotatsuSymbols: Sequence, - tachiyomiSymbols: Sequence, ): Int { if (sourcesWriter == null && factoryWriter == null) { return 0 } factoryWriter?.write( """ - package org.koitharu.kotatsu.parsers + package tsuki - import org.koitharu.kotatsu.parsers.model.MangaParserSource - import org.koitharu.kotatsu.parsers.core.MangaParserWrapper - import org.koitharu.kotatsu.parsers.compat.TachiyomiSourceAdapter + import tsuki.model.MangaParserSource + import tsuki.core.MangaParserWrapper internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) { @@ -84,7 +78,7 @@ class ParserProcessor( ) sourcesWriter?.write( """ - package org.koitharu.kotatsu.parsers.model + package tsuki.model public enum class MangaParserSource( override val title: String, @@ -103,18 +97,11 @@ class ParserProcessor( .onEach { it.accept(kotatsuVisitor, Unit) } .count() - // Process Tachiyomi source bridges - val tachiyomiVisitor = TachiyomiVisitor(sourcesWriter, factoryWriter, kotatsuVisitor.titles) - val tachiyomiCount = tachiyomiSymbols - .filter { it is KSClassDeclaration && it.validate() } - .onEach { it.accept(tachiyomiVisitor, Unit) } - .count() - factoryWriter?.write( - """ + $$""" }.let { require(it.source == this) { - "Cannot instantiate manga parser: ${'$'}name mapped to ${'$'}{it.source}" + "Cannot instantiate manga parser: $name mapped to ${it.source}" } MangaParserWrapper(it) } @@ -126,7 +113,7 @@ class ParserProcessor( } """.trimIndent(), ) - return kotatsuCount + tachiyomiCount + return kotatsuCount } private fun writeSummary(totalCount: Int) { @@ -197,91 +184,4 @@ class ParserProcessor( ) } } - - /** - * Visitor for @TachiyomiSource annotations. - * Generates factory entries that instantiate the Tachiyomi HttpSource and wrap it with TachiyomiSourceAdapter. - */ - private inner class TachiyomiVisitor( - private val sourcesWriter: Writer?, - private val factoryWriter: Writer?, - private val titles: HashMap, - ) : KSVisitorVoid() { - - override fun visitClassDeclaration( - classDeclaration: KSClassDeclaration, - data: Unit, - ) { - val annotation = classDeclaration.annotations.single { it.shortName.asString() == "TachiyomiSource" } - - // Get annotation arguments with defaults - val nameArg = annotation.arguments.find { it.name?.asString() == "name" }?.value as? String ?: "" - val titleArg = annotation.arguments.find { it.name?.asString() == "title" }?.value as? String ?: "" - val localeArg = annotation.arguments.find { it.name?.asString() == "locale" }?.value as? String ?: "" - val typeArg = annotation.arguments.find { it.name?.asString() == "type" }?.value?.toString() - val type = if (typeArg.isNullOrBlank() || typeArg == "null") "ContentType.MANGA" else typeArg - - // Auto-detect values from class if not provided - val className = classDeclaration.simpleName.asString() - val qualifiedClassName = checkNotNull(classDeclaration.qualifiedName?.asString()) { - "Class qualified name is null" - } - - // Generate source name from class name (UPPER_CASE) - val sourceName = nameArg.takeIf { it.isNotBlank() } - ?: className.uppercase().replace(Regex("[^A-Z0-9_]"), "_").let { - // Ensure it starts with a letter - if (it.firstOrNull()?.isLetter() == true) it else "SOURCE_$it" - } - - // Use provided title or class name - val sourceTitle = titleArg.takeIf { it.isNotBlank() } ?: className - - // Auto-detect locale from package name if not provided - val locale = localeArg.takeIf { it.isNotBlank() } - ?: detectLocaleFromPackage(qualifiedClassName) - - val localeString = "\"$locale\"" - val localeObj = if (locale.isEmpty()) null else Locale(locale) - val localeTitle = localeObj?.getDisplayLanguage(localeObj) - - if (localeObj != null && localeObj !in availableLocales) { - logger.error("Tachiyomi source $sourceName has wrong locale: $localeTitle") - } - - if (!sourceNamePattern.matches(sourceName)) { - logger.error("Tachiyomi source name must be uppercase: $sourceName") - } - - val prevTitleClass = titles.put(sourceTitle, qualifiedClassName) - if (prevTitleClass != null) { - logger.warn("Source title duplication: \"$sourceTitle\" is assigned to both $prevTitleClass and $qualifiedClassName") - } - - // Factory: wrap the Tachiyomi source with TachiyomiSourceAdapter - factoryWriter?.write( - "\tMangaParserSource.$sourceName -> TachiyomiSourceAdapter(context, MangaParserSource.$sourceName, $qualifiedClassName())\n" - ) - - // Enum entry - val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty() - sourcesWriter?.write( - "\t$sourceName(\"$sourceTitle\", $localeString$localeComment, $type, false),\n", - ) - } - - private fun detectLocaleFromPackage(qualifiedName: String): String { - // Common locale patterns in package names - val localePatterns = mapOf( - ".vi." to "vi", ".en." to "en", ".ja." to "ja", ".ko." to "ko", - ".zh." to "zh", ".es." to "es", ".fr." to "fr", ".de." to "de", - ".it." to "it", ".pt." to "pt", ".ru." to "ru", ".ar." to "ar", - ".th." to "th", ".id." to "id", ".tr." to "tr", ".pl." to "pl", - ".all." to "", ".multi." to "" - ) - - return localePatterns.entries.find { qualifiedName.contains(it.key) }?.value - ?: "" - } - } } From 68b313d007b1f864a50d0e1dbde123eb8bb43420 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:46:51 +0200 Subject: [PATCH 004/434] Delete src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi directory --- .../site/tachiyomi/en/mistscans/MistScans.kt | 26 - .../site/tachiyomi/en/qiscan/QiScan.kt | 58 - .../tachiyomi/en/qiscan/QiScansGenreFilter.kt | 68 - .../parsers/site/tachiyomi/en/scansgg/Dto.kt | 104 -- .../site/tachiyomi/en/scansgg/Filters.kt | 44 - .../site/tachiyomi/en/scansgg/ScansGG.kt | 185 --- .../site/tachiyomi/en/vortexscan/Constants.kt | 125 -- .../site/tachiyomi/en/vortexscan/Dto.kt | 75 -- .../site/tachiyomi/en/vortexscan/Filters.kt | 83 -- .../site/tachiyomi/en/vortexscan/Helpers.kt | 67 - .../site/tachiyomi/en/vortexscan/Mapper.kt | 116 -- .../tachiyomi/en/vortexscan/VortexScan.kt | 290 ---- .../site/tachiyomi/es/mangaoni/Filters.kt | 99 -- .../site/tachiyomi/es/mangaoni/MangaOni.kt | 226 ---- .../site/tachiyomi/es/olympusscan/Dto.kt | 170 --- .../tachiyomi/es/olympusscan/OlympusScan.kt | 338 ----- .../site/tachiyomi/es/submanhwa/SubManhwa.kt | 140 -- .../parsers/site/tachiyomi/parsers/Keyoapp.kt | 391 ------ .../parsers/site/tachiyomi/parsers/Madara.kt | 1179 ----------------- .../tachiyomi/parsers/ezmanhwa/EZManhwa.kt | 160 --- .../tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt | 124 -- .../parsers/ezmanhwa/EZManhwaFilters.kt | 27 - .../parsers/galleryadults/GalleryAdults.kt | 950 ------------- .../galleryadults/GalleryAdultsFilters.kt | 31 - .../galleryadults/GalleryAdultsUtils.kt | 148 --- .../tachiyomi/pt/astratoons/Astratoons.kt | 213 --- .../site/tachiyomi/pt/astratoons/Dto.kt | 31 - .../site/tachiyomi/pt/astratoons/Filters.kt | 80 -- 28 files changed, 5548 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt deleted file mode 100644 index 795f0bcf..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt +++ /dev/null @@ -1,26 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.mistscans - -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.Keyoapp -import eu.kanade.tachiyomi.source.model.SManga -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.TachiyomiSource - -@TachiyomiSource("MISTSCANS", "Mist Scans", "en") -class MistScans : Keyoapp() { - - override fun popularMangaSelector() = ".series-splide .splide__slide" - - override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - val a = element.selectFirst("a[href]")!! - title = a.attr("title") - setUrlWithoutDomain(a.attr("abs:href")) - thumbnail_url = element.getImageUrl("*[style*=background-image]") - } - - override val name: String - get() = "Mist Scans" - override val lang: String - get() = "en" - override val baseUrl: String - get() = "https://mistscans.com" -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt deleted file mode 100644 index 522caf2b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt +++ /dev/null @@ -1,58 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.qiscan - -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwa -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaSortFilter -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaStatusFilter -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaTypeFilter -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import org.koitharu.kotatsu.parsers.TachiyomiSource - - -@TachiyomiSource("QISCANS", "QiScans", "en") -class QiScans : EZManhwa("QiScans", "https://qimanga.com", "https://api.qimanga.com/api/v1") { - - override fun headersBuilder() = super.headersBuilder() - .set("Origin", baseUrl) - .set("Sec-Fetch-Dest", "empty") - .set("Sec-Fetch-Mode", "cors") - .set("Sec-Fetch-Site", "same-site") - - // QiScans search endpoint ignores filters — only send the query when searching. - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val isSearch = query.isNotBlank() - val endpoint = if (isSearch) "$apiUrl/series/search" else "$apiUrl/series" - val url = endpoint.toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "20") - if (isSearch) { - addQueryParameter("q", query) - } else { - var sortAdded = false - for (filter in filters) { - when (filter) { - is EZManhwaSortFilter -> { - addQueryParameter("sort", filter.value) - sortAdded = true - } - is EZManhwaStatusFilter -> if (filter.value.isNotBlank()) addQueryParameter("status", filter.value) - is EZManhwaTypeFilter -> if (filter.value.isNotBlank()) addQueryParameter("type", filter.value) - is QiScansGenreFilter -> if (filter.value.isNotBlank()) addQueryParameter("genre", filter.value) - else -> {} - } - } - if (!sortAdded) addQueryParameter("sort", "latest") - } - }.build() - return GET(url, headers) - } - - override fun getFilterList() = FilterList( - EZManhwaSortFilter(), - EZManhwaStatusFilter(), - EZManhwaTypeFilter(), - QiScansGenreFilter(), - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt deleted file mode 100644 index d4c29a0b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt +++ /dev/null @@ -1,68 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.qiscan - -import eu.kanade.tachiyomi.source.model.Filter - -class QiScansGenreFilter : Filter.Select("Genre", DISPLAY) { - val value get() = VALUES[state] - - companion object { - private val ENTRIES = arrayOf( - "All" to "", - "Acting" to "acting", - "Action" to "action", - "Action (Alt)" to "action-582", - "Adventure" to "adventure", - "Adventure (Alt)" to "adventure-589", - "Apocalypse" to "apocalypce", - "Comedy" to "comedy", - "Cooking" to "cooking", - "Crazy MC" to "crazy-mc", - "Cultivation" to "cultivation", - "Drama" to "drama", - "Ecchi" to "ecchi", - "Fantasy" to "fantasy", - "Fantasy (Alt)" to "fantasy-747", - "Fight" to "fight", - "Gender Bender" to "gender-bender", - "Harem" to "harem", - "Hidden" to "hidden", - "Historical" to "historical", - "Horror" to "horror", - "Josei" to "josei", - "Live" to "live", - "Magic" to "magic", - "Manhua" to "manhua", - "Martial Arts" to "martial-arts", - "Mature" to "mature", - "Mecha" to "mecha", - "Medieval Area" to "medieval-area", - "Munchkin" to "munchkin", - "Murim" to "murim", - "Mystery" to "mystery", - "Myth" to "myth", - "Politics" to "politics", - "Psychological" to "psychological", - "Reincarnation" to "reincarnation", - "Revenge" to "revenge", - "Romance" to "romance", - "School Life" to "school-life", - "Sci-Fi" to "sci-fi", - "Seinen" to "seinen", - "Shounen" to "shounen", - "Slice of Life" to "slice-of-life", - "Sports" to "sports", - "Supernatural" to "supernatural", - "Superpower" to "superpower", - "System" to "system", - "Taming" to "taming", - "Tower" to "tower", - "Tragedy" to "tragedy", - "Urban" to "urban", - "Vampires" to "vampiers", - "Virtual Reality" to "virtual-reality", - "Wuxia" to "wuxia", - ) - val DISPLAY = ENTRIES.map { it.first }.toTypedArray() - val VALUES = ENTRIES.map { it.second }.toTypedArray() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt deleted file mode 100644 index 031a927c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt +++ /dev/null @@ -1,104 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import java.text.SimpleDateFormat - -@Serializable -class ResponseDto( - val data: T, - val meta: MetaDto? = null, -) - -@Serializable -class MetaDto( - @SerialName("has_more") val hasMore: Boolean = false, -) - -@Serializable -class SeriesDto( - private val id: Int, - private val title: String, - private val summary: String? = null, - private val cover: String? = null, - private val author: List? = null, - private val artist: List? = null, - private val tags: List? = null, - private val status: Int? = null, -) { - fun toSManga(cdnUrl: String, tagsMap: Map? = null) = SManga.create().apply { - url = id.toString() - this.title = this@SeriesDto.title - thumbnail_url = cover?.let { "$cdnUrl/covers/$it" } - if (tagsMap != null) { - description = summary - author = this@SeriesDto.author?.joinToString() - artist = this@SeriesDto.artist?.joinToString() - genre = tags?.mapNotNull { tagsMap[it] }?.joinToString() - this.status = when (this@SeriesDto.status) { - 1 -> SManga.ONGOING - 2 -> SManga.COMPLETED - 3, 4, 5 -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - } -} - -@Serializable -class ChapterDto( - private val id: Int, - private val number: Float, - private val title: String? = null, - @SerialName("created_at") private val createdAt: String? = null, - @SerialName("group_id") private val groupId: Int? = null, - private val group: GroupDto? = null, -) { - fun toSChapter(seriesIdStr: String, dateFormat: SimpleDateFormat) = SChapter.create().apply { - // SChapter.url is used as a full API path to avoid passing multiple fields later - url = "/chapter-navigation?series_id=$seriesIdStr&chapter_id=$id&group_id=${groupId ?: 0}" - name = buildString { - append("Chapter ") - append(number.toString().removeSuffix(".0")) - if (!title.isNullOrEmpty()) { - append(" - ") - append(title) - } - } - date_upload = dateFormat.tryParse(createdAt) - scanlator = group?.title - } -} - -@Serializable -class GroupDto( - val title: String? = null, -) - -@Serializable -class PageListDto( - private val chapter: ChapterDataDto? = null, -) { - fun toPages(cdnUrl: String): List { - val chapterId = chapter?.id ?: return emptyList() - return chapter.pages?.map { - Page(it.position, imageUrl = "$cdnUrl/pages/$chapterId/${it.path}") - } ?: emptyList() - } -} - -@Serializable -class ChapterDataDto( - val id: Int? = null, // Capturing the chapter id to use it in the image path - val pages: List? = null, -) - -@Serializable -class PageDto( - val position: Int, - val path: String, -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt deleted file mode 100644 index 947dddf6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt +++ /dev/null @@ -1,44 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.source.model.Filter - -class TypeFilter : Filter.Group("Type", types.map { TypeCheckBox(it.first, it.second) }) -class TypeCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -private val types = listOf( - Pair("Comic", 1), - Pair("Manga", 2), - Pair("Manhwa", 3), - Pair("Manhua", 4), - Pair("Webtoon", 5), -) - -class StatusFilter : Filter.Group("Status", statuses.map { StatusCheckBox(it.first, it.second) }) -class StatusCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -private val statuses = listOf( - Pair("Ongoing", 1), - Pair("Completed", 2), - Pair("Hiatus", 3), - Pair("Cancelled", 4), - Pair("Dropped", 5), -) - -class TagFilter : Filter.Group("Tags", tagsMap.map { TagCheckBox(it.value, it.key) }) -class TagCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -val tagsMap = mapOf( - 49 to "Regression", 48 to "Male Protagonist", 47 to "Survival", - 46 to "Avant Garde", 45 to "Award Winning", 44 to "Lolicon", - 43 to "Mahou Shoujo", 42 to "Doujinshi", 41 to "Girls Love", - 40 to "Hentai", 39 to "Mecha", 38 to "Shotacon", 37 to "Ecchi", - 36 to "Music", 35 to "Smut", 34 to "Erotica", 33 to "Adult", - 32 to "Gourmet", 31 to "Yuri", 30 to "Shoujo Ai", 29 to "Yaoi", - 28 to "Shounen Ai", 27 to "Boys Love", 26 to "Harem", 25 to "Tragedy", - 24 to "Gender Bender", 23 to "Suspense", 22 to "Psychological", - 21 to "Mature", 20 to "Horror", 19 to "Mystery", 18 to "Martial Arts", - 17 to "Sci-fi", 16 to "Adventure", 15 to "Supernatural", 14 to "Sports", - 13 to "Shounen", 12 to "Historical", 11 to "Seinen", 10 to "Action", - 9 to "Josei", 8 to "Thriller", 7 to "School Life", 6 to "Slice Of Life", - 5 to "Drama", 4 to "Comedy", 3 to "Shoujo", 2 to "Romance", 1 to "Fantasy", -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt deleted file mode 100644 index 840c6eee..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt +++ /dev/null @@ -1,185 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone -import eu.kanade.tachiyomi.network.asObservable - -@TachiyomiSource("SCANSGG", "ScansGG", "en") -class ScansGG : HttpSource() { - - override val name = "ScansGG" - override val baseUrl = "https://scans.gg" - override val lang = "en" - override val supportsLatest = true - - private val apiUrl = "https://api.scans.gg" - private val cdnUrl = "https://cdn.scans.gg/uploads" - - override fun headersBuilder() = super.headersBuilder() - .add("Origin", baseUrl) - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - // ============================== Popular ============================== - - override fun popularMangaRequest(page: Int): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder() - .addQueryParameter("limit", POPULAR_LIMIT.toString()) - .addQueryParameter("offset", ((page - 1) * POPULAR_LIMIT).toString()) - .build() - return GET(url, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val dto = response.parseAs>>() - val mangas = dto.data.map { it.toSManga(cdnUrl) } - - // The /series endpoint doesn't return the meta pagination object, - // so we check if the returned items match our limit - return MangasPage(mangas, mangas.size == POPULAR_LIMIT) - } - - // ============================== Latest =============================== - - override fun latestUpdatesRequest(page: Int): Request { - val url = "$apiUrl/chapters".toHttpUrl().newBuilder() - .addQueryParameter("page", page.toString()) - .addQueryParameter("limit", LATEST_LIMIT.toString()) - .addQueryParameter("chapters", "true") - .addQueryParameter("series_details", "true") - .addQueryParameter("group_details", "true") - .addQueryParameter("sort", "date") - .build() - return GET(url, headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val dto = response.parseAs>>() - val mangas = dto.data.map { it.toSManga(cdnUrl) } - return MangasPage(mangas, dto.meta?.hasMore == true) - } - - // ============================== Search =============================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder().apply { - addQueryParameter("limit", POPULAR_LIMIT.toString()) - addQueryParameter("offset", ((page - 1) * POPULAR_LIMIT).toString()) - if (query.isNotEmpty()) { - addQueryParameter("q", query) - } - - val types = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - val statuses = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - val tags = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - - addQueryParameter("q_type", types.joinToString(",", "[", "]")) - addQueryParameter("q_status", statuses.joinToString(",", "[", "]")) - addQueryParameter("q_tags", tags.joinToString(",", "[", "]")) - }.build() - - return GET(url, headers) - } - - override fun searchMangaParse(response: Response) = popularMangaParse(response) - - // ============================== Details ============================== - - override fun getMangaUrl(manga: SManga): String = "$baseUrl/series/${manga.url}" - - override fun mangaDetailsRequest(manga: SManga): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder() - .addQueryParameter("id", manga.url) - .addQueryParameter("trackers", "true") - .addQueryParameter("sources", "true") - .build() - return GET(url, headers) - } - - override fun mangaDetailsParse(response: Response): SManga { - val dto = response.parseAs>() - return dto.data.toSManga(cdnUrl, tagsMap) - } - - // ============================= Chapters ============================== - -// ============================= Chapters ============================== - - override fun getChapterUrl(chapter: SChapter): String { - val url = (apiUrl + chapter.url).toHttpUrl() - val seriesId = url.queryParameter("series_id") ?: return baseUrl - return "$baseUrl/series/$seriesId" - } - - override fun fetchChapterList(manga: SManga): Observable> = - client.newCall(chapterListRequest(manga)) - .asObservable() - .map(::chapterListParseApi) - - override fun chapterListRequest(manga: SManga): Request { - val url = "$apiUrl/chapters".toHttpUrl().newBuilder() - .addQueryParameter("series_id", manga.url) - .addQueryParameter("limit", CHAPTER_LIMIT.toString()) - .addQueryParameter("page", "1") - .addQueryParameter("group_details", "true") - .build() - - return GET(url, headers) - } - - private fun chapterListParseApi(response: Response): List { - val mangaId = response.request.url.queryParameter("series_id") - ?: throw Exception("Missing series id") - - val dto = response.parseAs>>() - - return dto.data.map { - it.toSChapter(mangaId, dateFormat) - } - } - - override fun chapterListParse(response: Response): List = - throw UnsupportedOperationException() - // =============================== Pages =============================== - - override fun pageListRequest(chapter: SChapter): Request = GET(apiUrl + chapter.url, headers) - - override fun pageListParse(response: Response): List { - val dto = response.parseAs>() - return dto.data.toPages(cdnUrl) - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ============================== Filters ============================== - - override fun getFilterList() = FilterList( - TypeFilter(), - StatusFilter(), - TagFilter(), - ) - - companion object { - private const val POPULAR_LIMIT = 21 - private const val LATEST_LIMIT = 14 - private const val CHAPTER_LIMIT = 100 - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt deleted file mode 100644 index 0dfe0ea5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt +++ /dev/null @@ -1,125 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import java.util.Locale - -const val PER_PAGE = 18 - -const val SERIES_PATH_SEGMENT = "series" - -const val API_QUERY_PATH = "/api/query" -const val API_POST_PATH = "/api/post" -const val API_CHAPTER_PATH = "/api/chapter" - -const val STATUS_FILTER_KEY = "seriesStatus" -val STATUS_OPTIONS: Options = listOf( - "All" to "", - "Ongoing" to "ONGOING", - "Completed" to "COMPLETED", - "Cancelled" to "CANCELLED", - "Dropped" to "DROPPED", - "Coming soon" to "COMING_SOON", - "Mass released" to "MASS_RELEASED", -) - -const val TYPE_FILTER_KEY = "seriesType" -val TYPE_OPTIONS: Options = listOf( - "All" to "", - "Manga" to "MANGA", - "Manhua" to "MANHUA", - "Manhwa" to "MANHWA", -) - -const val SORT_FILTER_KEY = "orderBy" -val SORT_OPTIONS: Options = listOf( - "Last chapter added" to "lastChapterAddedAt", - "Views" to "totalViews", - "Created at" to "createdAt", - "Chapter count" to "chaptersCount", - "Alphabetical" to "postTitle", -) - -const val SORT_DIRECTION_FILTER_KEY = "orderDirection" -val SORT_DIRECTION_OPTIONS: Options = listOf( - "Descending" to "desc", - "Ascending" to "asc", -) - -const val GENRE_INCLUDE_FILTER_KEY = "genreIds" -const val GENRE_EXCLUDE_FILTER_KEY = "excludedGenreIds" - -val GENRE_OPTIONS: Options = listOf( - "Action" to "1", - "Drama" to "2", - "Shounen" to "3", - "Sports" to "4", - "Manhwa" to "5", - "Martial Arts" to "6", - "Comedy" to "7", - "Fantasy" to "8", - "Horror" to "9", - "Seinen" to "10", - "Supernatural" to "11", - "Mature" to "12", - "Adventure" to "13", - "Monsters" to "14", - "System" to "15", - "Reincarnation" to "16", - "Revenge" to "17", - "Slice Of Life" to "18", - "Historical" to "19", - "Romance" to "20", - "Josei" to "21", - "Shoujo" to "22", - "School Life" to "23", - "terror" to "24", - "elf" to "25", - "shojo" to "26", - "Video Games" to "27", - "Fantas" to "28", - "WEB COMIC" to "29", - "Webtoons" to "30", - "Murim" to "31", - "Restaurant" to "32", - "Webtoon" to "33", - "+100 Chapter" to "34", - "Tower" to "35", - "Legendary " to "36", - "Dungeons" to "37", - "bully" to "38", - "orphan" to "39", - "Sci-Fi" to "40", - "Gore" to "41", - "Isekai" to "42", - "magic" to "43", - "blood" to "44", - "war" to "45", - "magic and sword" to "46", - "academy" to "47", - "violence" to "48", - "Harem" to "49", - "Myth" to "50", - "OverpoweredMC" to "51", - "TerritoryManagement" to "52", - "Swordsman" to "53", - "Necromancer" to "54", - "Mage" to "55", - "JackOfAllTrades" to "56", - "Artifacts" to "57", - "CharacterGrowth" to "58", - "Mercenary" to "59", - "Elementals" to "60", - "Genius" to "61", - "Psychological" to "62", - "Tragedy" to "63", - "Gender Bender" to "64", -).sortedBy { it.first.trim().lowercase(Locale.ROOT) } - -const val POPULAR_ORDER_BY = "totalViews" -const val LATEST_ORDER_BY = "lastChapterAddedAt" -const val ORDER_DESC = "desc" - -const val SHOW_LOCKED_CHAPTERS_PREF_KEY = "pref_show_locked_chapters" -const val SHOW_LOCKED_CHAPTERS_DEFAULT = false - -const val MISSING_TITLE_MESSAGE = "Series title is missing" -const val LOCKED_CHAPTER_MESSAGE = "Unlock chapter in WebView" \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt deleted file mode 100644 index 6b296ff6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt +++ /dev/null @@ -1,75 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.JsonPrimitive - -@Serializable -class SearchResponseDto( - val posts: List, - val totalCount: Int, -) - -@Serializable -class PostSummaryDto( - val id: Int, - val slug: String, - val postTitle: String, - val featuredImage: String? = null, - val seriesStatus: String? = null, - val genres: List = emptyList(), -) - -@Serializable -class GenreDto( - val name: String, -) - -@Serializable -class PostResponseDto( - val post: PostDto, -) - -@Serializable -class PostDto( - val id: Int, - val slug: String, - val postTitle: String, - val postContent: String? = null, - val alternativeTitles: String? = null, - val author: String? = null, - val artist: String? = null, - val featuredImage: String? = null, - val seriesType: String? = null, - val seriesStatus: String? = null, - val genres: List = emptyList(), - val chapters: List = emptyList(), -) - -@Serializable -class PostChapterDto( - val id: Int, - val slug: String, - val number: JsonPrimitive, - val title: String? = null, - val createdAt: String, - val isLocked: Boolean? = null, - val isAccessible: Boolean? = null, -) - -@Serializable -class ChapterResponseDto( - val chapter: ChapterDto, -) - -@Serializable -class ChapterDto( - val isLocked: Boolean? = null, - val isAccessible: Boolean? = null, - val images: List = emptyList(), -) - -@Serializable -class ChapterImageDto( - val url: String, - val order: Int? = null, -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt deleted file mode 100644 index 217083bc..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt +++ /dev/null @@ -1,83 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import eu.kanade.tachiyomi.source.model.Filter -import okhttp3.HttpUrl - -typealias Options = List> - -interface UrlPartFilter { - fun addUrlParameter(url: HttpUrl.Builder) -} - -abstract class SelectFilter( - name: String, - private val urlParameter: String, - private val options: Options, - defaultValue: String? = null, -) : Filter.Select( - name, - options.map { it.first }.toTypedArray(), - options.indexOfFirst { it.second == defaultValue }.takeIf { it != -1 } ?: 0, -), - UrlPartFilter { - override fun addUrlParameter(url: HttpUrl.Builder) { - options[state].second.takeIf { it.isNotBlank() }?.let { - url.addQueryParameter(urlParameter, it) - } - } -} - -class StatusFilter( - title: String, - key: String, - options: Options, -) : SelectFilter(title, key, options) - -class TypeFilter( - title: String, - key: String, - options: Options, -) : SelectFilter(title, key, options) - -class SortFilter( - title: String, - key: String, - options: Options, - defaultValue: String? = null, -) : SelectFilter(title, key, options, defaultValue) - -class SortDirectionFilter( - title: String, - key: String, - options: Options, - defaultValue: String? = null, -) : SelectFilter(title, key, options, defaultValue) - -class GenreFilter( - name: String, - genres: Options, -) : Filter.Group( - name, - genres.map { GenreValue(it.first, it.second) }, -), - UrlPartFilter { - override fun addUrlParameter(url: HttpUrl.Builder) { - val included = state - .filter { it.state == Filter.TriState.STATE_INCLUDE } - .map { it.id } - - val excluded = state - .filter { it.state == Filter.TriState.STATE_EXCLUDE } - .map { it.id } - - if (included.isNotEmpty()) { - url.addQueryParameter(GENRE_INCLUDE_FILTER_KEY, included.joinToString(",")) - } - - if (excluded.isNotEmpty()) { - url.addQueryParameter(GENRE_EXCLUDE_FILTER_KEY, excluded.joinToString(",")) - } - } -} - -class GenreValue(name: String, val id: String) : Filter.TriState(name) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt deleted file mode 100644 index 4b6e49e2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt +++ /dev/null @@ -1,67 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull - -class DeepLink( - val mangaSlug: String, - val chapterSlug: String?, -) - -fun parseDeepLink(query: String, baseHost: String): DeepLink? { - val url = query.toHttpUrlOrNull() ?: return null - if (url.host != baseHost) return null - - val pathSegments = url.pathSegments - if (pathSegments.getOrNull(0) != SERIES_PATH_SEGMENT) return null - - val mangaSlug = pathSegments.getOrNull(1) - ?.takeIf { it.isNotBlank() } - ?: return null - - val chapterSlug = pathSegments.getOrNull(2) - ?.takeIf { it.isNotBlank() } - - return DeepLink(mangaSlug, chapterSlug) -} - -fun extractMangaSlug(rawUrl: String): String { - val pathSegments = rawUrl.substringBefore('#').trim('/').split('/').filter { it.isNotBlank() } - - return when { - pathSegments.size >= 2 && pathSegments[0] == SERIES_PATH_SEGMENT -> pathSegments[1] - pathSegments.isNotEmpty() -> pathSegments[0] - else -> throw Exception("Invalid manga url") - } -} - -fun extractChapterSlugs(rawUrl: String): Pair? { - val pathSegments = rawUrl.substringBefore('#').trim('/').split('/').filter { it.isNotBlank() } - - return when { - pathSegments.size >= 3 && pathSegments[0] == SERIES_PATH_SEGMENT -> pathSegments[1] to pathSegments[2] - pathSegments.size >= 2 -> pathSegments[pathSegments.size - 2] to pathSegments.last() - else -> null - } -} - -fun buildSlugSearchTerms(slug: String): List { - val withSpaces = slug.replace('-', ' ') - val withoutApostrophe = withSpaces.replace("'", " ") - - return listOf( - withSpaces, - withoutApostrophe, - collapseSpaces(withoutApostrophe), - slug, - ).map { it.trim() } - .filter { it.isNotBlank() } - .distinct() -} - -private fun collapseSpaces(text: String): String { - var value = text - while (value.contains(" ")) { - value = value.replace(" ", " ") - } - return value -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt deleted file mode 100644 index 3cde0ce5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt +++ /dev/null @@ -1,116 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import org.jsoup.Jsoup -import java.text.SimpleDateFormat - -fun PostSummaryDto.toSMangaSummary(): SManga { - val cleanTitle = postTitle.trim() - if (cleanTitle.isEmpty()) throw Exception(MISSING_TITLE_MESSAGE) - - return SManga.create().apply { - url = "$slug#$id" - title = cleanTitle - thumbnail_url = featuredImage - genre = genres.joinToString { it.name } - status = mapStatus(seriesStatus) - } -} - -fun PostDto.toSMangaDetailsModel(): SManga { - val cleanTitle = postTitle.trim() - if (cleanTitle.isEmpty()) throw Exception(MISSING_TITLE_MESSAGE) - - return SManga.create().apply { - url = "$slug#$id" - title = cleanTitle - description = buildDescription(this@toSMangaDetailsModel) - author = this@toSMangaDetailsModel.author?.trim()?.takeUnless { it.isEmpty() } - artist = this@toSMangaDetailsModel.artist?.trim()?.takeUnless { it.isEmpty() } - genre = buildGenre(seriesType, genres) - status = mapStatus(seriesStatus) - thumbnail_url = featuredImage - initialized = true - } -} - -fun PostChapterDto.toSChapterModel(mangaSlug: String, dateFormat: SimpleDateFormat): SChapter { - val rawNumber = number.toString().trim('"').trim() - val fallbackNumber = slug.substringAfter("chapter-", "") - val chapterNumberText = if (rawNumber.isNotEmpty()) rawNumber else fallbackNumber - val chapterTitle = title?.trim().orEmpty() - - return SChapter.create().apply { - url = "$mangaSlug/$slug#$id" - name = buildString { - if (isAccessible == false || isLocked == true) { - append("🔒 ") - } - - append("Chapter") - if (chapterNumberText.isNotEmpty()) { - append(' ') - append(chapterNumberText) - } - - if (chapterTitle.isNotEmpty()) { - append(" - ") - append(chapterTitle) - } - } - chapterNumberText.toFloatOrNull()?.let { chapter_number = it } - date_upload = dateFormat.tryParse(createdAt) - } -} - -private fun buildDescription(post: PostDto): String? { - val synopsis = post.postContent - ?.takeIf { it.isNotBlank() } - ?.replace("
", "\n") - ?.replace("
", "\n") - ?.replace("
", "\n") - ?.let { Jsoup.parse(it).text() } - ?.trim() - ?.takeIf { it.isNotEmpty() } - - val alternativeTitles = post.alternativeTitles - ?.lineSequence() - ?.map { it.trim() } - ?.filter { it.isNotEmpty() } - ?.toList() - ?.takeIf { it.isNotEmpty() } - ?.joinToString("\n") - - return buildString { - synopsis?.let { append(it) } - alternativeTitles?.let { - if (isNotEmpty()) append("\n\n") - append("Alternative titles:\n") - append(it) - } - }.takeIf { it.isNotBlank() } -} - -private fun buildGenre(seriesType: String?, genres: List): String? { - val values = mutableListOf() - - when (seriesType?.uppercase()) { - "MANGA" -> values += "Manga" - "MANHUA" -> values += "Manhua" - "MANHWA" -> values += "Manhwa" - } - - genres.mapTo(values) { it.name } - - val out = values.distinct().joinToString() - return out.takeIf { it.isNotBlank() } -} - -private fun mapStatus(status: String?): Int = when (status) { - "ONGOING", "COMING_SOON", "MASS_RELEASED" -> SManga.ONGOING - "COMPLETED" -> SManga.COMPLETED - "CANCELLED", "DROPPED" -> SManga.CANCELLED - else -> SManga.UNKNOWN -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt deleted file mode 100644 index 188e56c7..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt +++ /dev/null @@ -1,290 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("VORTEXSCANS", "Vortex Scans", "en") -class VortexScans : - HttpSource(), - ConfigurableSource { - - private lateinit var summaryOn: String - private lateinit var summaryOff: String - override val name = "Vortex Scans" - override val baseUrl = "https://vortexscans.org" - private val apiUrl = "https://api.vortexscans.org" - override val lang = "en" - override val supportsLatest = true - - private val preferences by getPreferencesLazy() - - private val baseHost = baseUrl.toHttpUrl().host - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) - private val showLockedChapters - get() = preferences.getBoolean(SHOW_LOCKED_CHAPTERS_PREF_KEY, SHOW_LOCKED_CHAPTERS_DEFAULT) - - override fun headersBuilder() = super.headersBuilder() - .set("Referer", "$baseUrl/") - - private val apiHeaders: Headers by lazy { - headersBuilder() - .set("Accept", "application/json, text/plain, */*") - .set("Origin", baseUrl) - .build() - } - - // ========================= Popular ========================= - - override fun popularMangaRequest(page: Int): Request = queryRequest( - page = page, - query = "", - orderBy = POPULAR_ORDER_BY, - orderDirection = ORDER_DESC, - ) - - override fun popularMangaParse(response: Response): MangasPage = searchMangaParse(response) - - // ========================= Latest ========================= - - override fun latestUpdatesRequest(page: Int): Request = queryRequest( - page = page, - query = "", - orderBy = LATEST_ORDER_BY, - orderDirection = ORDER_DESC, - ) - - override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response) - - // ========================= Search ========================= - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (page == 1) { - val deepLink = parseDeepLink(query, baseHost) - if (deepLink != null) { - val summary = findPostBySlug(deepLink.mangaSlug) - ?: throw Exception("Series not found") - - val manga = summary.toSMangaSummary() - - return fetchMangaDetails(manga) - .map { MangasPage(listOf(it), false) } - } - } - - return super.fetchSearchManga(page, query, filters) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = queryRequest( - page = page, - query = query.trim(), - filters = filters.filterIsInstance(), - ) - - override fun searchMangaParse(response: Response): MangasPage { - val payload = response.parseAs() - val currentPage = response.request.url.queryParameter("page")?.toIntOrNull() ?: 1 - val hasNextPage = payload.totalCount > (currentPage * PER_PAGE) - - val mangas = payload.posts.map { it.toSMangaSummary() } - return MangasPage(mangas, hasNextPage) - } - - // ========================= Filters ========================= - - override fun getFilterList() = FilterList( - StatusFilter("Status", STATUS_FILTER_KEY, STATUS_OPTIONS), - TypeFilter("Type", TYPE_FILTER_KEY, TYPE_OPTIONS), - SortFilter("Sort by", SORT_FILTER_KEY, SORT_OPTIONS), - SortDirectionFilter("Sort direction", SORT_DIRECTION_FILTER_KEY, SORT_DIRECTION_OPTIONS), - GenreFilter("Genres", GENRE_OPTIONS), - ) - - // ========================= Details ========================= - - override fun getMangaUrl(manga: SManga): String { - val slug = extractMangaSlug(manga.url) - return "$baseUrl/$SERIES_PATH_SEGMENT/$slug" - } - - override fun mangaDetailsRequest(manga: SManga): Request = postRequest(resolvePostId(manga)) - - override fun mangaDetailsParse(response: Response): SManga { - val payload = response.parseAs() - return payload.post.toSMangaDetailsModel() - } - - // ========================= Chapters ========================= - - override fun getChapterUrl(chapter: SChapter): String { - val slugs = extractChapterSlugs(chapter.url) - ?: return "$baseUrl/$SERIES_PATH_SEGMENT/${chapter.url.substringBefore('#').trim('/')}" - - return "$baseUrl/$SERIES_PATH_SEGMENT/${slugs.first}/${slugs.second}" - } - - override fun chapterListRequest(manga: SManga): Request = postRequest(resolvePostId(manga)) - - override fun chapterListParse(response: Response): List { - val payload = response.parseAs() - val mangaSlug = payload.post.slug - - return payload.post.chapters - .filter { chapter -> - showLockedChapters || (chapter.isAccessible != false && chapter.isLocked != true) - } - .map { it.toSChapterModel(mangaSlug, dateFormat) } - } - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_LOCKED_CHAPTERS_PREF_KEY - title = "Show inaccessible chapters" - summaryOn = "Locked/inaccessible chapters will be shown in the chapter list." - summaryOff = "Locked/inaccessible chapters will be hidden from the chapter list." - setDefaultValue(SHOW_LOCKED_CHAPTERS_DEFAULT) - }.also(screen::addPreference) - } - - // ========================= Pages ========================= - - override fun pageListRequest(chapter: SChapter): Request = chapterRequest(resolveChapterId(chapter)) - - override fun pageListParse(response: Response): List { - val payload = response.parseAs() - val chapter = payload.chapter - - if (chapter.isAccessible == false || chapter.isLocked == true) { - throw Exception(LOCKED_CHAPTER_MESSAGE) - } - - val images = chapter.images.sortedBy { it.order ?: Int.MAX_VALUE } - if (images.isEmpty()) { - throw Exception("No pages found") - } - - return images.mapIndexed { index, image -> - Page(index = index, imageUrl = image.url) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - private fun queryRequest( - page: Int, - query: String, - orderBy: String? = null, - orderDirection: String? = null, - filters: List = emptyList(), - ): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_QUERY_PATH) - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", PER_PAGE.toString()) - addQueryParameter("searchTerm", query) - - filters.forEach { it.addUrlParameter(this) } - - if (!orderBy.isNullOrBlank()) { - setQueryParameter(SORT_FILTER_KEY, orderBy) - } - - if (!orderDirection.isNullOrBlank()) { - setQueryParameter(SORT_DIRECTION_FILTER_KEY, orderDirection) - } - }.build() - - return GET(url, apiHeaders) - } - - private fun postRequest(postId: Int): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_POST_PATH) - addQueryParameter("postId", postId.toString()) - }.build() - - return GET(url, apiHeaders) - } - - private fun chapterRequest(chapterId: Int): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_CHAPTER_PATH) - addQueryParameter("chapterId", chapterId.toString()) - }.build() - - return GET(url, apiHeaders) - } - - private fun resolvePostId(manga: SManga): Int { - manga.url.substringAfter('#', "").toIntOrNull()?.let { return it } - - val slug = extractMangaSlug(manga.url) - val summary = findPostBySlug(slug) - return summary?.id ?: throw Exception("Unable to resolve series id") - } - - private fun resolveChapterId(chapter: SChapter): Int { - chapter.url.substringAfter('#', "").toIntOrNull()?.let { return it } - - val slugs = extractChapterSlugs(chapter.url) - ?: throw Exception("Unable to resolve chapter id") - - val summary = findPostBySlug(slugs.first) - ?: throw Exception("Unable to resolve series id") - - return try { - client.newCall(postRequest(summary.id)).execute().use { response -> - if (!response.isSuccessful) { - throw Exception("Unable to resolve chapter id") - } - - val payload = response.parseAs() - val chapterDto = payload.post.chapters.firstOrNull { it.slug == slugs.second } - ?: throw Exception("Unable to resolve chapter id") - - chapterDto.id - } - } catch (_: Exception) { - throw Exception("Unable to resolve chapter id") - } - } - - private fun findPostBySlug(slug: String): PostSummaryDto? { - val normalizedSlug = slug.trim().trim('/').lowercase() - val terms = buildSlugSearchTerms(normalizedSlug) - - for (term in terms) { - try { - client.newCall(queryRequest(page = 1, query = term)).execute().use { response -> - if (response.isSuccessful) { - val payload = response.parseAs() - val post = payload.posts.firstOrNull { it.slug.equals(normalizedSlug, ignoreCase = true) } - if (post != null) { - return post - } - } - } - } catch (_: Exception) { - } - } - - return null - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt deleted file mode 100644 index 6ad673c8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt +++ /dev/null @@ -1,99 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.mangaoni - -import eu.kanade.tachiyomi.source.model.Filter - -open class UriPartFilter(displayName: String, private val vals: Array>) : Filter.Select(displayName, vals.map { it.first }.toTypedArray()) { - fun toUriPart() = vals[state].second -} - -class StatusFilter : - UriPartFilter( - "Estado", - arrayOf( - Pair("Estado", "false"), - Pair("En desarrollo", "1"), - Pair("Completo", "0"), - ), - ) - -class TypeFilter : - UriPartFilter( - "Tipo", - arrayOf( - Pair("Todo", "false"), - Pair("Mangas", "0"), - Pair("Manhwas", "1"), - Pair("One Shot", "2"), - Pair("Manhuas", "3"), - Pair("Novelas", "4"), - ), - ) - -class GenreFilter : - UriPartFilter( - "Géneros", - arrayOf( - Pair("Todos", "false"), - Pair("Comedia", "1"), - Pair("Drama", "2"), - Pair("Acción", "3"), - Pair("Escolar", "4"), - Pair("Romance", "5"), - Pair("Ecchi", "6"), - Pair("Aventura", "7"), - Pair("Shōnen", "8"), - Pair("Shōjo", "9"), - Pair("Deportes", "10"), - Pair("Psicológico", "11"), - Pair("Fantasía", "12"), - Pair("Mecha", "13"), - Pair("Gore", "14"), - Pair("Yaoi", "15"), - Pair("Yuri", "16"), - Pair("Misterio", "17"), - Pair("Sobrenatural", "18"), - Pair("Seinen", "19"), - Pair("Ficción", "20"), - Pair("Harem", "21"), - Pair("Webtoon", "25"), - Pair("Histórico", "27"), - Pair("Músical", "30"), - Pair("Ciencia ficción", "31"), - Pair("Shōjo-ai", "32"), - Pair("Josei", "33"), - Pair("Magia", "34"), - Pair("Artes Marciales", "35"), - Pair("Horror", "36"), - Pair("Demonios", "37"), - Pair("Supervivencia", "38"), - Pair("Recuentos de la vida", "39"), - Pair("Shōnen ai", "40"), - Pair("Militar", "41"), - Pair("Eroge", "42"), - Pair("Isekai", "43"), - ), - ) - -class AdultContentFilter : - UriPartFilter( - "Contenido +18", - arrayOf( - Pair("Mostrar todo", "false"), - Pair("Mostrar solo +18", "1"), - Pair("No mostrar +18", "0"), - ), - ) - -class SortBy : - Filter.Sort( - "Ordenar por", - arrayOf("Visitas", "Recientes", "Alfabético"), - Selection(0, false), - ) { - fun toUriPart() = when (state?.index) { - 0 -> "visitas" - 1 -> "id" - 2 -> "nombre" - else -> "visitas" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt deleted file mode 100644 index 874747b2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt +++ /dev/null @@ -1,226 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.mangaoni - -import android.content.SharedPreferences -import android.net.Uri -import android.util.Base64 -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.tryParse -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import java.nio.charset.Charset -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("MANGAONI", "MangaOni", "es") -class MangaOni : - HttpSource(), - ConfigurableSource { - - override val name = "MangaOni" - - override val id: Long = 2202687009511923782 - - override val baseUrl = "https://manga-oni.com" - - override val lang = "es" - - override val supportsLatest = true - - private val preferences: SharedPreferences by getPreferencesLazy() - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - - override fun popularMangaRequest(page: Int) = GET( - url = "$baseUrl/directorio?genero=false&estado=false&filtro=visitas&tipo=false&adulto=${if (hideNSFWContent()) "0" else "false"}&orden=desc&p=$page", - headers = headers, - ) - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val mangas = document.select("#article-div a").map { element -> - SManga.create().apply { - setUrlWithoutDomain(element.attr("abs:href")) - thumbnail_url = element.select("img").attr("abs:data-src") - title = element.select("div:eq(1)").text() - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/recientes?p=$page", headers) - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - - val mangas = document.select("div._1bJU3").map { element -> - SManga.create().apply { - thumbnail_url = element.select("img").attr("abs:data-src") - element.selectFirst("a[data-test=latest-update-name]")?.also { - title = it.text() - setUrlWithoutDomain(it.attr("abs:href")) - } - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val uri = Uri.parse("$baseUrl/${if (query.isNotBlank()) "buscar" else "directorio"}").buildUpon() - - if (query.isNotBlank()) { - uri.appendQueryParameter("q", query) - } else { - val adultContent = filters.firstInstanceOrNull()?.toUriPart() - ?: if (hideNSFWContent()) "0" else "false" - uri.appendQueryParameter("adulto", adultContent) - - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("estado", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("tipo", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("genero", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { - uri.appendQueryParameter("filtro", it.toUriPart()) - uri.appendQueryParameter("orden", if (it.state?.ascending == true) "asc" else "desc") - } - } - uri.appendQueryParameter("p", page.toString()) - return GET(uri.toString(), headers) - } - - override fun searchMangaParse(response: Response): MangasPage { - if (!response.isSuccessful) throw Exception("Búsqueda fallida ${response.code}") - - val document = response.asJsoup() - - val mangas = if (document.location().startsWith("$baseUrl/directorio")) { - document.select("#article-div a").map { element -> - SManga.create().apply { - setUrlWithoutDomain(element.attr("abs:href")) - thumbnail_url = element.select("img").attr("abs:data-src") - title = element.select("div:eq(1)").text() - } - } - } else { - document.select("#article-div > div").map { element -> - SManga.create().apply { - thumbnail_url = element.select("img").attr("abs:src") - title = element.select("a").firstOrNull { it.text().isNotBlank() }?.text() - ?: throw Exception("Title not found") - element.selectFirst("a")?.also { - setUrlWithoutDomain(it.attr("abs:href")) - } - } - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun mangaDetailsParse(response: Response): SManga { - val document = response.asJsoup() - - return SManga.create().apply { - title = document.selectFirst("h1")?.text() ?: throw Exception("Title not found") - thumbnail_url = document.select("img[src*=cover]").attr("abs:src") - description = document.select("div#sinopsis").lastOrNull()?.ownText() - author = document.select("div#info-i").text().let { - if (it.contains("Autor", true)) { - it.substringAfter("Autor:").substringBefore("Fecha:").trim() - } else { - "N/A" - } - } - artist = author - genre = document.select("div#categ a").joinToString(", ") { it.text() } - status = when (document.selectFirst("strong:contains(Estado) + span")?.text()) { - "En desarrollo" -> SManga.ONGOING - "Finalizado" -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - } - } - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - - return document.select("div#c_list a").map { element -> - SChapter.create().apply { - name = element.text() - setUrlWithoutDomain(element.attr("abs:href")) - chapter_number = element.select("span").attr("data-num").toFloatOrNull() ?: -1f - date_upload = dateFormat.tryParse(element.select("span").attr("datetime")) - } - } - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - - val encoded = document.selectFirst("script:containsData(unicap)") - ?.data()?.substringAfter("'")?.substringBefore("'") - ?: throw Exception("unicap not found") - val drop = encoded.length % 4 - val decoded = Base64.decode(encoded.dropLast(drop), Base64.DEFAULT).toString(Charset.defaultCharset()) - val path = decoded.substringBefore("||") - - return decoded.substringAfter("[").substringBefore("]").split(",").mapIndexed { i, file -> - Page(i, imageUrl = path + file.removeSurrounding("\"")) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - override fun getFilterList(): FilterList { - val filterList = mutableListOf( - Filter.Header("NOTA: Se ignoran si se usa el buscador"), - Filter.Separator(), - SortBy(), - StatusFilter(), - TypeFilter(), - GenreFilter(), - ) - - if (!hideNSFWContent()) { - filterList.add(AdultContentFilter()) - } - return FilterList(filterList) - } - - override fun setupPreferenceScreen(screen: androidx.preference.PreferenceScreen) { - val contentPref = android.preference.CheckBoxPreference(screen.context).apply { - key = CONTENT_PREF - title = CONTENT_PREF_TITLE - summary = CONTENT_PREF_SUMMARY - setDefaultValue(CONTENT_PREF_DEFAULT_VALUE) - } - } - - private fun hideNSFWContent(): Boolean = preferences.getBoolean(CONTENT_PREF, CONTENT_PREF_DEFAULT_VALUE) - - companion object { - private const val CONTENT_PREF = "showNSFWContent" - private const val CONTENT_PREF_TITLE = "Ocultar contenido +18" - private const val CONTENT_PREF_SUMMARY = "Ocultar el contenido erótico en mangas populares y filtros, no funciona en los mangas recientes ni búsquedas textuales." - private const val CONTENT_PREF_DEFAULT_VALUE = false - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt deleted file mode 100644 index beef1a42..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt +++ /dev/null @@ -1,170 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.olympusscan - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import java.text.ParseException -import java.text.SimpleDateFormat - -@Serializable -class RankingDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class PayloadSeriesDto(val data: PayloadSeriesDataDto) - -@Serializable -class PayloadSeriesDataDto( - val series: SeriesDto, -) - -@Serializable -class SeriesDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class PayloadMangaDto(val data: List) - -@Serializable -class MangaDto( - val id: Int, - val name: String, - val slug: String, - private val cover: String? = null, - val type: String? = null, - private val summary: String? = null, - private val status: MangaStatusDto? = null, - private val genres: List? = null, -) { - fun toSManga() = SManga.create().apply { - title = name - url = id.toString() - thumbnail_url = cover - } - - fun toSMangaDetails() = toSManga().apply { - description = summary - status = parseStatus() - genre = genres?.joinToString { it.name.trim() } - } - - private fun parseStatus(): Int { - val status = this.status ?: return SManga.UNKNOWN - return when (status.id) { - 1 -> SManga.ONGOING - 3 -> SManga.ON_HIATUS - 4 -> SManga.COMPLETED - 5 -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } -} - -@Serializable -class NewChaptersDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class LatestMangaDto( - val id: Int, - private val name: String, - val slug: String, - private val cover: String? = null, - val type: String? = null, -) { - fun toSManga() = SManga.create().apply { - title = name - url = id.toString() - thumbnail_url = cover - } -} - -@Serializable -class MangaDetailDto( - var data: MangaDto, -) - -@Serializable -class PayloadChapterDto(var data: List, val meta: MetaDto) - -@Serializable -class ChapterDto( - private val id: Int, - private val name: String, - @SerialName("published_at") private val date: String, -) { - fun toSChapter(mangaId: String, dateFormat: SimpleDateFormat) = SChapter.create().apply { - name = "Capitulo ${this@ChapterDto.name}" - url = "$mangaId/$id" - date_upload = try { - dateFormat.parse(date)!!.time - } catch (e: ParseException) { - 0L - } - } -} - -@Serializable -class MetaDto(val total: Int) - -@Serializable -class PayloadPagesDto(val chapter: PageDto) - -@Serializable -class PageDto(val pages: List) - -@Serializable -class MangaStatusDto( - val id: Int, -) - -@Serializable -class GenresStatusesDto( - val genres: List, - val statuses: List, -) - -@Serializable -class FilterDto( - val id: Int, - val name: String, -) - -@Serializable -class BookmarksWrapperDto( - private val data: List = emptyList(), - val meta: BookmarksMetaDto, -) { - fun getBookmarks() = data.filter { it.type == "comic" && it.id != null && it.slug != null } -} - -@Serializable -class BookmarkDto( - val id: Int?, - val slug: String?, - val type: String?, -) - -@Serializable -class BookmarksMetaDto( - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt deleted file mode 100644 index 823932cd..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt +++ /dev/null @@ -1,338 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.olympusscan - -import android.content.SharedPreferences -import android.widget.Toast -import androidx.preference.EditTextPreference -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import org.koitharu.kotatsu.parsers.util.rateLimit -import keiyoushi.utils.getPreferences -import keiyoushi.utils.parseAs -import keiyoushi.utils.toJsonString -import kotlinx.serialization.SerializationException -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone -import kotlin.time.Duration.Companion.seconds - -@TachiyomiSource("OLYMPUSSCAN", "Olympus Scanlation", "es") -class OlympusScan : - HttpSource(), - ConfigurableSource { - private val fetchedDomainUrlHost by lazy { fetchedDomainUrl.toHttpUrl().host } - private val apiBaseUrlHost by lazy { apiBaseUrl.toHttpUrl().host } - - override val versionId = 3 - private val isCi = System.getenv("CI") == "true" - - override val baseUrl: String get() = when { - isCi -> defaultBaseUrl - else -> preferences.prefBaseUrl - } - - private val defaultBaseUrl: String = "https://olympusxyz.com" - - private val fetchedDomainUrl: String by lazy { - if (!preferences.fetchDomainPref()) return@lazy preferences.prefBaseUrl - try { - val initClient = network.client - val headers = super.headersBuilder().build() - val document = initClient.newCall(GET("https://olympus.pages.dev", headers)).execute().asJsoup() - val domain = document.selectFirst("meta[property=og:url]")?.attr("content") - ?: return@lazy preferences.prefBaseUrl - val host = initClient.newCall(GET(domain, headers)).execute().request.url.host - val newDomain = "https://$host" - preferences.prefBaseUrl = newDomain - newDomain - } catch (_: Exception) { - preferences.prefBaseUrl - } - } - - private val apiBaseUrl by lazy { - fetchedDomainUrl.replace("https://", "https://panel.") - } - - override val lang: String = "es" - override val name: String = "Olympus Scanlation" - - override val supportsLatest: Boolean = true - - private val preferences: SharedPreferences = getPreferences { - this.getString(DEFAULT_BASE_URL_PREF, null).let { domain -> - if (domain != defaultBaseUrl) { - this.edit() - .putString(BASE_URL_PREF, defaultBaseUrl) - .putString(DEFAULT_BASE_URL_PREF, defaultBaseUrl) - .apply() - } - } - } - - override val client by lazy { - val client = network.client.newBuilder() -// .rateLimit(1, 2.seconds) { it.host == fetchedDomainUrlHost } -// .rateLimit(2, 1.seconds) { it.host == apiBaseUrlHost } - .build() - - return@lazy client - } - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - @Volatile - private var seriesList: List = emptyList() - - @Volatile - private var lastFetchTime: Long = 0L - - @Synchronized - private fun fetchSeriesList() { - val now = System.currentTimeMillis() - - if (seriesList.isNotEmpty() && (now - lastFetchTime) < CACHE_DURATION_MS) { - return - } - - val result = client.newCall(GET("$baseUrl/api/series/list")).execute() - if (!result.isSuccessful) { - throw Exception("Failed to fetch series list: HTTP ${result.code}") - } - - val series = result.parseAs() - - val comics = series.data.asSequence() - .filter { it.type == "comic" } - .toList() - - seriesList = comics - lastFetchTime = now - - val newSlugMap = comics.associate { it.id to it.slug } - - preferences.slugMap += newSlugMap - } - - override fun fetchPopularManga(page: Int): Observable { - fetchSeriesList() - return super.fetchPopularManga(page) - } - - override fun popularMangaRequest(page: Int): Request { - val apiUrl = "$baseUrl/api/rankings?page=$page&period=total_ranking".toHttpUrl().newBuilder() - .build() - return GET(apiUrl, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val result = response.parseAs() - val slugMap = preferences.slugMap.toMutableMap() - val mangaList = result.data - .filter { it.type == "comic" } - .map { - slugMap[it.id] = it.slug - it.toSManga() - } - preferences.slugMap = slugMap - return MangasPage(mangaList, hasNextPage = result.hasNextPage()) - } - - override fun fetchLatestUpdates(page: Int): Observable { - fetchSeriesList() - return super.fetchLatestUpdates(page) - } - - override fun latestUpdatesRequest(page: Int): Request { - val apiUrl = "$baseUrl/api/new-chapters?page=$page".toHttpUrl().newBuilder() - .build() - return GET(apiUrl, headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val result = response.parseAs() - val slugMap = preferences.slugMap.toMutableMap() - val mangaList = result.data.filter { it.type == "comic" } - .map { - slugMap[it.id] = it.slug - it.toSManga() - } - preferences.slugMap = slugMap - return MangasPage(mangaList, result.hasNextPage()) - } - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - fetchSeriesList() - return Observable.just(parseSearchManga(page, query)) - } - - private fun parseSearchManga(page: Int, query: String): MangasPage { - val filteredList = seriesList.filter { it.name.contains(query, ignoreCase = true) } - val paginatedList = filteredList.drop((page - 1) * 20).take(20) - val hasNextPage = page * 20 < filteredList.size - return MangasPage(paginatedList.map { it.toSManga() }, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw Exception("La búsqueda no es compatible.") - - override fun searchMangaParse(response: Response) = throw Exception("La búsqueda no es compatible.") - - override fun getMangaUrl(manga: SManga): String { - val slug = preferences.slugMap[manga.url.toInt()]!! - return "$baseUrl/series/comic-$slug" - } - - override fun fetchMangaDetails(manga: SManga): Observable { - fetchSeriesList() - return super.fetchMangaDetails(manga) - } - - override fun mangaDetailsRequest(manga: SManga): Request { - val slug = preferences.slugMap[manga.url.toInt()]!! - - val apiUrl = "$baseUrl/api/series/$slug?type=comic" - return GET(url = apiUrl, headers = headers) - } - - override fun mangaDetailsParse(response: Response): SManga { - val result = response.parseAs() - return result.data.toSMangaDetails() - } - - override fun getChapterUrl(chapter: SChapter): String { - val mangaId = chapter.url.substringBefore("/") - val chapterId = chapter.url.substringAfter("/") - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - return "$baseUrl/capitulo/$chapterId/comic-$mangaSlug" - } - - override fun fetchChapterList(manga: SManga): Observable> { - fetchSeriesList() - return super.fetchChapterList(manga) - } - - override fun chapterListRequest(manga: SManga): Request { - val mangaId = manga.url - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - - return paginatedChapterListRequest(mangaSlug, mangaId, 1) - } - - private fun paginatedChapterListRequest(mangaSlug: String, mangaId: String, page: Int): Request = GET( - url = "$apiBaseUrl/api/series/$mangaSlug/chapters?page=$page&direction=desc&type=comic#$mangaId", - headers = headers, - ) - - override fun chapterListParse(response: Response): List { - val mangaId = response.request.url.fragment ?: "" - val slug = response.request.url.toString() - .substringAfter("/series/") - .substringBefore("/chapters") - - val data = response.parseAs() - var resultSize = data.data.size - var page = 2 - while (data.meta.total > resultSize) { - val newRequest = paginatedChapterListRequest(slug, mangaId, page) - val newResponse = client.newCall(newRequest).execute() - val newData = newResponse.parseAs() - data.data += newData.data - resultSize += newData.data.size - page += 1 - } - return data.data.map { it.toSChapter(mangaId, dateFormat) }.sortedBy { it.chapter_number } - } - - override fun pageListRequest(chapter: SChapter): Request { - val mangaId = chapter.url.substringBefore("/") - val chapterId = chapter.url.substringAfter("/") - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - - return GET("$baseUrl/api/capitulo/comic-$mangaSlug/$chapterId") - } - - override fun pageListParse(response: Response): List = response.parseAs().chapter.pages.mapIndexed { i, img -> - Page(i, imageUrl = img) - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = FETCH_DOMAIN_PREF - title = "Buscar dominio automáticamente" - summary = "Intenta buscar el dominio automáticamente al abrir la fuente." - setDefaultValue(FETCH_DOMAIN_PREF_DEFAULT) - }.also { screen.addPreference(it) } - - EditTextPreference(screen.context).apply { - key = BASE_URL_PREF - title = "Editar URL de la fuente" - summary = "Para uso temporal, si la extensión se actualiza se perderá el cambio." - dialogTitle = "Editar URL de la fuente" - setDefaultValue(defaultBaseUrl) - }.also { screen.addPreference(it) } - } - - private var cachedBaseUrl: String? = null - private var SharedPreferences.prefBaseUrl: String - get() { - if (cachedBaseUrl == null) { - cachedBaseUrl = getString(BASE_URL_PREF, defaultBaseUrl)!! - } - return cachedBaseUrl!! - } - set(value) { - cachedBaseUrl = value - edit().putString(BASE_URL_PREF, value).apply() - } - - private fun SharedPreferences.fetchDomainPref() = getBoolean(FETCH_DOMAIN_PREF, FETCH_DOMAIN_PREF_DEFAULT) - - private var slugMapCache: Map? = null - private var SharedPreferences.slugMap: Map - get() { - slugMapCache?.let { return it } - val json = getString(SLUG_MAP, "{}")!! - slugMapCache = try { - json.parseAs>() - } catch (_: SerializationException) { - emptyMap() - } - return slugMapCache!! - } - set(map) { - slugMapCache = map - edit().putString(SLUG_MAP, map.toJsonString()).apply() - } - - companion object { - private const val BASE_URL_PREF = "overrideBaseUrl" - private const val DEFAULT_BASE_URL_PREF = "defaultBaseUrl" - - private const val FETCH_DOMAIN_PREF = "fetchDomain" - private const val FETCH_DOMAIN_PREF_DEFAULT = true - - private const val SLUG_MAP = "slugMap" - - private const val CACHE_DURATION_MS = 60 * 60 * 1000L // 1 hour - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt deleted file mode 100644 index 96fef581..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt +++ /dev/null @@ -1,140 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.submanhwa - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.tryParse -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.TachiyomiSource -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("SUBMANHWA", "Submanhwa", "es") -class Submanhwa : HttpSource() { - - override val name = "Submanhwa" - override val baseUrl = "https://submanhwa.com" - override val lang = "es" - override val supportsLatest = true - - private val dateFormat = SimpleDateFormat("dd MMM. yyyy", Locale.ENGLISH) - - override fun headersBuilder() = super.headersBuilder() - .add("Accept-Language", "es-PE,es;q=0.9,en-US;q=0.8,en;q=0.7") - .add("Referer", "$baseUrl/") - - override fun popularMangaRequest(page: Int): Request = GET( - baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("filterList") - addQueryParameter("page", page.toString()) - addQueryParameter("sortBy", "views") - addQueryParameter("asc", "false") - }.build(), - headers, - ) - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(".series-card").map { element -> - SManga.create().apply { - title = element.selectFirst(".series-title")!!.text() - setUrlWithoutDomain(element.selectFirst("a")!!.absUrl("href")) - thumbnail_url = element.selectFirst("img")!!.absUrl("src") - } - } - val hasNextPage = document.selectFirst("li a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun latestUpdatesRequest(page: Int): Request = GET(baseUrl, headers) - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select("div[class^=manga-item]").map { element -> - SManga.create().apply { - title = element.selectFirst("h3[class^=manga-title] a")!!.text() - setUrlWithoutDomain(element.selectFirst("a")!!.absUrl("href")) - thumbnail_url = element.selectFirst("img")!!.absUrl("src") - } - } - - return MangasPage(mangas, false) - } - - override fun searchMangaRequest( - page: Int, - query: String, - filters: FilterList, - ): Request = GET( - baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("filterList") - addQueryParameter("page", page.toString()) - addQueryParameter("sortBy", "views") - addQueryParameter("asc", "false") - addQueryParameter("alpha", query) - }.build(), - headers, - ) - - override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response) - - override fun mangaDetailsParse(response: Response): SManga { - val document = response.asJsoup() - return SManga.create().apply { - title = document.selectFirst(".manga-title-centered")!!.text() - thumbnail_url = document.selectFirst("img")?.absUrl("src") - description = document.selectFirst("h5:contains(Resumen) + p")?.text() - - val box = document.selectFirst(".main-content > .boxed-modern") - - status = when (box?.selectFirst(".detail-label:contains(Estado) + .detail-value span")?.text()?.lowercase()) { - "completa" -> SManga.COMPLETED - "en curso" -> SManga.ONGOING - else -> SManga.UNKNOWN - } - - author = box?.selectFirst(".detail-label:contains(Autor) + .detail-value a")?.text() - artist = box?.selectFirst(".detail-label:contains(Artist) + .detail-value a")?.text() - genre = box?.select(".detail-label:contains(Categor) + .detail-value a")?.joinToString { it.text() } - } - } - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - return document.select(".chapters-grid [class^=chapter-card]").map { element -> - SChapter.create().apply { - val a = element.selectFirst("a.chapter-link")!! - name = a.text() - setUrlWithoutDomain(a.absUrl("href")) - - val date = element.selectFirst("span:has(i.glyphicon-time)")?.text() - ?: element.selectFirst(".chapter-preview-meta > span")?.text() - - date_upload = dateFormat.tryParse(date) - } - } - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - return document.select("#all img").mapIndexed { idx, img -> - Page(idx, imageUrl = img.imgAttr()) - } - } - - private fun Element.imgAttr(): String = when { - hasAttr("data-src") -> attr("abs:data-src") - else -> attr("abs:src") - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt deleted file mode 100644 index 1ec2ad2c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt +++ /dev/null @@ -1,391 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers - -import android.content.SharedPreferences -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.lib.i18n.Intl -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import keiyoushi.utils.tryParse -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.Locale - -abstract class Keyoapp : - HttpSource(), - ConfigurableSource { - - protected val preferences: SharedPreferences by getPreferencesLazy() - - override val supportsLatest = true - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - protected val intl = Intl( - language = lang, - baseLanguage = "en", - availableLanguages = setOf("ar", "en", "fr"), - classLoader = this::class.java.classLoader!!, - ) - - // ============================== Popular ============================== - - override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers) - - open val popularMangaTitleSelector = listOf( - "Popular", - "Popularie", - "Trending", - ) - - open fun popularMangaSelector(): String = selector( - "div:contains(%s) + div .group.overflow-hidden.grid", - popularMangaTitleSelector, - ) - - open fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - thumbnail_url = element.getImageUrl("*[style*=background-image]") - element.selectFirst("a[href]")!!.run { - title = attr("title") - setUrlWithoutDomain(attr("abs:href")) - } - } - - open fun popularMangaNextPageSelector(): String? = null - - override fun popularMangaParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - val mangas = document.select(popularMangaSelector()).map { popularMangaFromElement(it) } - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) } != null - return MangasPage(mangas, hasNextPage) - } - - // ============================== Latest =============================== - - override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/latest/", headers) - - open fun latestUpdatesSelector(): String = "div.grid > div.group" - - open fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element) - - open fun latestUpdatesNextPageSelector(): String? = null - - override fun latestUpdatesParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - val mangas = document.select(latestUpdatesSelector()).map { latestUpdatesFromElement(it) } - val hasNextPage = latestUpdatesNextPageSelector()?.let { document.selectFirst(it) } != null - return MangasPage(mangas, hasNextPage) - } - - // ============================== Search =============================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("series") - addPathSegment("") - if (query.isNotBlank()) { - addQueryParameter("q", query) - } - filters.firstInstanceOrNull()?.also { filter -> - filter.state - .filter { it.state } - .forEach { genre -> - addQueryParameter("genre", genre.id) - } - } - }.build() - - return GET(url, headers) - } - - open fun searchMangaSelector() = "#searched_series_page > button" - - open fun searchMangaFromElement(element: Element): SManga = popularMangaFromElement(element) - - open fun searchMangaNextPageSelector(): String? = null - - override fun searchMangaParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - - val query = response.request.url.queryParameter("q") ?: "" - val genres = response.request.url.queryParameterValues("genre").filterNotNull() - - val mangaList = document.select(searchMangaSelector()) - .filter { it.attr("title").contains(query, true) } - .filter { entry -> - val entryGenres = runCatching { - entry.attr("tags").parseAs>() - }.getOrDefault(emptyList()) - genres.all { genre -> entryGenres.any { it.equals(genre, true) } } - } - .map(::searchMangaFromElement) - - return MangasPage(mangaList, false) - } - - // ============================== Filters ============================== - - /** - * Automatically fetched genres from the source to be used in the filters. - */ - private var genresList: List = emptyList() - - /** - * Inner variable to control the genre fetching failed state. - */ - private var fetchGenresFailed: Boolean = false - - /** - * Inner variable to control how much tries the genres request was called. - */ - private var fetchGenresAttempts: Int = 0 - - class Genre(name: String, val id: String = name) : Filter.CheckBox(name) - - protected class GenreList(title: String, genres: List) : Filter.Group(title, genres) - - override fun getFilterList(): FilterList = if (genresList.isNotEmpty()) { - FilterList( - GenreList("Genres", genresList), - ) - } else { - FilterList( - Filter.Header("Press 'Reset' to attempt to show the genres"), - ) - } - - /** - * Fetch the genres from the source to be used in the filters. - */ - protected open fun fetchGenres() { - if (fetchGenresAttempts <= 3 && (genresList.isEmpty() || fetchGenresFailed)) { - val genres = runCatching { - client.newCall(genresRequest()).execute() - .use { parseGenres(it.asJsoup()) } - } - - fetchGenresFailed = genres.isFailure - genresList = genres.getOrNull().orEmpty() - fetchGenresAttempts++ - } - } - - protected open fun genresRequest(): Request = GET("$baseUrl/series/", headers) - - /** - * Get the genres from the search page document. - * - * @param document The search page document - */ - protected open fun parseGenres(document: Document): List = document.select("#series_tags_page > button") - .map { btn -> - Genre(btn.text(), btn.attr("tag")) - } - - // ============================== Details ============================== - protected open val descriptionSelector: String = "div:containsOwn(Synopsis) ~ div" - protected open val statusSelector: String = "div:has(span:containsOwn(Status)) ~ div" - protected open val authorSelector: String = "div:has(span:containsOwn(Author)) ~ div" - protected open val artistSelector: String = "div:has(span:containsOwn(Artist)) ~ div" - protected open val genreSelector: String = "div.grid:has(>h1) > div > a:not([title='Status'])" - - protected open val typeSelector: String = "div:has(span:containsOwn(Type)) ~ div" - protected open val dateSelector: String = ".text-xs" - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - open fun mangaDetailsParse(document: Document): SManga = SManga.create().apply { - title = document.selectFirst("div.grid > h1")!!.text() - thumbnail_url = document.getImageUrl("div[class*=photoURL]") - description = document.selectFirst(descriptionSelector)?.text() - status = document.selectFirst(statusSelector).parseStatus() - author = document.selectFirst(authorSelector)?.text() - artist = document.selectFirst(artistSelector)?.text() - genre = buildList { - document.selectFirst(typeSelector)?.text()?.replaceFirstChar { - if (it.isLowerCase()) { - it.titlecase( - Locale.ENGLISH, - ) - } else { - it.toString() - } - }?.let(::add) - document.select(genreSelector).forEach { add(it.text()) } - }.joinToString() - } - - protected fun Element?.parseStatus(): Int = when (this?.text()?.lowercase()) { - "ongoing" -> SManga.ONGOING - "dropped" -> SManga.CANCELLED - "paused" -> SManga.ON_HIATUS - "completed" -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - - // ============================= Chapters ============================== - protected open val paidChapterSelector: String = "img[alt~=Coin]" - - override fun chapterListParse(response: Response): List = response.asJsoup().select(chapterListSelector()).map { chapterFromElement(it) }.reversed() - - open fun chapterListSelector(): String { - if (!preferences.showPaidChapters) { - return "#chapters > a:not(:has(.text-sm span:matches(Upcoming))):not(:has($paidChapterSelector))" - } - return "#chapters > a:not(:has(.text-sm span:matches(Upcoming)))" - } - - open fun chapterFromElement(element: Element): SChapter = SChapter.create().apply { - setUrlWithoutDomain(element.selectFirst("a[href]")!!.attr("abs:href")) - name = element.selectFirst(".text-sm")!!.text() - element.selectFirst(dateSelector)?.run { - date_upload = text().trim().parseDate() - } - if (element.select(paidChapterSelector).isNotEmpty()) { - name = "🔒 $name" - } - } - - // =============================== Pages =============================== - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - open fun pageListParse(document: Document): List { - val cdnUrl = getCdnUrl(document) - document.select("#pages > img") - .map { it.attr("uid") } - .filter { it.isNotEmpty() } - .also { if (it.isNotEmpty() && cdnUrl == null) throw Exception(intl["chapter_page_url_not_found"]) } - .mapIndexed { index, img -> - Page(index, url = document.location(), imageUrl = "$cdnUrl/$img") - } - .takeIf { it.isNotEmpty() } - ?.also { return it } - - // Fallback, old method - return document.select("#pages > img") - .map { it.imgAttr() } - .filter { it.contains(oldImgCdnRegex) } - .mapIndexed { index, img -> - Page(index, url = document.location(), imageUrl = img) - } - } - - protected open fun getCdnUrl(document: Document): String? = document.select("script") - .firstOrNull { CDN_HOST_REGEX.containsMatchIn(it.html()) } - ?.let { - val cdnHost = CDN_HOST_REGEX.find(it.html()) - ?.groups?.get(1)?.value - ?.replace(CDN_CLEAN_REGEX, "") - "https://$cdnHost/uploads" - } - - private val oldImgCdnRegex = Regex("""^(https?:)?//cdn\d*\.keyoapp\.com""") - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ============================= Utilities ============================= - - // From mangathemesia - private fun Element.imgAttr(): String = when { - hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") - hasAttr("data-src") -> attr("abs:data-src") - else -> attr("abs:src") - } - - protected open fun Element.getImageUrl(selector: String): String? = this.selectFirst(selector)?.let { element -> - IMG_REGEX.find(element.attr("style"))?.groups?.get(1)?.value - ?.toHttpUrlOrNull()?.let { - it.newBuilder() - .setQueryParameter("w", "480") // Keyoapp returns the dynamic size of the thumbnail to any size - .build() - .toString() - } - } - - private fun String.parseDate(): Long = if (this.contains("ago")) { - this.parseRelativeDate() - } else { - dateFormat.tryParse(this) - } - - private fun String.parseRelativeDate(): Long { - val now = Calendar.getInstance().apply { - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - } - - val relativeDate = this.split(" ").firstOrNull() - ?.replace("one", "1") - ?.replace("a", "1") - ?.toIntOrNull() - ?: return 0L - - when { - "second" in this -> now.add(Calendar.SECOND, -relativeDate) - - // parse: 30 seconds ago - "minute" in this -> now.add(Calendar.MINUTE, -relativeDate) - - // parses: "42 minutes ago" - "hour" in this -> now.add(Calendar.HOUR, -relativeDate) - - // parses: "1 hour ago" and "2 hours ago" - "day" in this -> now.add(Calendar.DAY_OF_YEAR, -relativeDate) - - // parses: "2 days ago" - "week" in this -> now.add(Calendar.WEEK_OF_YEAR, -relativeDate) - - // parses: "2 weeks ago" - "month" in this -> now.add(Calendar.MONTH, -relativeDate) - - // parses: "2 months ago" - "year" in this -> now.add(Calendar.YEAR, -relativeDate) // parse: "2 years ago" - } - return now.timeInMillis - } - - private fun selector(selector: String, contains: List): String = contains.joinToString { selector.replace("%s", it) } - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_PAID_CHAPTERS_PREF - title = intl["pref_show_paid_chapter_title"] - setDefaultValue(SHOW_PAID_CHAPTERS_DEFAULT) - }.also(screen::addPreference) - } - - protected val SharedPreferences.showPaidChapters: Boolean - get() = getBoolean(SHOW_PAID_CHAPTERS_PREF, SHOW_PAID_CHAPTERS_DEFAULT) - - companion object { - private const val SHOW_PAID_CHAPTERS_PREF = "pref_show_paid_chap" - private const val SHOW_PAID_CHAPTERS_DEFAULT = false - val CDN_HOST_REGEX = """realUrl\s*=\s*`[^`]+//([^/]+)""".toRegex() - val CDN_CLEAN_REGEX = """\$\{[^}]*\}""".toRegex() - val IMG_REGEX = """url\(['"]?([^(['")])]+)""".toRegex() - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt deleted file mode 100644 index a7941e41..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt +++ /dev/null @@ -1,1179 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers - -import android.util.Base64 -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.lib.cryptoaes.CryptoAES -import keiyoushi.lib.i18n.Intl -import keiyoushi.utils.decodeHex -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import okhttp3.FormBody -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import rx.Observable -import uy.kohesive.injekt.injectLazy -import java.text.ParseException -import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.Locale - -abstract class Madara( - override val name: String, - override val baseUrl: String, - final override val lang: String, - protected val dateFormat: SimpleDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US), -) : HttpSource() { - - override val supportsLatest = true - - override val client = network.cloudflareClient - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - protected val xhrHeaders by lazy { - headersBuilder() - .set("X-Requested-With", "XMLHttpRequest") - .build() - } - - protected open val json: Json by injectLazy() - - protected val intl = Intl( - language = lang, - baseLanguage = "en", - availableLanguages = setOf("en", "pt-BR", "es"), - classLoader = this::class.java.classLoader!!, - ) - - /** - * If enabled, will attempt to remove non-manga items in popular and latest. - * The filter will not be used in search as the theme doesn't set the CSS class. - * Can be disabled if the source incorrectly sets the entry types. - */ - protected open val filterNonMangaItems = true - - /** - * The CSS selector used to filter manga items in popular and latest - * if `filterNonMangaItems` is set to `true`. Can be override if needed. - * If the flag is set to `false`, it will be empty by default. - */ - protected open val mangaEntrySelector: String by lazy { - if (filterNonMangaItems) ".manga" else "" - } - - /** - * Automatically fetched genres from the source to be used in the filters. - */ - protected open var genresList: List = emptyList() - - /** - * Whether genres have been fetched - */ - private var genresFetched: Boolean = false - - /** - * Inner variable to control how much tries the genres request was called. - */ - private var fetchGenresAttempts: Int = 0 - - /** - * Disable it if you don't want the genres to be fetched. - */ - protected open val fetchGenres: Boolean = true - - /** - * The path used in the URL for the manga pages. Can be - * changed if needed as some sites modify it to other words. - */ - protected open val mangaSubString = "manga" - - /** - * enable if the site use "madara_load_more" to load manga on the site - * Typically has "load More" instead of next/previous page - * - * with LoadMoreStrategy.AutoDetect it tries to detect if site uses `madara_load_more` - */ - protected open val useLoadMoreRequest = LoadMoreStrategy.AutoDetect - - enum class LoadMoreStrategy { - AutoDetect, - Always, - Never, - } - - /** - * internal variable to save if site uses load_more or not - */ - private var loadMoreRequestDetected = LoadMoreDetection.Pending - - private enum class LoadMoreDetection { - Pending, - True, - False, - } - - protected fun detectLoadMore(document: Document) { - if (useLoadMoreRequest == LoadMoreStrategy.AutoDetect && - loadMoreRequestDetected == LoadMoreDetection.Pending - ) { - loadMoreRequestDetected = when (document.selectFirst("nav.navigation-ajax") != null) { - true -> LoadMoreDetection.True - false -> LoadMoreDetection.False - } - } - } - - protected fun useLoadMoreRequest(): Boolean = when (useLoadMoreRequest) { - LoadMoreStrategy.Always -> true - LoadMoreStrategy.Never -> false - else -> loadMoreRequestDetected == LoadMoreDetection.True - } - - // Popular Manga - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val entries = document.select(popularMangaSelector()) - .map(::popularMangaFromElement) - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) } != null - - detectLoadMore(document) - - return MangasPage(entries, hasNextPage) - } - - // exclude/filter bilibili manga from list - protected open fun popularMangaSelector() = "div.page-item-detail:not(:has(a[href*='bilibilicomics.com']))$mangaEntrySelector , .manga__item" - - open val popularMangaUrlSelector = "div.post-title a" - open val popularMangaUrlSelectorImg = "img" - - protected open fun popularMangaFromElement(element: Element): SManga { - val manga = SManga.create() - - with(element) { - selectFirst(popularMangaUrlSelector)!!.let { - manga.setUrlWithoutDomain(it.attr("abs:href")) - manga.title = it.ownText() - } - - selectFirst(popularMangaUrlSelectorImg)?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - - return manga - } - - override fun popularMangaRequest(page: Int): Request = if (useLoadMoreRequest()) { - loadMoreRequest(page, popular = true) - } else { - GET("$baseUrl/$mangaSubString/${searchPage(page)}?m_orderby=views", headers) - } - - protected open fun popularMangaNextPageSelector(): String? = if (useLoadMoreRequest()) { - "body:not(:has(.no-posts))" - } else { - "div.nav-previous, nav.navigation-ajax, a.nextpostslink" - } - - // Related Manga - protected open fun relatedMangaSelector() = ".related-reading-wrap" - - override fun relatedMangaListParse(response: Response): List { - val document = response.asJsoup() - return document.select(relatedMangaSelector()) - .mapNotNull { manga -> - SManga.create().apply { - manga.selectFirst(".widget-title a")?.let { - setUrlWithoutDomain(it.attr("abs:href")) - title = it.ownText() - } ?: return@mapNotNull null - manga.selectFirst(".widget-thumbnail img")?.let { - thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - } - } - - // Latest Updates - - protected open fun latestUpdatesSelector() = popularMangaSelector() - - protected open fun latestUpdatesFromElement(element: Element): SManga { - // Even if it's different from the popular manga's list, the relevant classes are the same - return popularMangaFromElement(element) - } - - override fun latestUpdatesRequest(page: Int): Request = if (useLoadMoreRequest()) { - loadMoreRequest(page, popular = false) - } else { - GET("$baseUrl/$mangaSubString/${searchPage(page)}?m_orderby=latest", headers) - } - - protected open fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector() - - override fun latestUpdatesParse(response: Response): MangasPage { - val mp = popularMangaParse(response) - val mangas = mp.mangas.distinctBy { it.url } - return MangasPage(mangas, mp.hasNextPage) - } - - // load more - protected fun loadMoreRequest(page: Int, popular: Boolean): Request { - val formBody = FormBody.Builder().apply { - add("action", "madara_load_more") - add("page", (page - 1).toString()) - add("template", "madara-core/content/content-archive") - add("vars[orderby]", "meta_value_num") - add("vars[paged]", "1") - add("vars[post_type]", "wp-manga") - add("vars[post_status]", "publish") - add("vars[meta_key]", if (popular) "_wp_manga_views" else "_latest_update") - add("vars[order]", "desc") - add("vars[sidebar]", "right") - add("vars[manga_archives_item_layout]", "big_thumbnail") - }.build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, formBody) - } - - // Search Manga - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (query.startsWith("https://")) { - val url = query.toHttpUrl() - if (url.host != baseUrl.toHttpUrl().host) { - throw Exception("Unsupported url") - } - if (url.pathSegments.size < 2) { - throw Exception("Unsupported url") - } - val slug = url.pathSegments[1] - return fetchSearchManga(page, "$URL_SEARCH_PREFIX$slug", filters) - } - if (query.startsWith(URL_SEARCH_PREFIX)) { - val mangaUrl = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment(mangaSubString) - addPathSegment(query.substringAfter(URL_SEARCH_PREFIX)) - addPathSegment("") // add trailing slash - }.build() - return client.newCall(GET(mangaUrl, headers)) - .asObservableSuccess().map { response -> - val manga = mangaDetailsParse(response).apply { - setUrlWithoutDomain(mangaUrl.toString()) - initialized = true - } - - MangasPage(listOf(manga), false) - } - } - - return super.fetchSearchManga(page, query, filters) - } - - protected open fun searchPage(page: Int): String = if (page == 1) { - "" - } else { - "page/$page/" - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = if (useLoadMoreRequest()) { - searchLoadMoreRequest(page, query, filters) - } else { - searchRequest(page, query, filters) - } - - protected open fun searchRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/${searchPage(page)}".toHttpUrl().newBuilder() - url.addQueryParameter("s", query) - url.addQueryParameter("post_type", "wp-manga") - filters.forEach { filter -> - when (filter) { - is AuthorFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("author", filter.state) - } - } - - is ArtistFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("artist", filter.state) - } - } - - is YearFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("release", filter.state) - } - } - - is StatusFilter -> { - filter.state.forEach { - if (it.state) { - url.addQueryParameter("status[]", it.id) - } - } - } - - is OrderByFilter -> { - if (filter.state != 0) { - url.addQueryParameter("m_orderby", filter.toUriPart()) - } - } - - is AdultContentFilter -> { - url.addQueryParameter("adult", filter.toUriPart()) - } - - is GenreConditionFilter -> { - url.addQueryParameter("op", filter.toUriPart()) - } - - is GenreList -> { - filter.state - .filter { it.state } - .let { list -> - if (list.isNotEmpty()) { - list.forEach { genre -> url.addQueryParameter("genre[]", genre.id) } - } - } - } - - else -> {} - } - } - return GET(url.build(), headers) - } - - protected open fun searchLoadMoreRequest(page: Int, query: String, filters: FilterList): Request { - val formBody = FormBody.Builder().apply { - add("action", "madara_load_more") - add("page", (page - 1).toString()) - add("template", "madara-core/content/content-search") - add("vars[paged]", "1") - add("vars[template]", "archive") - add("vars[sidebar]", "right") - add("vars[post_type]", "wp-manga") - add("vars[post_status]", "publish") - add("vars[manga_archives_item_layout]", "big_thumbnail") - - if (filterNonMangaItems) { - add("vars[meta_query][0][key]", "_wp_manga_chapter_type") - add("vars[meta_query][0][value]", "manga") - } - - add("vars[s]", query) - - var metaQueryIdx = if (filterNonMangaItems) 1 else 0 - var taxQueryIdx = 0 - val genres = filters.filterIsInstance().firstOrNull()?.state - ?.filter { it.state } - ?.map { it.id } - .orEmpty() - - filters.forEach { filter -> - when (filter) { - is AuthorFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-author") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is ArtistFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-artist") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is YearFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-release") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is StatusFilter -> { - val statuses = filter.state - .filter { it.state } - .map { it.id } - - if (statuses.isNotEmpty()) { - add("vars[meta_query][$metaQueryIdx][key]", "_wp_manga_status") - - statuses.forEachIndexed { i, slug -> - add("vars[meta_query][$metaQueryIdx][value][$i]", slug) - } - - metaQueryIdx++ - } - } - - is OrderByFilter -> { - if (filter.state != 0) { - when (filter.toUriPart()) { - "latest" -> { - add("vars[orderby]", "meta_value_num") - add("vars[order]", "DESC") - add("vars[meta_key]", "_latest_update") - } - - "alphabet" -> { - add("vars[orderby]", "post_title") - add("vars[order]", "ASC") - } - - "rating" -> { - add("vars[orderby][query_average_reviews]", "DESC") - add("vars[orderby][query_total_reviews]", "DESC") - } - - "trending" -> { - add("vars[orderby]", "meta_value_num") - add("vars[meta_key]", "_wp_manga_week_views_value") - add("vars[order]", "DESC") - } - - "views" -> { - add("vars[orderby]", "meta_value_num") - add("vars[meta_key]", "_wp_manga_views") - add("vars[order]", "DESC") - } - - else -> { - add("vars[orderby]", "date") - add("vars[order]", "DESC") - } - } - } - } - - is AdultContentFilter -> { - if (filter.state != 0) { - add("vars[meta_query][$metaQueryIdx][key]", "manga_adult_content") - add( - "vars[meta_query][$metaQueryIdx][compare]", - if (filter.state == 1) "not exists" else "exists", - ) - - metaQueryIdx++ - } - } - - is GenreConditionFilter -> { - if (filter.state == 1 && genres.isNotEmpty()) { - add("vars[tax_query][$taxQueryIdx][operation]", "AND") - } - } - - is GenreList -> { - if (genres.isNotEmpty()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-genre") - add("vars[tax_query][$taxQueryIdx][field]", "slug") - - genres.forEachIndexed { i, slug -> - add("vars[tax_query][$taxQueryIdx][terms][$i]", slug) - } - - taxQueryIdx++ - } - } - - else -> {} - } - } - }.build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, formBody) - } - - protected open val statusFilterOptions: Map = - mapOf( - intl["status_filter_completed"] to "end", - intl["status_filter_ongoing"] to "on-going", - intl["status_filter_canceled"] to "canceled", - intl["status_filter_on_hold"] to "on-hold", - ) - - protected open val orderByFilterOptions: Map = mapOf( - intl["order_by_filter_relevance"] to "", - intl["order_by_filter_latest"] to "latest", - intl["order_by_filter_az"] to "alphabet", - intl["order_by_filter_rating"] to "rating", - intl["order_by_filter_trending"] to "trending", - intl["order_by_filter_views"] to "views", - intl["order_by_filter_new"] to "new-manga", - ) - - protected open val genreConditionFilterOptions: Map = - mapOf( - intl["genre_condition_filter_or"] to "", - intl["genre_condition_filter_and"] to "1", - ) - - protected open val adultContentFilterOptions: Map = - mapOf( - intl["adult_content_filter_all"] to "", - intl["adult_content_filter_none"] to "0", - intl["adult_content_filter_only"] to "1", - ) - - open class UriPartFilter(displayName: String, private val vals: Array>, state: Int = 0) : Filter.Select(displayName, vals.map { it.first }.toTypedArray(), state) { - fun toUriPart() = vals[state].second - } - - open class Tag(name: String, val id: String) : Filter.CheckBox(name) - - protected class AuthorFilter(title: String) : Filter.Text(title) - protected class ArtistFilter(title: String) : Filter.Text(title) - protected class YearFilter(title: String) : Filter.Text(title) - protected class StatusFilter(title: String, status: List) : Filter.Group(title, status) - - protected class OrderByFilter(title: String, options: List>, state: Int = 0) : UriPartFilter(title, options.toTypedArray(), state) - - protected class GenreConditionFilter(title: String, options: List>) : - UriPartFilter( - title, - options.toTypedArray(), - ) - - protected class AdultContentFilter(title: String, options: List>) : - UriPartFilter( - title, - options.toTypedArray(), - ) - - protected class GenreList(title: String, genres: List) : Filter.Group(title, genres.map { GenreCheckBox(it.name, it.id) }) - class GenreCheckBox(name: String, val id: String = name) : Filter.CheckBox(name) - class Genre(val name: String, val id: String = name) - - override fun getFilterList(): FilterList { - launchIO { fetchGenres() } - - val filters = mutableListOf( - AuthorFilter(intl["author_filter_title"]), - ArtistFilter(intl["artist_filter_title"]), - YearFilter(intl["year_filter_title"]), - StatusFilter( - title = intl["status_filter_title"], - status = statusFilterOptions.map { Tag(it.key, it.value) }, - ), - OrderByFilter( - title = intl["order_by_filter_title"], - options = orderByFilterOptions.toList(), - state = 0, - ), - AdultContentFilter( - title = intl["adult_content_filter_title"], - options = adultContentFilterOptions.toList(), - ), - ) - - if (genresList.isNotEmpty()) { - filters += listOf( - Filter.Separator(), - Filter.Header(intl["genre_filter_header"]), - GenreConditionFilter( - title = intl["genre_condition_filter_title"], - options = genreConditionFilterOptions.toList(), - ), - GenreList( - title = intl["genre_filter_title"], - genres = genresList, - ), - ) - } else if (fetchGenres) { - filters += listOf( - Filter.Separator(), - Filter.Header(intl["genre_missing_warning"]), - ) - } - - return FilterList(filters) - } - - override fun searchMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val entries = document.select(searchMangaSelector()) - .map(::searchMangaFromElement) - val hasNextPage = searchMangaNextPageSelector()?.let { document.selectFirst(it) } != null - - detectLoadMore(document) - - return MangasPage(entries, hasNextPage) - } - - protected open fun searchMangaSelector() = "div.c-tabs-item__content , .manga__item" - - protected open val searchMangaUrlSelector = "div.post-title a" - - protected open fun searchMangaFromElement(element: Element): SManga { - val manga = SManga.create() - - with(element) { - selectFirst(searchMangaUrlSelector)!!.let { - manga.setUrlWithoutDomain(it.attr("abs:href")) - manga.title = it.ownText() - } - selectFirst("img")?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - - return manga - } - - protected open fun searchMangaNextPageSelector() = popularMangaNextPageSelector() - - // Manga Details Parse - - protected val completedStatusList: Array = arrayOf( - "Completed", - "Completo", - "Completado", - "Concluído", - "Concluido", - "Finalizado", - "Achevé", - "Terminé", - "Hoàn Thành", - "مكتملة", - "مكتمل", - "已完结", - "Tamamlandı", - "Đã hoàn thành", - "Завершено", - "Tamamlanan", - "Complété", - ) - - protected val ongoingStatusList: Array = arrayOf( - "OnGoing", "Продолжается", "Updating", "Em Lançamento", "Em lançamento", "Em andamento", - "Em Andamento", "En cours", "En Cours", "En cours de publication", "Ativo", "Lançando", "Đang Tiến Hành", "Còn Nữa", "Devam Ediyor", - "Devam ediyor", "In Corso", "In Arrivo", "مستمرة", "مستمر", "En Curso", "En curso", "Emision", - "Curso", "En marcha", "Publicandose", "Publicándose", "En emision", "连载中", "Em Lançamento", "Devam Ediyo", - "Đang làm", "Em postagem", "Devam Eden", "Em progresso", "Em curso", "Atualizações Semanais", - ) - - protected val hiatusStatusList: Array = arrayOf( - "On Hold", - "Pausado", - "En espera", - "Durduruldu", - "Beklemede", - "Đang chờ", - "متوقف", - "En Pause", - "Заморожено", - "En attente", - ) - - protected val canceledStatusList: Array = arrayOf( - "Canceled", - "Cancelado", - "İptal Edildi", - "Güncel", - "Đã hủy", - "ملغي", - "Abandonné", - "Заброшено", - "Annulé", - ) - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - protected open fun mangaDetailsParse(document: Document): SManga { - val manga = SManga.create() - with(document) { - manga.title = selectFirst(mangaDetailsSelectorTitle)!!.ownText() - select(mangaDetailsSelectorAuthor).eachText().filter { - it.notUpdating() - }.joinToString().takeIf { it.isNotBlank() }?.let { - manga.author = it - } - select(mangaDetailsSelectorArtist).eachText().filter { - it.notUpdating() - }.joinToString().takeIf { it.isNotBlank() }?.let { - manga.artist = it - } - select(mangaDetailsSelectorDescription).let { - if (it.select("p").text().isNotEmpty()) { - manga.description = it.select("p").joinToString(separator = "\n\n") { p -> - p.text().replace("
", "\n") - } - } else { - manga.description = it.text() - } - } - selectFirst(mangaDetailsSelectorThumbnail)?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it)) - } - select(mangaDetailsSelectorStatus).last()?.let { - manga.status = with(it.text().filter { ch -> ch.isLetterOrDigit() || ch.isWhitespace() }.trim()) { - when { - containsIn(completedStatusList) -> SManga.COMPLETED - containsIn(ongoingStatusList) -> SManga.ONGOING - containsIn(hiatusStatusList) -> SManga.ON_HIATUS - containsIn(canceledStatusList) -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - } - val genres = select(mangaDetailsSelectorGenre) - .mapTo(ArrayList()) { element -> element.text() } - - if (mangaDetailsSelectorTag.isNotEmpty()) { - select(mangaDetailsSelectorTag).forEach { element -> - if ( - element.text().length <= 25 && - element.text().contains("read", true).not() && - element.text().contains(name, true).not() && - element.text().contains(name.replace(" ", ""), true).not() && - element.text().contains(manga.title, true).not() && - element.text().contains(altName, true).not() - ) { - genres.add(element.text()) - } - } - } - - // add manga/manhwa/manhua thinggy to genre - document.selectFirst(seriesTypeSelector)?.ownText()?.let { - if (it.isEmpty().not() && it.notUpdating() && it != "-") { - genres.add(it) - } - } - - manga.genre = genres.distinctBy(String::lowercase).joinToString() - - // add alternative name to manga description - document.selectFirst(altNameSelector)?.ownText()?.let { - if (it.isBlank().not() && it.notUpdating()) { - manga.description = when { - manga.description.isNullOrBlank() -> "$altName $it" - else -> "${manga.description}\n\n$altName $it" - } - } - } - } - - return manga - } - - // Manga Details Selector - open val mangaDetailsSelectorTitle = "div.post-title h3, div.post-title h1, #manga-title > h1" - open val mangaDetailsSelectorAuthor = "div.author-content > a, div.manga-authors > a" - open val mangaDetailsSelectorArtist = "div.artist-content > a" - open val mangaDetailsSelectorStatus = "div.summary-content, div.summary-heading:contains(Status) + div" - open val mangaDetailsSelectorDescription = "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt" - open val mangaDetailsSelectorThumbnail = "div.summary_image img" - open val mangaDetailsSelectorGenre = "div.genres-content a" - open val mangaDetailsSelectorTag = "div.tags-content a" - - open val seriesTypeSelector = ".post-content_item:contains(Type) .summary-content" - open val altNameSelector = ".post-content_item:contains(Alt) .summary-content" - open val altName = intl["alt_names_heading"] - open val updatingRegex = "Updating|Atualizando".toRegex(RegexOption.IGNORE_CASE) - - fun String.notUpdating(): Boolean = this.contains(updatingRegex).not() - - private fun String.containsIn(array: Array): Boolean = this.lowercase() in array.map { it.lowercase() } - - protected open fun imageFromElement(element: Element): String? = when { - element.hasAttr("data-src") -> element.attr("abs:data-src") - element.hasAttr("data-lazy-src") -> element.attr("abs:data-lazy-src") - element.hasAttr("srcset") -> element.attr("abs:srcset").getSrcSetImage() - element.hasAttr("data-cfsrc") -> element.attr("abs:data-cfsrc") - element.hasAttr("data-manga-src") -> element.attr("abs:data-manga-src") - else -> element.attr("abs:src") - } - - /** - * Get the best image quality available from srcset - */ - protected open fun String.getSrcSetImage(): String? = this.split(" ") - .filter(URL_REGEX::matches) - .maxOfOrNull(String::toString) - - /** - * Apply any additional processing to the thumbnail URL if needed. - */ - protected open fun processThumbnail(url: String?, fromSearch: Boolean = false): String? = url - - /** - * Set it to true if the source uses the new AJAX endpoint to - * fetch the manga chapters instead of the old admin-ajax.php one. - */ - protected open val useNewChapterEndpoint: Boolean = false - - /** - * Internal attribute to control if it should always use the - * new chapter endpoint after a first check if useNewChapterEndpoint is - * set to false. Using a separate variable to still allow the other - * one to be overridable manually in each source. - */ - private var oldChapterEndpointDisabled: Boolean = false - - protected open fun oldXhrChaptersRequest(mangaId: String): Request { - val form = FormBody.Builder() - .add("action", "manga_get_chapters") - .add("manga", mangaId) - .build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, form) - } - - protected open fun xhrChaptersRequest(mangaUrl: String): Request = POST("$mangaUrl/ajax/chapters", xhrHeaders) - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - - launchIO { countViews(document) } - - val chaptersWrapper = document.select("div[id^=manga-chapters-holder]") - - var chapterElements = document.select(chapterListSelector()) - - if (chapterElements.isEmpty() && !chaptersWrapper.isNullOrEmpty()) { - val mangaUrl = document.location().removeSuffix("/") - val mangaId = chaptersWrapper.attr("data-id") - - var xhrRequest = if (useNewChapterEndpoint || oldChapterEndpointDisabled) { - xhrChaptersRequest(mangaUrl) - } else { - oldXhrChaptersRequest(mangaId) - } - var xhrResponse = client.newCall(xhrRequest).execute() - - // Newer Madara versions throws HTTP 400 when using the old endpoint. - if (!useNewChapterEndpoint && xhrResponse.code == 400) { - xhrResponse.close() - // Set it to true so following calls will be made directly to the new endpoint. - oldChapterEndpointDisabled = true - - xhrRequest = xhrChaptersRequest(mangaUrl) - xhrResponse = client.newCall(xhrRequest).execute() - } - - chapterElements = xhrResponse.asJsoup().select(chapterListSelector()) - xhrResponse.close() - } - - return chapterElements.map(::chapterFromElement) - } - - protected open fun chapterListSelector() = "li.wp-manga-chapter" - - protected open fun chapterDateSelector() = "span.chapter-release-date" - - open val chapterUrlSelector = "a" - - // can cause some issue for some site. blocked by cloudflare when opening the chapter pages - open val chapterUrlSuffix = "?style=list" - - protected open fun chapterFromElement(element: Element): SChapter { - val chapter = SChapter.create() - - with(element) { - selectFirst(chapterUrlSelector)!!.let { urlElement -> - chapter.url = urlElement.attr("abs:href").let { - it.substringBefore("?style=paged") + if (!it.endsWith(chapterUrlSuffix)) chapterUrlSuffix else "" - } - chapter.name = urlElement.text() - } - // Dates can be part of a "new" graphic or plain text - // Added "title" alternative - chapter.date_upload = selectFirst("img:not(.thumb)")?.attr("alt")?.let { parseRelativeDate(it) } - ?: selectFirst("span a")?.attr("title")?.let { parseRelativeDate(it) } - ?: parseChapterDate(selectFirst(chapterDateSelector())?.text()) - } - - return chapter - } - - open fun parseChapterDate(date: String?): Long { - date ?: return 0 - - fun SimpleDateFormat.tryParse(string: String): Long = try { - parse(string)?.time ?: 0 - } catch (_: ParseException) { - 0 - } - - return when { - // Handle 'yesterday' and 'today', using midnight - WordSet("yesterday", "يوم واحد").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("today").startsWith(date) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("يومين").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -2) // day before yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("ago", "atrás", "önce", "قبل", "trước").endsWith(date) -> { - parseRelativeDate(date) - } - - WordSet("hace", "năm", "tháng", "tuần", "ngày", "giờ", "phút", "giây").startsWith(date) -> { - parseRelativeDate(date) - } - - // Handle "jour" with a number before it - date.contains(Regex("""\b\d+ jour""")) -> { - parseRelativeDate(date) - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - } - .let { dateFormat.tryParse(it.joinToString(" ")) } - } - - else -> dateFormat.tryParse(date) - } - } - - // Parses dates in this form: - // 21 horas ago - protected open fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - - return when { - WordSet("hari", "gün", "jour", "día", "dia", "day", "วัน", "ngày", "giorni", "أيام", "天").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("jam", "saat", "heure", "hora", "hour", "ชั่วโมง", "giờ", "ore", "ساعة", "小时").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("menit", "dakika", "min", "minute", "minuto", "นาที", "دقائق", "phút").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("detik", "segundo", "second", "วินาที", "giây").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("week", "semana", "tuần").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - WordSet("month", "mes", "tháng").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "año", "năm").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0 - } - } - - override fun pageListRequest(chapter: SChapter): Request { - if (chapter.url.startsWith("http")) { - return GET(chapter.url, headers) - } - return super.pageListRequest(chapter) - } - - open val pageListParseSelector = "div.page-break, li.blocks-gallery-item, .reading-content .text-left:not(:has(.blocks-gallery-item)) img" - - open val chapterProtectorSelector = "#chapter-protector-data" - open val chapterProtectorPasswordPrefix = "wpmangaprotectornonce='" - open val chapterProtectorDataPrefix = "chapter_data='" - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - protected open fun pageListParse(document: Document): List { - launchIO { countViews(document) } - - val chapterProtector = document.selectFirst(chapterProtectorSelector) - ?: return document.select(pageListParseSelector).mapIndexed { index, element -> - val imageUrl = element.selectFirst("img")?.let { imageFromElement(it) } - Page(index, document.location(), imageUrl) - } - val chapterProtectorHtml = chapterProtector.attr("src") - .takeIf { it.startsWith("data:text/javascript;base64,") } - ?.substringAfter("data:text/javascript;base64,") - ?.let { Base64.decode(it, Base64.DEFAULT).toString(Charsets.UTF_8) } - ?: chapterProtector.html() - val password = chapterProtectorHtml - .substringAfter(chapterProtectorPasswordPrefix) - .substringBefore("';") - val chapterData = json.parseToJsonElement( - chapterProtectorHtml - .substringAfter(chapterProtectorDataPrefix) - .substringBefore("';") - .replace("\\/", "/"), - ).jsonObject - - val unsaltedCiphertext = Base64.decode(chapterData["ct"]!!.jsonPrimitive.content, Base64.DEFAULT) - val salt = chapterData["s"]!!.jsonPrimitive.content.decodeHex() - val ciphertext = salted + salt + unsaltedCiphertext - - val rawImgArray = CryptoAES.decrypt(Base64.encodeToString(ciphertext, Base64.DEFAULT), password) - val imgArrayString = json.parseToJsonElement(rawImgArray).jsonPrimitive.content - val imgArray = json.parseToJsonElement(imgArrayString).jsonArray - - return imgArray.mapIndexed { idx, it -> - Page(idx, document.location(), it.jsonPrimitive.content) - } - } - - override fun imageRequest(page: Page): Request = GET(page.imageUrl!!, headers.newBuilder().set("Referer", page.url).build()) - - override fun imageUrlParse(response: Response) = throw UnsupportedOperationException() - - /** - * Set it to false if you want to disable the extension reporting the view count - * back to the source website through admin-ajax.php. - */ - protected open val sendViewCount: Boolean = true - - protected open fun countViewsRequest(document: Document): Request? { - val wpMangaData = document.selectFirst("script#wp-manga-js-extra") - ?.data() ?: return null - - val wpMangaInfo = wpMangaData - .substringAfter("var manga = ") - .substringBeforeLast(";") - - val wpManga = json.parseToJsonElement(wpMangaInfo).jsonObject - - if (wpManga["enable_manga_view"]?.jsonPrimitive?.content == "1") { - val formBuilder = FormBody.Builder() - .add("action", "manga_views") - .add("manga", wpManga["manga_id"]!!.jsonPrimitive.content) - - if (wpManga["chapter_slug"] != null) { - formBuilder.add("chapter", wpManga["chapter_slug"]!!.jsonPrimitive.content) - } - - val formBody = formBuilder.build() - - val newHeaders = headersBuilder() - .set("Referer", document.location()) - .build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", newHeaders, formBody) - } - - return null - } - - /** - * Send the view count request to the Madara endpoint. - * - * @param document The response document with the wp-manga data - */ - protected fun countViews(document: Document) { - if (!sendViewCount) { - return - } - - try { - val request = countViewsRequest(document) ?: return - client.newCall(request).execute().close() - } catch (_: Exception) { } - } - - /** - * Fetch the genres from the source to be used in the filters. - */ - protected fun fetchGenres() { - if (fetchGenres && fetchGenresAttempts < 3 && !genresFetched) { - try { - client.newCall(genresRequest()).execute() - .use { parseGenres(it.asJsoup()) } - .also { - genresFetched = true - } - .takeIf { it.isNotEmpty() } - ?.also { - genresList = it - } - } catch (_: Exception) { - } finally { - fetchGenresAttempts++ - } - } - } - - /** - * The request to the search page (or another one) that have the genres list. - */ - protected open fun genresRequest(): Request = GET("$baseUrl/?s=genre&post_type=wp-manga", headers) - - /** - * Get the genres from the search page document. - * - * @param document The search page document - */ - protected open fun parseGenres(document: Document): List = document.selectFirst("div.checkbox-group") - ?.select("div.checkbox") - .orEmpty() - .map { li -> - Genre( - li.selectFirst("label")!!.text(), - li.selectFirst("input[type=checkbox]")!!.`val`(), - ) - } - - protected val salted = "Salted__".toByteArray(Charsets.UTF_8) - - private val scope = CoroutineScope(Dispatchers.IO) - - protected fun launchIO(block: () -> Unit) = scope.launch { block() } - - companion object { - const val URL_SEARCH_PREFIX = "slug:" - val URL_REGEX = """^(https?://[^\s/$.?#].[^\s]*)${'$'}""".toRegex() - } -} - -class WordSet(private vararg val words: String) { - fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) } - fun startsWith(dateString: String): Boolean = words.any { dateString.startsWith(it, ignoreCase = true) } - fun endsWith(dateString: String): Boolean = words.any { dateString.endsWith(it, ignoreCase = true) } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt deleted file mode 100644 index d5fbc0e8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt +++ /dev/null @@ -1,160 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response - -abstract class EZManhwa( - override val name: String, - override val baseUrl: String, - val apiUrl: String, - final override val lang: String = "en", -) : HttpSource(), - ConfigurableSource { - - override val supportsLatest = true - - private val preferences by getPreferencesLazy() - - override fun headersBuilder() = super.headersBuilder() - .set("Accept", "application/json, text/plain, */*") - .set("Referer", "$baseUrl/") - - override fun getMangaUrl(manga: SManga) = "$baseUrl/series/${manga.url}" - - // Chapter URLs are stored as: series/{seriesSlug}/chapters/{chapterSlug} - // The website drops "chapters/" from the path, so we strip it here. - override fun getChapterUrl(chapter: SChapter) = "$baseUrl/${chapter.url.replace("/chapters/", "/")}" - - // ── Browse ─────────────────────────────────────────────────────────────── - - override fun popularMangaRequest(page: Int) = GET("$apiUrl/series?page=$page&perPage=20&sort=popular", headers) - - override fun latestUpdatesRequest(page: Int) = GET("$apiUrl/series?page=$page&perPage=20&sort=latest", headers) - - override fun popularMangaParse(response: Response) = parseSeriesList(response) - override fun latestUpdatesParse(response: Response) = parseSeriesList(response) - - private fun parseSeriesList(response: Response): MangasPage { - val dto = response.parseAs() - val mangas = dto.data.mapNotNull { if (it.type != "NOVEL") it.toSManga() else null } - return MangasPage(mangas, dto.currentPage < dto.totalPages) - } - - // ── Search ─────────────────────────────────────────────────────────────── - - // Base implementation sends filters only during browse. - // Override if the source's search endpoint behaviour differs. - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val isSearch = query.isNotBlank() - val endpoint = if (isSearch) "$apiUrl/series/search" else "$apiUrl/series" - val url = endpoint.toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "20") - if (isSearch) { - addQueryParameter("q", query) - } else { - var sortAdded = false - for (filter in filters) { - when (filter) { - is EZManhwaSortFilter -> { - addQueryParameter("sort", filter.value) - sortAdded = true - } - is EZManhwaStatusFilter -> if (filter.value.isNotBlank()) addQueryParameter("status", filter.value) - is EZManhwaTypeFilter -> if (filter.value.isNotBlank()) addQueryParameter("type", filter.value) - else -> {} - } - } - if (!sortAdded) addQueryParameter("sort", "latest") - } - }.build() - return GET(url, headers) - } - - override fun searchMangaParse(response: Response) = parseSeriesList(response) - - // ── Details ────────────────────────────────────────────────────────────── - - override fun mangaDetailsRequest(manga: SManga) = GET("$apiUrl/series/${manga.url}", headers) - - override fun mangaDetailsParse(response: Response) = response.parseAs().toSManga().apply { initialized = true } - - // ── Chapters ───────────────────────────────────────────────────────────── - - override fun chapterListRequest(manga: SManga) = GET("$apiUrl/series/${manga.url}/chapters?page=1&perPage=100&sort=desc", headers) - - override fun chapterListParse(response: Response): List { - // pathSegments for /api/v1/series/{slug}/chapters: [api, v1, series, slug, chapters] - val seriesSlug = response.request.url.pathSegments[3] - val initialData = response.parseAs() - val chapters = mutableListOf() - - fun parsePage(dto: EZManhwaChapterListDto) { - dto.data.mapNotNullTo(chapters) { - if (shouldShowChapter(it)) it.toSChapter(seriesSlug) else null - } - } - - parsePage(initialData) - var curr = initialData.currentPage - while (curr < initialData.totalPages) { - curr++ - val nextUrl = response.request.url.newBuilder() - .setQueryParameter("page", curr.toString()).build() - parsePage( - client.newCall(GET(nextUrl, headers)).execute() - .parseAs(), - ) - } - return chapters - } - - open fun shouldShowChapter(chapter: EZManhwaChapterDto): Boolean = chapter.requiresPurchase != true || - preferences.getBoolean(SHOW_LOCKED_CHAPTER_PREF_KEY, false) - - // ── Pages ──────────────────────────────────────────────────────────────── - - override fun pageListRequest(chapter: SChapter) = GET("$apiUrl/${chapter.url}", headers) - - override fun pageListParse(response: Response): List { - val data = response.parseAs() - if (data.requiresPurchase == true) { - throw Exception( - "Chapter requires purchase (${data.totalImages} pages). " + - "Log in via webview and purchase to read.", - ) - } - return data.images?.mapIndexed { i, img -> Page(i, imageUrl = img.url) } - ?: throw Exception("No images found. Chapter may be locked or require login via webview.") - } - - override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used") - - override fun getFilterList() = FilterList(EZManhwaSortFilter(), EZManhwaStatusFilter(), EZManhwaTypeFilter()) - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_LOCKED_CHAPTER_PREF_KEY - title = "Show locked chapters" - summary = "Show chapters requiring coins. Note: They only load if owned/logged in via webview." - setDefaultValue(false) - }.also(screen::addPreference) - } - - companion object { - const val SHOW_LOCKED_CHAPTER_PREF_KEY = "pref_show_locked_chapters" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt deleted file mode 100644 index 4a4d0e6e..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt +++ /dev/null @@ -1,124 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.jsoup.Jsoup -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone - -internal val EZMANHWA_DATE_FORMAT by lazy { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH).apply { - timeZone = TimeZone.getTimeZone("UTC") - } -} - -internal val CHAPTER_PREFIX_REGEX = Regex("^(?i)(chapter|ch\\.?|episode|ep\\.?)\\s*.*") - -@Serializable -data class EZManhwaSeriesListDto( - val data: List, - val totalPages: Int, - @SerialName("current") val currentPage: Int, -) - -// Used for both list (partial) and details (full) responses. -// Fields absent in the list response deserialize as null and are -// only populated after mangaDetailsParse sets initialized = true. -@Serializable -data class EZManhwaSeriesDto( - val slug: String, - val title: String, - val cover: String? = null, - val type: String? = null, - val status: String? = null, - val alternativeTitles: String? = null, - val description: String? = null, - val author: String? = null, - val artist: String? = null, - val genres: List? = null, -) { - fun toSManga() = SManga.create().apply { - url = slug - title = this@EZManhwaSeriesDto.title - thumbnail_url = cover - author = this@EZManhwaSeriesDto.author?.trim()?.takeIf { it.isNotBlank() } - artist = this@EZManhwaSeriesDto.artist?.trim()?.takeIf { it.isNotBlank() } - description = buildString { - this@EZManhwaSeriesDto.description?.let { append(Jsoup.parse(it).text()) } - if (!alternativeTitles.isNullOrBlank()) { - append("\n\nAlternative Titles: $alternativeTitles") - } - }.trim().takeIf { it.isNotBlank() } - genre = genres?.joinToString { it.name } - status = when (this@EZManhwaSeriesDto.status) { - "ONGOING", "MASS_RELEASED" -> SManga.ONGOING - "COMPLETED" -> SManga.COMPLETED - "DROPPED" -> SManga.CANCELLED - "HIATUS" -> SManga.ON_HIATUS - else -> SManga.UNKNOWN - } - } -} - -@Serializable -data class EZManhwaGenreDto(val name: String) - -@Serializable -data class EZManhwaChapterListDto( - val data: List, - val totalPages: Int = 1, - @SerialName("current") val currentPage: Int = 1, -) - -@Serializable -data class EZManhwaChapterDto( - val slug: String, - val number: Double? = null, - val title: String? = null, - val requiresPurchase: Boolean? = null, - val createdAt: String? = null, -) { - fun toSChapter(seriesSlug: String) = SChapter.create().apply { - // Unified URL format: series/{seriesSlug}/chapters/{chapterSlug} - // pageListRequest uses: $apiUrl/${chapter.url} - // getChapterUrl uses: $baseUrl/${chapter.url.replace("/chapters/", "/")} - url = "series/$seriesSlug/chapters/$slug" - val prefix = if (requiresPurchase == true) "\uD83D\uDD12 " else "" - val numStr = number?.let { - if (it % 1.0 == 0.0) it.toInt().toString() else it.toString() - } ?: "" - - val parsedTitle = title?.trim()?.takeIf { it.isNotBlank() } - - val chapterName = if (numStr.isBlank()) { - parsedTitle ?: "Chapter" - } else { - when { - parsedTitle == null || parsedTitle == numStr -> "Chapter $numStr" - isChapterTitle(parsedTitle, numStr) -> parsedTitle - parsedTitle.startsWith("-") || parsedTitle.startsWith(":") -> "Chapter $numStr $parsedTitle" - else -> "Chapter $numStr - $parsedTitle" - } - } - - name = prefix + chapterName - chapter_number = number?.toFloat() ?: -1f - date_upload = EZMANHWA_DATE_FORMAT.tryParse(createdAt) - } - - private fun isChapterTitle(title: String, numStr: String): Boolean = title.contains(numStr) && CHAPTER_PREFIX_REGEX.matches(title) -} - -@Serializable -data class EZManhwaPageListDto( - val images: List? = null, - val requiresPurchase: Boolean? = null, - val totalImages: Int? = null, -) - -@Serializable -data class EZManhwaImageDto(val url: String) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt deleted file mode 100644 index 027a93c8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt +++ /dev/null @@ -1,27 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import eu.kanade.tachiyomi.source.model.Filter - -class EZManhwaSortFilter : Filter.Select("Sort", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("latest", "popular", "newest", "alphabetical") - val DISPLAY = arrayOf("Latest", "Popular", "Newest", "Alphabetical") - } -} - -class EZManhwaStatusFilter : Filter.Select("Status", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("", "ONGOING", "COMPLETED", "HIATUS", "DROPPED") - val DISPLAY = arrayOf("All", "Ongoing", "Completed", "Hiatus", "Dropped") - } -} - -class EZManhwaTypeFilter : Filter.Select("Type", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("", "MANGA", "MANHWA", "MANHUA") - val DISPLAY = arrayOf("All", "Manga", "Manhwa", "Manhua") - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt deleted file mode 100644 index 1d685b12..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt +++ /dev/null @@ -1,950 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import android.content.SharedPreferences -import android.util.Log -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.model.UpdateStrategy -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.serialization.SerializationException -import kotlinx.serialization.json.JsonObject -import okhttp3.FormBody -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements -import rx.Observable -import java.text.SimpleDateFormat -import kotlin.collections.iterator - -abstract class GalleryAdults( - override val name: String, - override val baseUrl: String, - override val lang: String = "all", - protected open val mangaLang: String = LANGUAGE_MULTI, - protected val simpleDateFormat: SimpleDateFormat? = null, -) : HttpSource(), - ConfigurableSource { - - protected open val xhrHeaders = headers.newBuilder() - .add("X-Requested-With", "XMLHttpRequest") - .build() - - /* Preferences */ - protected val preferences: SharedPreferences by getPreferencesLazy() - - protected open val useShortTitlePreference = true - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = PREF_SHORT_TITLE - title = "Display Short Titles" - setDefaultValue(false) - }.also(screen::addPreference) - } - - protected val SharedPreferences.shortTitle - get() = getBoolean(PREF_SHORT_TITLE, false) - - /* List detail */ - protected class SMangaDto( - val title: String, - val url: String, - val thumbnail: String?, - val lang: String, - ) - - protected open fun Element.mangaTitle(selector: String = ".caption"): String? = mangaFullTitle(selector).let { - if (preferences.shortTitle) it?.shortenTitle() else it - } - - protected open fun Element.mangaFullTitle(selector: String) = selectFirst(selector)?.text() - - protected open fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() - - protected open val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") - - protected open fun Element.mangaUrl() = selectFirst(".inner_thumb a")?.attr("abs:href") - - protected open fun Element.mangaThumbnail() = selectFirst(".inner_thumb img")?.imgAttr() - - // Overwrite this to filter other languages' manga from search result. - // Default to [mangaLang] won't filter anything - protected open fun Element.mangaLang() = mangaLang - - protected open fun HttpUrl.Builder.addPageUri(page: Int): HttpUrl.Builder { - val url = toString() - if (!url.endsWith('/') && !url.contains('?')) { - addPathSegment("") // trailing slash (/) - } - addQueryParameter("page", page.toString()) - return this - } - - /* Popular */ - override fun popularMangaRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - if (mangaLang.isNotBlank()) addPathSegments("language/$mangaLang") - if (supportsLatest) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(popularMangaSelector()).map { element -> - popularMangaFromElement(element) - } - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - return MangasPage(mangas, hasNextPage) - } - - protected open fun popularMangaSelector() = "div.thumb" - - protected open fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - title = element.mangaTitle()!! - setUrlWithoutDomain(element.mangaUrl()!!) - thumbnail_url = element.mangaThumbnail() - } - - protected open fun popularMangaNextPageSelector(): String? = ".pagination li.active + li:not(.disabled)" - - /* Latest */ - override fun latestUpdatesRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - if (mangaLang.isNotBlank()) addPathSegments("language/$mangaLang") - addPageUri(page) - } - return GET(url.build(), headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(latestUpdatesSelector()).map { element -> - latestUpdatesFromElement(element) - } - val hasNextPage = latestUpdatesNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - return MangasPage(mangas, hasNextPage) - } - - protected open fun latestUpdatesSelector() = popularMangaSelector() - - protected open fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element) - - protected open fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector() - - /* Search */ - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (query.startsWith("https://")) { - val url = query.toHttpUrl() - if (url.host != baseUrl.toHttpUrl().host) { - throw Exception("Unsupported url") - } - val id = url.pathSegments.getOrNull(1) - ?: throw Exception("Unsupported url") - return fetchSearchManga(page, "$PREFIX_ID_SEARCH$id", filters) - } - - val randomEntryFilter = filters.firstInstanceOrNull() - - return when { - randomEntryFilter?.state == true -> { - client.newCall(randomEntryRequest()) - .asObservableSuccess() - .map { response -> randomEntryParse(response) } - } - - query.startsWith(PREFIX_ID_SEARCH) -> { - val id = query.removePrefix(PREFIX_ID_SEARCH) - client.newCall(searchMangaByIdRequest(id)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, id) } - } - - query.toIntOrNull() != null -> { - client.newCall(searchMangaByIdRequest(query)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, query) } - } - - else -> { - client.newCall(searchMangaRequest(page, query, filters)) - .asObservableSuccess() - .map { response -> searchMangaParse(response) } - } - } - } - - protected open fun randomEntryRequest(): Request = GET("$baseUrl/random/", headers) - - protected open fun randomEntryParse(response: Response): MangasPage { - val document = response.asJsoup() - - val url = response.request.url.toString() - val id = url.removeSuffix("/").substringAfterLast('/') - return MangasPage( - listOf( - SManga.create().apply { - title = document.mangaTitle("h1")!! - setUrlWithoutDomain("$baseUrl/$idPrefixUri/$id/") - thumbnail_url = document.getCover() - }, - ), - false, - ) - } - - /** - * Manga URL: $baseUrl/$idPrefixUri// - */ - protected open val idPrefixUri = "gallery" - - protected open fun searchMangaByIdRequest(id: String): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment(idPrefixUri) - addPathSegments("$id/") - } - return GET(url.build(), headers) - } - - protected open fun searchMangaByIdParse(response: Response, id: String): MangasPage { - val details = mangaDetailsParse(response.asJsoup()) - details.url = "/$idPrefixUri/$id/" - return MangasPage(listOf(details), false) - } - - protected open val useIntermediateSearch: Boolean = false - protected open val supportAdvancedSearch: Boolean = false - protected open val supportSpeechless: Boolean = false - protected open val useBasicSearch: Boolean - get() = !useIntermediateSearch - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - val favoriteFilter = filters.firstInstanceOrNull() - - // Speechless - val speechlessFilter = filters.firstInstanceOrNull() - - // Advanced search - val advancedSearchFilters = filters.filterIsInstance() - - return when { - favoriteFilter?.state == true -> - favoriteFilterSearchRequest(page, query, filters) - - supportSpeechless && speechlessFilter?.state == true -> - speechlessFilterSearchRequest(page, query, filters) - - supportAdvancedSearch && advancedSearchFilters.any { it.state.isNotBlank() } -> - advancedSearchRequest(page, query, filters) - - selectedGenres.size == 1 && query.isBlank() -> - tagBrowsingSearchRequest(page, query, filters) - - useIntermediateSearch -> - intermediateSearchRequest(page, query, filters) - - useBasicSearch && (selectedGenres.size > 1 || query.isNotBlank()) -> - basicSearchRequest(page, query, filters) - - sortOrderFilter?.state == 1 -> - latestUpdatesRequest(page) - - else -> - popularMangaRequest(page) - } - } - - protected open val basicSearchKey = "q" - - /** - * Basic Search: support query string with multiple-genres filter by adding genres to query string. - */ - protected open fun basicSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegments("search/") - addEncodedQueryParameter(basicSearchKey, buildQueryString(selectedGenres.map { it.name }, query)) - if (sortOrderFilter?.state == 0) addQueryParameter("sort", "popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - protected open val intermediateSearchKey = "key" - - /** - * This supports filter query search with languages, categories (manga, doujinshi...) - * with additional sort orders. - */ - protected open fun intermediateSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Intermediate search - val categoryFilters = filters.firstInstanceOrNull() - - // Only for query string or multiple tags - val url = "$baseUrl/search/".toHttpUrl().newBuilder().apply { - getSortOrderURIs().forEachIndexed { index, pair -> - addQueryParameter(pair.second, toBinary(sortOrderFilter?.state == index)) - } - categoryFilters?.state?.forEach { - addQueryParameter(it.uri, toBinary(it.state)) - } - getLanguageURIs().forEach { pair -> - addQueryParameter( - pair.second, - toBinary(mangaLang == pair.first || mangaLang == LANGUAGE_MULTI), - ) - } - addEncodedQueryParameter(intermediateSearchKey, buildQueryString(selectedGenres.map { it.name }, query)) - addPageUri(page) - } - return GET(url.build(), headers) - } - - protected open val advancedSearchKey = "key" - protected open val advancedSearchUri = "advsearch" - - /** - * Advanced Search normally won't support search for string but allow to include/exclude specific - * tags/artists/groups/parodies/characters - */ - protected open fun advancedSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Intermediate search - val categoryFilters = filters.firstInstanceOrNull() - // Advanced search - val advancedSearchFilters = filters.filterIsInstance() - - val url = "$baseUrl/$advancedSearchUri/".toHttpUrl().newBuilder().apply { - getSortOrderURIs().forEachIndexed { index, pair -> - addQueryParameter(pair.second, toBinary(sortOrderFilter?.state == index)) - } - categoryFilters?.state?.forEach { - addQueryParameter(it.uri, toBinary(it.state)) - } - getLanguageURIs().forEach { pair -> - addQueryParameter( - pair.second, - toBinary( - mangaLang == pair.first || - mangaLang == LANGUAGE_MULTI, - ), - ) - } - - // Build this query string: +tag:"bat+man"+-tag:"cat"+artist:"Joe"... - // +tag must be encoded into %2Btag while the rest are not needed to encode - val keys = emptyList().toMutableList() - keys.addAll(selectedGenres.map { "%2Btag:\"${it.name}\"" }) - advancedSearchFilters.forEach { filter -> - val key = when (filter) { - is TagsFilter -> "tag" - is ParodiesFilter -> "parody" - is ArtistsFilter -> "artist" - is CharactersFilter -> "character" - is GroupsFilter -> "group" - else -> null - } - if (key != null) { - keys.addAll( - filter.state.trim() - .replace(regexSpaceNotAfterComma, "+") - .replace(" ", "") - .split(',') - .mapNotNull { - val match = regexExcludeTerm.find(it) - match?.groupValues?.let { groups -> - "${if (groups[1].isNotBlank()) "-" else "%2B"}$key:\"${groups[2]}\"" - } - }, - ) - } - } - addEncodedQueryParameter(advancedSearchKey, keys.joinToString("+")) - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Convert space( ) typed in search-box into plus(+) in URL. Then: - * - uses plus(+) to search for exact match - * - use comma(, ) for separate terms, as AND condition. - * Plus(+) after comma(, ) doesn't have any effect. - */ - protected open fun buildQueryString(tags: List, query: String): String = (tags + query).filter(String::isNotBlank) - .joinToString(",") { - // any space except after a comma (we're going to replace spaces only between words) - it.trim() - .replace(regexSpaceNotAfterComma, "+") - .replace(" ", "") - } - - protected open fun tagBrowsingSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Browsing single tag's catalog - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("tag") - addPathSegment(selectedGenres.single().uri) - if (sortOrderFilter?.state == 0) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Browsing speechless titles. Some sites exclude speechless titles from normal search and - * allow browsing separately. - */ - protected open fun speechlessFilterSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("language") - addPathSegment(LANGUAGE_SPEECHLESS) - if (sortOrderFilter?.state == 0) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Browsing user's personal favorites saved on site. This requires login in view WebView. - */ - protected open fun favoriteFilterSearchRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/$favoritePath".toHttpUrl().newBuilder() - return POST( - url.build().toString(), - xhrHeaders, - FormBody.Builder() - .add("page", page.toString()) - .build(), - ) - } - - protected open val favoritePath = "user/fav_pags.php" - - protected open fun loginRequired(document: Document, url: String): Boolean = ( - url.contains("/login/") && - document.select("input[value=Login]").isNotEmpty() - ) - - override fun searchMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - if (loginRequired(document, response.request.url.toString())) { - throw Exception("Log in via WebView to view favorites") - } else { - val hasNextPage = searchMangaNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - val mangas = document.select(searchMangaSelector()) - .searchMangaFromElements(hasNextPage) - return MangasPage(mangas, hasNextPage) - } - } - - protected open fun Elements.searchMangaFromElements(hasNextPage: Boolean): List = mapNotNull { - SMangaDto( - title = it.mangaTitle() ?: return@mapNotNull null, - url = it.mangaUrl() ?: return@mapNotNull null, - thumbnail = it.mangaThumbnail(), - lang = it.mangaLang(), - ) - } - .let { unfiltered -> - val results = unfiltered.filter { mangaLang.isBlank() || it.lang == mangaLang } - // return at least 1 title if all mangas in current page is of other languages - if (results.isEmpty() && hasNextPage) { - unfiltered.firstOrNull()?.let(::listOf) ?: emptyList() - } else { - results - } - } - .map { - SManga.create().apply { - title = it.title - setUrlWithoutDomain(it.url) - thumbnail_url = it.thumbnail - } - } - - protected open fun searchMangaSelector() = popularMangaSelector() - - protected open fun searchMangaNextPageSelector(): String? = popularMangaNextPageSelector() - - /* Related titles */ - protected open fun relatedMangaSelector() = popularMangaSelector() - - override fun relatedMangaListParse(response: Response): List = response.asJsoup() - .select(relatedMangaSelector()) - .searchMangaFromElements(hasNextPage = false) - - /* Details */ - protected open val mangaDetailInfoSelector = ".gallery_top" - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - protected open fun mangaDetailsParse(document: Document): SManga = document.selectFirst(mangaDetailInfoSelector)!!.run { - SManga.create().apply { - update_strategy = UpdateStrategy.ONLY_FETCH_ONCE - status = SManga.COMPLETED - mangaTitle("h1")?.let { title = it } - thumbnail_url = getCover() - genre = getInfo("Tags") - author = getInfo("Artists") - description = getDescription(document) - } - } - - protected open fun Element.getCover() = selectFirst(".cover img")?.imgAttr() - - protected val regexTag = Regex("Tags?") - - /** - * Parsing document to extract info related to [tag]. - */ - protected abstract fun Element.getInfo(tag: String): String - - protected open fun Element.getDescription(document: Document? = null): String = ( - listOf("Parodies", "Characters", "Languages", "Categories", "Category") - .mapNotNull { tag -> - getInfo(tag) - .takeIf { it.isNotBlank() } - ?.let { "$tag: $it" } - } + - listOfNotNull( - getInfoPages(document), - getInfoAlternativeTitle(), - getInfoFullTitle(), - ) - ) - .joinToString("\n\n") - - protected open fun Element.getInfoPages(document: Document? = null): String? = document?.inputIdValueOf(totalPagesSelector) - ?.takeIf { it.isNotBlank() } - ?.let { "Pages: $it" } - - protected open fun Element.getInfoAlternativeTitle(): String? = selectFirst("h1 + h2, .subtitle")?.ownText() - .takeIf { !it.isNullOrBlank() } - ?.let { "Alternative title: $it" } - - protected open fun Element.getInfoFullTitle(): String? = if (preferences.shortTitle) "Full title: ${mangaFullTitle("h1")}" else null - - protected open fun Element.getTime(): Long = selectFirst(".uploaded") - ?.ownText() - .toDate(simpleDateFormat) - - /* Chapters */ - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - return listOf( - SChapter.create().apply { - name = "Chapter" - scanlator = document.selectFirst(mangaDetailInfoSelector) - ?.getInfo("Groups") - date_upload = document.getTime() - setUrlWithoutDomain(response.request.url.encodedPath) - }, - ) - } - - /* Pages */ - protected open fun Element.inputIdValueOf(string: String): String = select("input[id=$string]").attr("value") - - protected open val pagesRequest = "inc/thumbs_loader.php" - protected open val galleryIdSelector = "gallery_id" - protected open val loadIdSelector = "load_id" - protected open val loadDirSelector = "load_dir" - protected open val totalPagesSelector = "load_pages" - protected open val serverSelector = "load_server" - - protected open fun pageRequestForm(document: Document, totalPages: String, loadedPages: Int): FormBody { - val token = document.select("[name=csrf-token]").attr("content") - val serverNumber = document.serverNumber() - - return FormBody.Builder() - .add("u_id", document.inputIdValueOf(galleryIdSelector)) - .add("g_id", document.inputIdValueOf(loadIdSelector)) - .add("img_dir", document.inputIdValueOf(loadDirSelector)) - .add("visible_pages", loadedPages.toString()) - .add("total_pages", totalPages) - .add("type", "2") // 1 would be "more", 2 is "all remaining" - .apply { - if (token.isNotBlank()) add("_token", token) - if (serverNumber != null) add("server", serverNumber) - } - .build() - } - - protected open val thumbnailSelector = ".gallery_thumb" - - protected open fun Element.getServer(): String { - val domain = baseUrl.toHttpUrl().host - return serverNumber() - ?.let { "m$it.$domain" } - ?: getCover()!!.toHttpUrl().host - } - - protected open fun Element.serverNumber(): String? = inputIdValueOf(serverSelector) - .takeIf { it.isNotBlank() } - - protected open fun Element.parseJson(): String? = selectFirst("script:containsData(parseJSON)")?.data() - ?.substringAfter("$.parseJSON('") - ?.substringBefore("');")?.trim() - - /** - * Page URL: $baseUrl/$pageUri// - */ - protected open val pageUri = "g" - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - protected open fun pageListParse(document: Document): List { - val json = document.parseJson() - - if (json != null) { - val loadDir = document.inputIdValueOf(loadDirSelector) - val loadId = document.inputIdValueOf(loadIdSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val server = document.getServer() - val imagesUri = "https://$server/$loadDir/$loadId" - - try { - val pages = mutableListOf() - val images = json.parseAs() - - // JSON string in this form: {"1":"j,1100,1148","2":"j,728,689",... - for (image in images) { - val ext = image.value.toString().replace("\"", "").split(",")[0] - val imageExt = when (ext) { - "p" -> "png" - "b" -> "bmp" - "g" -> "gif" - "w" -> "webp" - else -> "jpg" - } - val idx = image.key.toInt() - pages.add( - Page( - index = idx, - imageUrl = "$imagesUri/${image.key}.$imageExt", - url = "$pageUrl/$idx/", - ), - ) - } - return pages - } catch (_: SerializationException) { - Log.e("GalleryAdults", "Failed to decode JSON") - return this.pageListParseAlternative(document) - } - } else { - return this.pageListParseAlternative(document) - } - } - - /** - * Overwrite this to force extension not blindly converting thumbnails to full image - * by simply removing the trailing "t" from file name. Instead, it will open each page, - * one by one, then parsing for actual image's URL. - * This will be much slower but guaranteed work. - * - * This only apply if site doesn't provide 'parseJSON'. - */ - protected open val parsingImagePageByPage: Boolean = false - - /** - * Either: - * - Load all thumbnails then convert thumbnails to full images. - * - Or request then parse for a list of manga's page's URL, - * which will then request one by one to parse for page's image's URL using [imageUrlParse]. - */ - protected open fun pageListParseAlternative(document: Document): List { - val totalPages = document.inputIdValueOf(totalPagesSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val pages = document.select("$thumbnailSelector a") - .mapNotNull { - if (parsingImagePageByPage) { - it.absUrl("href") - } else { - it.selectFirst("img")?.imgAttr() ?: return@mapNotNull null - } - } - .toMutableList() - - if (totalPages.isNotBlank() && totalPages.toInt() > pages.size) { - val form = pageRequestForm(document, totalPages, pages.size) - - val morePages = client.newCall(POST("$baseUrl/$pagesRequest", xhrHeaders, form)) - .execute().use { it -> - it.asJsoup() - .select("a") - .mapNotNull { - if (parsingImagePageByPage) { - it.absUrl("href") - } else { - it.selectFirst("img")?.imgAttr() ?: return@mapNotNull null - } - } - } - if (morePages.isNotEmpty()) { - pages.addAll(morePages) - } else { - return pageListParseDummy(document) - } - } - - return pages.mapIndexed { idx, url -> - if (parsingImagePageByPage) { - Page(idx, url) - } else { - Page( - index = idx, - imageUrl = url.thumbnailToFull(), - url = "$pageUrl/$idx/", - ) - } - } - } - - /** - * Generate all images using `totalPages`. Supposedly they are sequential. - * Use in case any extension doesn't know how to request for "All thumbnails" - */ - protected open fun pageListParseDummy(document: Document): List { - val loadDir = document.inputIdValueOf(loadDirSelector) - val loadId = document.inputIdValueOf(loadIdSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val server = document.getServer() - val imagesUri = "https://$server/$loadDir/$loadId" - - val images = document.select("$thumbnailSelector img") - val thumbUrls = images.map { it.imgAttr() }.toMutableList() - - val totalPages = document.inputIdValueOf(totalPagesSelector) - - if (totalPages.isNotBlank() && totalPages.toInt() > thumbUrls.size) { - val imagesExt = images.first()?.imgAttr()!! - .substringAfterLast('.') - - thumbUrls.addAll( - listOf((images.size + 1)..totalPages.toInt()).flatten().map { - "$imagesUri/${it}t.$imagesExt" - }, - ) - } - return thumbUrls.mapIndexed { idx, url -> - Page( - index = idx, - imageUrl = url.thumbnailToFull(), - url = "$pageUrl/$idx/", - ) - } - } - - override fun imageUrlParse(response: Response): String = imageUrlParse(response.asJsoup()) - - protected open fun imageUrlParse(document: Document): String = document.selectFirst("img#gimg, img#fimg")?.imgAttr() ?: "" - - /* Filters */ - private val scope = CoroutineScope(Dispatchers.IO) - private fun launchIO(block: () -> Unit) = scope.launch { block() } - private var tagsFetched = false - private var tagsFetchAttempt = 0 - - /** - * List of tags in pairs - */ - protected var genres: MutableMap = mutableMapOf() - - protected open fun tagsRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegments("tags/popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Parsing [document] to return a list of tags in pairs. - */ - protected open fun tagsParser(document: Document): List = document.select("a.tag_btn") - .mapNotNull { - Genre( - it.select(".list_tag, .tag_name").text(), - it.attr("href") - .removeSuffix("/").substringAfterLast('/'), - ) - } - - protected open fun requestTags() { - if (!tagsFetched && tagsFetchAttempt < 3) { - launchIO { - val tags = mutableListOf() - try { - (1..5).forEach { page -> - runCatching { - client.newCall(tagsRequest(page)) - .execute().use { it.asJsoup().let(::tagsParser) } - }.onSuccess { parsed -> - tags.addAll(parsed) - } - } - - tags.sortedWith(compareBy { it.name }) - .forEach { genres[it.name] = it.uri } - - tagsFetched = true - } finally { - tagsFetchAttempt++ - } - } - } - } - - override fun getFilterList(): FilterList { - requestTags() - val filters = emptyList>().toMutableList() - if (useIntermediateSearch) { - filters.add(Filter.Header("HINT: Separate search term with comma (,)")) - } - - filters.add(SortOrderFilter(getSortOrderURIs())) - - if (genres.isEmpty()) { - filters.add(Filter.Header("Press 'reset' to attempt to load tags")) - } else { - filters.add(GenresFilter(genres)) - } - - if (useIntermediateSearch || supportAdvancedSearch) { - filters.addAll( - listOf( - Filter.Separator(), - CategoryFilters(getCategoryURIs()), - ), - ) - } - - if (supportAdvancedSearch) { - filters.addAll( - listOf( - Filter.Separator(), - Filter.Header("Advanced filters will ignore query search. Separate terms by comma (,) and precede term with minus (-) to exclude."), - TagsFilter(), - ParodiesFilter(), - ArtistsFilter(), - CharactersFilter(), - GroupsFilter(), - ), - ) - } - - filters.add(Filter.Separator()) - - if (supportSpeechless) { - filters.add(SpeechlessFilter()) - } - filters.add(FavoriteFilter()) - - filters.add(RandomEntryFilter()) - - return FilterList(filters) - } - - protected open fun getSortOrderURIs() = listOf( - Pair("Popular", "pp"), - Pair("Latest", "lt"), - ) + if (useIntermediateSearch || supportAdvancedSearch) { - listOf( - Pair("Downloads", "dl"), - Pair("Top Rated", "tr"), - ) - } else { - emptyList() - } - - protected open fun getCategoryURIs() = listOf( - SearchFlagFilter("Manga", "m"), - SearchFlagFilter("Doujinshi", "d"), - SearchFlagFilter("Western", "w"), - SearchFlagFilter("Image Set", "i"), - SearchFlagFilter("Artist CG", "a"), - SearchFlagFilter("Game CG", "g"), - ) - - protected open fun getLanguageURIs() = listOf( - Pair(LANGUAGE_ENGLISH, "en"), - Pair(LANGUAGE_JAPANESE, "jp"), - Pair(LANGUAGE_SPANISH, "es"), - Pair(LANGUAGE_FRENCH, "fr"), - Pair(LANGUAGE_KOREAN, "kr"), - Pair(LANGUAGE_GERMAN, "de"), - Pair(LANGUAGE_RUSSIAN, "ru"), - ) - - companion object { - const val PREFIX_ID_SEARCH = "id:" - - private const val PREF_SHORT_TITLE = "pref_short_title" - - // references to be used in factory - const val LANGUAGE_MULTI = "" - const val LANGUAGE_ENGLISH = "english" - const val LANGUAGE_JAPANESE = "japanese" - const val LANGUAGE_CHINESE = "chinese" - const val LANGUAGE_KOREAN = "korean" - const val LANGUAGE_SPANISH = "spanish" - const val LANGUAGE_FRENCH = "french" - const val LANGUAGE_GERMAN = "german" - const val LANGUAGE_RUSSIAN = "russian" - const val LANGUAGE_SPEECHLESS = "speechless" - const val LANGUAGE_TRANSLATED = "translated" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt deleted file mode 100644 index f4a0c0c3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt +++ /dev/null @@ -1,31 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import eu.kanade.tachiyomi.source.model.Filter - -class Genre(name: String, val uri: String) : Filter.CheckBox(name) -class GenresFilter(genres: Map) : - Filter.Group( - "Tags", - genres.map { Genre(it.key, it.value) }, - ) - -class SortOrderFilter(sortOrderURIs: List>) : Filter.Select("Sort By", sortOrderURIs.map { it.first }.toTypedArray()) - -class FavoriteFilter : Filter.CheckBox("Show favorites only (login via WebView)", false) - -class RandomEntryFilter : Filter.CheckBox("Random manga", false) - -// Speechless -class SpeechlessFilter : Filter.CheckBox("Show speechless items only", false) - -// Intermediate search -class SearchFlagFilter(name: String, val uri: String, state: Boolean = true) : Filter.CheckBox(name, state) -class CategoryFilters(flags: List) : Filter.Group("Categories", flags) - -// Advance search -abstract class AdvancedTextFilter(name: String) : Filter.Text(name) -class TagsFilter : AdvancedTextFilter("Tags") -class ParodiesFilter : AdvancedTextFilter("Parodies") -class ArtistsFilter : AdvancedTextFilter("Artists") -class CharactersFilter : AdvancedTextFilter("Characters") -class GroupsFilter : AdvancedTextFilter("Groups") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt deleted file mode 100644 index 04c21f5a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt +++ /dev/null @@ -1,148 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import keiyoushi.utils.tryParse -import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.Calendar - -// any space except after a comma (we're going to replace spaces only between words) -val regexSpaceNotAfterComma = Regex("""(? absUrl("data-cfsrc") - hasAttr("data-src") -> absUrl("data-src") - hasAttr("data-lazy-src") -> absUrl("data-lazy-src") - hasAttr("srcset") -> absUrl("srcset").substringBefore(" ") - else -> absUrl("src") -} - -fun Element.cleanTag(): String = text().cleanTag() -fun String.cleanTag(): String = replace(regexTagCountNumber, "").trim() - -// convert thumbnail URLs to full image URLs -fun String.thumbnailToFull(): String { - val ext = substringAfterLast(".") - return replace("t.$ext", ".$ext") -} - -fun String?.toDate(simpleDateFormat: SimpleDateFormat?): Long { - this ?: return 0L - - return if (simpleDateFormat != null) { - if (contains(regexDateSuffix)) { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - split(" ").map { - if (it.contains(regexDate)) { - it.replace(regexNotNumber, "") - } else { - it - } - } - .let { simpleDateFormat.tryParse(it.joinToString(" ")) } - } else { - simpleDateFormat.tryParse(this) - } - } else { - parseDate(this) - } -} - -private fun parseDate(date: String?): Long { - date ?: return 0L - - return when { - // Handle 'yesterday' and 'today', using midnight - WordSet("yesterday", "يوم واحد").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("today", "just now").startsWith(date) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("يومين").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -2) // day before yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("ago", "atrás", "önce", "قبل").endsWith(date) -> { - parseRelativeDate(date) - } - - WordSet("hace").startsWith(date) -> { - parseRelativeDate(date) - } - - else -> 0L - } -} - -// Parses dates in this form: 21 hours ago OR "2 days ago (Updated 19 hours ago)" -private fun parseRelativeDate(date: String): Long { - val number = regexRelativeDateTime.find(date)?.value?.toIntOrNull() - ?: date.split(" ").firstOrNull() - ?.replace("one", "1") - ?.replace("a", "1") - ?.toIntOrNull() - ?: return 0L - val now = Calendar.getInstance() - - // Sort by order - return when { - WordSet("detik", "segundo", "second", "วินาที").anyWordIn(date) -> - now.apply { add(Calendar.SECOND, -number) }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minuto", "นาที", "دقائق").anyWordIn(date) -> - now.apply { add(Calendar.MINUTE, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "hour", "ชั่วโมง", "giờ", "ore", "ساعة", "小时").anyWordIn(date) -> - now.apply { add(Calendar.HOUR, -number) }.timeInMillis - - WordSet("hari", "gün", "jour", "día", "dia", "day", "วัน", "ngày", "giorni", "أيام", "天").anyWordIn(date) -> - now.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("week", "semana").anyWordIn(date) -> - now.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - - WordSet("month", "mes").anyWordIn(date) -> - now.apply { add(Calendar.MONTH, -number) }.timeInMillis - - WordSet("year", "año").anyWordIn(date) -> - now.apply { add(Calendar.YEAR, -number) }.timeInMillis - - else -> 0L - } -} - -class WordSet(private vararg val words: String) { - fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) } - fun startsWith(dateString: String): Boolean = words.any { dateString.startsWith(it, ignoreCase = true) } - fun endsWith(dateString: String): Boolean = words.any { dateString.endsWith(it, ignoreCase = true) } -} - -fun toBinary(boolean: Boolean) = if (boolean) "1" else "0" \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt deleted file mode 100644 index f0078e74..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt +++ /dev/null @@ -1,213 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.TachiyomiSource -import okhttp3.OkHttpClient - -@TachiyomiSource("ASTRATOONS", "Astratoons", "pt") -class Astratoons : HttpSource() { - - override val name = "Astratoons" - - override val baseUrl = "https://new.astratoons.com" - - override val lang = "pt-BR" - - override val supportsLatest = true - - override val client: OkHttpClient = network.client.newBuilder() - .connectionPool( - okhttp3.ConnectionPool(5,5,java.util.concurrent.TimeUnit.MINUTES) - ) - .build() - - override val versionId: Int = 2 - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - // ======================== Popular ========================== - - override fun popularMangaRequest(page: Int): Request { - - val url = "$baseUrl/api/comics" - .toHttpUrl() - .newBuilder() - .addQueryParameter("sortBy", "views") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "40") - .build() - - return GET(url, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - - val dto = response.parseAs() - - return MangasPage( - dto.data.map { - it.toSManga(baseUrl) - }, - dto.currentPage < dto.lastPage - ) - } - - // ======================== Latest ========================== - - override fun latestUpdatesRequest(page: Int): Request { - val url = "$baseUrl/api/comics" - .toHttpUrl() - .newBuilder() - .addQueryParameter("sortBy", "updated_at") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "40") - .build() - - return GET(url, headers) - } - override fun latestUpdatesParse(response: Response): MangasPage { - val dto = response.parseAs() - val mangas = dto.data.map { it.toSManga(baseUrl) } - return MangasPage(mangas, dto.currentPage < dto.lastPage) - } - - // ======================== Search ========================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/api/comics".toHttpUrl().newBuilder() - .addQueryParameter("page", page.toString()) - - if (query.isNotEmpty()) { - url.addQueryParameter("search", query) - } - - filters.firstInstanceOrNull()?.let { - url.addQueryParameter("sortBy", it.toQuery()) - } - - filters.firstInstanceOrNull()?.toQuery() - ?.takeIf { it.isNotEmpty() } - ?.let { url.addQueryParameter("status", it) } - - filters.firstInstanceOrNull()?.state - ?.filter { it.state } - ?.forEach { url.addQueryParameter("types[]", it.value) } - - filters.firstInstanceOrNull()?.state - ?.filter { it.state } - ?.forEach { url.addQueryParameter("tags[]", it.value) } - - return GET(url.build(), headers) - } - - override fun searchMangaParse(response: Response) = latestUpdatesParse(response) - - // ======================== Details ========================= - - override fun mangaDetailsParse(response: Response) = SManga.create().apply { - val document = response.asJsoup() - title = document.selectFirst("h1")?.text() ?: "Unknown" - thumbnail_url = document.selectFirst("img[class*=object-cover]")?.absUrl("src") - description = document.selectFirst("div.space-y-4 > p")?.text() - ?: document.selectFirst("div:has(>h1) + div")?.text() - genre = document.select("h3:contains(Tags) + div a").joinToString { it.text() } - author = document.selectFirst("span:contains(Autor) > span")?.text() - artist = document.selectFirst("span:contains(Artista) > span")?.text() - - val statusText = document.selectFirst("h3:contains(Informações) + div span.capitalize")?.text() - status = when (statusText?.lowercase()) { - "em andamento", "em dia" -> SManga.ONGOING - "completo" -> SManga.COMPLETED - "hiato" -> SManga.ON_HIATUS - "cancelado", "dropado" -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - - // ======================== Chapter ========================= - - override fun chapterListRequest(manga: SManga): Request = mangaDetailsRequest(manga) - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - val html = document.html() - val mangaId = MANGA_ID.find(html)?.groupValues?.get(1) - ?: throw Exception("Could not find manga id") - - var page = 1 - var hasMore = true - val chapters = mutableListOf() - - while (hasMore) { - val url = "$baseUrl/api/comics/$mangaId/chapters".toHttpUrl().newBuilder() - .addQueryParameter("search", "") - .addQueryParameter("order", "desc") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "100") - .build() - - val res = client.newCall(GET(url, headers)).execute() - val dto = res.parseAs() - - val fragment = Jsoup.parseBodyFragment(dto.html, baseUrl) - chapters += fragment.select("a").map { element -> - SChapter.create().apply { - name = element.selectFirst(".text-lg")?.text() ?: "Chapter" - setUrlWithoutDomain(element.absUrl("href")) - } - } - - hasMore = dto.hasMore - page++ - } - - return chapters - } - - // ======================== Pages =========================== - - override fun imageRequest(page: Page): Request { - val imageHeaders = headersBuilder() - .set("Referer", page.url) - .build() - - return GET(page.imageUrl!!, imageHeaders) - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - return document.select("#reader-container img[src], #reader-container canvas[data-src]") - .mapIndexed { index, element -> - val imageUrl = element.absUrl("src").ifEmpty { element.absUrl("data-src") } - Page(index, document.location(), imageUrl) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ======================== Filters ========================= - - override fun getFilterList() = FilterList( - SortFilter(), - StatusFilter(), - TypeFilter(), - TagFilter(), - ) - - companion object { - val MANGA_ID = """comicId:\s*(\d+)""".toRegex() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt deleted file mode 100644 index 2dcfdf18..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt +++ /dev/null @@ -1,31 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.source.model.SManga -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -class ChapterListDto( - val hasMore: Boolean, - val html: String, -) - -@Serializable -class ComicsResponseDto( - val data: List, - @SerialName("current_page") val currentPage: Int, - @SerialName("last_page") val lastPage: Int, -) - -@Serializable -class ComicDto( - @SerialName("title") private val title: String, - @SerialName("slug") private val slug: String, - @SerialName("cover_image") private val coverImage: String, -) { - fun toSManga(baseUrl: String) = SManga.create().apply { - this.title = this@ComicDto.title - this.thumbnail_url = "$baseUrl/storage/$coverImage" - this.url = "/comics/$slug" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt deleted file mode 100644 index 295abf64..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt +++ /dev/null @@ -1,80 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.source.model.Filter - -internal class CheckBox(name: String, val value: String) : Filter.CheckBox(name) - -internal class SortFilter : Filter.Select("Ordenar por", arrayOf("Mais Recente", "Título", "Nº de Capítulos")) { - fun toQuery() = when (state) { - 0 -> "updated_at" - 1 -> "title" - 2 -> "chapters_count" - else -> "updated_at" - } -} - -internal class StatusFilter : Filter.Select("Status", arrayOf("Todo Status", "Em dia", "Em andamento", "Completo", "Cancelado", "Hiato", "Dropado")) { - fun toQuery() = when (state) { - 1 -> "em dia" - 2 -> "em andamento" - 3 -> "completo" - 4 -> "cancelado" - 5 -> "hiato" - 6 -> "dropado" - else -> "" - } -} - -internal class TypeFilter : - Filter.Group( - "Tipos", - listOf( - CheckBox("Manga", "Manga"), - CheckBox("Manhwa", "Manhwa"), - CheckBox("Manhua", "Manhua"), - CheckBox("Webtoon", "Webtoon"), - ), - ) - -internal class TagFilter : - Filter.Group( - "Gêneros", - listOf( - CheckBox("Ação", "acao"), - CheckBox("Artes Marciais", "artes-marciais"), - CheckBox("Aventura", "aventura"), - CheckBox("Comédia", "comedia"), - CheckBox("Crime", "crime"), - CheckBox("Cultivo", "cultivo"), - CheckBox("Dança", "danca"), - CheckBox("Drama", "drama"), - CheckBox("Ecchi", "ecchi"), - CheckBox("Esporte", "esporte"), - CheckBox("Fantasia", "fantasia"), - CheckBox("Harém", "harem"), - CheckBox("Histórico", "historico"), - CheckBox("Isekai", "isekai"), - CheckBox("Iskai", "iskai"), - CheckBox("Magia", "magia"), - CheckBox("Mistério", "misterio"), - CheckBox("Monstros", "monstros"), - CheckBox("Murim", "murim"), - CheckBox("Reencarnação", "reencarnacao"), - CheckBox("Regressão", "regressao"), - CheckBox("Romance", "romance"), - CheckBox("Sci-fi", "sci-fi"), - CheckBox("Seinen", "seinen"), - CheckBox("Shounen", "shounen"), - CheckBox("Silence", "silence"), - CheckBox("Slice of Life", "slice-of-life"), - CheckBox("Sobrenatural", "sobrenatural"), - CheckBox("Super poderes", "super-poderes"), - CheckBox("Suspense", "suspense"), - CheckBox("Tragédia", "tragedia"), - CheckBox("Troca de gênero", "troca-de-genero"), - CheckBox("Viagem no tempo", "viagem-no-tempo"), - CheckBox("Vida Cotidiana", "vida-cotidiana"), - CheckBox("Vida Escolar", "vida-escolar"), - CheckBox("Zumbis", "zumbis"), - ), - ) \ No newline at end of file From 081e60191e0743f8689edce1f81f8e9fea9cde49 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 11:49:58 +0200 Subject: [PATCH 005/434] Delete src/main/kotlin/org/koitharu/kotatsu/parsers/util directory --- .../koitharu/kotatsu/parsers/util/Context.kt | 7 ------ .../kotatsu/parsers/util/MangaParserExt.kt | 23 ------------------- .../kotatsu/parsers/util/WebViewHelper.kt | 20 ---------------- 3 files changed, 50 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt deleted file mode 100644 index 46a98888..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt +++ /dev/null @@ -1,7 +0,0 @@ -package org.koitharu.kotatsu.parsers.util - -import android.app.Application -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get - -val applicationContext: Application get() = Injekt.get() \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt deleted file mode 100644 index ddacdcb5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt +++ /dev/null @@ -1,23 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.util - -import org.koitharu.kotatsu.parsers.model.MangaListFilter - -/** - * Extension properties to help with smart-casting across modules. - * These return a local copy of the property value. - */ -public val MangaListFilter.queryOrNull: String? - get() = query - -public val MangaListFilter.authorOrNull: String? - get() = author - -public inline fun T?.ifNotNull(block: (T) -> Unit) { - if (this != null) block(this) -} - -public fun String?.nullIfEmpty(): String? = if (isNullOrEmpty()) null else this - - diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt deleted file mode 100644 index 08962d96..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt +++ /dev/null @@ -1,20 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.util - -import org.koitharu.kotatsu.parsers.MangaLoaderContext - -public class WebViewHelper( - private val context: MangaLoaderContext, -) { - - public suspend fun getLocalStorageValue(domain: String, key: String): String? { - return context.evaluateJs("https://$domain/", "window.localStorage.getItem(\"$key\")") - } - - public suspend fun getUrlValue(url: String, value: String): String? { - return context.evaluateJs(url, "new URLSearchParams(window.location.search).get('$value');") - } -} - - From 86bd5c25dd9bddbf9942f2fd69abeb0e6f18feb9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 12:01:24 +0200 Subject: [PATCH 006/434] Delete src/main/kotlin/org/koitharu/kotatsu/parsers/compat directory --- .../parsers/compat/TachiyomiSourceAdapter.kt | 560 ------------------ 1 file changed, 560 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt deleted file mode 100644 index 3f418bae..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt +++ /dev/null @@ -1,560 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.compat - -import android.app.Application -import eu.kanade.tachiyomi.network.NetworkHelper -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import okhttp3.Headers -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.InternalParsersApi -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import uy.kohesive.injekt.Injekt - -/** - * Adapter that wraps a Tachiyomi [HttpSource] and exposes it as a parser via [PagedMangaParser]. - * Enables Tachiyomi extension sources to be used within Usagi and Kotatsu ecosystem (if possible). - */ -@OptIn(InternalParsersApi::class) -open class TachiyomiSourceAdapter( - context: MangaLoaderContext, - source: MangaParserSource, - protected val tachiyomiSource: HttpSource, -) : PagedMangaParser(context, source, DEFAULT_PAGE_SIZE) { - - /** Disable login button, it's useless */ - override val authorizationProvider: MangaParserAuthProvider? = null - - /** Default for source settings. */ - protected open val splitByTranslationsKey = ConfigKey.SplitByTranslations(false) - protected open val showSuspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) - protected open val preferredImageServerKey = ConfigKey.PreferredImageServer(emptyMap(), null) - - init { - /** Register Application context so Injekt.get() works. */ - runCatching { - val activityThreadClass = Class.forName("android.app.ActivityThread") - val currentApplicationMethod = activityThreadClass.getMethod("currentApplication") - val app = currentApplicationMethod.invoke(null) as? Application - if (app != null) { - Injekt.addSingleton(app) - } else { - Injekt.addSingleton(Application()) - } - Injekt.addSingleton(context) - }.onFailure { - Injekt.addSingleton(Application()) - Injekt.addSingleton(context) - } - - /** Setup intercepted client for domain and User-Agent swapping */ - val originalDomain = runCatching { java.net.URI(tachiyomiSource.baseUrl).host }.getOrNull() - val interceptedClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - var request = chain.request() - val builder = request.newBuilder() - val customUa = config[userAgentKey] - if (originalDomain != null && request.url.host == originalDomain) { - val newUrl = request.url.newBuilder() - .host(domain) - .build() - builder.url(newUrl) - } - if (customUa.isNotBlank()) { - builder.header("User-Agent", customUa) - } - chain.proceed(builder.build()) - } - .build() - - runCatching { - tachiyomiSource.javaClass.getMethod("setClient", okhttp3.OkHttpClient::class.java) - .invoke(tachiyomiSource, interceptedClient) - } - - NetworkHelper.setClient(interceptedClient) - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val configKeyDomain: ConfigKey.Domain by lazy { - val host = runCatching { - java.net.URI(tachiyomiSource.baseUrl).host - ?: tachiyomiSource.baseUrl.toHostOnly() - }.getOrDefault(tachiyomiSource.baseUrl.toHostOnly()) - ConfigKey.Domain(host) - } - - // ========================== Sort / Order =============================== - override val availableSortOrders: Set = buildSet { - add(SortOrder.UPDATED) - add(SortOrder.POPULARITY) - if (tachiyomiSource.supportsLatest) { - add(SortOrder.NEWEST) - } - } - - /** - * Lazily computed: true if the source's filter list contains any Filter.Text entry, - * which typically means author/artist/year text inputs are supported. - */ - private val hasTextFilter: Boolean by lazy { - tachiyomiSource.getFilterList().any { it is eu.kanade.tachiyomi.source.model.Filter.Text } - } - - /** Mapping filters, allow to search + use all filters through tags */ - @InternalParsersApi - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isAuthorSearchSupported = hasTextFilter, - ) - - - // ============================== List =================================== - - open override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - - val mangasPage: MangasPage = withContext(Dispatchers.IO) { - when { - !query.isNullOrEmpty() || filter.hasNonSearchOptions() -> { - val tachiyomiFilters = tachiyomiSource.getFilterList() - - // --- Map Kotatsu tag filters → Tachiyomi CheckBox / Group --- - filter.tags.forEach { tag -> - tachiyomiFilters.forEach { f -> - if (f is eu.kanade.tachiyomi.source.model.Filter.CheckBox && f.name == tag.title) { - f.state = true - } else if (f is eu.kanade.tachiyomi.source.model.Filter.Group<*>) { - f.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox && sub.name == tag.title) { - sub.state = true - } - } - } else if (f is eu.kanade.tachiyomi.source.model.Filter.Select<*> && !isStatusFilterSelect(f)) { - // For generic Select filters, select the matching option by name - val idx = f.values.indexOfFirst { it.toString() == tag.title } - if (idx >= 0) f.state = idx - } - } - } - - // --- Map Kotatsu state filters → Tachiyomi StatusFilter (Group or Select) --- - val states = filter.states - if (states.isNotEmpty()) { - tachiyomiFilters.forEach { f -> - when { - f is eu.kanade.tachiyomi.source.model.Filter.Group<*> && isStatusFilterGroup(f) -> { - f.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox) { - val mappedState = mapNameToState(sub.name) - if (mappedState != null && mappedState in states) { - sub.state = true - } - } - } - } - f is eu.kanade.tachiyomi.source.model.Filter.Select<*> && isStatusFilterSelect(f) -> { - // Pick the first matching option for single-select status - val values = f.values - for (i in 1 until values.size) { - val mappedState = mapNameToState(values[i].toString()) - if (mappedState != null && mappedState in states) { - f.state = i - break - } - } - } - else -> {} - } - } - } - - tachiyomiSource.fetchSearchManga(page, query ?: "", tachiyomiFilters) - .toBlocking().single() - } - order == SortOrder.NEWEST && tachiyomiSource.supportsLatest -> { - tachiyomiSource.fetchLatestUpdates(page).toBlocking().single() - } - else -> { - tachiyomiSource.fetchPopularManga(page).toBlocking().single() - } - } - } - - return mangasPage.mangas.map { it.toManga() } - } - - // ============================== Details ================================ - - open override suspend fun getDetails(manga: Manga): Manga { - val sManga = manga.toSManga() - - return withContext(Dispatchers.IO) { - val details = tachiyomiSource.fetchMangaDetails(sManga).toBlocking().single() - val chapters = tachiyomiSource.fetchChapterList(sManga).toBlocking().single() - - /** Auto-detect chapter ordering: Tachiyomi default is newest-first, - but some sources return oldest-first. Detect by comparing chapter_number - of first two entries. If not available, fall back to date comparison. */ - val isNewestFirst = detectNewestFirst(chapters) - - manga.copy( - title = details.title.ifEmpty { manga.title }, - altTitles = emptySet(), - coverUrl = details.thumbnail_url ?: manga.coverUrl, - largeCoverUrl = details.thumbnail_url, - description = details.description, - authors = setOfNotNull(details.author, details.artist) - .filter { it.isNotBlank() }.toSet(), - tags = parseGenreTags(details.genre), - state = mapStatus(details.status), - chapters = chapters.mapChapters { i, sChapter -> - sChapter.toMangaChapter(i, chapters.size, isNewestFirst) - }.sortedBy { it.number }, - ) - } - } - - /** - * Detects whether [chapters] are ordered newest-first (index 0 = latest). - * Priority: - * 1. chapter_number: if first > second → newest-first - * 2. date_upload: if first > second → newest-first - * 3. Default: true (Tachiyomi convention) - * It helps rearrange the chapter list in asc order, avoiding confusion. - */ - private fun detectNewestFirst(chapters: List): Boolean { - if (chapters.size < 2) return true - val a = chapters[0] - val b = chapters[1] - if (a.chapter_number > 0f && b.chapter_number > 0f) { - return a.chapter_number > b.chapter_number - } - if (a.date_upload > 0L && b.date_upload > 0L) { - return a.date_upload > b.date_upload - } - return true - } - - // ============================== Pages ================================== - - open override suspend fun getPages(chapter: MangaChapter): List { - val sChapter = SChapter.create().apply { - url = chapter.url - name = chapter.title ?: "" - chapter_number = chapter.number - } - - val pages = withContext(Dispatchers.IO) { - tachiyomiSource.fetchPageList(sChapter).toBlocking().single() - } - - return pages.map { page -> - val imgUrl = page.imageUrl ?: page.url - MangaPage( - id = generateUid(imgUrl.ifEmpty { "${chapter.url}#${page.index}" }), - url = imgUrl, - preview = null, - source = source, - ) - } - } - - open override suspend fun getPageUrl(page: MangaPage): String { - val url = page.url - if (url.startsWith("http://") || url.startsWith("https://")) return url - return "https://$domain$url" - } - - // ============================== Filter Options ========================= - - open override suspend fun getFilterOptions(): MangaListFilterOptions = withContext(Dispatchers.IO) { - val tachiyomiFilters = tachiyomiSource.getFilterList() - val tags = mutableSetOf() - val states = mutableSetOf() - - tachiyomiFilters.forEach { filter -> - when (filter) { - is eu.kanade.tachiyomi.source.model.Filter.CheckBox -> { - tags.add(MangaTag(title = filter.name, key = filter.name.lowercase(), source = source)) - } - - // --- Group (multi-checkbox list, e.g. GenreList or StatusFilter) --- - is eu.kanade.tachiyomi.source.model.Filter.Group<*> -> { - if (isStatusFilterGroup(filter)) { - // Map each child checkbox to a MangaState - filter.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox) { - mapNameToState(sub.name)?.let { states.add(it) } - } - } - } else { - // Generic group → flatten children as tags - filter.state.forEach { sub -> - when (sub) { - is eu.kanade.tachiyomi.source.model.Filter.CheckBox -> { - tags.add( - MangaTag( - title = sub.name, - key = sub.name.lowercase(), - source = source, - ), - ) - } - - else -> {} // ignore nested selects / text inside groups - } - } - } - } - - // --- Select (single-choice dropdown, e.g. GenreFilter / StatusFilter in fastscan) --- - is eu.kanade.tachiyomi.source.model.Filter.Select<*> -> { - if (isStatusFilterSelect(filter)) { - // Map each option to MangaState (skip index 0 which is usually "All") - val values = filter.values - for (i in 1 until values.size) { - mapNameToState(values[i].toString())?.let { states.add(it) } - } - } else { - // Generic select → extract non-"All" options as tags - val values = filter.values - for (i in 1 until values.size) { - val name = values[i].toString() - if (name.isNotBlank()) { - tags.add(MangaTag(title = name, key = name.lowercase(), source = source)) - } - } - } - } - - // Filter.Text / Header / Separator — intentionally ignored for tag/state population - else -> {} - } - } - - MangaListFilterOptions(availableTags = tags, availableStates = states) - } - - /** - * Heuristic: returns true when a [eu.kanade.tachiyomi.source.model.Filter.Group] represents a status/state selector. - * Matches group names containing words like "status", "tình trạng", "trang thai", etc. - */ - private fun isStatusFilterGroup(filter: eu.kanade.tachiyomi.source.model.Filter.Group<*>): Boolean { - val name = filter.name.lowercase() - return STATUS_KEYWORDS.any { name.contains(it) } - } - - /** - * Heuristic: returns true when a [eu.kanade.tachiyomi.source.model.Filter.Select] represents a status/state selector. - */ - private fun isStatusFilterSelect(filter: eu.kanade.tachiyomi.source.model.Filter.Select<*>): Boolean { - val name = filter.name.lowercase() - return STATUS_KEYWORDS.any { name.contains(it) } - } - - /** - * Maps common status label strings (in multiple languages) to a [MangaState]. - * Returns null if no mapping is found. - */ - protected open fun mapNameToState(name: String): MangaState? { - val lower = name.lowercase() - return when { - ONGOING_KEYWORDS.any { lower.contains(it) } -> MangaState.ONGOING - FINISHED_KEYWORDS.any { lower.contains(it) } -> MangaState.FINISHED - PAUSED_KEYWORDS.any { lower.contains(it) } -> MangaState.PAUSED - ABANDONED_KEYWORDS.any { lower.contains(it) } -> MangaState.ABANDONED - else -> null - } - } - - // ============================== Headers ================================ - - open override fun getRequestHeaders(): Headers = tachiyomiSource.headers - - // ============================== Interceptor ============================ - - open override fun intercept(chain: Interceptor.Chain): Response = chain.proceed(chain.request()) - - // ============================== Private Helpers ======================== - - protected open fun SManga.toManga(): Manga { - val mangaUrl = this.url - val publicUrl = if (mangaUrl.startsWith("http")) mangaUrl - else "${tachiyomiSource.baseUrl}$mangaUrl" - - return Manga( - id = generateUid(mangaUrl), - title = this.title, - altTitles = emptySet(), - url = mangaUrl, - publicUrl = publicUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = this.thumbnail_url, - tags = parseGenreTags(this.genre), - state = mapStatus(this.status), - authors = setOfNotNull(this.author, this.artist) - .filter { it.isNotBlank() }.toSet(), - source = source, - ) - } - - protected open fun Manga.toSManga(): SManga = SManga.create().apply { - url = this@toSManga.url - title = this@toSManga.title - thumbnail_url = this@toSManga.coverUrl - } - - protected open fun SChapter.toMangaChapter(index: Int, total: Int, isNewestFirst: Boolean = true): MangaChapter { - // When chapter_number is set by the source, use it directly. - // Otherwise derive from position: newest-first → total-index, oldest-first → index+1 - val number = if (this.chapter_number > 0f) { - this.chapter_number - } else if (isNewestFirst) { - (total - index).toFloat() // index 0 = newest = highest number - } else { - (index + 1).toFloat() // index 0 = oldest = chapter 1 - } - return MangaChapter( - id = generateUid(this.url), - title = this.name.ifBlank { null }, - number = number, - volume = 0, - url = this.url, - scanlator = this.scanlator, - uploadDate = this.date_upload, - branch = null, - source = source, - ) - } - - protected open fun parseGenreTags(genreStr: String?): Set { - if (genreStr.isNullOrBlank()) return emptySet() - return genreStr.split(",") - .map { it.trim() } - .filter { it.isNotBlank() } - .mapTo(mutableSetOf()) { genre -> - MangaTag(title = genre, key = genre.lowercase(), source = source) - } - } - - protected open fun mapStatus(status: Int): MangaState? = when (status) { - SManga.ONGOING -> MangaState.ONGOING - SManga.COMPLETED, SManga.PUBLISHING_FINISHED -> MangaState.FINISHED - SManga.ON_HIATUS -> MangaState.PAUSED - SManga.CANCELLED -> MangaState.ABANDONED - else -> null - } - - protected open fun String.toHostOnly(): String = - removePrefix("https://").removePrefix("http://").substringBefore("/") - - companion object { - const val DEFAULT_PAGE_SIZE = 20 - - /** Filter names (lowercased) that indicate a status/state filter. */ - private val STATUS_KEYWORDS = listOf( - "status", - "tình trạng", - "tinh trang", - "trạng thái", - "trang thai", - "durum", - "état", - "estado", - "zustand", - ) - - /** Label fragments (lowercased) that map to [MangaState.ONGOING]. */ - private val ONGOING_KEYWORDS = listOf( - "ongoing", - "on-going", - "on going", - "updating", - "publishing", - "airing", - "đang", // Vietnamese "đang tiến hành" - "dang tien hanh", - "devam", // Turkish - "en cours", // French - "en curso", // Spanish - "lançando", // Portuguese - "выходит", // Russian - "продолжается", - ) - - /** Label fragments (lowercased) that map to [MangaState.FINISHED]. */ - private val FINISHED_KEYWORDS = listOf( - "completed", - "complete", - "finished", - "end", - "full", - "hoàn", // Vietnamese "hoàn thành" / "hoàn tất" - "hoan thanh", - "trọn bộ", - "tron bo", - "tamamland", // Turkish - "achevé", - "terminé", - "завершено", // Russian - ) - - /** Label fragments (lowercased) that map to [MangaState.PAUSED]. */ - private val PAUSED_KEYWORDS = listOf( - "hiatus", - "on hold", - "on-hold", - "paused", - "beklemede", // Turkish - "durduruldu", - "en pause", - "заморожено", // Russian - ) - - /** Label fragments (lowercased) that map to [MangaState.ABANDONED]. */ - private val ABANDONED_KEYWORDS = listOf( - "canceled", - "cancelled", - "dropped", - "drop", - "abandoned", - "discontinued", - "iptal", // Turkish - "đã hủy", - "da huy", - "abandonné", - "заброшено", // Russian - ) - } -} - From 9fb51e785eecd38fd33aab768bd1f45c8ceaf286 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:11:31 +0200 Subject: [PATCH 007/434] Delete src/main/kotlin directory --- .../parsers/parsers/DoujinDesuParser.kt | 373 ------ .../parsers/parsers/GalleryAdultsParser.kt | 224 ---- .../kotatsu/parsers/parsers/GalleryParser.kt | 124 -- .../kotatsu/parsers/parsers/HeanCmsParser.kt | 213 ---- .../kotatsu/parsers/parsers/HiperParser.kt | 380 ------ .../parsers/parsers/LikeMangaParser.kt | 276 ----- .../kotatsu/parsers/parsers/LilianaParser.kt | 239 ---- .../kotatsu/parsers/parsers/MadaraParser.kt | 896 -------------- .../kotatsu/parsers/parsers/MadthemeParser.kt | 416 ------- .../kotatsu/parsers/parsers/Manga18Parser.kt | 248 ---- .../parsers/parsers/MangaReaderParser.kt | 417 ------- .../kotatsu/parsers/parsers/MangaThemesia.kt | 391 ------ .../kotatsu/parsers/parsers/MangaboxParser.kt | 462 ------- .../parsers/parsers/ZeistMangaParser.kt | 356 ------ .../site/kotatsu/all/MangaDexParser.kt | 543 --------- .../parsers/site/kotatsu/all/MangaFire.kt | 504 -------- .../parsers/site/kotatsu/all/MangaPlus.kt | 293 ----- .../parsers/site/kotatsu/all/Mangaball.kt | 912 -------------- .../parsers/site/kotatsu/all/ProjectSuki.kt | 502 -------- .../parsers/site/kotatsu/all/Webtoons.kt | 455 ------- .../site/kotatsu/all/hentais/AsmHentai.kt | 44 - .../site/kotatsu/all/hentais/BuonDua.kt | 18 - .../site/kotatsu/all/hentais/HentaiEnvy.kt | 81 -- .../site/kotatsu/all/hentais/HentaiFox.kt | 133 -- .../kotatsu/all/hentais/HitomiLaParser.kt | 796 ------------ .../site/kotatsu/all/hentais/IMHentai.kt | 271 ----- .../site/kotatsu/all/hentais/Kiutaku.kt | 34 - .../site/kotatsu/all/hentais/MangaCrazy.kt | 14 - .../site/kotatsu/all/hentais/Misskon.kt | 165 --- .../site/kotatsu/all/hentais/Multporn.kt | 254 ---- .../site/kotatsu/all/hentais/NHentaiParser.kt | 1076 ----------------- .../kotatsu/all/hentais/NHentaiXXXParser.kt | 184 --- .../site/kotatsu/all/hentais/Novelcrow.kt | 13 - .../parsers/site/kotatsu/ar/TeamXNovel.kt | 272 ----- .../parsers/site/kotatsu/en/AquaManga.kt | 195 --- .../parsers/site/kotatsu/en/AsuraScan.kt | 349 ------ .../parsers/site/kotatsu/en/Atsumaru.kt | 457 ------- .../parsers/site/kotatsu/en/Batcave.kt | 251 ---- .../parsers/site/kotatsu/en/CoffeeManga.kt | 10 - .../parsers/site/kotatsu/en/DemonicScan.kt | 244 ---- .../parsers/site/kotatsu/en/EnThunderScans.kt | 24 - .../parsers/site/kotatsu/en/FlameComics.kt | 263 ---- .../parsers/site/kotatsu/en/KaliScan.kt | 230 ---- .../parsers/site/kotatsu/en/KingofShojo.kt | 10 - .../parsers/site/kotatsu/en/LikeManga.kt | 27 - .../parsers/site/kotatsu/en/MangaFox.kt | 236 ---- .../parsers/site/kotatsu/en/MangaGG.kt | 156 --- .../parsers/site/kotatsu/en/MangaGeko.kt | 242 ---- .../parsers/site/kotatsu/en/MangaJinx.kt | 219 ---- .../kotatsu/parsers/site/kotatsu/en/MangaK.kt | 340 ------ .../parsers/site/kotatsu/en/MangaKatana.kt | 396 ------ .../parsers/site/kotatsu/en/MangaTown.kt | 375 ------ .../parsers/site/kotatsu/en/Mangadotnet.kt | 470 ------- .../parsers/site/kotatsu/en/Mangago.kt | 685 ----------- .../parsers/site/kotatsu/en/Mangaread.kt | 13 - .../parsers/site/kotatsu/en/Mangataro.kt | 861 ------------- .../parsers/site/kotatsu/en/Manhuaus.kt | 78 -- .../parsers/site/kotatsu/en/ManhwaNex.kt | 14 - .../parsers/site/kotatsu/en/ManhwaTop.kt | 10 - .../parsers/site/kotatsu/en/ManhwaZ.kt | 103 -- .../kotatsu/parsers/site/kotatsu/en/Mgread.kt | 295 ----- .../parsers/site/kotatsu/en/Novel24h.kt | 16 - .../parsers/site/kotatsu/en/TopManhua.kt | 14 - .../parsers/site/kotatsu/en/WeebCentral.kt | 378 ------ .../parsers/site/kotatsu/en/ZinManga.kt | 59 - .../site/kotatsu/en/hentais/AllPornComic.kt | 16 - .../site/kotatsu/en/hentais/ComicLand.kt | 286 ----- .../site/kotatsu/en/hentais/HentaiNexus.kt | 397 ------ .../site/kotatsu/en/hentais/HentaiXComic.kt | 28 - .../site/kotatsu/en/hentais/HeyToon.kt | 293 ----- .../site/kotatsu/en/hentais/Hiperdex.kt | 19 - .../site/kotatsu/en/hentais/MadaraDex.kt | 176 --- .../site/kotatsu/en/hentais/Manga18.kt | 11 - .../site/kotatsu/en/hentais/MangaDistrict.kt | 106 -- .../site/kotatsu/en/hentais/Manhwa18CC.kt | 162 --- .../site/kotatsu/en/hentais/Manhwa18Com.kt | 240 ---- .../site/kotatsu/en/hentais/Manhwa210.kt | 259 ---- .../site/kotatsu/en/hentais/OmegaScan.kt | 10 - .../site/kotatsu/en/mangabox/MangaBats.kt | 392 ------ .../site/kotatsu/en/mangabox/ManganatoGG.kt | 392 ------ .../parsers/site/kotatsu/id/AinzScans.kt | 301 ----- .../parsers/site/kotatsu/id/MGKomik.kt | 178 --- .../kotatsu/parsers/site/kotatsu/id/ReYume.kt | 177 --- .../parsers/site/kotatsu/id/Westmanga.kt | 257 ---- .../site/kotatsu/id/hentais/BacaKomikpics.kt | 184 --- .../site/kotatsu/id/hentais/DoujinDesu.kt | 21 - .../parsers/site/kotatsu/pt/HiperToon.kt | 10 - .../parsers/site/kotatsu/pt/Roxinha.kt | 218 ---- .../site/kotatsu/pt/hentais/LerHentais.kt | 17 - .../parsers/site/kotatsu/vi/TruyenTuoiTho.kt | 13 - .../kotatsu/vi/hentai/TruyenTranhDamMyy.kt | 43 - .../parsers/site/kotatsu/zh/hentai/Xiutaku.kt | 16 - 92 files changed, 22821 deletions(-) delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt delete mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt deleted file mode 100644 index cabf4992..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt +++ /dev/null @@ -1,373 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.net.URLDecoder -import java.time.Instant -import java.util.* -import kotlin.time.Duration.Companion.milliseconds - -internal abstract class DoujinDesuParser( - context: MangaLoaderContext, - source: MangaParserSource -): PagedMangaParser(context, source, pageSize = 18) { - - protected abstract val defaultTypes: String - protected abstract val availableContentTypes: Set - - override val configKeyDomain = ConfigKey.Domain("doujin.desu.xxx") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - @Volatile - private var genresCache: Set? = null - private val genresMutex = Mutex() - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - override val defaultSortOrder: SortOrder = SortOrder.UPDATED - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isAuthorSearchSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = availableContentTypes, - ) - - private suspend fun getOrFetchGenres(): Set { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - genresCache = fetchAvailableTags() - genresCache!! - } - } - } - - private suspend fun fetchAvailableTags(): Set { - val url = "/api/terms?taxonomy=genre".toAbsoluteUrl(domain) - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val array = JSONArray(decrypted) - return (0 until array.length()).mapTo(mutableSetOf()) { i -> - val obj = array.getJSONObject(i) - MangaTag(key = obj.getString("slug"), title = obj.getString("name"), source = source) - } - } - - override fun getRequestHeaders() = Headers.Builder() - .add("X-Requested-With", "XMLHttpRequest") - .add("Referer", "https://$domain/") - .add("X-App-Secret", "dfdf72051dbfdc7d76889ebd31324e74") - .build() - - private suspend fun executeWithCloudflareRetry(url: String, headers: Headers): okhttp3.Response { - var response = webClient.httpGet(url, headers) - if (response.code == 403) { - val bodyString = response.body?.string().orEmpty() - if ("Unauthorized" in bodyString || "Access denied" in bodyString) { - response.close() - refreshCloudflare() - response = webClient.httpGet(url, headers) - } - } - if (response.code in 500..599) { - response.close() - kotlinx.coroutines.delay(1000L.milliseconds) - response = webClient.httpGet(url, headers) - } - return response - } - - private suspend fun refreshCloudflare() { - webClient.httpGet("https://$domain/", getRequestHeaders()).close() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val limit = pageSize - val offset = (page - 1) * limit - - val url = urlBuilder().apply { - addPathSegment("api") - addPathSegment("manga") - addQueryParameter("limit", limit.toString()) - addQueryParameter("offset", offset.toString()) - - filter.query?.let { addQueryParameter("search", it) } - - val sortParam = when (order) { - SortOrder.POPULARITY -> "rating" - SortOrder.ALPHABETICAL -> "title_asc" - SortOrder.NEWEST -> "newest" - SortOrder.UPDATED -> "latest_chapter" - else -> "latest_chapter" - } - addQueryParameter("sort", sortParam) - - filter.types.oneOrThrowIfMany()?.let { - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.DOUJINSHI -> "doujinshi" - else -> null - }?.let { type -> addQueryParameter("type", type) } - } - - filter.states.oneOrThrowIfMany()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> null - }?.let { status -> addQueryParameter("status", status) } - } - - if (filter.tags.isNotEmpty()) { - addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) - } - }.build() - - val jsonResponse = executeWithCloudflareRetry(url.toString(), getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val array = JSONArray(decrypted) - - return (0 until array.length()).map { i -> - val obj = array.getJSONObject(i) - val slug = obj.getString("slug") - val href = "/manga/$slug" - Manga( - id = generateUid(href), - title = obj.getString("title"), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = obj.optDouble("rating", 0.0).toFloat() / 10f, - contentRating = ContentRating.ADULT, - coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> - if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) - }, - tags = emptySet(), - state = null, - authors = emptySet(), - largeCoverUrl = null, - description = null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val slug = manga.url.removePrefix("/manga/").removeSuffix("/") - val url = "/api/manga/$slug".toAbsoluteUrl(domain) - - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val obj = JSONObject(decrypted) - - val state = when (obj.optString("status")) { - "completed" -> MangaState.FINISHED - "ongoing" -> MangaState.ONGOING - else -> null - } - - val author = obj.optString("author").takeIf { it.isNotEmpty() && it != "null" } - - val tags = mutableSetOf() - val genresArray = obj.optJSONArray("manga_genres") - if (genresArray != null) { - for (i in 0 until genresArray.length()) { - val genreObj = genresArray.getJSONObject(i).getJSONObject("genres") - tags.add( - MangaTag( - key = genreObj.getString("slug"), - title = genreObj.getString("name"), - source = source, - ) - ) - } - } - - val chaptersArray = obj.getJSONArray("chapters") - val chapters = ArrayList(chaptersArray.length()) - for (i in 0 until chaptersArray.length()) { - val chapObj = chaptersArray.getJSONObject(i) - val chNum = chapObj.optDouble("chapter_number", 0.0).toFloat() - val chTitle = if (chNum == chNum.toLong().toFloat()) { - "Chapter ${chNum.toLong()}" - } else { - "Chapter $chNum" - } - chapters.add( - MangaChapter( - id = generateUid("/reader/${chapObj.getString("id")}"), - title = chTitle, - number = chNum, - volume = 0, - url = "/reader/${chapObj.getString("id")}", - scanlator = null, - uploadDate = runCatching { Instant.parse(chapObj.optString("created_at")).toEpochMilli() }.getOrDefault(0L), - branch = null, - source = source, - ) - ) - } - - val rawDesc = obj.optString("description").takeIf { it.isNotEmpty() && it != "null" } - val cleanDesc = rawDesc?.let { html -> - val document = org.jsoup.Jsoup.parseBodyFragment(html) - val fullText = document.text() - val cutoffRegex = Regex("""download\s+(batch|volume)""", RegexOption.IGNORE_CASE) - val cutoffIndex = cutoffRegex.find(fullText)?.range?.first - val textBeforeDownload = if (cutoffIndex != null) fullText.substring(0, cutoffIndex) else fullText - textBeforeDownload - .replace(Regex("""^Sinopsis:\s*""", RegexOption.IGNORE_CASE), "") - .replace(Regex("""https?://\S+"""), "") - .replace(Regex("""\[url=.*?]|\[/url]"""), "") - .trim() - } - - val coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> - if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) - } - - val fullManga = manga.copy( - authors = setOfNotNull(author), - description = cleanDesc, - state = state, - rating = obj.optDouble("rating", 0.0).toFloat() / 10f, - tags = tags, - coverUrl = coverUrl ?: manga.coverUrl, - chapters = chapters.reversed() - ) - - detailsCache[manga.url] = fullManga - return fullManga - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chId = chapter.url.removePrefix("/reader/").removeSuffix("/") - val url = "/api/chapters/$chId".toAbsoluteUrl(domain) - - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val obj = JSONObject(decrypted) - - val pagesList = mutableListOf() - val contentUrls = obj.optJSONArray("content_urls") - if (contentUrls != null && contentUrls.length() > 0) { - for (i in 0 until contentUrls.length()) { - pagesList.add( - MangaPage( - id = generateUid(contentUrls.getString(i)), - url = transformPageUrl(contentUrls.getString(i)), - preview = null, - source = source, - ) - ) - } - } else { - val signedUrlsStr = obj.optString("signed_content_urls") - if (signedUrlsStr.isNotEmpty()) { - val signedUrls = JSONArray(signedUrlsStr) - for (i in 0 until signedUrls.length()) { - pagesList.add( - MangaPage( - id = generateUid(signedUrls.getString(i)), - url = transformPageUrl(signedUrls.getString(i)), - preview = null, - source = source, - ) - ) - } - } - } - - return pagesList - } - - private fun toSigned32(x: Long): Int { - var y = x and 0xFFFFFFFFL - if (y >= 0x80000000L) y -= 0x100000000L - return y.toInt() - } - - private fun wH(e: Int): String { - val t = "doujindesu-scrapers-cannot-read-this-super-secret-salt-2026-v2_$e" - var a = 0 - for (ch in t) { - a = (a shl 5) - a + ch.code - a = toSigned32(a.toLong()) - } - var l = if (a != 0) kotlin.math.abs(a).toLong() else 123456789L - return buildString { - repeat(32) { - l = (l * 1664525L + 1013904223L) % 4294967296L - append((33 + (l % 93)).toInt().toChar()) - } - } - } - - private fun lU(): List { - val now = System.currentTimeMillis() - val t = now / 3600000L - return listOf(wH(t.toInt()), wH((t - 1).toInt()), wH((t + 1).toInt())) - } - - private fun yre(encryptedHex: String, key: String): String { - val bytes = encryptedHex.chunked(2).mapNotNull { it.toIntOrNull(16) } - var d = 42 - val keyLen = key.length - return buildString { - for ((idx, byteVal) in bytes.withIndex()) { - val keyChar = key[idx % keyLen].code - val k = byteVal xor keyChar xor (idx * 13) xor d - append((k and 0xFF).toChar()) - d = (d + byteVal) % 256 - } - } - } - - private fun decrypt(encryptedHex: String): String { - for (key in lU()) { - try { - val decoded = yre(encryptedHex, key) - return URLDecoder.decode(decoded, "UTF-8") - } catch (_: Exception) { - } - } - throw RuntimeException("Decryption failed") - } - - private fun transformPageUrl(page: String): String = when { - page.contains("/uploads/") && !page.contains("/storage/uploads/") -> - page.replace("/uploads/", "/storage/uploads/") - page.contains("/upload/") && !page.contains("/storage/upload/") -> - page.replace("/upload/", "/storage/upload/") - else -> page - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt deleted file mode 100644 index bff94429..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt +++ /dev/null @@ -1,224 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import androidx.collection.ArraySet -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* - -internal abstract class GalleryAdultsParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = setOf( - Locale.ENGLISH, - Locale.FRENCH, - Locale.JAPANESE, - Locale.CHINESE, - Locale("es"), - Locale("ru"), - Locale("ko"), - Locale.GERMAN, - Locale("id"), - Locale.ITALIAN, - Locale("pt"), - Locale("tr"), - Locale("th"), - Locale("vi"), - ), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("/search/?q=") - append(query?.urlEncoded()) - append("&") - } - - else -> { - val tag = filter.tags.oneOrThrowIfMany() - val lang = filter.locale - if (tag != null && lang != null) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) - } - if (tag != null) { - append("/tag/") - append(tag.key) - append("/?") - } else if (filter.locale != null) { - val locale = filter.locale - append("/language/") - append(locale?.toLanguagePath()) - append("/?") - } else { - append("/?") - } - } - } - append("page=") - append(page) - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - protected open val selectGallery = ".thumb" - protected open val selectGalleryLink = ".inner_thumb a" - protected open val selectGalleryImg = "img" - protected open val selectGalleryTitle = "h2" - private val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") - private val regexSpaces = Regex("\\s+") - - protected open fun parseMangaList(doc: Document): List { - return doc.select(selectGallery).map { div -> - val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(selectGalleryTitle).text().cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = div.selectFirst(selectGalleryImg)?.src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. - private suspend fun fetchAvailableTags(): Set { - return coroutineScope { - (1..3).map { page -> - async { getTags(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } - - protected open val pathTagUrl = "/tags/popular/?page=" - protected open val selectTags = ".tags_page ul.tags li" - - private suspend fun getTags(page: Int): Set { - val url = "https://$domain$pathTagUrl$page" - val root = webClient.httpGet(url).parseHtml().selectFirstOrThrow(selectTags) - return root.parseTags() - } - - protected open fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.html().substringBefore("<") - MangaTag( - key = key, - title = name.toTitleCase(), - source = source, - ) - } - - protected open val selectTitle = "h1.title" - protected open val selectTag = "div.tags:contains(Tags:) .tag_list" - protected open val selectAuthor = "ul.artists a.tag_btn" - protected open val selectLanguageChapter = "div.tags:contains(Languages:) .tag_list a span.tag" - protected open val selectUrlChapter = "#cover a, .cover a, .left_cover a, .g_thumb a, .gallery_left a, .gt_left a" - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val urlChapters = doc.selectFirstOrThrow(selectUrlChapter).attr("href") - val tag = doc.selectFirst(selectTag)?.parseTags() - val branch = doc.select(selectLanguageChapter).joinToString(separator = " / ") { - it.html().substringBefore("<") - } - val author = doc.selectFirst(selectAuthor)?.html()?.substringBefore(" { - return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) - } - - protected open val selectTotalPage = ".total_pages, .num-pages, .tp" - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() - val rawUrl = chapter.url.removeSuffix("/").substringBeforeLast("/") + "/" - return (1..totalPages).map { - val url = "$rawUrl$it/" - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - protected open val idImg = "gimg" - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.requireElementById(idImg).requireSrc() - } - - protected fun String.cleanupTitle() = replace(regexBrackets, "") - .replace(regexSpaces, " ") - .trim() - - protected open fun Locale.toLanguagePath() = getDisplayLanguage(Locale.ENGLISH).lowercase() -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt deleted file mode 100644 index a2d0eff4..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt +++ /dev/null @@ -1,124 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class GalleryParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set - get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = urlBuilder().apply { - when { - !query.isNullOrEmpty() -> addQueryParameter("search", query) - filter.tags.isNotEmpty() -> addPathSegments(filter.tags.first().key) - order == SortOrder.POPULARITY -> addPathSegment("hot") - } - - addQueryParameter("start", (page*pageSize).toString()) - }.build() - - val content = webClient.httpGet(url).parseHtml() - return content.select("div.items-row").map { el -> - val titleEl = el.selectFirstOrThrow("div.page-header a.item-link") - val relUrl = titleEl.attrOrThrow("href") - Manga( - id = generateUid(relUrl), - url = relUrl, - title = titleEl.text(), - altTitles = emptySet(), - publicUrl = relUrl.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = el.selectFirst("div.item-thumb img")?.attr("src"), - tags = el.select("div.item-tags > a.tag").mapNotNullToSet { tagEl -> - MangaTag( - title = tagEl.text(), - key = tagEl.attrAsRelativeUrlOrNull("href") - ?.removePrefix("/") ?: return@mapNotNullToSet null, - source = source, - ) - }, - state = MangaState.FINISHED, - authors = emptySet(), - largeCoverUrl = null, - description = null, - chapters = null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val content = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val description = content.selectFirst("div.article-info")?.text()?.trim() ?: "" - val df = SimpleDateFormat("HH:mm dd-MM-yyyy") - val time = content.selectFirst("div.article-info > small")?.text()?.trim() - val chapters = content.selectFirst("nav.pagination")?.select("a.pagination-link") - ?.mapChapters { index, element -> - val relUrl = element.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(relUrl), - title = null, - number = index + 1f, - volume = 0, - url = relUrl, - scanlator = null, - uploadDate = df.parseSafe(time), - branch = null, - source = source, - ) - }.orEmpty() - return manga.copy(chapters = chapters, description = description) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val content = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return content.selectFirstOrThrow("div.article-fulltext").select("p > img").mapNotNull { el -> - val url = el.attrOrNull("src") ?: return@mapNotNull null - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - protected suspend fun fetchTags(): Set { - val root = webClient.httpGet("https://$domain").parseHtml() - return root.select("div#navbar-main a.navbar-item").map { a -> - MangaTag( - title = a.text(), - key = a.attr("href").removePrefix("/"), - source = source, - ) - }.toSet() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt deleted file mode 100644 index 3e72bf24..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt +++ /dev/null @@ -1,213 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class HeanCmsParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.POPULARITY, - SortOrder.POPULARITY_ASC, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), - ) - - protected open val pathManga = "series" - protected open val apiPath - get() = "api.$domain" - - protected open val paramsUpdated = "latest" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(apiPath) - append("/query?query_string=") - - query?.let { - append(query.urlEncoded()) - } - - append("&series_type=Comic&perPage=$pageSize") - - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - MangaState.ABANDONED -> "Dropped" - MangaState.PAUSED -> "Hiatus" - else -> "" - }, - ) - } - - append("&orderBy=") - when (order) { - SortOrder.POPULARITY -> append("total_views&order=desc") - SortOrder.POPULARITY_ASC -> append("total_views&order=asc") - SortOrder.UPDATED -> append("$paramsUpdated&order=desc") - SortOrder.UPDATED_ASC -> append("$paramsUpdated&order=asc") - SortOrder.NEWEST -> append("created_at&order=desc") - SortOrder.NEWEST_ASC -> append("created_at&order=asc") - SortOrder.ALPHABETICAL -> append("title&order=asc") - SortOrder.ALPHABETICAL_DESC -> append("title&order=desc") - else -> append("latest&order=desc") - } - append("&tags_ids=") - append("[".urlEncoded()) - append(filter.tags.joinToString(",") { it.key }) - append("]".urlEncoded()) - - append("&page=") - append(page.toString()) - } - - - return parseMangaList(webClient.httpGet(url).parseJson()) - } - - protected open val cdn = "api.$domain/" - - private fun parseMangaList(response: JSONObject): List { - return response.getJSONArray("data").mapJSON { it -> - val id = it.getLong("id") - val publicUrl = "/series/${it.getString("series_slug")}" - val title = it.getString("title") - val cover = if (it.getString("thumbnail").startsWith("https://")) { - it.getString("thumbnail") - } else { - "https://$cdn${it.getString("thumbnail")}" - } - - Manga( - id = generateUid(id), - url = id.toString(), - title = title, - altTitles = setOfNotNull(it.getString("alternative_names").takeIf { it.isNotBlank() }), - publicUrl = publicUrl.toAbsoluteUrl(domain), - description = it.getString("description"), - rating = it.getFloatOrDefault("rating", RATING_UNKNOWN) / 5f, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = cover, - tags = emptySet(), - state = when (it.getString("status")) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Dropped" -> MangaState.ABANDONED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = emptySet(), - source = source, - ) - } - } - - protected open val datePattern = "yyyy-MM-dd" - - protected open fun reqUrl(seriesId: Long): String { - return "https://$apiPath/chapter/query?page=1&perPage=9999&series_id=$seriesId" - } - - override suspend fun getDetails(manga: Manga): Manga { - val seriesId = manga.url.toLongOrNull() ?: manga.id // backward compatibility - val url = reqUrl(seriesId) // keep old API url, can replace it - val response = webClient.httpGet(url).parseJson() - val data = response.getJSONArray("data").asTypedList() - val dateFormat = SimpleDateFormat(datePattern, Locale.ENGLISH) - return manga.copy( - chapters = data.mapChapters(reversed = true) { i, it -> - val chapterUrl = - "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" - MangaChapter( - id = generateUid(it.getLong("id")), - title = it.getString("chapter_name"), - number = i + 1f, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = dateFormat.parseSafe(it.getString("created_at").substringBefore("T")), - branch = null, - source = source, - ) - }, - ) - } - - protected open val selectPages = ".flex > img:not([alt])" - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select(selectPages).map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/comics").parseHtml() - val regex = Regex("\"tags\\\\?\":\\s*\\[(.+?)]\\s*[},]") - val tags = doc.select("script").joinToString("") { it.html() } - .let { fullHtml -> - regex.find(fullHtml)?.groupValues?.getOrNull(1) - } - ?.unescapeJson() - ?.replace(Regex(""""]\)\s*self\.__next_f\.push\(\[\d+,""""), "") - ?.let { "[$it]" } - ?: return emptySet() - - return JSONArray(tags).mapJSONToSet { - MangaTag( - key = it.getInt("id").toString(), - title = it.getString("name").toTitleCase(sourceLocale), - source = source, - ) - } - } - -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt deleted file mode 100644 index 4b3b2a74..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt +++ /dev/null @@ -1,380 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import okhttp3.Headers -import okhttp3.Request -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -abstract class HiperParser( - context: MangaLoaderContext, - source: MangaParserSource, - protected val domainName: String, -) : PagedMangaParser(context, source, pageSize = 30) { - - protected open val availableContentTypes: Set = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA) - - protected open val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED) - - protected open val availableContentRating: Set = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) - - protected open val mangaPath = "manga" - - override val configKeyDomain = ConfigKey.Domain(domainName) - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.RELEVANCE - ) - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) - - val client: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request() - val newRequest = request.newBuilder() - .header("Referer", "https://$domainName/") - .header("Origin", "https://$domainName") - .build() - var response = chain.proceed(newRequest) - if (response.code == 401) { - response.close() - val acceptHeaders = Headers.Builder() - .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - .build() - val homeRequest = Request.Builder() - .url("https://$domainName/") - .headers(acceptHeaders) - .build() - chain.proceed(homeRequest).close() - response = chain.proceed(request) - } - response - } - .build() - OkHttpWebClient(httpClient, source) - } - - @Volatile - private var genresListCache: List? = null - - override suspend fun getFilterOptions(): MangaListFilterOptions { - if (genresListCache == null) { - try { - val input = "%7B%220%22%3A%7B%22json%22%3Anull%2C%22meta%22%3A%7B%22values%22%3A%5B%22undefined%22%5D%7D%7D%7D" - val res = client.httpGet(apiUrl("search.genres", input), getRequestHeaders()) - val body = res.body?.string() - if (body != null) { - val root = JSONArray(body) - val jsonArray = root.optJSONObject(0) - ?.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") - if (jsonArray != null) { - val tags = mutableListOf() - for (i in 0 until jsonArray.length()) { - val obj = jsonArray.optJSONObject(i) ?: continue - val name = obj.optString("name", "") - if (name.isNotEmpty()) tags.add(MangaTag(name, name, source)) - } - if (tags.isNotEmpty()) genresListCache = tags - } - } - } catch (_: Exception) {} - } - return MangaListFilterOptions( - availableTags = genresListCache?.toSet() ?: emptySet(), - availableContentTypes = availableContentTypes, - availableStates = availableStates, - availableContentRating = availableContentRating, - ) - } - - private fun apiUrl(trpcPath: String, input: String) = "https://$domainName/api/trpc/$trpcPath?batch=1&input=${input.urlEncoded()}" - - private fun searchPayload(query: String, sort: String, limit: Int, offset: Int, filter: MangaListFilter): String { - val genres = filter.tags.map { it.key } - val types = filter.types.map { it.name.lowercase() } - val statusStr = filter.states.firstOrNull()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "cancelled" - else -> null - } - } - val contentRatingStr = filter.contentRating.firstOrNull()?.let { - when (it) { - ContentRating.SAFE -> "safe" - ContentRating.SUGGESTIVE -> "suggestive" - ContentRating.ADULT -> "pornographic" - } - } - return JSONObject().apply { - put("0", JSONObject().apply { - put("json", JSONObject().apply { - put("q", query) - put("sort", sort) - put("filters", JSONObject().apply { - if (genres.isNotEmpty()) put("genres", JSONArray(genres)) else put("genres", JSONObject.NULL) - if (types.isNotEmpty()) put("type", types.first()) else put("type", JSONObject.NULL) - if (statusStr != null) put("status", statusStr) else put("status", JSONObject.NULL) - if (contentRatingStr != null) put("contentRating", contentRatingStr) else put("contentRating", JSONObject.NULL) - put("author", JSONObject.NULL) - put("artist", JSONObject.NULL) - put("year", JSONObject.NULL) - }) - put("limit", limit) - put("offset", offset) - put("maxRating", "pornographic") - }) - put("meta", JSONObject().apply { - put("values", JSONObject().apply { - if (genres.isEmpty()) put("filters.genres", JSONArray().put("undefined")) - if (types.isEmpty()) put("filters.type", JSONArray().put("undefined")) - if (statusStr == null) put("filters.status", JSONArray().put("undefined")) - if (contentRatingStr == null) put("filters.contentRating", JSONArray().put("undefined")) - put("filters.author", JSONArray().put("undefined")) - put("filters.artist", JSONArray().put("undefined")) - put("filters.year", JSONArray().put("undefined")) - }) - }) - }) - }.toString() - } - - private fun detailsPayload(slug: String) = JSONObject().apply { - put("0", JSONObject().apply { - put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { put("slug", slug) }) - }) - }.toString() - - private fun chaptersPayload(mangaId: Long) = JSONObject().apply { - put("0", JSONObject().apply { - put("json", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { - put("seriesId", mangaId) - put("sort", "best") - put("page", 1) - put("limit", 20) - }) - put("meta", JSONObject().apply { - put("values", JSONObject().apply { put("chapterId", JSONArray().put("undefined")) }) - }) - }) - put("2", JSONObject().apply { - put("json", JSONObject().apply { put("seriesId", mangaId) }) - }) - }.toString() - - private fun pagesPayload(slug: String, number: Float) = JSONObject().apply { - put("0", JSONObject().apply { - put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { put("slug", slug) }) - }) - put("2", JSONObject().apply { - put("json", JSONObject().apply { - put("seriesSlug", slug) - put("chapterNumber", number.toDouble()) - }) - }) - put("3", JSONObject().apply { - put("json", JSONObject().apply { put("position", "footer_bottom") }) - }) - }.toString() - - private fun parseLastItemJsonArray(body: String): JSONArray? = try { - val root = JSONArray(body) - root.optJSONObject(root.length() - 1) - ?.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") - } catch (_: Exception) { null } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val q = filter.query?.trim().orEmpty() - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "recent" - SortOrder.NEWEST -> "newest" - SortOrder.NEWEST_ASC -> "oldest" - SortOrder.RATING -> "score" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "relevance" - else -> "newest" - } - val input = searchPayload(q, sort, pageSize, (page - 1) * pageSize, filter) - val res = client.httpGet(apiUrl("search.query", input), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val root = JSONArray(body) - val first = root.optJSONObject(0) ?: return emptyList() - val hits = first.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONObject("json") - ?.optJSONArray("hits") ?: return emptyList() - return (0 until hits.length()).mapNotNull { i -> - hits.optJSONObject(i)?.let { obj -> - val id = obj.optLong("id", -1).takeIf { it > 0 } ?: return@mapNotNull null - val slug = obj.optString("slug", "").takeIf { it.isNotEmpty() } ?: return@mapNotNull null - val title = obj.optString("title", "") - val cover = obj.optString("coverUrl", "") - Manga( - id = generateUid("/$mangaPath/$slug#$id"), - url = "/$mangaPath/$slug#$id", - publicUrl = "https://$domainName/$mangaPath/$slug#$id", - title = title, coverUrl = cover, - altTitles = emptySet(), authors = emptySet(), tags = emptySet(), - rating = RATING_UNKNOWN, state = null, - contentRating = ContentRating.ADULT, source = source - ) - } - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#") - .takeIf { it.isNotBlank() } ?: return manga - val res = client.httpGet(apiUrl("auth.me,series.bySlugWithGenres", detailsPayload(slug)), getRequestHeaders()) - val body = res.body?.string() ?: return manga - val detailJson = try { - val arr = JSONArray(body) - arr.getJSONObject(arr.length() - 1) - .optJSONObject("result")?.optJSONObject("data")?.optJSONObject("json") - } catch (_: Exception) { null } ?: return manga - - val title = detailJson.optString("title", manga.title) - val desc = detailJson.optString("synopsis", manga.description) - val cover = detailJson.optString("coverUrl", manga.coverUrl) - val authors = detailJson.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - val genres = detailJson.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - val status = detailJson.optString("status", null) - val chapters = loadChapters(manga) - - return manga.copy( - title = title, description = desc, coverUrl = cover, - authors = authors.toSet(), - tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), - state = when (status?.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled" -> MangaState.ABANDONED - else -> null - }, - chapters = chapters, - ) - } - - private suspend fun loadChapters(manga: Manga): List { - val mangaId = manga.url.substringAfterLast("#").toLongOrNull() ?: return emptyList() - val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#").ifBlank { return emptyList() } - val res = client.httpGet(apiUrl("auth.me,comments.list,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val chaptersArray = parseLastItemJsonArray(body) ?: return emptyList() - return (0 until chaptersArray.length()).mapNotNull { i -> - chaptersArray.optJSONObject(i)?.let { obj -> - val number = obj.optDouble("number", 0.0).toFloat() - val createdAt = obj.optString("createdAt", null) - val uploadDate = createdAt?.let { dateFormat.parse(it)?.time } ?: 0L - MangaChapter( - id = generateUid("/$mangaPath/$slug/$number"), - url = "/$mangaPath/$slug/$number", - title = "Chapter ${number.formatForTitle()}", - number = number, volume = 0, uploadDate = uploadDate, - scanlator = null, branch = null, source = source - ) - } - }.sortedBy { it.number } - } - - // triyng first tRPC API if it fails then chapter html scraping - override suspend fun getPages(chapter: MangaChapter): List { - val path = chapter.url.removePrefix("/$mangaPath/") - val slug = path.substringBefore("/") - val num = path.substringAfter("/").toFloatOrNull() ?: return emptyList() - - val input = pagesPayload(slug, num) - val res = client.httpGet(apiUrl("auth.me,series.bySlug,reader.chapterPages", input), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val apiPages = parsePages(body) - if (!apiPages.isNullOrEmpty()) return apiPages - - val chapterUrl = "https://$domainName/$mangaPath/$slug/$num" - val doc = client.httpGet(chapterUrl).parseHtml() - - // search in json results image URLs - val script = doc.select("script").find { it.data().contains("\"webpUrl\"") } - ?: doc.select("script").find { it.data().contains("\"avifUrl\"") } - if (script != null) { - val arrayRegex = Regex("""\[\s*\{.*?\"(?:webp|avif)Url\".*?\}\s*\]""", RegexOption.DOT_MATCHES_ALL) - val match = arrayRegex.find(script.data())?.value - if (match != null) { - return try { - val jsonArray = JSONArray(match) - (0 until jsonArray.length()).mapNotNull { i -> - val page = jsonArray.optJSONObject(i) ?: return@mapNotNull null - val url = page.optString("webpUrl", "").ifEmpty { - page.optString("avifUrl", "") - }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } catch (_: Exception) { emptyList() } - } - } - - return emptyList() - } - - private fun parsePages(body: String): List? { - val elements = try { JSONArray(body) } catch (_: Exception) { return null } - for (i in 0 until elements.length()) { - if (elements.optJSONObject(i)?.has("error") == true) return null - } - val last = elements.optJSONObject(elements.length() - 1) ?: return null - val items = last.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") ?: return null - return (0 until items.length()).mapNotNull { i -> - val page = items.optJSONObject(i) ?: return@mapNotNull null - val url = page.optString("webpUrl", "").ifEmpty { - page.optString("avifUrl", "") - }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage) = page.url - - private fun Float.formatForTitle() = if (this == toLong().toFloat()) toLong().toString() else toString() - - companion object { - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt deleted file mode 100644 index 5ff71ada..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt +++ /dev/null @@ -1,276 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class LikeMangaParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 36, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = - EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.POPULARITY_TODAY, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - append("/?act=search") - - query?.let { - append("&f") - append("[keyword]".urlEncoded()) - append("=") - append(query.urlEncoded()) - } - - append("&f") - append("[sortby]".urlEncoded()) - append("=") - when (order) { - SortOrder.UPDATED -> append("lastest-chap") - SortOrder.NEWEST -> append("lastest-manga") - SortOrder.POPULARITY -> append("top-manga") - SortOrder.POPULARITY_TODAY -> append("top-day") - SortOrder.POPULARITY_WEEK -> append("top-week") - SortOrder.POPULARITY_MONTH -> append("top-month") - else -> append("lastest-chap") - } - - if (filter.tags.isNotEmpty()) { - append("&f") - append("[genres]".urlEncoded()) - append("=") - filter.tags.oneOrThrowIfMany()?.let { - append(it.key) - } - } - - filter.states.oneOrThrowIfMany()?.let { - append("&f") - append("[status]".urlEncoded()) - append("=") - append( - when (it) { - MangaState.ONGOING -> "in-process" - MangaState.FINISHED -> "complete" - MangaState.PAUSED -> "pause" - else -> "all" - }, - ) - } - - if (page > 1) { - append("&pageNum=") - append(page) - } - - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.card-body div.video").map { div -> - val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.selectFirstOrThrow("p.title-manga").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = div.selectFirstOrThrow("img").src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/genres/").parseHtml() - return doc.select("ul.nav-genres li:not(.text-center) a").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val mangaId = manga.url.toAbsoluteUrl(domain).removeSuffix("/").substringAfterLast("-").toInt() - val maxPageChapterSelect = doc.getElementById("nav_list_chapter_id_detail")?.select("a:not(.next)") - var maxPageChapter = 1 - if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { - val i = it.text().toInt() - if (i > maxPageChapter) { - maxPageChapter = i - } - } - } - val author = doc.selectLast("li.author p")?.textOrNull() - return manga.copy( - altTitles = setOfNotNull(doc.selectFirst(".list-info li.othername h2")?.textOrNull()), - tags = doc.select("li.kind a").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - }, - authors = setOfNotNull(author), - description = doc.requireElementById("summary_shortened").html(), - chapters = run { - if (maxPageChapter == 1) { - parseChapters(doc) - } else { - coroutineScope { - val result = ArrayList(parseChapters(doc)) - result.ensureCapacity(result.size * maxPageChapter) - (2..maxPageChapter).map { i -> - async { - loadChapters(mangaId, i) - } - }.awaitAll() - .flattenTo(result) - result - } - } - }.reversed(), - ) - } - - private suspend fun loadChapters(mangaId: Int, page: Int): List { - val json = - webClient.httpGet( - "https://$domain/?act=ajax&code=load_list_chapter&manga_id=$mangaId&page_num=$page&chap_id=0&keyword=", - ) - .parseJson().getString("list_chap") - val doc = Jsoup.parse(json) - return parseChapters(doc) - } - - private val dateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) - private fun parseChapters(root: Element): List { - return root.select("li.wp-manga-chapter") - .map { li -> - val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href") - val dateText = if (li.selectFirstOrThrow(".chapter-release-date").text() == "New") { - "today" - } else { - li.selectFirstOrThrow(".chapter-release-date").text() - } - val chapNum = url.substringAfter("chapter-").substringBefore("-") - - MangaChapter( - id = generateUid(url), - title = li.selectFirstOrThrow("a").text(), - number = chapNum.toFloatOrNull() ?: 0f, - volume = 0, - url = url, - scanlator = null, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - branch = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val testJson = doc.selectFirst("div.reading input#next_img_token") - if (testJson != null) { - val jsonRaw = testJson.attr("value").split(".")[1] - val jsonData = JSONObject(context.decodeBase64(jsonRaw).toString(Charsets.UTF_8)) - val jsonImg = context.decodeBase64(jsonData.getString("data")).toString(Charsets.UTF_8) - val images = jsonImg.replace("\\", "").replace("[", "").replace("]", "").replace("\"", "").split(",") - val baseUrl = doc.selectFirstOrThrow(".reading-detail img").src() - val cdn = baseUrl?.substringBefore("manga/", "")?.ifEmpty { - baseUrl.toHttpUrl().resolve("/").toString() - } - return images.map { img -> - val url = concatUrl(cdn.orEmpty(), img) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - } else { - return doc.select(".reading-detail img").map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - } - - } - - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - val d = date?.lowercase() ?: return 0 - return when { - d.startsWith("today") -> Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - - else -> dateFormat.parseSafe(date) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt deleted file mode 100644 index 5db210ed..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt +++ /dev/null @@ -1,239 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import androidx.collection.scatterSetOf -import kotlinx.coroutines.coroutineScope -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault -import java.util.* - -internal abstract class LilianaParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 24, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.POPULARITY_MONTH, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_TODAY, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.RATING, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( - availableTags = getAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED - ), - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - append("/search/") - append(page) - append("/?keyword=") - append((filter.query?.urlEncoded() ?: "")) - } - - else -> { - append("/filter/") - append(page) - - append("/?sort=") - when (order) { - SortOrder.UPDATED -> append("latest-updated") - SortOrder.POPULARITY -> append("views") - SortOrder.POPULARITY_MONTH -> append("views_month") - SortOrder.POPULARITY_WEEK -> append("views_week") - SortOrder.POPULARITY_TODAY -> append("views_day") - SortOrder.ALPHABETICAL -> append("az") - SortOrder.ALPHABETICAL_DESC -> append("za") - SortOrder.NEWEST -> append("new") - SortOrder.NEWEST_ASC -> append("old") - SortOrder.RATING -> append("score") - else -> append("latest-updated") - } - - append("&genres=") - filter.tags.joinTo(this, ",") { it.key } - - append("¬Genres=") - filter.tagsExclude.joinTo(this, ",") { it.key } - - if (filter.states.isNotEmpty()) { - append("&status=") - append( - when (filter.states.first()) { - MangaState.ONGOING -> "on-going" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "on-hold" - MangaState.ABANDONED -> "canceled" - else -> "all" - }, - ) - } - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div#main div.grid > div").map { parseSearchManga(it) } - } - - private fun parseSearchManga(element: Element): Manga { - val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") - return Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = element.selectFirst("img")?.src(), - title = element.selectFirst(".text-center a")?.text().orEmpty(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - - @JvmField - protected val ongoing = scatterSetOf( - "on-going", "đang tiến hành", "進行中", - ) - - @JvmField - protected val finished = scatterSetOf( - "completed", "hoàn thành", "完了", - ) - - @JvmField - protected val abandoned = scatterSetOf( - "canceled", "đã huỷ bỏ", "キャンセル", - ) - - @JvmField - protected val paused = scatterSetOf( - "on-hold", "tạm dừng", "一時停止", - ) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val author = doc.selectFirst("div.y6x11p i.fas.fa-user + span.dt")?.textOrNull()?.takeUnless { - it.equals("updating", true) - } - return manga.copy( - description = doc.selectFirst("div#syn-target")?.html(), - largeCoverUrl = doc.selectFirst(".a1 > figure img")?.src(), - tags = doc.select(".a2 div > a[rel='tag'].label").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast('/'), - title = a.text().toTitleCase(sourceLocale), - source = source, - ) - }, - authors = setOfNotNull(author), - state = when (doc.selectFirst("div.y6x11p i.fas.fa-rss + span.dt")?.text()?.lowercase().orEmpty()) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in paused -> MangaState.PAUSED - in abandoned -> MangaState.ABANDONED - else -> null - }, - chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = true) { i, element -> - val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - title = element.selectFirst("a")?.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - uploadDate = element.selectFirst("time[datetime]")?.attr("datetime")?.toLongOrNull()?.times(1000) - ?: 0L, - branch = null, - source = source, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val script = doc.selectFirstOrThrow("script:containsData(const CHAPTER_ID)").data() - - val chapterId = script.substringAfter("const CHAPTER_ID = ", "").substringBefore(';', "") - check(chapterId.isNotEmpty()) { "Failed to get chapter id" } - - val ajaxUrl = buildString { - append("https://") - append(domain) - append("/ajax/image/list/chap/") - append(chapterId) - } - - val responseJson = webClient.httpGet(ajaxUrl).parseJson() - - check(responseJson.getBooleanOrDefault("status", false)) { responseJson.getString("msg") } - - val pageListDoc = Jsoup.parse(responseJson.getString("html")) - - return pageListDoc.selectOrThrow("div").map { - val url = it.selectFirstOrThrow("img").attr("src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - protected open suspend fun getAvailableTags(): Set = coroutineScope { - val doc = webClient.httpGet("https://$domain/filter").parseHtml() - doc.select("div.advanced-genres > div > .advance-item").mapToSet { element -> - MangaTag( - key = element.selectFirstOrThrow("span[data-genre]").attr("data-genre"), - title = element.text().toTitleCase(sourceLocale), - source = source, - ) - } - } -} - diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt deleted file mode 100644 index 482ac579..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt +++ /dev/null @@ -1,896 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import androidx.collection.scatterSetOf -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONObject -import org.jsoup.HttpStatusException -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.AuthRequiredException -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.net.HttpURLConnection -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class MadaraParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 12, -) : PagedMangaParser(context, source, pageSize), MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - // Change these values only if the site does not support manga listings via ajax - protected open val withoutAjax = false - protected open val authorSearchSupported = false - - override val availableSortOrders: Set = setupAvailableSortOrders() - - private fun setupAvailableSortOrders(): Set { - return if (!withoutAjax) { - EnumSet.of( - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - SortOrder.POPULARITY, - SortOrder.POPULARITY_ASC, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.RATING, - SortOrder.RATING_ASC, - SortOrder.RELEVANCE, - ) - } else { - EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.RELEVANCE, - ) - } - } - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = !withoutAjax, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = true, - isAuthorSearchSupported = authorSearchSupported, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - MangaState.PAUSED, - MangaState.UPCOMING, - ), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), - ) - - override val authUrl: String - get() = "https://${domain}" - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name.contains("wordpress_logged_in") - } - } - - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${domain}/").parseHtml().body() - return body.selectFirst(".c-user_name")?.text() - ?: run { - throw if (body.selectFirst("#loginform") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } - - protected open val tagPrefix = "manga-genre/" - protected open val datePattern = "MMMM d, yyyy" - protected open val stylePage = "?style=list" - protected open val postReq = false - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - protected fun Element.tableValue(): Element { - for (p in parents()) { - val children = p.children() - if (children.size == 2) { - return children[1] - } - } - parseFailed("Cannot find tableValue for node ${text()}") - } - - @JvmField - protected val ongoing = scatterSetOf( - "مستمرة", - "en curso", - "ongoing", - "on going", - "OnGoing", - "ativo", - "en cours", - "en cours \uD83D\uDFE2", - "en cours de publication", - "activo", - "đang tiến hành", - "em lançamento", - "онгоінг", - "publishing", - "devam ediyor", - "em andamento", - "in corso", - "güncel", - "berjalan", - "продолжается", - "updating", - "lançando", - "in arrivo", - "emision", - "en emision", - "مستمر", - "curso", - "en marcha", - "publicandose", - "publicando", - "连载中", - "đang làm", - "em postagem", - "devam eden", - "em progresso", - "atualizações semanais", - "em lançamento", - "devam ediyo", - ) - - @JvmField - protected val finished = scatterSetOf( - "completed", - "complete", - "completo", - "complété", - "fini", - "achevé", - "terminé", - "terminé ⚫", - "tamamlandı", - "đã hoàn thành", - "hoàn thành", - "مكتملة", - "завершено", - "завершен", - "finished", - "finalizado", - "completata", - "one-shot", - "bitti", - "tamat", - "completado", - "concluído", - "concluido", - "已完结", - "bitmiş", - "end", - "منتهية", - "tamamlanan", - "مكتمل", - ) - - @JvmField - protected val abandoned = scatterSetOf( - "canceled", - "cancelled", - "cancelado", - "cancellato", - "cancelados", - "dropped", - "discontinued", - "abandonné", - "iptal edildi", - "đã hủy", - "ملغي", - "заброшено", - "annulé", - ) - - @JvmField - protected val paused = scatterSetOf( - "hiatus", - "on hold", - "pausado", - "en espera", - "en pause", - "en attente", - "durduruldu", - "beklemede", - "đang chờ", - "متوقف", - "заморожено", - ) - - @JvmField - protected val upcoming = scatterSetOf( - "upcoming", - "لم تُنشَر بعد", - "prochainement", - "à venir", - "in arrivo", - ) - - // can be changed to retrieve tags see getTags - protected open val listUrl = "manga/" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - if (withoutAjax) { - val pages = page + 1 - - val url = buildString { - append("https://") - append(domain) - - if (pages > 1) { - append("/page/") - append(pages.toString()) - } - append("/?s=") - append(filter.query?.urlEncoded() ?: "") - - append("&post_type=wp-manga") - - // Known bug: in some cases, if there are no manga with the associated tags, the source returns the full list of manga - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { - append("&genre[]=") - append(it.key) - } - } - - filter.states.forEach { - append("&status[]=") - when (it) { - MangaState.ONGOING -> append("on-going") - MangaState.FINISHED -> append("end") - MangaState.ABANDONED -> append("canceled") - MangaState.PAUSED -> append("on-hold") - MangaState.UPCOMING -> append("upcoming") - else -> throw IllegalArgumentException("$it not supported") - } - } - - filter.contentRating.oneOrThrowIfMany()?.let { - append("&adult=") - append( - when (it) { - ContentRating.SAFE -> "0" - ContentRating.ADULT -> "1" - else -> "" - }, - ) - } - - if (filter.year != 0) { - append("&release=") - append(filter.year.toString()) - } - - filter.author?.takeIf { it.isNotEmpty() }?.let { - append("&author=") - // should be like "minamida-usuke" - append(it.lowercase().replace(" ", "-")) - } - - // Support artist - //filter.artist?.let { - // append("&artist=") - // append(filter.artist) - //} - - - append("&m_orderby=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.NEWEST -> append("new-manga") - SortOrder.ALPHABETICAL -> append("alphabet") - SortOrder.RATING -> append("rating") - SortOrder.RELEVANCE -> {} - else -> {} - } - } - val html = try { - webClient.httpGet(url).parseHtml() - } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", url) - } - } - return parseMangaList(html) - } else { - - val payload = createRequestTemplate() - - payload["page"] = page.toString() - - filter.query?.takeIf { it.isNotEmpty() }?.let { - payload["vars[s]"] = it.urlEncoded() - } - - if (filter.tags.isNotEmpty()) { - payload["vars[tax_query][0][taxonomy]"] = "wp-manga-genre" - payload["vars[tax_query][0][field]"] = "slug" - filter.tags.forEachIndexed { i, it -> - payload["vars[tax_query][0][terms][$i]"] = it.key - } - payload["vars[tax_query][0][operator]"] = "IN" - } - - if (filter.tagsExclude.isNotEmpty()) { - payload["vars[tax_query][1][taxonomy]"] = "wp-manga-genre" - payload["vars[tax_query][1][field]"] = "slug" - filter.tagsExclude.forEachIndexed { i, it -> - payload["vars[tax_query][1][terms][$i]"] = it.key - } - payload["vars[tax_query][1][operator]"] = "NOT IN" - } - - if (filter.year != 0) { - payload["vars[tax_query][2][taxonomy]"] = "wp-manga-release" - payload["vars[tax_query][2][field]"] = "slug" - payload["vars[tax_query][2][terms][]"] = filter.year.toString() - } - - filter.author?.takeIf { it.isNotEmpty() }?.let { - payload["vars[tax_query][3][taxonomy]"] = "wp-manga-author" - payload["vars[tax_query][3][field]"] = "name" - payload["vars[tax_query][3][terms][0]"] = it - payload["vars[tax_query][3][operator]"] = "IN" - } - - // Support artist - // filter.artist.let { - // payload["vars[tax_query][4][taxonomy]"] = "wp-manga-artist" - // payload["vars[tax_query][4][field]"] = "name" - // payload["vars[tax_query][4][terms][0]"] = filter.artist - // payload["vars[tax_query][4][operator]"] = "IN" - //} - - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.year != 0) { - payload["vars[tax_query][relation]"] = "AND" - } - - when (order) { - SortOrder.POPULARITY -> { - payload["vars[meta_key]"] = "_wp_manga_views" - payload["vars[orderby]"] = "meta_value_num" - payload["vars[order]"] = "desc" - } - - SortOrder.POPULARITY_ASC -> { - payload["vars[meta_key]"] = "_wp_manga_views" - payload["vars[orderby]"] = "meta_value_num" - payload["vars[order]"] = "asc" - } - - SortOrder.UPDATED -> { - payload["vars[meta_key]"] = "_latest_update" - payload["vars[orderby]"] = "meta_value_num" - payload["vars[order]"] = "desc" - } - - SortOrder.UPDATED_ASC -> { - payload["vars[meta_key]"] = "_latest_update" - payload["vars[orderby]"] = "meta_value_num" - payload["vars[order]"] = "asc" - } - - SortOrder.NEWEST -> { - payload["vars[orderby]"] = "date" - payload["vars[order]"] = "desc" - } - - SortOrder.NEWEST_ASC -> { - payload["vars[orderby]"] = "date" - payload["vars[order]"] = "asc" - } - - SortOrder.ALPHABETICAL -> { - payload["vars[orderby]"] = "post_title" - payload["vars[order]"] = "asc" - } - - SortOrder.ALPHABETICAL_DESC -> { - payload["vars[orderby]"] = "post_title" - payload["vars[order]"] = "desc" - } - - SortOrder.RATING -> { - payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" - payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - - payload["vars[orderby][query_avarage_reviews]"] = "DESC" - payload["vars[orderby][query_total_reviews]"] = "DESC" - } - - SortOrder.RATING_ASC -> { - payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" - payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - - payload["vars[orderby][query_avarage_reviews]"] = "ASC" - payload["vars[orderby][query_total_reviews]"] = "ASC" - } - - SortOrder.RELEVANCE -> { - payload["vars[orderby]"] = "" - } - - else -> payload["vars[orderby]"] = "" - } - - filter.states.forEach { - payload["vars[meta_query][0][0][key]"] = "_wp_manga_status" - payload["vars[meta_query][0][0][compare]"] = "IN" - payload["vars[meta_query][0][0][value][]"] = - when (it) { - MangaState.ONGOING -> "on-going" - MangaState.FINISHED -> "end" - MangaState.ABANDONED -> "canceled" - MangaState.PAUSED -> "on-hold" - MangaState.UPCOMING -> "upcoming" - else -> throw IllegalArgumentException("$it not supported") - } - } - - filter.contentRating.oneOrThrowIfMany()?.let { - payload["vars[meta_query][0][1][key]"] = "manga_adult_content" - payload["vars[meta_query][0][1][value]"] = - when (it) { - ContentRating.SAFE -> "" - ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" - else -> "" - } - } - - val html = try { - webClient.httpPost( - "https://$domain/wp-admin/admin-ajax.php", - payload, - ).parseHtml() - } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", domain) - } - } - return parseMangaList(html) - } - } - - protected open fun parseMangaList(doc: Document): List { - val elements = doc.select("div.row.c-tabs-item__content").ifEmpty { - doc.select("div.page-item-detail, div.manga__item") - } - - // Avoid "Content not found or removed" errors - if (elements.isEmpty()) { - return emptyList() - } - - return elements.map { div -> - val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") - val author = summary?.selectFirst(".mg_author, .mg_artists")?.selectFirst("a")?.ownText() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src(), - title = (summary?.selectFirst("h3, h4") - ?: div.selectFirst(".manga-name, .post-title h2 a, .post-title"))?.text() - .orEmpty(), - altTitles = emptySet(), - rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), - source = source, - ) - }.orEmpty(), - authors = setOfNotNull(author), - state = when ( - summary?.selectFirst(".mg_status") - ?.selectFirst(".summary-content") - ?.ownText()?.lowercase() - .orEmpty() - ) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in abandoned -> MangaState.ABANDONED - in paused -> MangaState.PAUSED - in upcoming -> MangaState.UPCOMING - else -> null - }, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - protected open suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() - val body = doc.body() - val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") - val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") - if (root1 == null && root2 == null) { - doc.parseFailed("Root not found") - } - val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix('/').substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().ifEmpty { - a.selectFirst(".menu-image-title")?.textOrNull() - }?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null, - source = source, - ) - } - } - - protected open val selectDesc = - "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" - protected open val selectGenre = "div.genres-content a" - protected open val selectTestAsync = "div.listing-chapters_wrap" - protected open val selectState = - "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + - "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + - "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(Статус)," + - "div.post-content_item:contains(状态), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" - protected open val selectAlt = - ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" - - protected open suspend fun createMangaTag(a: Element): MangaTag? { - return MangaTag( - key = a.attr("href").removeSuffix("/").substringAfterLast('/'), - title = a.text().toTitleCase(), - source = source, - ) - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val href = doc.selectFirst("head meta[property='og:url']")?.attr("content")?.toRelativeUrl(domain) ?: manga.url - val testCheckAsync = doc.select(selectTestAsync) - val chaptersDeferred = if (testCheckAsync.isEmpty()) { - async { loadChapters(href, doc) } - } else { - async { getChapters(manga, doc) } - } - - val desc = doc.select(selectDesc).html() - - val stateDiv = doc.selectFirst(selectState)?.selectLast("div.summary-content") - - val state = stateDiv?.let { - when (it.text().lowercase()) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in abandoned -> MangaState.ABANDONED - in paused -> MangaState.PAUSED - else -> null - } - } - - val alt = doc.body().select(selectAlt).firstOrNull()?.tableValue()?.textOrNull() - - manga.copy( - title = doc.selectFirst("h1")?.textOrNull() ?: manga.title, - url = href, - publicUrl = href.toAbsoluteUrl(domain), - tags = doc.body().select(selectGenre).mapToSet { a -> createMangaTag(a) }.filterNotNull().toSet(), - description = desc, - altTitles = setOfNotNull(alt), - state = state, - chapters = chaptersDeferred.await(), - contentRating = if (doc.selectFirst(".adult-confirm") != null || isNsfwSource) { - ContentRating.ADULT - } else { - ContentRating.SAFE - }, - ) - } - - - protected open val selectDate = "span.chapter-release-date i" - protected open val selectChapter = "li.wp-manga-chapter" - - protected open suspend fun getChapters(manga: Manga, doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() - MangaChapter( - id = generateUid(href), - title = name, - number = i + 1f, - volume = 0, - url = link, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - protected open val postDataReq = "action=manga_get_chapters&manga=" - - protected open suspend fun loadChapters(mangaUrl: String, document: Document): List { - val doc = if (postReq) { - val mangaId = document.select("div#manga-chapters-holder").attr("data-id") - val url = "https://$domain/wp-admin/admin-ajax.php" - val postData = postDataReq + mangaId - webClient.httpPost(url, postData).parseHtml() - } else { - val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" - webClient.httpPost(url, emptyMap()).parseHtml() - } - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() - MangaChapter( - id = generateUid(href), - url = link, - title = name, - number = i + 1f, - volume = 0, - branch = null, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - scanlator = null, - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow(".related-manga") - return root.select("div.related-reading-wrap").mapNotNull { div -> - val a = div.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitles = emptySet(), - title = div.selectFirstOrThrow(".widget-title").text(), - authors = emptySet(), - coverUrl = div.selectFirst("img")?.src(), - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - protected open val selectBodyPage = "div.main-col-inner div.reading-content" - protected open val selectPage = "div.page-break, div.page-box" - protected open val selectRequiredLogin = ".content-blocked, .login-required" - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val chapterProtector = doc.getElementById("chapter-protector-data") - if (chapterProtector == null) { - throw if (doc.selectFirst(selectRequiredLogin) != null) { - AuthRequiredException(source) - } else { - val root = doc.body().selectFirst(selectBodyPage) ?: throw ParseException( - "No image found, try to log in", - fullUrl, - ) - return root.select(selectPage).flatMap { div -> - div.selectOrThrow("img").map { img -> - val url = img.requireSrc().toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - } - } else { - - val chapterProtectorHtml = chapterProtector.attr("src") - .takeIf { it.startsWith("data:text/javascript;base64,") } - ?.substringAfter("data:text/javascript;base64,") - ?.let { - Base64.getDecoder().decode(it).decodeToString() - } - ?: chapterProtector.html() - - val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';") - val chapterData = JSONObject( - chapterProtectorHtml.substringAfter("chapter_data='").substringBefore("';").replace("\\/", "/"), - ) - val unsaltedCiphertext = context.decodeBase64(chapterData.getString("ct")) - val salt = chapterData.getString("s").decodeHex() - val ciphertext = "Salted__".toByteArray(Charsets.UTF_8) + salt + unsaltedCiphertext - - val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password) - val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' } - - return imgArrayString.split(",").map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - } - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - val d = date?.lowercase() ?: return 0 - return when { - - WordSet( - " ago", "atrás", " hace", " publicado", " назад", " önce", " trước", "مضت", "قبل", - " h", " d", " días", " jour", " horas", " heure", " mins", " minutos", " minute", " mois", - ).endsWith(d) -> { - parseRelativeDate(d) - } - - WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút", "giây").startsWith(d) -> { - parseRelativeDate(d) - } - - WordSet("yesterday", "يوم واحد").startsWith(d) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("today").startsWith(d) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("يومين").startsWith(d) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -2) // day before yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - d.contains(Regex("""\b\d+ jour""")) -> { - parseRelativeDate(d) - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - }.let { dateFormat.parseSafe(it.joinToString(" ")) } - - else -> dateFormat.parseSafe(date) - } - } - - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - return when { - WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان") - .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minutes", "minuto", "mins", "นาที", "دقائق", "phút", "минут", "دقيقة") - .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "horas", "hour", "hours", "h", "ชั่วโมง", "giờ", "ore", "ساعة", "小时", "ساعات") - .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - - WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "d", "วัน", "ngày", "giorni", "أيام", "天", "день") - .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(date) -> - cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - - WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois") - .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - - WordSet("year", "año", "năm") - .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - - else -> 0 - } - } - - private companion object { - - private fun createRequestTemplate() = - ("action=madara_load_more&page=0&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=AND&vars%5Bpost_type%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmanga_archives_item_layout%5D=default").split( - '&', - ).map { - val pos = it.indexOf('=') - it.substring(0, pos) to it.substring(pos + 1) - }.toMutableMap() - - fun String.decodeHex(): ByteArray { - check(length % 2 == 0) { "Must have an even length" } - - return chunked(2).map { it.toInt(16).toByte() }.toByteArray() - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt deleted file mode 100644 index deb6ba44..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt +++ /dev/null @@ -1,416 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import okhttp3.Headers -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.util.* -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class MadthemeParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 48, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - SortOrder.RATING, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - - @JvmField - protected val ongoing: Set = setOf( - "on going", - "ongoing", - ) - - @JvmField - protected val finished: Set = setOf( - "completed", - ) - - protected open val listUrl = "search/" - protected open val datePattern = "MMM dd, yyyy" - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - override fun intercept(chain: okhttp3.Interceptor.Chain): Response { - val request = chain.request() - val url = request.url - val fragment = url.fragment - - if (fragment != null && (fragment.startsWith("https://") || fragment.startsWith("http://"))) { - val cleanUrl = url.newBuilder().fragment(null).build() - val response = chain.proceed(request.newBuilder().url(cleanUrl).build()) - if (response.isSuccessful) { - return response - } - response.close() - return chain.proceed(request.newBuilder().url(fragment).build()) - } - - val response = chain.proceed(request) - if (!response.isSuccessful && fragment == IMAGE_REQUEST_FRAGMENT) { - response.close() - val fallbackUrl = url.newBuilder() - .host(IMAGE_FALLBACK_HOST) - .encodedPath(url.encodedPath.replaceFirst("/res/", "/")) - .fragment(null) - .build() - return chain.proceed(request.newBuilder().url(fallbackUrl).build()) - } - return response - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - append('/') - append(listUrl) - - append("?page=") - append(page.toString()) - - query?.let { - append("&q=") - append(query.urlEncoded()) - } - - append("&sort=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("updated_at") - SortOrder.ALPHABETICAL -> append("name") // On some sites without tags or searches, the alphabetical option is empty. - SortOrder.NEWEST -> append("created_at") - SortOrder.RATING -> append("rating") - else -> append("updated_at") - } - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { - append("&") - append("genre[]".urlEncoded()) - append("=") - append(it.key) - } - } - - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> "all" - }, - ) - } - } - - val doc = webClient.httpGet(url).parseHtml() - - return doc.select("div.book-item").map { div -> - val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src(), - title = div.selectFirst("div.meta")?.selectFirst("div.title")?.text().orEmpty(), - altTitles = emptySet(), - rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = doc.body().select("div.meta div.genres span").mapToSet { span -> - MangaTag( - key = span.attr("class"), - title = span.text().toTitleCase(), - source = source, - ) - }, - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() - return doc.select("div.genres .checkbox").mapNotNullToSet { checkbox -> - val key = checkbox.selectFirstOrThrow("input").attr("value") - val name = checkbox.selectFirst("span.radio__label")?.text() ?: key - MangaTag( - key = key, - title = name, - source = source, - ) - } - } - - protected open val selectDesc = "div.section-body.summary p.content" - protected open val selectState = "div.detail p:contains(Status) span" - protected open val selectAlt = "div.detail div.name h2" - protected open val selectTag = "div.detail p:contains(Genres) a" - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val chaptersDeferred = async { getChapters(doc) } - - val desc = doc.selectFirst(selectDesc)?.html() - - val stateDiv = doc.selectFirst(selectState) - - val state = stateDiv?.let { - when (it.text().lowercase()) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - else -> null - } - } - - val alt = doc.body().select(selectAlt).text() - - val nsfw = doc.getElementById("adt-warning") != null - - manga.copy( - tags = doc.body().select(selectTag).mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().toTitleCase().replace(",", ""), - source = source, - ) - }, - description = desc?.nullIfEmpty(), - altTitles = setOfNotNull(alt.nullIfEmpty()), - state = state, - chapters = chaptersDeferred.await(), - contentRating = if (nsfw || manga.contentRating == ContentRating.ADULT) { - ContentRating.ADULT - } else { - ContentRating.SAFE - }, - ) - } - - - protected open val selectDate = "div .chapter-update" - protected open val selectChapter = "ul#chapter-list li" - - protected open suspend fun getChapters(doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val slug = doc.selectFirstOrThrow("script:containsData(bookSlug)").data().substringAfter("bookSlug = \"") - .substringBefore("\";") - val docChapter = webClient.httpGet("https://$domain/api/manga/$slug/chapters?source=detail").parseHtml() - return docChapter.select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val dateText = li.selectFirst(selectDate)?.text() - MangaChapter( - id = generateUid(href), - title = li.selectFirst(".chapter-title")?.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - protected open val selectPage = "div#chapter-images img" - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.normalizedChapterUrl() - val doc = webClient.httpGet(fullUrl).parseHtml() - val known = HashSet() - val result = ArrayList() - // html parisng - doc.select(selectPage).forEach { img -> - result.addPage(known, img.resolveImageUrl()) - } - // js parsing - val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]") - val mainServer = doc.select("script").firstNotNullOfOrNull { script -> - Regex("mainServer\\s*=\\s*\"(.*?)\"").find(script.html())?.groupValues?.getOrNull(1) - } - val schemePrefix = if (mainServer?.startsWith("//") == true) "https:" else "" - val pages = doc.select("script").firstNotNullOfOrNull { script -> - regexPages.find(script.html())?.groupValues?.getOrNull(1) - }?.split(',') - pages?.forEach { url -> - val pageUrl = if (mainServer.isNullOrEmpty()) { - url.resolveChapterImageUrl() - } else { - "$schemePrefix$mainServer$url" - } - result.addPage(known, pageUrl) - } - return result - } - - override suspend fun getPageUrl(page: MangaPage): String { - val url = page.url.toAbsoluteUrl(domain) - return if ('#' in url) url else "$url#$IMAGE_REQUEST_FRAGMENT" - } - - private fun MutableList.addPage(known: MutableSet, url: String) { - if (known.add(url)) { - add( - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ), - ) - } - } - - private fun Element.resolveImageUrl(): String { - val primary = requireSrc() - val rawFallback = attr("onerror") - .substringAfter("this.src='", "") - .substringBefore("'") - .takeIf { it.isNotBlank() } - ?: return primary - val fallback = rawFallback.resolveChapterImageUrl() - .takeIf { it.startsWith("https://") || it.startsWith("http://") } - ?: return primary - - return if ("://s20." in primary) fallback else "$primary#$fallback" - } - - private fun String.resolveChapterImageUrl(): String { - val value = trim() - return when { - value.startsWith("https://") || value.startsWith("http://") -> value - value.startsWith("//") -> "https:$value" - value.contains("/manga/") && !value.contains("/wp-content/") -> - "https://$IMAGE_FALLBACK_HOST/manga${value.substringAfter("/manga")}" - value.startsWith("/") -> "https://$domain$value" - else -> value.toAbsoluteUrl(domain) - } - } - - private fun String.normalizedChapterUrl(): String { - val value = trim() - if (value.startsWith("https://") || value.startsWith("http://")) { - val schemeEnd = value.indexOf("://") - val pathStart = value.indexOf('/', startIndex = schemeEnd + 3) - if (pathStart == -1) return value - val prefix = value.substring(0, pathStart) - val path = value.substring(pathStart).replace(Regex("/{2,}"), "/") - return prefix + path - } - return value.toAbsoluteUrl(domain) - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - val d = date?.lowercase() ?: return 0 - return when { - - WordSet(" ago", " h", " d").endsWith(d) -> { - parseRelativeDate(d) - } - - WordSet("today").startsWith(d) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - }.let { dateFormat.parseSafe(it.joinToString(" ")) } - - else -> dateFormat.parseSafe(date) - } - } - - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - return when { - WordSet("second") - .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - - WordSet("min", "minute", "minutes") - .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - - WordSet("hour", "hours", "h") - .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - - WordSet("day", "days") - .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("month", "months") - .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - - WordSet("year") - .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - - else -> 0 - } - } - - private companion object { - private const val IMAGE_REQUEST_FRAGMENT = "image-request" - private const val IMAGE_FALLBACK_HOST = "sb.mbcdn.xyz" - } - -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt deleted file mode 100644 index 31033fd4..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt +++ /dev/null @@ -1,248 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class Manga18Parser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - ) - - @Volatile - private var genresCache: Set? = null - private val genresMutex = Mutex() - - private suspend fun getOrFetchGenres(): Set { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - genresCache = fetchAvailableTags() - genresCache!! - } - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/$listUrl/").parseHtml() - return doc.select("div.grid_cate li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix('/').substringAfterLast('/') - MangaTag( - key = href, - title = a.text(), - source = source, - ) - } - } - - override fun getRequestHeaders() = Headers.Builder() - .add("Referer", "https://$domain/") - .build() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - ) - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - protected open val ongoing: Set = setOf("On Going") - protected open val finished: Set = setOf("Completed") - - protected open val listUrl = "list-manga/" - protected open val tagUrl = "manga-list/" - protected open val datePattern = "dd-MM-yyyy" - - protected open val selectDesc = "div.detail_reviewContent" - protected open val selectState = "div.item:contains(Status) div.info_value" - protected open val selectAlt = "div.item:contains(Other name) div.info_value" - protected open val selectTag = "div.info_value > a[href*=/manga-list/]" - protected open val selectAuthor = "div.info_label:contains(author) + div.info_value, div.info_label:contains(autor) + div.info_value" - protected open val selectArtist = "div.info_label:contains(artist) + div.info_value" - protected open val selectDate = "div.item p" - protected open val selectChapter = "div.chapter_box li" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim().orEmpty() - val tag = filter.tags.firstOrNull() - - val url = buildString(120) { - append("https://") - append(domain) - append('/') - - when { - tag != null && query.isNotEmpty() -> { - throw IllegalArgumentException("Search is not supported with tags") - } - tag != null -> { - append(tagUrl) - append(tag.key) - append('/') - append(page) - } - query.isNotEmpty() -> { - append(listUrl) - append(page) - append("?search=") - append(query.urlEncoded()) - append("&order_by=latest") - } - else -> { - append(listUrl) - append(page) - } - } - if (query.isEmpty()) { - append("?order_by=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.ALPHABETICAL -> append("name") - else -> append("latest") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - protected open fun parseMangaList(doc: Document): List { - return doc.select("div.story_item").map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirst("div.mg_info") - ?.selectFirst("div.mg_name a") - ?.text() - ?: a.attr("title") - val cover = div.selectFirst("img")?.src() - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = cover, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val info = doc.body().selectFirstOrThrow("div.detail_listInfo") - val chaptersDeferred = async { getChapters(doc) } - val desc = doc.selectFirst(selectDesc)?.html()?.nullIfEmpty() - val state = info.selectFirst(selectState)?.text()?.let { - when (it) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - else -> null - } - } - val alt = info.selectFirst(selectAlt)?.textOrNull()?.takeUnless { it == "Updating" } - val author = info.selectFirst(selectAuthor)?.textOrNull()?.takeUnless { it == "Updating" } - val artist = info.selectFirst(selectArtist)?.textOrNull()?.takeUnless { it == "Updating" } - val tags = doc.body().select(selectTag).mapToSet { a -> - val key = a.attr("href").removeSuffix('/').substringAfterLast('/') - MangaTag( - key = key, - title = a.text().toTitleCase(), - source = source, - ) - } - manga.copy( - tags = tags, - description = desc.nullIfEmpty(), - altTitles = setOfNotNull(alt), - authors = setOfNotNull(author, artist), - state = state, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun getChapters(doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val dateText = li.selectFirst(selectDate)?.text() - MangaChapter( - id = generateUid(href), - title = a.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = dateFormat.parseSafe(dateText), - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val script = doc.selectFirstOrThrow("script:containsData(slides_p_path)") - val encoded = script.data() - .substringAfter('[') - .substringBefore(",]") - .replace("\"", "") - .split(",") - - return encoded.map { encodedUrl -> - val decoded = context.decodeBase64(encodedUrl).toString(Charsets.UTF_8) - val absoluteUrl = decoded.toAbsoluteUrl(domain) - MangaPage( - id = generateUid(absoluteUrl), - url = absoluteUrl, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt deleted file mode 100644 index 438354da..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt +++ /dev/null @@ -1,417 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import androidx.collection.ArrayMap -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.* -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.internal.closeQuietly -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.Base64 -import java.util.EnumSet - -internal abstract class MangaReaderParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int, - searchPageSize: Int, -) : PagedMangaParser(context, source, pageSize, searchPageSize), Interceptor { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set - get() = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.NEWEST, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrCreateTagMap().values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ContentType.NOVEL, - ), - ) - - protected open val listUrl = "/manga" - protected open val datePattern = "MMMM d, yyyy" - protected open val isNetShieldProtected = false - - protected var tagCache: ArrayMap? = null - protected val mutex = Mutex() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - when { - - !query.isNullOrEmpty() -> { - append("/page/") - append(page.toString()) - append("/?s=") - append(query.urlEncoded()) - } - - else -> { - append(listUrl) - - append("/?order=") - append( - when (order) { - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - SortOrder.NEWEST -> "latest" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "update" - else -> "" - }, - ) - - filter.tags.forEach { - append("&") - append("genre[]".urlEncoded()) - append("=") - append(it.key) - } - - filter.tagsExclude.forEach { - append("&") - append("genre[]".urlEncoded()) - append("=-") - append(it.key) - } - - if (filter.states.isNotEmpty()) { - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - when (it) { - MangaState.ONGOING -> append("ongoing") - MangaState.FINISHED -> append("completed") - MangaState.PAUSED -> append("hiatus") - else -> append("") - } - } - } - - filter.types.oneOrThrowIfMany()?.let { - append("&type=") - append( - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - }, - ) - } - - append("&page=") - append(page.toString()) - } - } - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - protected open val selectMangaList = ".postbody .listupd .bs .bsx" - protected open val selectMangaListImg = "img.ts-post-image" - protected open val selectMangaListTitle = "div.tt" - - protected open fun parseMangaList(docs: Document): List { - return docs.select(selectMangaList).mapNotNull { - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - val rating = it.selectFirst(".numscore")?.text() - ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - title = it.selectFirst(selectMangaListTitle)?.text() ?: a.attr("title"), - altTitles = emptySet(), - publicUrl = a.attrAsAbsoluteUrl("href"), - rating = rating, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = it.selectFirst(selectMangaListImg)?.src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - protected open val selectChapter = "#chapterlist > ul > li" - override suspend fun getDetails(manga: Manga): Manga { - val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(url), - title = element.selectFirst(".chapternum")?.textOrNull(), - url = url, - number = index + 1f, - volume = 0, - scanlator = null, - uploadDate = dateFormat.parseSafe(element.selectFirst(".chapterdate")?.text()), - branch = null, - source = source, - ) - } - return parseInfo(docs, manga, chapters) - } - - protected open val detailsDescriptionSelector = "div.entry-content" - - open suspend fun parseInfo(docs: Document, manga: Manga, chapters: List): Manga { - /// set if is table - val tableMode = - docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr") - ?: docs.selectFirst("div.seriestucon") - - val tagMap = getOrCreateTagMap() - - val selectTag = if (tableMode != null) { - docs.select(".seriestugenre > a") - } else { - docs.select(".wd-full .mgen > a") - } - - val tags = selectTag.mapNotNullToSet { tagMap[it.text()] } - - val stateSelect = if (tableMode != null) { - tableMode.selectFirst(".infotable td:contains(Status)") - ?: tableMode.selectFirst(".infotable td:contains(Statut)") - ?: tableMode.selectFirst(".infotable td:contains(حالة العمل)") - ?: tableMode.selectFirst(".infotable td:contains(الحالة)") - ?: tableMode.selectFirst(".infotable td:contains(Estado)") - ?: tableMode.selectFirst(".infotable td:contains(สถานะ)") - ?: tableMode.selectFirst(".infotable td:contains(Stato )") - ?: tableMode.selectFirst(".infotable td:contains(Durum)") - ?: tableMode.selectFirst(".infotable td:contains(Statüsü)") - - } else { - docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)") - ?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)") - ?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )") - ?: docs.selectFirst(".tsinfo div:contains(Durum)") ?: docs.selectFirst(".tsinfo div:contains(Statüsü)") - ?: docs.selectFirst(".tsinfo div:contains(Statü)") ?: docs.selectFirst(".tsinfo div:contains(الحالة)") - } - - val state = if (tableMode != null) { - stateSelect?.lastElementSibling() - - } else { - stateSelect?.lastElementChild() - } - - val mangaState = state?.let { - when (it.text()) { - - "مستمرة", "En curso", "En Curso", "Ongoing", "OnGoing", "On going", "Ativo", "En Cours", "En cours", "En cours \uD83D\uDFE2", - "En cours de publication", "Đang tiến hành", "Em lançamento", "em lançamento", "Em Lançamento", "Онгоінг", "Publishing", - "Devam Ediyor", "Em Andamento", "In Corso", "Güncel", "Berjalan", "Продолжается", "Updating", "Lançando", "In Arrivo", "Emision", - "En emision", "مستمر", "Curso", "En marcha", "Publicandose", "Publicando", "连载中", "Devam ediyor", "Devam Etmekte", - -> MangaState.ONGOING - - "Completed", "Completo", "Complété", "Fini", "Achevé", "Terminé", "Terminé ⚫", "Tamamlandı", "Đã hoàn thành", "Hoàn Thành", - "مكتملة", "Завершено", "Finished", "Finalizado", "Completata", "One-Shot", "Bitti", "Tamat", "Completado", "Concluído", - "Concluido", "已完结", "Bitmiş", - -> MangaState.FINISHED - - "Canceled", "Cancelled", "Cancelado", "cancellato", "Cancelados", "Dropped", "Discontinued", "abandonné", "Abandonné", - -> MangaState.ABANDONED - - "Hiatus", "On Hold", "Pausado", "En espera", "En pause", "En Pause", "En attente", - -> MangaState.PAUSED - - else -> null - } - } - - val author = tableMode?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.textOrNull() - - val nsfw = docs.selectFirst(".restrictcontainer") != null - || docs.selectFirst(".info-right .alr") != null - || docs.selectFirst(".postbody .alr") != null - - val title = docs.selectFirst("h1.entry-title")?.text() - ?: docs.selectFirst("h1")?.text() - ?: docs.selectFirst(".entry-title")?.text() - ?: docs.selectFirst(".seriestucontent h1")?.text() - ?: docs.selectFirst(".postbody h1")?.text() - ?: docs.selectFirst("title")?.text()?.substringBefore(" - ")?.trim() - ?: manga.title - - return manga.copy( - title = title, - description = docs.selectFirst(detailsDescriptionSelector)?.text(), - state = mangaState, - authors = setOfNotNull(author), - contentRating = if (manga.contentRating == ContentRating.ADULT || nsfw) { - ContentRating.ADULT - } else { - ContentRating.SAFE - }, - tags = tags, - chapters = chapters, - ) - } - - override suspend fun getRelatedManga(seed: Manga): List { - return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) - } - - protected open val encodedSrc = false - protected open val selectScript = "div.wrapper script" - protected open val selectPage = "div#readerarea img" - - protected open val selectTestScript = "script:containsData(ts_reader)" - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - val docs = webClient.httpGet(chapterUrl).parseHtml() - - val test = docs.select(selectTestScript) - if (test.isEmpty() and !encodedSrc) { - return docs.select(selectPage).map { img -> - val url = img.requireSrc().toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } else { - val images = if (encodedSrc) { - val script = docs.select(selectScript) - var decode = "" - for (i in script) { - if (i.attr("src").startsWith("data:text/javascript;base64,")) { - decode = Base64.getDecoder().decode(i.attr("src").replace("data:text/javascript;base64,", "")) - .decodeToString() - if (decode.startsWith("ts_reader.run")) { - break - } - } - } - JSONObject(decode.substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - - } else { - val script = docs.selectFirstOrThrow(selectTestScript) - JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - } - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - pages.add( - MangaPage( - id = generateUid(images.getString(i)), - url = images.getString(i), - preview = null, - source = source, - ), - ) - } - return pages - } - } - - protected open suspend fun getOrCreateTagMap(): Map = mutex.withLock { - tagCache?.let { return@withLock it } - val url = listUrl.toAbsoluteUrl(domain) - val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") - val tagMap = ArrayMap(tagElements.size) - for (el in tagElements) { - tagMap[el.text()] = MangaTag( - title = el.textOrNull()?.toTitleCase(sourceLocale) ?: continue, - key = el.selectFirst("input")?.attrOrNull("value") ?: continue, - source = source, - ) - } - tagCache = tagMap - return@withLock tagMap - } - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (!isNetShieldProtected) { - return response - } - val contentType = response.mimeType - if ( - contentType?.endsWith("/html") != false && - context.cookieJar.getCookies(domain).none { it.name.contains("NetShield") } - ) { - val cookie = runBlocking { response.copy().parseHtml().getNetShieldCookie() } ?: return response - context.cookieJar.insertCookie(domain, cookie) - return chain.proceed(response.request.newBuilder().build()).also { - response.closeQuietly() - } - } - return response - } - - @Suppress("DEPRECATION") - private suspend fun Document.getNetShieldCookie(): Cookie? = runCatchingCancellable { - val script = select("script").firstNotNullOfOrNull { s -> - s.html().takeIf { x -> x.contains("slowAES.decrypt") } - } ?: return@runCatchingCancellable null - val min = webClient.httpGet("https://$domain/min.js").parseRaw() - val res = context.evaluateJs(min + "\n\n" + script.replace(Regex("document.cookie\\s*=\\s*"), "return ")) - res?.let { - Cookie.parse(baseUri().toHttpUrl(), it) - } - }.getOrNull() - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val mangaSlug = link.pathSegments[1].nullIfEmpty() ?: return null - val url = "/manga/$mangaSlug/" - return resolver.resolveManga( - this, url, generateUid(url), - ) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt deleted file mode 100644 index 86436d32..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt +++ /dev/null @@ -1,391 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -abstract class MangaThemesia( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://$domain/") - .build() - - protected open val mangaDirectory = "manga" - protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US) - protected open val withoutAjax = false - protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" - protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" - protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num), div.bxcl li:not(a[data-bs-target='#lockedChapterModal'])" - protected open val pageSelector = "div#readerarea img" - - override val availableSortOrders: EnumSet = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.UPDATED, - SortOrder.ADDED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - @Volatile - private var genreCache: Set? = null - private val genreMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ContentType.NOVEL, - ), - ) - - private suspend fun getOrFetchGenres(): Set { - genreCache?.let { return it } - return genreMutex.withLock { - genreCache ?: fetchGenres().also { genreCache = it } - } - } - - protected open suspend fun fetchGenres(): Set { - val doc = webClient.httpGet("https://$domain/$mangaDirectory/").parseHtml() - val genrez = doc.selectFirst("ul.genrez") ?: return emptySet() - return genrez.select("li").mapNotNull { li -> - val label = li.selectFirst("label")?.text()?.trim() ?: return@mapNotNull null - val input = li.selectFirst("input[type=checkbox]") ?: return@mapNotNull null - val value = input.attr("value").trim() - if (label.isBlank() || value.isBlank()) null - else MangaTag(key = value, title = label, source = source) - }.toSet() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim().orEmpty() - val url = buildString { - append("https://$domain/$mangaDirectory/") - if (query.isNotEmpty()) { - append("?s=${query.urlEncoded()}") - } else { - append("?page=$page") - } - filter.states.firstOrNull()?.let { - append("&status=") - append(when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "dropped" - else -> "" - }) - } - filter.types.firstOrNull()?.let { - append("&type=") - append(when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - }) - } - filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } - filter.tagsExclude.forEach { append("&genre[]=-${it.key.urlEncoded()}") } - append("&order=") - append(when (order) { - SortOrder.UPDATED -> "update" - SortOrder.POPULARITY -> "popular" - SortOrder.ADDED -> "latest" - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - else -> "update" - }) - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - protected open fun parseMangaList(doc: Document): List { - return doc.select(searchSelector).mapNotNull { element -> - parseMangaElement(element) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val title = doc.selectFirst("h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1") - ?.text() ?: manga.title - - val description = doc.selectFirst( - ".desc, .entry-content[itemprop=description], .summary__content, .sinopsis, .entry-content p" - )?.text()?.trim().orEmpty() - - val coverUrl = doc.selectFirst( - ".thumb img, .infomanga img, .summary_image img, .cover img, img.wp-post-image" - )?.imgAttr() ?: manga.coverUrl - - val table = doc.selectFirst("table.infotable") - val hasTable = table != null - - val authors = if (hasTable) { - val authorCell = table.selectFirst("tr:has(td:contains(Author)) td:last-child") - val artistCell = table.selectFirst("tr:has(td:contains(Artist)) td:last-child") - listOfNotNull( - authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, - artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } - ).toSet() - } else { - doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Author) i, .spe span:contains(Author) a") - .map { it.text().trim() } - .filter { it.isNotBlank() } - .toSet() - } - - val tags = doc.select(".mgen a, .gnr a, .seriestugenre a, .genres-content a") - .mapNotNull { a -> - val text = a.text().trim() - if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null - }.toSet() - - val statusText = if (hasTable) { - table.selectFirst("tr:has(td:contains(Status)) td:last-child")?.text() - } else { - doc.select(".imptdt:contains(Status) i, .tsinfo .imptdt:contains(Status) a, .fmed b:contains(Status)+span span") - .text() - .ifEmpty { - doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() - } - } - val state = parseStatus(statusText) - - val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() - ?: doc.selectFirst(".num")?.text()?.toFloatOrNull() - val normalizedRating = if (rating != null && rating > 0) rating / 10f else RATING_UNKNOWN - - val chapters = loadChapters(doc, manga.url) - - return manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - tags = tags, - state = state, - rating = normalizedRating, - chapters = chapters, - ) - } - - protected open fun parseStatus(text: String?): MangaState? { - val status = text?.lowercase() ?: return null - return when (status) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in paused -> MangaState.PAUSED - in abandoned -> MangaState.ABANDONED - else -> null - } - } - - protected open suspend fun loadChapters(doc: Document, mangaUrl: String): List { - val chapterElements = doc.select(chapterListSelector) - if (chapterElements.isEmpty() && !withoutAjax) { - return loadChaptersAjax(mangaUrl) - } - return parseChapters(chapterElements) - } - - protected open suspend fun loadChaptersAjax(mangaUrl: String): List { - val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/" - val doc = webClient.httpPost(ajaxUrl, emptyMap()).parseHtml() - return parseChapters(doc.select(chapterListSelector)) - } - - private fun parseChapters(elements: Elements): List { - return elements.mapChapters(reversed = true) { i, element -> - val a = element.selectFirst("a") ?: return@mapChapters null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() - val dateStr = element.selectFirst(".chapterdate")?.text() - val number = extractChapterNumber(name) ?: (i + 1).toFloat() - MangaChapter( - id = generateUid(href), - url = href, - title = name, - number = number, - volume = 0, - uploadDate = parseChapterDate(dateStr), - scanlator = null, - branch = null, - source = source, - ) - } - } - - private fun extractChapterNumber(title: String): Float? { - return Regex("""(?i)(?:chapter|ch\.?)\s*([0-9]+(?:\.[0-9]+)?)""") - .find(title) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - - protected open fun parseChapterDate(date: String?): Long { - if (date.isNullOrBlank()) return 0L - val clean = date.trim() - return when { - WordSet(" ago", "atrás", " hace", " назад", " önce", " trước", "مضت", "قبل").endsWith(clean) -> parseRelativeDate(clean) - WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút").startsWith(clean) -> parseRelativeDate(clean) - WordSet("yesterday", "يوم واحد").startsWith(clean) -> yesterday() - WordSet("today").startsWith(clean) -> today() - WordSet("يومين").startsWith(clean) -> dayBeforeYesterday() - else -> dateFormat.parseSafe(clean) - } - } - - private fun parseRelativeDate(text: String): Long { - val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0L - val cal = Calendar.getInstance() - return when { - WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان").anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут").anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giờ", "ore", "ساعة", "小时").anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois").anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "año", "năm").anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0L - } - } - - private fun yesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - private fun today() = Calendar.getInstance().apply { set(Calendar.HOUR_OF_DAY, 0); set(Calendar.MINUTE, 0) }.timeInMillis - private fun dayBeforeYesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -2) }.timeInMillis - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - - // json image list - val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) - val jsonMatch = jsonRegex.find(doc.html()) - if (jsonMatch != null) { - val jsonArray = JSONArray(jsonMatch.groupValues[1]) - return (0 until jsonArray.length()).map { i -> - val url = jsonArray.getString(i) - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - // direct - return doc.select(pageSelector).mapNotNull { img -> - val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - override suspend fun getRelatedManga(seed: Manga): List { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(relatedSelector).mapNotNull { element -> - parseMangaElement(element) - } - } - - protected open fun parseMangaElement(element: Element): Manga? { - val a = element.selectFirst("a") ?: return null - val href = a.attrAsRelativeUrl("href") - val title = a.attr("title").ifBlank { a.text() } - val coverUrl = element.selectFirst("img")?.imgAttr() - return makeManga(href, title, coverUrl) - } - - private fun makeManga(href: String, title: String, coverUrl: String?): Manga { - return Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = title, - altTitles = emptySet(), - authors = emptySet(), - coverUrl = coverUrl, - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - - protected fun Element.imgAttr(): String { - for (attr in listOf("data-src", "data-lazy-src", "data-original", "data-cfsrc", "data-image", "src")) { - val value = attr(attr).trim().takeIf { it.isNotEmpty() } ?: continue - return value.toAbsoluteUrl(domain).substringBefore("?") // remove WP resize params - } - val srcset = attr("srcset") - if (srcset.isNotBlank()) { - return srcset.split(",").last().trim().split(" ").first().toAbsoluteUrl(domain) - } - return "" - } - - private fun SimpleDateFormat.parseSafe(date: String): Long = - runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) - - companion object { - val ongoing = setOf( - "ongoing", "on going", "publishing", "updating", "en curso", "ativo", "en cours", - "đang tiến hành", "em lançamento", "devam ediyor", "in corso", "güncel", "berjalan", - "продолжается", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة", - ) - val finished = setOf( - "completed", "complete", "finished", "finalizado", "terminé", "tamamlandı", - "đã hoàn thành", "hoàn thành", "مكتملة", "завершено", "completato", "one-shot", - "bitti", "tamat", "concluído", "已完结", "bitmiş", "achevé", - ) - val paused = setOf( - "hiatus", "on hold", "pausado", "en espera", "en pause", "en attente", - "durduruldu", "beklemede", "đang chờ", "متوقف", "заморожено", - ) - val abandoned = setOf( - "canceled", "cancelled", "cancelado", "cancellato", "dropped", "discontinued", - "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено", - ) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt deleted file mode 100644 index 7f29cc7a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt +++ /dev/null @@ -1,462 +0,0 @@ -@file:Suppress("DEPRECATION") - -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.FlexiblePagedMangaParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Exclude -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.AUTHOR -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.EnumSet -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* - - -internal abstract class MangaboxParser( - context: MangaLoaderContext, - source: MangaParserSource, - pageSize: Int = 24, -) : FlexiblePagedMangaParser(context, source, pageSize) { - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) - - @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) - override val searchQueryCapabilities: MangaSearchQueryCapabilities - get() = MangaSearchQueryCapabilities( - SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class, Exclude::class), - isMultiple = true, - ), - SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), - isMultiple = false, - ), - SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = AUTHOR, - criteriaTypes = setOf(Include::class), - isMultiple = false, - isExclusive = true, - ), - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - @JvmField - protected val ongoing: Set = setOf( - "ongoing", - ) - - @JvmField - protected val finished: Set = setOf( - "completed", - ) - - protected open val listUrl = "/manga-list/latest-manga" - protected open val searchUrl = "/search/story/" - protected open val datePattern = "MMM dd,yy" - - override suspend fun getListPage(query: MangaSearchQuery, page: Int): List { - val titleQuery = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TITLE_NAME } - ?.value - ?.toString() - ?.trim() - - if (!titleQuery.isNullOrBlank()) { - val url = "https://$domain${searchUrl.trimEnd('/')}/${normalizeSearchQuery(titleQuery)}?page=$page" - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - val tagCriterion = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TAG } - - val tagKey = tagCriterion?.values?.firstOrNull()?.let { value -> - when (value) { - is MangaTag -> value.key - else -> value.toString().replace(" ", "-").lowercase(sourceLocale) - } - } - - if (!tagKey.isNullOrBlank()) { - val url = "https://$domain/genre/$tagKey?page=$page" - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - val url = "https://$domain$listUrl?page=$page" - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - protected open fun parseSearchResults(doc: Document): List { - val elements = doc.select(".panel_story_list .story_item") - .ifEmpty { doc.select(".story_item") } - .ifEmpty { doc.select("div.list-truyen-item-wrap") } - .ifEmpty { doc.select("div.list-comic-item-wrap") } - .ifEmpty { doc.select("div.content-genres-item") } - .ifEmpty { doc.select("div.itemupdate") } - .ifEmpty { doc.select("div.manga-item") } - .ifEmpty { doc.select(".item") } - .ifEmpty { doc.select("a[href*='/manga/']").mapNotNull { it.parent() ?: it } } - - return elements.mapNotNull { div -> - val linkElement = div.selectFirst(".story_name a") - ?: div.selectFirst("h3 a") - ?: div.selectFirst("h2 a") - ?: div.selectFirst(".slide-caption h3 a") - ?: div.selectFirst("a.cover") - ?: div.selectFirst("a[href*='/manga/']") - ?: if (div.tagName() == "a") div else null - - linkElement ?: return@mapNotNull null - - val href = linkElement.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - if (!href.contains("/manga/")) return@mapNotNull null - - val title = linkElement.text().trim() - .takeIf { it.isNotEmpty() } - ?: linkElement.attr("title").trim().takeIf { it.isNotEmpty() } - ?: div.selectFirst("h3, h2, h1")?.text()?.trim()?.takeIf { it.isNotEmpty() } - ?: return@mapNotNull null - - val img = linkElement.selectFirst("img") ?: div.selectFirst("img") - val coverUrl = img?.attr("data-src")?.nullIfEmpty() - ?: img?.attr("data-lazy-src")?.nullIfEmpty() - ?: img?.src() - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = coverUrl, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = sourceContentRating, - ) - } - } - - protected open fun normalizeSearchQuery(query: String): String { - return query.lowercase(sourceLocale) - .replace("[àáạảãâầấậẩẫăằắặẳẵ]".toRegex(), "a") - .replace("[èéẹẻẽêềếệểễ]".toRegex(), "e") - .replace("[ìíịỉĩ]".toRegex(), "i") - .replace("[òóọỏõôồốộổỗơờớợởỡ]".toRegex(), "o") - .replace("[ùúụủũưừứựửữ]".toRegex(), "u") - .replace("[ỳýỵỷỹ]".toRegex(), "y") - .replace("đ".toRegex(), "d") - .replace("""[^\p{L}\p{N}]+""".toRegex(), "_") - .replace("_+".toRegex(), "_") - .trim('_') - } - - protected open val selectTagMap = "div.panel-genres-list a:not(.genres-select)" - - protected open suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain$listUrl").parseHtml() - val tags = doc.select(selectTagMap).drop(1) - - return tags.mapToSet { a -> - val key = a.attr("href").removeSuffix('/').substringAfterLast('/') - val name = a.attr("title").replace(" Manga", "") - MangaTag( - key = key, - title = name, - source = source, - ) - } - } - - protected open val selectDesc = "div#noidungm, div#panel-story-info-description" - protected open val selectState = "li:contains(status), td:containsOwn(status) + td" - protected open val selectAlt = ".story-alternative, tr:has(.info-alternative) h2" - protected open val selectAut = "li:contains(author) a, td:contains(author) + td a" - protected open val selectTag = "div.manga-info-top li:contains(genres) a , td:containsOwn(genres) + td a" - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val chaptersDeferred = async { getChapters(doc) } - val desc = doc.selectFirst(selectDesc)?.html() - val stateDiv = doc.select(selectState).text() - val state = stateDiv.let { - when (it.lowercase()) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - else -> null - } - } - val alt = doc.body().select(selectAlt).text().replace("Alternative : ", "").nullIfEmpty() - val authors = doc.body().select(selectAut).mapToSet { it.text() } - - manga.copy( - tags = doc.body().select(selectTag).mapToSet { a -> - val href = a.attr("href") - MangaTag( - key = href.substringAfterLast("category=").substringBefore("&") - .takeIf { it != href } - ?: href.removeSuffix("/").substringAfterLast("/"), - title = a.text().toTitleCase(), - source = source, - ) - }, - description = desc, - altTitles = setOfNotNull(alt), - authors = authors, - state = state, - chapters = chaptersDeferred.await(), - ) - } - - protected open val selectDate = "span" - protected open val selectChapter = "div.chapter-list div.row, ul.row-content-chapter li" - - protected open suspend fun getChapters(doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val dateText = li.select(selectDate).last()?.text() - - MangaChapter( - id = generateUid(href), - title = a.text(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - source = source, - scanlator = null, - branch = null, - ) - } - } - - protected open val selectPage = "div#vungdoc img, div.container-chapter-reader img" - protected open val otherDomain = "" - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val scriptPages = parseScriptPages(doc) - if (scriptPages.isNotEmpty()) { - return scriptPages - } - - val pages = doc.select(selectPage).mapNotNull { img -> - val url = img.requireSrc().toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - if (pages.isNotEmpty() || otherDomain.isBlank()) { - return pages - } - - val fallbackUrl = fullUrl.replace(domain, otherDomain) - val fallbackDoc = webClient.httpGet(fallbackUrl).parseHtml() - - return fallbackDoc.select(selectPage).map { img -> - val url = img.requireSrc().toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun parseScriptPages(doc: Document): List { - val content = doc.select("script:containsData(cdns =), script:containsData(chapterImages =)") - .joinToString("\n") { it.data() } - - if (content.isBlank()) { - return emptyList() - } - - val cdns = extractArray(content, cdnsRegex) + extractArray(content, backupImageRegex) - val chapterImages = extractArray(content, chapterImagesRegex) - - if (cdns.isEmpty() || chapterImages.isEmpty()) { - return emptyList() - } - - val baseUrl = cdns.first().toHttpUrlOrNull() ?: return emptyList() - - return chapterImages.map { imagePath -> - val url = baseUrl.newBuilder() - .encodedPath("/$imagePath".replace("//", "/")) - .build() - .toString() - - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun extractArray(scriptContent: String, regex: Regex): List { - val match = regex.find(scriptContent) ?: return emptyList() - return match.groupValues[1] - .split(",") - .mapNotNull { raw -> - raw.trim() - .removeSurrounding("\"") - .removeSurrounding("'") - .replace("\\/", "/") - .removeSuffix("/") - .nullIfEmpty() - } - } - - protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - val d = date?.lowercase() ?: return 0 - return when { - WordSet(" ago", " h", " d").endsWith(d) -> { - parseRelativeDate(d) - } - - WordSet("today").startsWith(d) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - }.let { dateFormat.parseSafe(it.joinToString(" ")) } - - else -> dateFormat.parseSafe(date) - } - } - - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - return when { - WordSet("second") - .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - - WordSet("min", "minute", "minutes") - .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - - WordSet("hour", "hours", "h") - .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - - WordSet("day", "days") - .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("month", "months") - .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - - WordSet("year") - .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - - else -> 0 - } - } - - @Deprecated("Use getRequestHeaders() instead", ReplaceWith("getRequestHeaders()")) - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8") - .add("Accept-Encoding", "gzip, deflate, br") - .add("Accept-Language", "en-US,en;q=0.9") - .build() - - companion object { - - private val cdnsRegex = Regex("""cdns\s*=\s*\[([^]]+)]""") - private val backupImageRegex = Regex("""backupImage\s*=\s*\[([^]]+)]""") - private val chapterImagesRegex = Regex("""chapterImages\s*=\s*\[([^]]+)]""") - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt deleted file mode 100644 index bbd41656..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt +++ /dev/null @@ -1,356 +0,0 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class ZeistMangaParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 12, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED), - ) - - @JvmField - protected val ongoing: Set = hashSetOf( - "ongoing", - "en curso", - "ativo", - "lançando", - "مستمر", - "devam ediyor", - "güncel", - "en emisión", - ) - - @JvmField - protected val finished: Set = hashSetOf( - "completed", - "completo", - "tamamlandı", - "finalizado", - ) - - @JvmField - protected val abandoned: Set = hashSetOf( - "cancelled", - "dropped", - "dropado", - "abandonado", - "cancelado", - "suspendido", - ) - - @JvmField - protected val paused: Set = hashSetOf( - "hiatus", - ) - - protected open val sateOngoing: String = "Ongoing" - protected open val sateFinished: String = "Completed" - protected open val sateAbandoned: String = "Cancelled" - - protected open val maxMangaResults: Int = 20 - - protected open val mangaCategory: String = "Series" - - protected open val datePattern = "yyyy-MM-dd" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query - val url = buildString { - append("https://") - append(domain) - append("/feeds/posts/default/-/") - when { - - !query.isNullOrEmpty() -> { - append(mangaCategory) - append("?alt=json&orderby=published&max-results=") - append((maxMangaResults + 1).toString()) - append("&start-index=") - append(startIndex.toString()) - append("&q=label:") - append(mangaCategory) - append("+") - append(query.urlEncoded()) - } - - else -> { - - - if (filter.tags.isNotEmpty() && filter.states.isNotEmpty()) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED) - } - - if (filter.tags.isNotEmpty()) { - append(filter.tags.oneOrThrowIfMany()?.key.orEmpty()) - } else if (filter.states.isNotEmpty()) { - append( - filter.states.oneOrThrowIfMany().let { - when (it) { - MangaState.ONGOING -> sateOngoing - MangaState.FINISHED -> sateFinished - MangaState.ABANDONED -> sateAbandoned - else -> mangaCategory - } - }, - ) - } else { - append(mangaCategory) - } - - append("?alt=json&orderby=published&max-results=") - append((maxMangaResults + 1).toString()) - append("&start-index=") - append(startIndex.toString()) - } - } - } - - val json = webClient.httpGet(url).parseJson().getJSONObject("feed") - - return if (json.toString().contains("\"entry\":")) { - parseMangaList(json.getJSONArray("entry")) - } else { - emptyList() - } - } - - protected open fun parseMangaList(json: JSONArray): List { - return json.mapJSON { j -> - val name = j.getJSONObject("title").getString("\$t") - val href = - j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" } - .getString("href") - val urlImg = if (j.toString().contains("media\$thumbnail")) { - j.getJSONObject("media\$thumbnail").getStringOrNull("url") - ?.replace("""/s.+?-c/""".toRegex(), "/w600/") - ?.replace("""=s(?!.*=s).+?-c$""".toRegex(), "=w600") - ?.replace("""/s.+?-c-rw/""".toRegex(), "/w600/") - ?.replace("""=s(?!.*=s).+?-c-rw$""".toRegex(), "=w600") - } else { - Jsoup.parse(j.getJSONObject("content").getString("\$t")).selectFirst("img")?.attr("src") - } - Manga( - id = generateUid(href), - url = href.toRelativeUrl(domain), - publicUrl = href, - coverUrl = urlImg.orEmpty(), - title = name, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - protected open suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain").parseHtml() - return doc.selectFirstOrThrow("div.filter").select("ul li").mapToSet { - MangaTag( - key = it.selectFirstOrThrow("input").attr("value"), - title = it.selectFirstOrThrow("label").text().toTitleCase(sourceLocale), - source = source, - ) - } - } - - protected open val selectTags = "article div.mt-15 a, .info-genre a, dl:contains(Genre) dd a" - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val state = - doc.selectFirst("div.y6x11p:contains(Status) .dt") - ?: doc.selectFirst("div.y6x11p:contains(Estado) .dt") - ?: doc.selectFirst("ul.infonime li:contains(Status) span") - ?: doc.selectFirst("ul.infonime li:contains(Estado) span") - ?: doc.selectFirst("span.status-novel") - ?: doc.selectFirst("span[data-status]") - val mangaState = state?.text()?.lowercase().let { - when (it) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in abandoned -> MangaState.ABANDONED - in paused -> MangaState.PAUSED - else -> null - } - } - val author = doc.selectFirst("div.y6x11p:contains(الكاتب) .dt") - ?: doc.selectFirst("div.y6x11p:contains(Author) .dt") - ?: doc.selectFirst("dl:contains(Author) dd") - ?: doc.selectFirst("div.y6x11p:contains(Autor) .dt") - ?: doc.selectFirst("div.y6x11p:contains(Yazar) .dt") - ?: doc.selectFirst("ul.infonime li:contains(Author) span") - - - val desc = doc.getElementById("synopsis") ?: doc.getElementById("Sinopse") ?: doc.getElementById("sinopas") - ?: doc.selectFirst(".sinopsis") ?: doc.selectFirst(".sinopas") - val chaptersDeferred = async { loadChapters(manga.url, doc) } - manga.copy( - authors = author?.text()?.let { setOf(it) } ?: emptySet(), - tags = doc.select(selectTags).mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast("label/").substringBefore("?"), - title = a.text().toTitleCase(), - source = source, - ) - }, - description = desc?.text().orEmpty(), - state = mangaState, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun loadChapters(mangaUrl: String, doc: Document): List { - - val feed = if (doc.getElementById("myUL") != null) { - doc.requireElementById("myUL").selectFirstOrThrow("script").attr("src") - .substringAfterLast("/-/").substringBefore("?").urlDecode() - - } else if (doc.selectFirst("#latest > script") != null) { - val chapterRegex = """label\s*=\s*'([^']+)'""".toRegex() - val scriptSelector = "#latest > script" - val script = doc.selectFirstOrThrow(scriptSelector) - chapterRegex - .find(script.html()) - ?.groupValues?.get(1) - ?: doc.parseFailed("Failed to find chapter feed") - - } else if (doc.selectFirst("#clwd > script") != null) { - val chapterRegex = """clwd\.run\('([^']+)'""".toRegex() - val scriptSelector = "#clwd > script" - val script = doc.selectFirstOrThrow(scriptSelector) - chapterRegex - .find(script.html()) - ?.groupValues?.get(1) - ?: doc.parseFailed("Failed to find chapter feed") - - } else if (doc.selectFirst("#chapterlist") != null) { - doc.selectFirstOrThrow("#chapterlist").attr("data-post-title") - } else { - doc.selectFirstOrThrow("script:containsData(var label_chapter)").data() - .substringAfter("label_chapter = \"").substringBefore("\"") - } - - val url = buildString { - append("https://") - append(domain) - append("/feeds/posts/default/-/") - append(feed) - append("?alt=json&orderby=published&max-results=9999") - } - val json = - webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList() - .reversed() - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return json.mapIndexedNotNull { i, j -> - val name = j.getJSONObject("title").getString("\$t") - val href = - j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" } - .getString("href") - val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") - val slug = mangaUrl.substringAfterLast('/') - val slugChapter = href.substringAfterLast('/') - if (slug == slugChapter) { - return@mapIndexedNotNull null - } - MangaChapter( - id = generateUid(href), - url = href, - title = name, - number = i + 1f, - volume = 0, - branch = null, - uploadDate = dateFormat.parseSafe(dateText), - scanlator = null, - source = source, - ) - } - } - - protected open val selectPage = - "div.check-box img, article#reader .separator img, article.container .separator img, #readarea img, #reader img, #readerarea img" - - override suspend fun getPages(chapter: MangaChapter): List { - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - - return if (doc.selectFirst("script:containsData(chapterImage =)") != null) { - doc.selectFirstOrThrow("script:containsData(chapterImage =)").data() - .substringAfter("[").substringBefore("]") - .replace(" ", "").replace("\"", "") - .split(",").map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - } else if (doc.selectFirst("script:containsData(const content = )") != null) { - doc.selectFirstOrThrow("script:containsData(const content = )").data() - .substringAfter("`").substringBefore("`;").split("src=\"").drop(1) - - .map { img -> - val url = img.substringBefore("\"") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - - } else { - doc.select(selectPage).map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt deleted file mode 100644 index fa3d1fc6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt +++ /dev/null @@ -1,543 +0,0 @@ -@file:Suppress("DEPRECATION") - -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import okhttp3.HttpUrl -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.FlexibleMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.* -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField -import org.koitharu.kotatsu.parsers.model.search.SearchableField.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.text.SimpleDateFormat -import java.util.* -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.network.CommonHeaders - -private const val PAGE_SIZE = 20 -private const val CHAPTERS_FIRST_PAGE_SIZE = 120 -private const val CHAPTERS_MAX_PAGE_SIZE = 500 -private const val CHAPTERS_PARALLELISM = 3 -private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug -private const val LOCALE_FALLBACK = "en" -private const val SERVER_DATA = "data" -private const val SERVER_DATA_SAVER = "data-saver" -private const val USER_AGENT = "Usagi/1 (Android)" - -@MangaSourceParser("MANGADEXORG", "MangaDex") -internal class MangaDexParser(context: MangaLoaderContext) : FlexibleMangaParser(context, MangaParserSource.MANGADEXORG) { - - override val userAgentKey = ConfigKey.UserAgent( - USER_AGENT, - - ) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = request.newBuilder() - .header(CommonHeaders.USER_AGENT, USER_AGENT) - .build() - - return chain.proceed(newRequest) - } - - override val configKeyDomain = ConfigKey.Domain("mangadex.org") - - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - SERVER_DATA to "Original quality", - SERVER_DATA_SAVER to "Compressed quality", - ), - defaultValue = SERVER_DATA, - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(preferredServerKey) - } - - override val availableSortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - SortOrder.POPULARITY, - SortOrder.POPULARITY_ASC, - SortOrder.RATING, - SortOrder.RATING_ASC, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.ADDED, - SortOrder.ADDED_ASC, - SortOrder.RELEVANCE, - ) - - @Deprecated("Too complex. Use filterCapabilities instead") - override val searchQueryCapabilities: MangaSearchQueryCapabilities - get() = MangaSearchQueryCapabilities( - SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class, Exclude::class), - isMultiple = true, - ), - SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), - isMultiple = false, - ), - SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = AUTHOR, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = CONTENT_TYPE, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = CONTENT_RATING, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = DEMOGRAPHIC, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = ORIGINAL_LANGUAGE, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = LANGUAGE, - criteriaTypes = setOf(Include::class), - isMultiple = true, - ), - SearchCapability( - field = PUBLICATION_YEAR, - criteriaTypes = setOf(Match::class), - isMultiple = false, - ), - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions = coroutineScope { - val localesDeferred = async { fetchAvailableLocales() } - val tagsDeferred = async { fetchAvailableTags() } - MangaListFilterOptions( - availableTags = tagsDeferred.await(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED, - ), - availableContentRating = EnumSet.allOf(ContentRating::class.java), - availableDemographics = EnumSet.of( - Demographic.SHOUNEN, - Demographic.SHOUJO, - Demographic.SEINEN, - Demographic.JOSEI, - Demographic.NONE, - ), - availableLocales = localesDeferred.await(), - ) - } - - private fun SearchableField.toParamName(): String = when (this) { - TITLE_NAME -> "title" - TAG -> "includedTags[]" - AUTHOR -> "authors[]" - STATE -> "status[]" - CONTENT_TYPE -> "contentType[]" - CONTENT_RATING -> "contentRating[]" - DEMOGRAPHIC -> "publicationDemographic[]" - ORIGINAL_LANGUAGE -> "originalLanguage[]" - LANGUAGE -> "availableTranslatedLanguage[]" - PUBLICATION_YEAR -> "year" - } - - private fun Any?.toQueryParam(): String = when (this) { - is String -> urlEncoded() - is Locale -> if (language == "in") "id" else language - is MangaTag -> key - is MangaState -> when (this) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.ABANDONED -> "cancelled" - MangaState.PAUSED -> "hiatus" - else -> "" - } - - is ContentRating -> when (this) { - ContentRating.SAFE -> "safe" - // quick fix for double value - ContentRating.SUGGESTIVE -> "suggestive&contentRating[]=erotica" - ContentRating.ADULT -> "pornographic" - } - - is Demographic -> when (this) { - Demographic.SHOUNEN -> "shounen" - Demographic.SHOUJO -> "shoujo" - Demographic.SEINEN -> "seinen" - Demographic.JOSEI -> "josei" - Demographic.NONE -> "none" - else -> "" - } - - is SortOrder -> when (this) { - SortOrder.UPDATED -> "[latestUploadedChapter]=desc" - SortOrder.UPDATED_ASC -> "[latestUploadedChapter]=asc" - SortOrder.RATING -> "[rating]=desc" - SortOrder.RATING_ASC -> "[rating]=asc" - SortOrder.ALPHABETICAL -> "[title]=asc" - SortOrder.ALPHABETICAL_DESC -> "[title]=desc" - SortOrder.NEWEST -> "[year]=desc" - SortOrder.NEWEST_ASC -> "[year]=asc" - SortOrder.POPULARITY -> "[followedCount]=desc" - SortOrder.POPULARITY_ASC -> "[followedCount]=asc" - SortOrder.ADDED -> "[createdAt]=desc" - SortOrder.ADDED_ASC -> "[createdAt]=asc" - SortOrder.RELEVANCE -> "&order[relevance]=desc" - else -> "[latestUploadedChapter]=desc" - } - - else -> this.toString().urlEncoded() - } - - private fun StringBuilder.appendCriterion(field: SearchableField, value: Any?, paramName: String? = null) { - val param = paramName ?: field.toParamName() - if (param.isNotBlank()) { - append("&$param=") - append(value.toQueryParam()) - } - } - - @Deprecated("Too complex. Use getList with filter instead") - override suspend fun getList(query: MangaSearchQuery): List { - val url = buildString { - append("https://api.$domain/manga?limit=$PAGE_SIZE&offset=${query.offset}") - .append("&includes[]=cover_art&includes[]=author&includes[]=artist&includedTagsMode=AND&excludedTagsMode=OR") - - var hasContentRating = false - - query.criteria.forEach { criterion -> - when (criterion) { - is Include<*> -> { - if (criterion.field == CONTENT_RATING) { - hasContentRating = true - } - criterion.values.forEach { appendCriterion(criterion.field, it) } - } - - is Exclude<*> -> { - criterion.values.forEach { appendCriterion(criterion.field, it, "excludedTags[]") } - } - - is Match<*> -> { - appendCriterion(criterion.field, criterion.value) - } - - else -> { - // Not supported - } - } - } - - // If contentRating is not provided, add default values - if (!hasContentRating) { - append("&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic") - } - - append("&order") - append((query.order ?: defaultSortOrder).toQueryParam()) - } - - val json = webClient.httpGet(url).parseJson().getJSONArray("data") - return json.mapJSON { jo -> jo.fetchManga(null) } - } - - override suspend fun getDetails(manga: Manga): Manga { - val mangaId = manga.url.removePrefix("/") - return getDetails(mangaId) - } - - override suspend fun resolveLink(link: HttpUrl): Manga? { - val regex = Regex("[0-9a-f\\-]{10,}", RegexOption.IGNORE_CASE) - val mangaId = link.pathSegments.find { regex.matches(it) } ?: return null - return getDetails(mangaId) - } - - private suspend fun getDetails(mangaId: String): Manga = coroutineScope { - val jsonDeferred = async { - webClient.httpGet( - "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", - ).parseJson().getJSONObject("data") - } - val feedDeferred = async { loadChapters(mangaId) } - jsonDeferred.await().fetchManga(mapChapters(feedDeferred.await())) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val json = webClient.httpGet( - "https://api.$domain/at-home/server/${chapter.url}?forcePort443=false", - ).parseJson() - val chapterJson = json.getJSONObject("chapter") - val server = config[preferredServerKey] ?: SERVER_DATA - val pages = chapterJson.getJSONArray( - if (server == SERVER_DATA_SAVER) "dataSaver" else "data", - ) - val prefix = "${json.getString("baseUrl")}/$server/${chapterJson.getString("hash")}/" - return List(pages.length()) { i -> - val url = prefix + pages.getString(i) - MangaPage( - id = generateUid(url), - url = url, - preview = null, // TODO prefix + dataSaver.getString(i), - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set { - val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() - .getJSONArray("data") - return tags.mapJSONToSet { jo -> - MangaTag( - title = jo.getJSONObject("attributes").getJSONObject("name") - .firstStringValue() - .toTitleCase(Locale.ENGLISH), - key = jo.getString("id"), - source = source, - ) - } - } - - private suspend fun fetchAvailableLocales(): Set { - val head = webClient.httpGet("https://$domain/").parseHtml().head() - return head.getElementsByAttributeValue("property", "og:locale:alternate") - .mapNotNullToSet { meta -> - val raw = meta.attrOrNull("content") ?: return@mapNotNullToSet null - Locale(raw.substringBefore('_'), raw.substringAfter('_', "")) - } - } - - private fun JSONObject.fetchManga(chapters: List?): Manga { - val id = getString("id") - val attrs = getJSONObject("attributes") - val relations = getJSONArray("relationships").associateByKey("type") - val cover = relations["cover_art"] - ?.firstOrNull() - ?.getJSONObject("attributes") - ?.getString("fileName") - ?.let { - "https://uploads.$domain/covers/$id/$it" - } - val authors: Set = (relations["author"] ?: relations["artist"]) - ?.mapNotNullToSet { - it.getJSONObject("attributes")?.getStringOrNull("name") - }.orEmpty() - - return Manga( - id = generateUid(id), - title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { - "Title should not be null" - }, - altTitles = setOfNotNull(attrs.optJSONArray("altTitles")?.flatten()?.selectByLocale()), // TODO - url = id, - publicUrl = "https://$domain/title/$id", - rating = RATING_UNKNOWN, - contentRating = when (attrs.getStringOrNull("contentRating")) { - "pornographic" -> ContentRating.ADULT - "erotica", "suggestive" -> ContentRating.SUGGESTIVE - "safe" -> ContentRating.SAFE - else -> null - }, - coverUrl = cover?.plus(".256.jpg"), - largeCoverUrl = cover, - description = attrs.optJSONObject("description")?.selectByLocale(), - tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> - MangaTag( - title = tag.getJSONObject("attributes") - .getJSONObject("name") - .firstStringValue() - .toTitleCase(), - key = tag.getString("id"), - source = source, - ) - }, - state = when (attrs.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled" -> MangaState.ABANDONED - else -> null - }, - authors = authors, - chapters = chapters, - source = source, - ) - } - - private fun JSONObject.firstStringValue() = entries().first().value - - private fun JSONObject.selectByLocale(): String? { - val preferredLocales = context.getPreferredLocales() - for (locale in preferredLocales) { - getStringOrNull(locale.language)?.let { return it } - getStringOrNull(locale.toLanguageTag())?.let { return it } - } - return getStringOrNull(LOCALE_FALLBACK) ?: entries().firstOrNull()?.value?.nullIfEmpty() - } - - private fun JSONArray.flatten(): JSONObject { - val result = JSONObject() - repeat(length()) { i -> - val jo = optJSONObject(i) - if (jo != null) { - for (key in jo.keys()) { - result.put(key, jo.get(key)) - } - } - } - return result - } - - private suspend fun loadChapters(mangaId: String): List { - val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) - if (firstPage.size >= firstPage.total) { - return firstPage.data - } - val tail = coroutineScope { - val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size - val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() - val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) - List(pages) { page -> - val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size - async(dispatcher) { - loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) - } - }.awaitAll() - } - val result = ArrayList(firstPage.total) - result += firstPage.data - tail.flatMapTo(result) { it.data } - return result - } - - private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { - val limitedLimit = when { - offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) - offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset - else -> limit - } - val url = buildString { - append("https://api.") - append(domain) - append("/manga/") - append(mangaId) - append("/feed") - append("?limit=") - append(limitedLimit) - append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") - append(offset) - append("&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic") - } - val json = webClient.httpGet(url).parseJson() - if (json.getString("result") == "ok") { - return Chapters( - data = json.optJSONArray("data")?.asTypedList().orEmpty(), - total = json.getInt("total"), - ) - } else { - val error = json.optJSONArray("errors").mapJSON { jo -> - jo.getString("detail") - }.joinToString("\n") - throw ParseException(error, url) - } - } - - private fun mapChapters(list: List): List { - // 2022-01-02T00:27:11+00:00 - val dateFormat = SimpleDateFormat( - "yyyy-MM-dd'T'HH:mm:ss'+00:00'", - Locale.ROOT, - ) - val chaptersBuilder = ChaptersListBuilder(list.size) - val branchedChapters = HashMap, MangaChapter>>() - for (jo in list) { - val id = jo.getString("id") - val attrs = jo.getJSONObject("attributes") - if (!attrs.isNull("externalUrl")) { - continue - } - val number = attrs.getFloatOrDefault("chapter", 0f) - val volume = attrs.getIntOrDefault("volume", 0) - val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } - val lc = locale?.getDisplayName(locale)?.toTitleCase(locale) - val relations = jo.getJSONArray("relationships").associateByKey("type") - val team = - relations["scanlation_group"]?.firstOrNull()?.optJSONObject("attributes")?.getStringOrNull("name") - val branch = (list.indices).firstNotNullOf { i -> - val b = if (i == 0) lc else "$lc ($i)" - if (branchedChapters[b]?.get(volume to number) == null) b else null - } - val chapter = MangaChapter( - id = generateUid(id), - title = attrs.getStringOrNull("title"), - number = number, - volume = volume, - url = id, - scanlator = team, - uploadDate = dateFormat.parseSafe(attrs.getString("publishAt")), - branch = branch, - source = source, - ) - if (chaptersBuilder.add(chapter)) { - branchedChapters.getOrPut(branch, ::HashMap)[volume to number] = chapter - } - } - return chaptersBuilder.toList() - } - - private fun JSONArray.associateByKey(key: String): Map> { - val destination = LinkedHashMap>(length()) - repeat(length()) { i -> - val item = getJSONObject(i) - val keyValue = item.getString(key) - destination.getOrPut(keyValue) { mutableListOf() }.add(item) - } - return destination - } - - private class Chapters( - val data: List, - val total: Int, - ) { - - val size: Int - get() = data.size - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt deleted file mode 100644 index c9f84c20..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt +++ /dev/null @@ -1,504 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import okhttp3.Interceptor -import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.parseJson -import java.util.EnumSet -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope - -internal abstract class MangaFireParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val siteLang: String, -) : PagedMangaParser(context, source, 30), Interceptor, MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain("mangafire.to") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, // chapter update - SortOrder.POPULARITY, // most views - SortOrder.RATING, // rating score - SortOrder.NEWEST, // created manga - SortOrder.ALPHABETICAL, // title asc - SortOrder.RELEVANCE, // relevance sc - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - ) - - private val apiClient by lazy { - val newHttpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request().newBuilder() - .header("Referer", "https://$domain/") - .header("Accept", "application/json") - .build() - chain.proceed(request) - } - .build() - OkHttpWebClient(newHttpClient, source) - } - - override fun intercept(chain: Interceptor.Chain) = chain.proceed( - chain.request().newBuilder() - .header("Referer", "https://$domain/") - .build() - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearRangeSupported = true, - ) - - companion object { - val GENRE_MAP = mapOf( - "Action" to "1", - "Adult" to "268929", - "Adventure" to "78", - "Avant Garde" to "3", - "Boys Love" to "4", - "Comedy" to "5", - "Crime" to "268921", - "Demons" to "77", - "Drama" to "6", - "Ecchi" to "7", - "Fantasy" to "79", - "Girls Love" to "9", - "Gourmet" to "10", - "Harem" to "11", - "Hentai" to "268930", - "Historical" to "268922", - "Horror" to "530", - "Isekai" to "13", - "Iyashikei" to "531", - "Josei" to "15", - "Kids" to "532", - "Magic" to "539", - "Magical Girls" to "268923", - "Mahou Shoujo" to "533", - "Martial Arts" to "534", - "Mature" to "268931", - "Mecha" to "19", - "Medical" to "268924", - "Military" to "535", - "Music" to "21", - "Mystery" to "22", - "Parody" to "23", - "Philosophical" to "268925", - "Psychological" to "536", - "Reverse Harem" to "25", - "Romance" to "26", - "School" to "73", - "Sci-Fi" to "28", - "Seinen" to "537", - "Shoujo" to "30", - "Shounen" to "31", - "Slice of Life" to "538", - "Smut" to "268932", - "Space" to "33", - "Sports" to "34", - "Super Power" to "75", - "Superhero" to "268926", - "Supernatural" to "76", - "Suspense" to "37", - "Thriller" to "38", - "Tragedy" to "268927", - "Vampire" to "39", - "Wuxia" to "268928" - ) - } - - private val tags by lazy { - GENRE_MAP.entries.map { (title, id) -> - MangaTag(title, id, source) - }.toSet() - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of( - MangaState.ONGOING, MangaState.FINISHED, - MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER, - ), - availableDemographics = EnumSet.of(Demographic.SHOUNEN, Demographic.SHOUJO, Demographic.SEINEN, Demographic.JOSEI), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val urlBuilder = okhttp3.HttpUrl.Builder() - .scheme("https") - .host(domain) - .addPathSegments("api/titles") - .addQueryParameter("page", page.toString()) - .addQueryParameter("limit", "50") - - if (!filter.query.isNullOrBlank()) { - urlBuilder.addQueryParameter("keyword", filter.query) - } - - if (filter.yearFrom > 0) { - urlBuilder.addQueryParameter("year_from", filter.yearFrom.toString()) - } - if (filter.yearTo > 0) { - urlBuilder.addQueryParameter("year_to", filter.yearTo.toString()) - } - - filter.types.forEach { type -> - val value = when (type) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.OTHER -> "other" - else -> null - } - value?.let { urlBuilder.addQueryParameter("types[]", it) } - } - - filter.demographics.forEach { demo -> - val id = when (demo) { - Demographic.JOSEI -> "268919" - Demographic.SEINEN -> "268920" - Demographic.SHOUJO -> "268917" - Demographic.SHOUNEN -> "268918" - else -> null - } - id?.let { urlBuilder.addQueryParameter("demographics[]", it) } - } - - filter.tags.forEach { urlBuilder.addQueryParameter("genres_in[]", it.key) } - filter.tagsExclude.forEach { urlBuilder.addQueryParameter("genres_ex[]", it.key) } - - filter.states.forEach { state -> - val apiState = when (state) { - MangaState.ONGOING -> "releasing" - MangaState.FINISHED -> "finished" - MangaState.ABANDONED -> "discontinued" - MangaState.PAUSED -> "on_hiatus" - MangaState.UPCOMING -> "not_yet_released" - else -> null - } - apiState?.let { urlBuilder.addQueryParameter("statuses[]", it) } - } - - val sortParam = when (order) { - SortOrder.UPDATED -> "chapter_updated_at" to "desc" - SortOrder.POPULARITY -> "views_total" to "desc" - SortOrder.RATING -> "score" to "desc" - SortOrder.NEWEST -> "created_at" to "desc" - SortOrder.ALPHABETICAL -> "title" to "asc" - SortOrder.RELEVANCE -> "relevance" to "desc" - SortOrder.POPULARITY_WEEK -> "views_7d" to "desc" - SortOrder.POPULARITY_MONTH -> "views_30d" to "desc" - else -> null - } - - sortParam?.let { (field, dir) -> - urlBuilder.addQueryParameter("order[$field]", dir) - } - - val url = urlBuilder.build().toString() - - val response = apiClient.httpGet(url).parseJson() - val items = response.getJSONArray("items") - val mangas = mutableListOf() - for (i in 0 until items.length()) { - val obj = items.getJSONObject(i) - val hid = obj.getString("hid") - val slug = obj.optString("slug", null) - val title = obj.getString("title") - val poster = obj.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") - ?: poster?.optString("small") ?: "" - val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" - mangas.add( - Manga( - id = generateUid(urlPath), - url = urlPath, - publicUrl = "https://$domain$urlPath", - title = title, - coverUrl = cover, - source = source, - altTitles = emptySet(), - largeCoverUrl = null, - authors = emptySet(), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - tags = emptySet(), - ) - ) - } - return mangas - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val result = coroutineScope { - val hid = extractHid(manga.url) - - val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() - val data = detailsJson.getJSONObject("data") - - val hasVolumes = data.optBoolean("hasVolumes", false) - - val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } - - val title = data.getString("title") - val poster = data.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") - ?: poster?.optString("small") - val synopsisHtml = data.optString("synopsisHtml", null) - val status = data.optString("status", null) - val type = data.optString("type", null) - val authorsList = data.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val artistsList = data.optJSONArray("artists")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val genres = data.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val themes = data.optJSONArray("themes")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - - val rawRating = data.optDouble("rating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" - - val genreList = buildList { - type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } - genres?.let { addAll(it) } - themes?.let { addAll(it) } - } - val genreTags = genreList.mapNotNull { name -> - tags.find { it.title == name } - }.toSet() - - val chapters = chaptersDeferred.await() - - manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - authors = (authorsList + artistsList).toSet(), - description = synopsisText.trim(), - rating = rating, - state = when (status?.lowercase()) { - "releasing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - "discontinued" -> MangaState.ABANDONED - "on_hiatus" -> MangaState.PAUSED - "not_yet_released" -> MangaState.UPCOMING - else -> null - }, - tags = genreTags, - altTitles = altTitlesArray.toSet(), - chapters = chapters, - ) - } - - detailsCache.put(manga.url, result) - return result - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { - return size > 10 - } - } - - private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List { - val chapters = mutableListOf() - val base = "https://$domain/api/titles/$hid" - - var page = 1 - var lastPage: Int - do { - val json = apiClient.httpGet( - "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" - ).parseJson() - val items = json.getJSONArray("items") - val meta = json.optJSONObject("meta") - lastPage = meta?.optInt("lastPage", 1) ?: 1 - - for (i in 0 until items.length()) { - val ch = items.getJSONObject(i) - if (ch.getString("language") != siteLang) continue - - val id = ch.getInt("id") - val number = ch.getDouble("number").toFloat() - val name = ch.optString("name", null) - val createdAt = ch.optLong("createdAt", 0L) * 1000L - val type = ch.getString("type") - val chapterUrl = "/title/$hid/$id" - val displayName = buildString { - append("Ch. ") - append(number.toString().removeSuffix(".0")) - if (!name.isNullOrBlank()) append(" - $name") - } - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = displayName, - number = number, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = createdAt, - branch = type, - source = source, - ) - ) - } - page++ - } while (page <= lastPage) - - if (hasVolumes) { - try { - val volJson = apiClient.httpGet("$base/volumes?language=$siteLang").parseJson() - val volItems = volJson.getJSONArray("items") - for (i in 0 until volItems.length()) { - val vol = volItems.getJSONObject(i) - if (vol.getString("language") != siteLang) continue - - val volId = vol.getInt("id") - val volNumber = vol.getDouble("number").toFloat() - val volName = vol.optString("name", "").takeIf { it.isNotBlank() } - val chapterCount = vol.optInt("chapterCount", 0) - - val title = buildString { - append("Vol. ") - append(volNumber.toString().removeSuffix(".0")) - if (volName != null) append(" - $volName") - } - val name = if (chapterCount > 0) "$chapterCount chapters" else "" - - chapters.add( - MangaChapter( - id = generateUid("/title/$hid/vol/$volId"), - title = title, - number = volNumber, - volume = 0, - url = "/title/$hid/vol/$volId", - scanlator = name, - uploadDate = 0L, - branch = "Volume", - source = source, - ) - ) - } - } catch (_: Exception) { - } - } - - val distinctBranches = chapters.map { it.branch }.distinct() - val useGroups = distinctBranches.size > 1 - - return chapters - .map { chapter -> - chapter.copy( - branch = if (useGroups) (chapter.branch ?: "").replaceFirstChar { it.uppercase() } else null - ) - } - .sortedBy { it.number } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterId = chapter.url.substringAfterLast("/") // numeric ID - val isVolume = chapter.url.contains("/vol/") - val endpoint = if (isVolume) "volumes" else "chapters" - - val response = apiClient.httpGet("https://$domain/api/$endpoint/$chapterId").parseJson() - val pagesArray = response.getJSONObject("data").getJSONArray("pages") - val pages = ArrayList(pagesArray.length()) - for (i in 0 until pagesArray.length()) { - val pageObj = pagesArray.getJSONObject(i) - val url = pageObj.getString("url") - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - ) - } - return pages - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - override val authUrl: String get() = "https://$domain" - override suspend fun isAuthorized(): Boolean = true - override suspend fun getUsername(): String = "" - - - private fun extractHid(url: String): String { - val lastPart = url.removeSuffix("/").substringAfterLast("/") - return when { - lastPart.contains(".") -> lastPart.substringAfterLast(".") - lastPart.contains("-") -> lastPart.substringBefore("-") - else -> lastPart - } - } - - @MangaSourceParser("MANGAFIRE_EN", "MangaFire (English)", "en") - class English(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_EN, "en") - - @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") - - @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire Spanish (Latin)", "es") - class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") - - @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") - class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") - - @MangaSourceParser("MANGAFIRE_JA", "MangaFire (Japanese)", "ja") - class Japanese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_JA, "ja") - - @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") - - @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire Portuguese (Brazil)", "pt") - class PortugueseBR(context: MangaLoaderContext) : - MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt deleted file mode 100644 index bf7ea2fc..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt +++ /dev/null @@ -1,293 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.Response -import okhttp3.ResponseBody.Companion.toResponseBody -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.SinglePageMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy -import java.util.* - -internal abstract class MangaPlusParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val sourceLang: String -): - SinglePageMangaParser(context, source), Interceptor { - - private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" - override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - private val extraHeaders = Headers.headersOf("Session-Token", UUID.randomUUID().toString()) - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - return when { - filter.query.isNullOrEmpty() -> { - when (order) { - SortOrder.POPULARITY -> getPopularList() - SortOrder.UPDATED -> getLatestList() - else -> getAllTitleList() - } - } - else -> getAllTitleList(filter.query) - } - } - - private suspend fun getPopularList(): List { - val json = apiCall("/title_list/ranking") - - return json.getJSONObject("titleRankingView") - .getJSONArray("titles") - .asTypedList() - .toMangaList() - } - - private suspend fun getLatestList(): List { - val json = apiCall("/title_list/updated") - - return json.getJSONObject("titleUpdatedView") - .getJSONArray("latestTitle") - .mapJSON { it.getJSONObject("title") } - .toMangaList() - } - - private val allTitleCache = suspendLazy { - apiCall("/title_list/allV2") - .getJSONObject("allTitlesViewV2") - .getJSONArray("AllTitlesGroup") - .mapJSON { it.getJSONArray("titles").asTypedList() } - .flatten() - } - - private suspend fun getAllTitleList(query: String? = null): List { - return allTitleCache.get().toMangaList(query) - } - - private fun List.toMangaList(query: String? = null): List { - return mapNotNull { - val language = it.getStringOrNull("language") ?: "ENGLISH" - - if (language != sourceLang) { - return@mapNotNull null - } - - val name = it.getString("name") - val author = it.getString("author") - .split('/') - .joinToString(transform = String::trim) - - if (query != null && !(name.contains(query, true) || author.contains(query, true))) { - return@mapNotNull null - } - - val titleId = it.getInt("titleId").toString() - - Manga( - id = generateUid(titleId), - url = titleId, - publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), - title = name, - coverUrl = it.getString("portraitImageUrl"), - altTitles = emptySet(), - authors = setOf(author), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - source = source, - tags = emptySet(), - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val json = apiCall("/title_detailV3?title_id=${manga.url}") - .getJSONObject("titleDetailView") - val title = json.getJSONObject("title") - - val completed = json.getJSONObject("titleLabels") - .getString("releaseSchedule").let { - it == "DISABLED" || it == "COMPLETED" - } - - val hiatus = json.getStringOrNull("nonAppearanceInfo")?.contains("on a hiatus") == true - val author = title.getString("author") - .split("/").joinToString(transform = String::trim) - - return manga.copy( - title = title.getString("name"), - publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain), - coverUrl = title.getString("portraitImageUrl"), - authors = setOf(author), - description = buildString { - json.getString("overview").let(::append) - json.getStringOrNull("viewingPeriodDescription") - ?.takeIf { !completed } - ?.let { append("

", it) } - }, - chapters = parseChapters( - json.getJSONArray("chapterListGroup"), - title.getStringOrNull("language") ?: "ENGLISH", - ), - state = when { - completed -> MangaState.FINISHED - hiatus -> MangaState.PAUSED - else -> MangaState.ONGOING - }, - ) - } - - private fun parseChapters(chapterListGroup: JSONArray, language: String): List { - val chapterList = chapterListGroup - .asTypedList() - .flatMap { - it.optJSONArray("firstChapterList")?.asTypedList().orEmpty() + - it.optJSONArray("lastChapterList")?.asTypedList().orEmpty() - } - - return chapterList.mapChapters { _, chapter -> - val chapterId = chapter.getInt("chapterId").toString() - val subtitle = chapter.getStringOrNull("subTitle") ?: return@mapChapters null - - MangaChapter( - id = generateUid(chapterId), - url = chapterId, - title = subtitle, - number = chapter.getString("name") - .substringAfter("#") - .toFloatOrNull() ?: -1f, - volume = 0, - uploadDate = chapter.getInt("startTimeStamp") * 1000L, - branch = when (language) { - "PORTUGUESE_BR" -> "Portuguese (Brazil)" - else -> language.lowercase().toTitleCase() - }, - scanlator = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high") - .getJSONObject("mangaViewer") - .getJSONArray("pages") - - return pages.mapJSONNotNull { - val mangaPage = it.optJSONObject("mangaPage") - ?: return@mapJSONNotNull null - val url = mangaPage.getString("imageUrl") - val encryptionKey = mangaPage.getStringOrNull("encryptionKey") - MangaPage( - id = generateUid(url), - url = url + if (encryptionKey == null) "" else "#$encryptionKey", - preview = null, - source = source, - ) - } - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - val encryptionKey = request.url.fragment - - if (encryptionKey.isNullOrEmpty()) { - return response - } - - return response.map { responseBody -> - val contentType = response.headers["Content-Type"] ?: "image/jpeg" - val image = responseBody.bytes().decodeXorCipher(encryptionKey) - image.toResponseBody(contentType.toMediaTypeOrNull()) - } - } - - private fun ByteArray.decodeXorCipher(key: String): ByteArray { - val keyStream = key.chunked(2) - .map { it.toInt(16) } - - return mapIndexed { i, byte -> byte.toInt() xor keyStream[i % keyStream.size] } - .map(Int::toByte) - .toByteArray() - } - - private suspend fun apiCall(url: String): JSONObject { - val newUrl = "$apiUrl$url".toHttpUrl().newBuilder() - .addQueryParameter("format", "json") - .build() - val response = webClient.httpGet(newUrl, extraHeaders).parseJson() - - val success = response.optJSONObject("success") - - return checkNotNull(success) { - val error = response.getJSONObject("error") - val reason = error.getJSONArray("popups") - .asTypedList() - .firstOrNull { it.getStringOrNull("language") == null } - - if (reason?.getStringOrNull("subject") == "Not Found" && url.contains("manga_viewer")) { - "This chapter has expired" - } else { - reason?.getStringOrNull("body") ?: "Unknown Error" - } - } - } - - @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") - class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH") - - @MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH") - - @MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus (French)", "fr") - class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH") - - @MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus (Indonesian)", "id") - class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN") - - @MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR") - - @MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus (Russian)", "ru") - class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN") - - @MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus (Thai)", "th") - class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI") - - @MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus (Vietnamese)", "vi") - class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE") - - @MangaSourceParser("MANGAPLUSPARSER_DE", "MANGA Plus (German)", "de") - class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN") -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt deleted file mode 100644 index 34b940db..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt +++ /dev/null @@ -1,912 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.HttpStatusException -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.network.WebClient -import org.koitharu.kotatsu.parsers.util.LinkResolver -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.insertCookies -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.ownTextOrNull -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -private const val DOMAIN = "mangaball.net" - -internal abstract class MangaBallParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val siteLanguages: Set, -) : PagedMangaParser(context, source, pageSize = 20), Interceptor { - - override val configKeyDomain = ConfigKey.Domain(DOMAIN) - - private val showSuspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) - private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } - private val client: WebClient by lazy { - OkHttpWebClient( - context.httpClient.newBuilder() - .addInterceptor(this) - .build(), - source, - ) - } - - @Volatile - private var csrfToken: String? = null - - private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT) - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.RELEVANCE, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(showSuspiciousContentKey) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = TAG_DEFINITIONS.mapTo(LinkedHashSet(TAG_DEFINITIONS.size)) { - MangaTag(key = it.id, title = it.title, source = source) - }, - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ), - availableDemographics = EnumSet.of( - Demographic.SHOUNEN, - Demographic.SHOUJO, - Demographic.SEINEN, - Demographic.JOSEI, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - ensureAdultCookie() - val query = filter.query?.trim().orEmpty() - if (page == 1 && query.startsWith("http")) { - return resolveSearchUrl(query) - } - val isQueryOnly = query.isNotEmpty() && !filter.hasNonSearchOptions() - return when { - isQueryOnly && page == 1 -> smartSearch(query) - isQueryOnly -> advancedSearch(page - 1, order, filter) - else -> advancedSearch(page, order, filter) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - ensureAdultCookie() - val doc = client.httpGet(getMangaUrl(manga.url)).parseHtml() - updateCsrf(doc) - - val tags = doc.select("#comicDetail span[data-tag-id]").mapTo(LinkedHashSet()) { tag -> - MangaTag( - key = tag.attr("data-tag-id"), - title = tag.ownTextOrNull().orEmpty().trim(), - source = source, - ) - } - val altTitles = doc.select("div.alternate-name-container").text() - .split('/') - .mapNotNullTo(LinkedHashSet()) { it.trim().nullIfEmpty() } - val authors = doc.select("#comicDetail span[data-person-id]") - .mapNotNullTo(LinkedHashSet()) { it.text().trim().nullIfEmpty() } - val status = when (doc.selectFirst("span.badge-status")?.text()?.trim()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Hiatus" -> MangaState.PAUSED - "Cancelled" -> MangaState.ABANDONED - else -> null - } - val title = doc.selectFirst("#comicDetail h6")?.ownTextOrNull()?.trim()?.nullIfEmpty() ?: manga.title - val cover = doc.selectFirst("img.featured-cover")?.absUrl("src")?.nullIfEmpty() ?: manga.coverUrl - val description = doc.selectFirst("#descriptionContent p")?.wholeText()?.trim()?.nullIfEmpty() - val contentRating = when { - tags.any { it.title in ADULT_TAG_TITLES } -> ContentRating.ADULT - else -> manga.contentRating - } - - val result = manga.copy( - title = title, - altTitles = if (altTitles.isEmpty()) manga.altTitles else altTitles, - publicUrl = getMangaUrl(manga.url), - coverUrl = cover, - largeCoverUrl = cover, - tags = tags, - state = status, - authors = authors, - description = description, - chapters = getChapterList(manga.url), - contentRating = contentRating, - ) - detailsCache[manga.url] = result - return result - } - - override suspend fun getPages(chapter: MangaChapter): List { - ensureAdultCookie() - val doc = client.httpGet(getChapterUrl(chapter)).parseHtml() - updateCsrf(doc) - val script = doc.select("script") - .firstOrNull { it.data().contains("chapterImages") } - ?.data() - .orEmpty() - val rawImages = CHAPTER_IMAGES_REGEX.find(script)?.groupValues?.getOrNull(1).orEmpty() - if (rawImages.isEmpty()) { - return emptyList() - } - val images = JSONArray(rawImages) - return List(images.length()) { index -> - val imageUrl = images.getString(index) - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - if (link.host != domain) { - return null - } - val pathType = link.pathSegments.firstOrNull() ?: return null - val slug = when (pathType) { - "title-detail" -> link.pathSegments.getOrNull(1) - "chapter-detail" -> resolveChapterSlug(link.toString()) - else -> null - } ?: return null - return getDetails(seedManga(slug)) - } - - override fun intercept(chain: Interceptor.Chain): Response { - ensureAdultCookie() - val request = chain.request() - var builder = request.newBuilder() - if ( - (request.url.host == domain || request.url.host.endsWith(".poke-black-and-white.net")) && - request.header("Referer").isNullOrEmpty() - ) { - builder = builder.header("Referer", "https://$domain/") - } - if (request.url.host == domain && request.url.pathSegments.firstOrNull() == "api") { - builder = builder - .header("X-Requested-With", "XMLHttpRequest") - .header("X-CSRF-TOKEN", getCsrf()) - } - var response = chain.proceed(builder.build()) - if (response.code == 403 && request.url.host == domain && request.url.pathSegments.firstOrNull() == "api") { - response.close() - refreshCsrf() - response = chain.proceed( - request.newBuilder() - .header("Referer", "https://$domain/") - .header("X-Requested-With", "XMLHttpRequest") - .header("X-CSRF-TOKEN", getCsrf()) - .build(), - ) - } - return response - } - - private suspend fun smartSearch(query: String): List { - val payload = mapOf("search_input" to query) - val json = postApi("$BASE_URL/api/v1/smart-search/search/".toHttpUrl(), payload).parseJson() - val mangaArray = json.getJSONObject("data").getJSONArray("manga") - return List(mangaArray.length()) { index -> - val item = mangaArray.getJSONObject(index) - val slug = extractSlug(item.getString("url")) - Manga( - id = generateUid(slug), - title = item.getString("title"), - altTitles = emptySet(), - url = slug, - publicUrl = getMangaUrl(slug), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = item.getString("img").nullIfEmpty(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - private suspend fun advancedSearch(page: Int, order: SortOrder, filter: MangaListFilter): List { - if (page < 1) { - return emptyList() - } - val payload = buildAdvancedPayload(page, order, filter) - val json = postApi("$BASE_URL/api/v1/title/search-advanced/".toHttpUrl(), payload).parseJson() - val data = json.optJSONArray("data") ?: return emptyList() - return List(data.length()) { index -> - val item = data.getJSONObject(index) - val slug = extractSlug(item.getString("url")) - Manga( - id = generateUid(slug), - title = item.getString("name"), - altTitles = emptySet(), - url = slug, - publicUrl = getMangaUrl(slug), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = item.optString("cover").nullIfEmpty(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - private fun buildAdvancedPayload(page: Int, order: SortOrder, filter: MangaListFilter): String { - val includedTags = LinkedHashSet() - includedTags += filter.tags.map { it.key } - filter.types.oneOrThrowIfMany()?.let { type -> - includedTags += TYPE_TAG_IDS.getValue(type) - } - - return buildList { - addEncoded("search_input", filter.query?.trim().orEmpty()) - addEncoded("filters[sort]", sortValue(order)) - addEncoded("filters[page]", page.toString()) - includedTags.forEach { addEncoded("filters[tag_included_ids][]", it) } - addEncoded("filters[tag_included_mode]", "and") - filter.tagsExclude.forEach { addEncoded("filters[tag_excluded_ids][]", it.key) } - addEncoded("filters[tag_excluded_mode]", "or") - addEncoded("filters[contentRating]", "any") - addEncoded("filters[demographic]", demographicValue(filter.demographics.oneOrThrowIfMany())) - addEncoded("filters[person]", "any") - addEncoded("filters[publicationYear]", filter.year.takeIf { it > 0 }?.toString().orEmpty()) - addEncoded("filters[publicationStatus]", stateValue(filter.states.oneOrThrowIfMany())) - siteLanguages.forEach { addEncoded("filters[translatedLanguage][]", it) } - }.joinToString("&") - } - - private suspend fun getChapterList(mangaSlug: String): List { - val titleId = mangaSlug.substringAfterLast('-') - val json = postApi( - "$BASE_URL/api/v1/chapter/chapter-listing-by-title-id/".toHttpUrl(), - mapOf("title_id" to titleId), - ).parseJson() - val containers = json.getJSONArray("ALL_CHAPTERS") - val chapterCandidates = ArrayList(containers.length()) - for (i in 0 until containers.length()) { - val container = containers.getJSONObject(i) - val number = container.optDouble("number_float", 0.0).toFloat() - val translations = container.getJSONArray("translations") - for (j in 0 until translations.length()) { - val translation = translations.getJSONObject(j) - val language = translation.optString("language") - if (language !in siteLanguages) { - continue - } - val group = translation.getJSONObject("group") - val groupId = group.optString("_id") - val groupName = group.optString("name").trim() - val scanlator = buildString { - append(groupName) - if (!GROUP_ID_REGEX.matches(groupId)) { - append(" (") - append(groupId) - append(')') - } - }.nullIfEmpty() - val volume = translation.optInt("volume", 0) - val translationName = translation.optString("name").trim() - val numberText = number.toString().removeSuffix(".0") - val chapter = MangaChapter( - id = generateUid(translation.getString("id")), - title = buildChapterTitle(numberText, volume, translationName), - number = number, - volume = volume, - url = translation.getString("id"), - scanlator = scanlator, - uploadDate = chapterDateFormat.parseSafe(translation.optString("date")), - branch = language.takeIf { siteLanguages.size > 1 }, - source = source, - ) - chapterCandidates += ChapterCandidate(language = language, chapter = chapter) - } - } - val scanlatorCounts = chapterCandidates.groupingBy { - "${it.language}|${it.chapter.scanlator.orEmpty()}" - }.eachCount() - return chapterCandidates - .groupBy { "${it.language}|${it.chapter.number}" } - .values - .mapNotNull { candidates -> - candidates.maxWithOrNull { left, right -> - compareChapterCandidates( - left = left, - right = right, - scanlatorCounts = scanlatorCounts, - ) - }?.chapter - } - .sortedWith( - compareBy { it.number } - .thenBy { if (it.volume > 0) 1 else 0 } - .thenBy { it.uploadDate } - .thenBy { it.branch.orEmpty() } - .thenBy { it.title.orEmpty() }, - ) - } - - private fun buildChapterTitle(numberText: String, volume: Int, rawName: String): String { - val baseTitle = buildString { - if (volume > 0) { - append("Vol. ") - append(volume) - append(' ') - } - append("Chapter ") - append(numberText) - } - val name = rawName.trim() - if (name.isEmpty()) { - return baseTitle - } - val compactName = name.replace(Regex("\\s+"), " ") - val chapterPrefixRegex = Regex("^(Vol\\.?\\s*\\d+\\s+)?Ch(?:apter)?\\.?\\s*", RegexOption.IGNORE_CASE) - val normalized = chapterPrefixRegex.find(compactName)?.let { match -> - val volumePrefix = match.groups[1]?.value?.trim().orEmpty() - buildString { - if (volumePrefix.isNotEmpty()) { - append(volumePrefix.replaceFirst(Regex("(?i)^vol\\.?"), "Vol.")) - append(' ') - } - append("Chapter ") - append(compactName.removeRange(match.range)) - } - }?.trim() ?: compactName.trim() - return if (normalized.startsWith("Chapter $numberText", ignoreCase = true) || - normalized.startsWith("Vol. $volume Chapter $numberText", ignoreCase = true) - ) { - normalized - } else { - "$baseTitle: $normalized" - } - } - - private fun compareChapterCandidates( - left: ChapterCandidate, - right: ChapterCandidate, - scanlatorCounts: Map, - ): Int { - val leftChapter = left.chapter - val rightChapter = right.chapter - val leftScore = scanlatorCounts["${left.language}|${leftChapter.scanlator.orEmpty()}"] ?: 0 - val rightScore = scanlatorCounts["${right.language}|${rightChapter.scanlator.orEmpty()}"] ?: 0 - if (leftScore != rightScore) { - return leftScore.compareTo(rightScore) - } - if ((leftChapter.volume > 0) != (rightChapter.volume > 0)) { - return if (leftChapter.volume == 0) 1 else -1 - } - val leftTitle = leftChapter.title.orEmpty() - val rightTitle = rightChapter.title.orEmpty() - if (leftTitle.length != rightTitle.length) { - return rightTitle.length.compareTo(leftTitle.length) - } - if (leftChapter.uploadDate != rightChapter.uploadDate) { - return leftChapter.uploadDate.compareTo(rightChapter.uploadDate) - } - return rightChapter.id.compareTo(leftChapter.id) - } - - private suspend fun resolveSearchUrl(query: String): List { - val slug = when { - query.startsWith("https://") || query.startsWith("http://") -> { - val link = query.toHttpUrl() - when (link.pathSegments.firstOrNull()) { - "title-detail" -> link.pathSegments.getOrNull(1) - "chapter-detail" -> resolveChapterSlug(query) - else -> null - } - } - else -> null - } ?: return emptyList() - return listOf(getDetails(seedManga(slug))) - } - - private suspend fun resolveChapterSlug(chapterUrl: String): String? { - val doc = client.httpGet(chapterUrl).parseHtml() - updateCsrf(doc) - val yoastRaw = doc.selectFirst("script.yoast-schema-graph")?.data()?.nullIfEmpty() ?: return null - val graph = JSONObject(yoastRaw).optJSONArray("@graph") ?: return null - for (i in 0 until graph.length()) { - val item = graph.getJSONObject(i) - if (item.optString("@type") == "WebPage") { - val url = item.optString("url").nullIfEmpty() ?: continue - return url.toHttpUrl().pathSegments.getOrNull(1) - } - } - return null - } - - private fun seedManga(slug: String): Manga = Manga( - id = generateUid(slug), - title = slug.substringBeforeLast('-').replace('-', ' ').trim(), - altTitles = emptySet(), - url = slug, - publicUrl = getMangaUrl(slug), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - - private fun extractSlug(url: String): String = url.toAbsoluteUrl(domain).toHttpUrl().pathSegments[1] - - private fun getMangaUrl(slug: String): String = "$BASE_URL/title-detail/$slug/" - - private fun getChapterUrl(chapter: MangaChapter): String = "$BASE_URL/chapter-detail/${chapter.url}/" - - private fun getHeadersWithReferer(): Headers = getRequestHeaders().newBuilder() - .add("Referer", "$BASE_URL/") - .build() - - private fun getApiHeaders(): Headers = getHeadersWithReferer().newBuilder() - .add("X-Requested-With", "XMLHttpRequest") - .add("X-CSRF-TOKEN", getCsrf()) - .build() - - private suspend fun postApi(url: HttpUrl, form: Map): Response { - return try { - client.httpPost(url, form, getApiHeaders()) - } catch (e: HttpStatusException) { - if (e.statusCode != 403) throw e - refreshCsrf() - client.httpPost(url, form, getApiHeaders()) - } - } - - private suspend fun postApi(url: HttpUrl, payload: String): Response { - return try { - client.httpPost(url, payload, getApiHeaders()) - } catch (e: HttpStatusException) { - if (e.statusCode != 403) throw e - refreshCsrf() - client.httpPost(url, payload, getApiHeaders()) - } - } - - @Synchronized - private fun ensureAdultCookie() { - context.cookieJar.insertCookies(domain, "show18PlusContent=${config[showSuspiciousContentKey]}") - } - - @Synchronized - private fun getCsrf(): String { - if (csrfToken.isNullOrEmpty()) { - refreshCsrf() - if (csrfToken.isNullOrEmpty()) { - refreshCsrf() - } - if (csrfToken.isNullOrEmpty()) { - error("CSRF token not found after two attempts") - } - } - return csrfToken!! - } - - @Synchronized - private fun refreshCsrf(document: Document? = null) { - ensureAdultCookie() - if (document != null) { - updateCsrf(document) - if (!csrfToken.isNullOrEmpty()) return - } - - val request = Request.Builder() - .url(BASE_URL) - .headers(getHeadersWithReferer()) - .header("Cache-Control", "no-cache") - .build() - val response = rawHttpClient.newCall(request).execute() - updateCsrf(response.parseHtml()) - response.close() - } - - @Synchronized - private fun updateCsrf(document: Document) { - document.selectFirst("meta[name=csrf-token]")?.attr("content")?.trim()?.nullIfEmpty()?.let { - csrfToken = it - } - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - private fun MutableList.addEncoded(key: String, value: String) { - add("${key.urlEncoded()}=${value.urlEncoded()}") - } - - private fun sortValue(order: SortOrder): String = when (order) { - SortOrder.UPDATED -> "updated_chapters_desc" - SortOrder.POPULARITY -> "views_desc" - SortOrder.NEWEST -> "created_at_desc" - SortOrder.ALPHABETICAL -> "name_asc" - SortOrder.ALPHABETICAL_DESC -> "name_desc" - SortOrder.RELEVANCE -> "updated_chapters_desc" - else -> "updated_chapters_desc" - } - - private fun stateValue(state: MangaState?): String = when (state) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "cancelled" - else -> "any" - } - - private fun demographicValue(demographic: Demographic?): String = when (demographic) { - Demographic.SHOUNEN -> "shounen" - Demographic.SHOUJO -> "shoujo" - Demographic.SEINEN -> "seinen" - Demographic.JOSEI -> "josei" - else -> "any" - } - - private data class ChapterCandidate( - val language: String, - val chapter: MangaChapter, - ) - - companion object { - - private const val BASE_URL = "https://$DOMAIN" - private val GROUP_ID_REGEX = Regex("[a-z0-9]{24}") - private val CHAPTER_IMAGES_REGEX = Regex("""const\s+chapterImages\s*=\s*JSON\.parse\(`([^`]+)`\)""") - private val ADULT_TAG_TITLES = setOf( - "Adult", - "Ecchi", - "Mature", - "Smut", - "Hentai", - "Manhwa 18+", - "Sexual Violence", - ) - - private val TYPE_TAG_IDS = mapOf( - ContentType.COMICS to "68ecab8507ec62d87e62780f", - ContentType.MANGA to "68ecab1e07ec62d87e627806", - ContentType.MANHUA to "68ecab4807ec62d87e62780b", - ContentType.MANHWA to "68ecab3b07ec62d87e627809", - ) - - private data class TagDef( - val id: String, - val title: String, - ) - - private val TAG_DEFINITIONS = listOf( - TagDef("685148d115e8b86aae68e4f3", "Gore"), - TagDef("685146c5f3ed681c80f257e7", "Sexual Violence"), - TagDef("685148d115e8b86aae68e4ec", "4-Koma"), - TagDef("685148cf15e8b86aae68e4de", "Adaptation"), - TagDef("685148e915e8b86aae68e558", "Anthology"), - TagDef("685148fe15e8b86aae68e5a7", "Award Winning"), - TagDef("6851490e15e8b86aae68e5da", "Doujinshi"), - TagDef("6851498215e8b86aae68e704", "Fan Colored"), - TagDef("685148d615e8b86aae68e502", "Full Color"), - TagDef("685148d915e8b86aae68e517", "Long Strip"), - TagDef("6851493515e8b86aae68e64a", "Official Colored"), - TagDef("685148eb15e8b86aae68e56c", "Oneshot"), - TagDef("6851492e15e8b86aae68e633", "Self-Published"), - TagDef("685148d715e8b86aae68e50d", "Web Comic"), - TagDef("685146c5f3ed681c80f257e3", "Action"), - TagDef("689371f0a943baf927094f03", "Adult"), - TagDef("685146c5f3ed681c80f257e6", "Adventure"), - TagDef("685148ef15e8b86aae68e573", "Boys' Love"), - TagDef("685146c5f3ed681c80f257e5", "Comedy"), - TagDef("685148da15e8b86aae68e51f", "Crime"), - TagDef("685148cf15e8b86aae68e4dd", "Drama"), - TagDef("6892a73ba943baf927094e37", "Ecchi"), - TagDef("685146c5f3ed681c80f257ea", "Fantasy"), - TagDef("685148da15e8b86aae68e524", "Girls' Love"), - TagDef("685148db15e8b86aae68e527", "Historical"), - TagDef("685148da15e8b86aae68e520", "Horror"), - TagDef("685146c5f3ed681c80f257e9", "Isekai"), - TagDef("6851490d15e8b86aae68e5d4", "Magical Girls"), - TagDef("68932d11a943baf927094e7b", "Mature"), - TagDef("6851490c15e8b86aae68e5d2", "Mecha"), - TagDef("6851494e15e8b86aae68e66e", "Medical"), - TagDef("685148d215e8b86aae68e4f4", "Mystery"), - TagDef("685148e215e8b86aae68e544", "Philosophical"), - TagDef("685148d715e8b86aae68e507", "Psychological"), - TagDef("685148cf15e8b86aae68e4db", "Romance"), - TagDef("685148cf15e8b86aae68e4da", "Sci-Fi"), - TagDef("689f0ab1f2e66744c6091524", "Shounen Ai"), - TagDef("685148d015e8b86aae68e4e3", "Slice of Life"), - TagDef("689371f2a943baf927094f04", "Smut"), - TagDef("685148f515e8b86aae68e588", "Sports"), - TagDef("6851492915e8b86aae68e61c", "Superhero"), - TagDef("685148d915e8b86aae68e51e", "Thriller"), - TagDef("685148db15e8b86aae68e529", "Tragedy"), - TagDef("68932c3ea943baf927094e77", "User Created"), - TagDef("6851490715e8b86aae68e5c3", "Wuxia"), - TagDef("68932f68a943baf927094eaa", "Yaoi"), - TagDef("6896a885a943baf927094f66", "Yuri"), - TagDef("6851490d15e8b86aae68e5d5", "Aliens"), - TagDef("685148e715e8b86aae68e54b", "Animals"), - TagDef("68bf09ff8fdeab0b6a9bc2b7", "Comics"), - TagDef("685148d215e8b86aae68e4f8", "Cooking"), - TagDef("685148df15e8b86aae68e534", "Crossdressing"), - TagDef("685148d915e8b86aae68e519", "Delinquents"), - TagDef("685146c5f3ed681c80f257e4", "Demons"), - TagDef("685148d715e8b86aae68e505", "Genderswap"), - TagDef("685148d615e8b86aae68e501", "Ghosts"), - TagDef("685148d015e8b86aae68e4e8", "Gyaru"), - TagDef("685146c5f3ed681c80f257e8", "Harem"), - TagDef("68bfceaf4dbc442a26519889", "Hentai"), - TagDef("685148f215e8b86aae68e584", "Incest"), - TagDef("685148d715e8b86aae68e506", "Loli"), - TagDef("685148d915e8b86aae68e518", "Mafia"), - TagDef("685148d715e8b86aae68e509", "Magic"), - TagDef("68f5f5ce5f29d3c1863dec3a", "Manhwa 18+"), - TagDef("6851490615e8b86aae68e5c2", "Martial Arts"), - TagDef("685148e215e8b86aae68e541", "Military"), - TagDef("685148db15e8b86aae68e52c", "Monster Girls"), - TagDef("685146c5f3ed681c80f257e2", "Monsters"), - TagDef("685148d015e8b86aae68e4e4", "Music"), - TagDef("685148d715e8b86aae68e508", "Ninja"), - TagDef("685148d315e8b86aae68e4fd", "Office Workers"), - TagDef("6851498815e8b86aae68e714", "Police"), - TagDef("685148e215e8b86aae68e540", "Post-Apocalyptic"), - TagDef("685146c5f3ed681c80f257e1", "Reincarnation"), - TagDef("685148df15e8b86aae68e533", "Reverse Harem"), - TagDef("6851490415e8b86aae68e5b9", "Samurai"), - TagDef("685148d015e8b86aae68e4e7", "School Life"), - TagDef("685148d115e8b86aae68e4ed", "Shota"), - TagDef("685148db15e8b86aae68e528", "Supernatural"), - TagDef("685148cf15e8b86aae68e4dc", "Survival"), - TagDef("6851490c15e8b86aae68e5d1", "Time Travel"), - TagDef("6851493515e8b86aae68e645", "Traditional Games"), - TagDef("685148f915e8b86aae68e597", "Vampires"), - TagDef("685148e115e8b86aae68e53c", "Video Games"), - TagDef("6851492115e8b86aae68e602", "Villainess"), - TagDef("68514a1115e8b86aae68e83e", "Virtual Reality"), - TagDef("6851490c15e8b86aae68e5d3", "Zombies"), - ) - } - - @MangaSourceParser("MANGABALL_AR", "Manga Ball (Arabic)", "ar") - class Arabic(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_AR, setOf("ar")) - - @MangaSourceParser("MANGABALL_BG", "Manga Ball (Bulgarian)", "bg") - class Bulgarian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_BG, setOf("bg")) - - @MangaSourceParser("MANGABALL_BN", "Manga Ball (Bengali)", "bn") - class Bengali(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_BN, setOf("bn")) - - @MangaSourceParser("MANGABALL_CA", "Manga Ball (Catalan)", "ca") - class Catalan(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_CA, - setOf("ca", "ca-ad", "ca-es", "ca-fr", "ca-it", "ca-pt"), - ) - - @MangaSourceParser("MANGABALL_CS", "Manga Ball (Czech)", "cs") - class Czech(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_CS, setOf("cs")) - - @MangaSourceParser("MANGABALL_DA", "Manga Ball (Danish)", "da") - class Danish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_DA, setOf("da")) - - @MangaSourceParser("MANGABALL_DE", "Manga Ball (German)", "de") - class German(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_DE, setOf("de")) - - @MangaSourceParser("MANGABALL_EL", "Manga Ball (Greek)", "el") - class Greek(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_EL, setOf("el")) - - @MangaSourceParser("MANGABALL_EN", "Manga Ball (English)", "en") - class English(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_EN, setOf("en")) - - @MangaSourceParser("MANGABALL_ES", "Manga Ball (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_ES, - setOf("es", "es-ar", "es-mx", "es-es", "es-la", "es-419"), - ) - - @MangaSourceParser("MANGABALL_FA", "Manga Ball (Persian)", "fa") - class Persian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FA, setOf("fa")) - - @MangaSourceParser("MANGABALL_FI", "Manga Ball (Finnish)", "fi") - class Finnish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FI, setOf("fi")) - - @MangaSourceParser("MANGABALL_FR", "Manga Ball (French)", "fr") - class French(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FR, setOf("fr")) - - @MangaSourceParser("MANGABALL_HE", "Manga Ball (Hebrew)", "he") - class Hebrew(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HE, setOf("he")) - - @MangaSourceParser("MANGABALL_HI", "Manga Ball (Hindi)", "hi") - class Hindi(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HI, setOf("hi")) - - @MangaSourceParser("MANGABALL_HU", "Manga Ball (Hungarian)", "hu") - class Hungarian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HU, setOf("hu")) - - @MangaSourceParser("MANGABALL_ID", "Manga Ball (Indonesian)", "id") - class Indonesian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_ID, setOf("id")) - - @MangaSourceParser("MANGABALL_IT", "Manga Ball (Italian)", "it") - class Italian(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_IT, - setOf("it", "it-it"), - ) - - @MangaSourceParser("MANGABALL_IS", "Manga Ball (Icelandic)", "is") - class Icelandic(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_IS, - setOf("ib", "ib-is", "is"), - ) - - @MangaSourceParser("MANGABALL_JA", "Manga Ball (Japanese)", "ja") - class Japanese(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_JA, setOf("jp")) - - @MangaSourceParser("MANGABALL_KO", "Manga Ball (Korean)", "ko") - class Korean(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_KO, setOf("kr")) - - @MangaSourceParser("MANGABALL_KN", "Manga Ball (Kannada)", "kn") - class Kannada(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_KN, - setOf("kn", "kn-in", "kn-my", "kn-sg", "kn-tw"), - ) - - @MangaSourceParser("MANGABALL_ML", "Manga Ball (Malayalam)", "ml") - class Malayalam(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_ML, - setOf("ml", "ml-in", "ml-my", "ml-sg", "ml-tw"), - ) - - @MangaSourceParser("MANGABALL_MS", "Manga Ball (Malay)", "ms") - class Malay(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_MS, setOf("ms")) - - @MangaSourceParser("MANGABALL_NE", "Manga Ball (Nepali)", "ne") - class Nepali(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_NE, setOf("ne")) - - @MangaSourceParser("MANGABALL_NL", "Manga Ball (Dutch)", "nl") - class Dutch(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_NL, - setOf("nl", "nl-be"), - ) - - @MangaSourceParser("MANGABALL_NO", "Manga Ball (Norwegian)", "no") - class Norwegian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_NO, setOf("no")) - - @MangaSourceParser("MANGABALL_PL", "Manga Ball (Polish)", "pl") - class Polish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_PL, setOf("pl")) - - @MangaSourceParser("MANGABALL_PTBR", "Manga Ball (Portuguese)", "pt") - class PortugueseBrazil(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_PTBR, - setOf("pt-br", "pt-pt"), - ) - - @MangaSourceParser("MANGABALL_RO", "Manga Ball (Romanian)", "ro") - class Romanian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_RO, setOf("ro")) - - @MangaSourceParser("MANGABALL_RU", "Manga Ball (Russian)", "ru") - class Russian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_RU, setOf("ru")) - - @MangaSourceParser("MANGABALL_SK", "Manga Ball (Slovak)", "sk") - class Slovak(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SK, setOf("sk")) - - @MangaSourceParser("MANGABALL_SL", "Manga Ball (Slovenian)", "sl") - class Slovenian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SL, setOf("sl")) - - @MangaSourceParser("MANGABALL_SQ", "Manga Ball (Albanian)", "sq") - class Albanian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SQ, setOf("sq")) - - @MangaSourceParser("MANGABALL_SR", "Manga Ball (Serbian)", "sr") - class Serbian(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_SR, - setOf("sr", "sr-cyrl"), - ) - - @MangaSourceParser("MANGABALL_SV", "Manga Ball (Swedish)", "sv") - class Swedish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SV, setOf("sv")) - - @MangaSourceParser("MANGABALL_TA", "Manga Ball (Tamil)", "ta") - class Tamil(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_TA, setOf("ta")) - - @MangaSourceParser("MANGABALL_TH", "Manga Ball (Thai)", "th") - class Thai(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_TH, - setOf("th", "th-hk", "th-kh", "th-la", "th-my", "th-sg"), - ) - - @MangaSourceParser("MANGABALL_TR", "Manga Ball (Turkish)", "tr") - class Turkish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_TR, setOf("tr")) - - @MangaSourceParser("MANGABALL_UK", "Manga Ball (Ukrainian)", "uk") - class Ukrainian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_UK, setOf("uk")) - - @MangaSourceParser("MANGABALL_VI", "Manga Ball (Vietnamese)", "vi") - class Vietnamese(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_VI, setOf("vi")) - - @MangaSourceParser("MANGABALL_ZH", "Manga Ball (Chinese)", "zh") - class Chinese(context: MangaLoaderContext) : MangaBallParser( - context, - MangaParserSource.MANGABALL_ZH, - setOf("zh", "zh-cn", "zh-hk", "zh-mo", "zh-sg", "zh-tw"), - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt deleted file mode 100644 index 08fd7947..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt +++ /dev/null @@ -1,502 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import android.util.LruCache -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import java.util.EnumSet -import java.util.LinkedHashMap -import java.util.LinkedHashSet -import java.util.Locale -import java.text.SimpleDateFormat -import java.util.Calendar - -@MangaSourceParser("PROJECTSUKI", "ProjectSuki") -internal class ProjectSuki(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.PROJECTSUKI, pageSize = 30) { - - override val configKeyDomain = ConfigKey.Domain("projectsuki.com") - private val pagesCache = LruCache>(50) - override val defaultSortOrder: SortOrder = SortOrder.UPDATED - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - ) - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("Referer", "https://$domain/") - .build() - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - return when { - !filter.query.isNullOrBlank() || !filter.isEmpty() -> search(page, filter) - order == SortOrder.UPDATED -> parseBookList( - webClient.httpGet("https://$domain/", getRequestHeaders()).parseHtml() - ) - else -> parseBookList( - webClient.httpGet( - "https://$domain/browse/${(page - 1).coerceAtLeast(0)}", - getRequestHeaders() - ).parseHtml() - ) - } - } - - private suspend fun search(page: Int, filter: MangaListFilter): List { - val url = "https://$domain/search".toHttpUrl().newBuilder() - .addQueryParameter("page", (page - 1).coerceAtLeast(0).toString()) - .addQueryParameter("q", filter.query.orEmpty()) - - filter.states.firstOrNull()?.let { state -> - url.addQueryParameter("adv", "1") - url.addQueryParameter( - "status", - when (state) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "cancelled" - else -> return@let - } - ) - } - - return parseBookList( - webClient.httpGet(url.build(), getRequestHeaders()).parseHtml() - ) - } - - private fun parseBookList(document: Document): List { - val result = LinkedHashMap() - - document.select("div.browse, div.item, div.book-item, .row .col-md-3, .row .col-sm-4, .row .col-xs-6") - .forEach { container -> - val bookAnchor = container.select("a[href]").firstOrNull { anchor -> - anchor.absUrl("href").toBookId() != null - } ?: return@forEach - - val bookId = bookAnchor.absUrl("href").toBookId() ?: return@forEach - if (bookId in result) return@forEach - - val title = extractBookTitle(container, bookAnchor, bookId) - val cover = extractBookCover(container, bookId) - val url = "/book/$bookId" - - val manga = Manga( - id = generateUid(url), - title = title, - altTitles = emptySet(), - url = url, - publicUrl = url.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - result[bookId] = manga - } - - return result.values.toList() - } - - private fun extractBookTitle(container: Element, anchor: Element, bookId: String): String { - return sequenceOf( - container.select(".details h4 a[href], h4 a[href], .details a[itemprop=title], a[itemprop=title]") - .firstOrNull { it.absUrl("href").toBookId() == bookId } - ?.text(), - container.select("h1, h2, h3, h4, .title, [itemprop=name]").firstOrNull()?.text(), - anchor.ownText(), - anchor.text(), - container.selectFirst("img[title]")?.attr("title"), - container.selectFirst("img[alt]")?.attr("alt"), - ).firstOrNull { it != null && it.isValidBookTitle(bookId) } - ?.trim() - ?: bookId - } - - private fun extractBookCover(container: Element, bookId: String): String { - return container.select("img").firstNotNullOfOrNull { it.imageSrc() } - ?: bookThumbnailUrl(bookId) - } - - private fun String.isValidBookTitle(bookId: String): Boolean { - val value = trim() - return value.isNotEmpty() && - !value.equals(bookId, ignoreCase = true) && - !value.equals("show more", ignoreCase = true) && - !value.all(Char::isDigit) - } - - override suspend fun getDetails(manga: Manga): Manga { - val document = webClient.httpGet( - manga.url.toAbsoluteUrl(domain), - getRequestHeaders() - ).parseHtml() - - val bookId = manga.url.toBookId() ?: document.location().toBookId() ?: manga.url.substringAfterLast('/') - - val fallbackTimestamp = document.selectFirst("meta[property='og:updated_time']") - ?.attr("content") - ?.toLongOrNull() - ?.times(1000L) - - val details = parseDetailsTable(document) - val title = document.selectFirst("h2[itemprop=title]")?.text()?.nullIfEmpty() - ?: document.selectFirst("h2")?.text()?.nullIfEmpty() - ?: manga.title - val cover = document.select("img").firstNotNullOfOrNull { it.imageSrc() }?.takeIf { bookId in it } - ?: manga.coverUrl - ?: bookThumbnailUrl(bookId) - - val description = buildDescription(document, details) - val authors = parseAuthors(document) - val state = parseState(details["status"]) - val genres = parseGenres(document) - val altTitles = setOfNotNull(details["alt titles"] ?: details["alternative titles"]) - val chapters = parseChapters(document, fallbackTimestamp) - - return manga.copy( - title = title, - coverUrl = cover, - description = description, - altTitles = altTitles, - authors = authors, - state = state, - tags = genres, - contentRating = ContentRating.SAFE, - chapters = chapters, - ) - } - - private fun parseDetailsTable(document: Document): Map { - val map = LinkedHashMap() - document.select("div, li, tr, .row .col-md-6, .col-sm-6").forEach { row -> - val children = row.children() - if (children.size < 2) return@forEach - val key = children[0].text().trim().trim(':').lowercase(Locale.ROOT) - if (key !in DETAIL_KEYS) return@forEach - val value = children[1].text().trim().nullIfEmpty() ?: return@forEach - map[key] = value - } - return map - } - - private fun buildDescription(document: Document, details: Map): String? { - val body = document.selectFirst("#descriptionCollapse")?.wholeText()?.trim() - ?: document.select(".description").joinToString("\n\n") { it.wholeText().trim() } - .nullIfEmpty() - - val detailText = details.entries.joinToString("\n") { (key, value) -> - "${key.replaceFirstChar { it.titlecase(Locale.ROOT) }}: $value" - }.nullIfEmpty() - - return listOfNotNull(body, detailText).joinToString("\n\n").nullIfEmpty() - } - - private fun parseAuthors(document: Document): Set { - return document.select("a[href]").mapNotNullTo(LinkedHashSet()) { anchor -> - val href = anchor.absUrl("href") - if ("author=" in href || "artist=" in href) { - anchor.text().nullIfEmpty() - } else null - } - } - - private fun parseGenres(document: Document): Set { - return document.select("a[href]").mapNotNullTo(LinkedHashSet()) { anchor -> - val url = anchor.absUrl("href").toHttpUrlOrNullSafe() ?: return@mapNotNullTo null - val segments = url.pathSegments.filter { it.isNotBlank() } - if (segments.firstOrNull() != "genre") return@mapNotNullTo null - val key = segments.getOrNull(1)?.nullIfEmpty() ?: return@mapNotNullTo null - MangaTag( - key = key, - title = anchor.text().ifBlank { key }.replace('-', ' ').replaceFirstChar { it.titlecase(Locale.ROOT) }, - source = source, - ) - } - } - - private fun parseState(value: String?): MangaState? { - return when (value?.trim()?.lowercase(Locale.ROOT)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled", "canceled" -> MangaState.ABANDONED - else -> null - } - } - - private fun parseChapters(document: Document, fallbackTimestamp: Long? = null): List { - val chaptersMap = LinkedHashMap() - - val table = document.select("table").firstOrNull { table -> - table.select("tr").firstOrNull()?.let { row -> - row.select("td, th").any { cell -> - cell.text().trim().matches(Regex("added|date", RegexOption.IGNORE_CASE)) - } - } ?: false - } - - if (table != null) { - val headerRow = table.select("tr").first() - val headers = headerRow!!.select("td, th").map { it.text().trim().lowercase() } - - val dateCol = headers.indexOfFirst { it == "added" || it == "date" } - val chapterCol = headers.indexOfFirst { it.matches(Regex("chapter|ch\\.|name|#")) } - .takeIf { it >= 0 } ?: 0 - - val dataRows = table.select("tbody tr").ifEmpty { table.select("tr").drop(1) } - for (row in dataRows) { - val cells = row.select("td, th") - if (cells.size <= maxOf(chapterCol, dateCol)) continue - - val chapterCell = cells[chapterCol] - val link = chapterCell.selectFirst("a[href]") ?: continue - val parts = link.absUrl("href").toChapterParts() ?: continue - val key = "${parts.bookId}/${parts.chapterId}" - if (key in chaptersMap) continue - - val title = link.text().trim().nullIfEmpty() - ?: chapterCell.text().trim().nullIfEmpty() - val number = parseChapterNumber(title ?: "") - - val dateText = if (dateCol >= 0) cells[dateCol].text().trim() else "" - val uploadDate = parseChapterDate(dateText) ?: fallbackTimestamp ?: 0L - - chaptersMap[key] = MangaChapter( - id = generateUid(key), - title = title, - number = number, - volume = 0, - url = "/read/${parts.bookId}/${parts.chapterId}/1", - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ) - } - } else { - document.select("a[href*=/read/]").forEach { anchor -> - val parts = anchor.absUrl("href").toChapterParts() ?: return@forEach - val key = "${parts.bookId}/${parts.chapterId}" - if (key in chaptersMap) return@forEach - - val title = anchor.text().trim().nullIfEmpty() - val number = parseChapterNumber(title ?: "") - chaptersMap[key] = MangaChapter( - id = generateUid(key), - title = title, - number = number, - volume = 0, - url = "/read/${parts.bookId}/${parts.chapterId}/1", - scanlator = null, - uploadDate = fallbackTimestamp ?: 0L, - branch = null, - source = source, - ) - } - } - - return chaptersMap.values.sortedBy { it.number } - } - - private fun parseChapterDate(text: String): Long? { - if (text.isBlank()) return null - - val relativeRegex = Regex("""(\d+)\s+(years?|months?|weeks?|days?|hours?|mins?|minutes?|seconds?|sec)\s+ago""", RegexOption.IGNORE_CASE) - relativeRegex.matchEntire(text.trim())?.let { match -> - val number = match.groupValues[1].toInt() - val unit = match.groupValues[2].lowercase() - val cal = Calendar.getInstance() - when { - unit.startsWith("year") -> cal.add(Calendar.YEAR, -number) - unit.startsWith("month") -> cal.add(Calendar.MONTH, -number) - unit.startsWith("week") -> cal.add(Calendar.DAY_OF_MONTH, -number * 7) - unit.startsWith("day") -> cal.add(Calendar.DAY_OF_MONTH, -number) - unit.startsWith("hour") -> cal.add(Calendar.HOUR, -number) - unit.startsWith("min") -> cal.add(Calendar.MINUTE, -number) - unit.startsWith("sec") -> cal.add(Calendar.SECOND, -number) - } - return cal.timeInMillis - } - - val formats = listOf( - SimpleDateFormat("MMMM dd, yyyy", Locale.US), - SimpleDateFormat("MMM dd, yyyy", Locale.US), - SimpleDateFormat("MMM d, yyyy", Locale.US), - SimpleDateFormat("yyyy-MM-dd", Locale.US), - ) - for (fmt in formats) { - try { - return fmt.parse(text)?.time - } catch (_: Exception) {} - } - - return null - } - - private fun parseChapterNumber(text: String): Float { - val match = CHAPTER_NUMBER_REGEX.find(text) ?: return 0f - val main = match.groupValues.getOrNull(1)?.toFloatOrNull() ?: return 0f - val sub = match.groupValues.getOrNull(2)?.toFloatOrNull() ?: 0f - return main + if (sub > 0f) sub / 10f.powDigits() else 0f - } - - private fun Float.powDigits(): Float { - var value = this - var divisor = 1f - while (value >= 1f) { - divisor *= 10f - value /= 10f - } - return divisor - } - - override suspend fun getPages(chapter: MangaChapter): List { - val cached = pagesCache.get(chapter.url) - if (cached != null) return cached - - val parts = chapter.url.toChapterParts() ?: return emptyList() - - val payload = JSONObject() - .put("bookid", parts.bookId) - .put("chapterid", parts.chapterId) - .put("first", true) - - val headers = getRequestHeaders().newBuilder() - .set("X-Requested-With", "XMLHttpRequest") - .set("Content-Type", "application/json;charset=UTF-8") - .build() - - val response = webClient.httpPost( - "https://$domain/callpage".toHttpUrl(), - payload, - headers - ).parseJson() - - val src = response.optString("src") - if (src.isEmpty()) return emptyList() - - val base = "https://$domain/images/gallery/" - val regex = Regex("""$base[^"'\s]+""") - - val seen = HashSet(16) - val pages = ArrayList(16) - - regex.findAll(src).forEach { match -> - val url = match.value - if (!seen.add(url)) return@forEach - - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - ) - } - - val result = pages.toList() - pagesCache.put(chapter.url, result) - - return result - } - - - private fun Element.imageSrc(): String? { - for (attr in IMAGE_ATTRS) { - val value = attr("abs:$attr").nullIfEmpty() ?: continue - return value.substringBefore(' ') - } - return attributes().firstOrNull { attr -> - "src" in attr.key && IMAGE_EXTENSIONS.any { ext -> ext in attr.value.lowercase(Locale.ROOT) } - }?.value?.substringBefore(' ')?.toAbsoluteUrl(domain) - } - - private fun bookThumbnailUrl(bookId: String): String = - "https://$domain/images/gallery/$bookId/thumb" - - private fun String.toBookId(): String? { - val url = toHttpUrlOrNullSafe() ?: return null - val segments = url.pathSegments.filter { it.isNotBlank() } - return if (segments.size >= 2 && segments[0].equals("book", ignoreCase = true)) { - segments[1] - } else null - } - - private fun String.toChapterParts(): ChapterParts? { - val url = runCatching { - if (startsWith("http")) this else "https://$domain/${trimStart('/')}" - }.getOrNull()?.toHttpUrlOrNullSafe() ?: return null - val segments = url.pathSegments.filter { it.isNotBlank() } - if (segments.size < 3 || !segments[0].equals("read", ignoreCase = true)) return null - return ChapterParts(segments[1], segments[2]) - } - - private fun String.toHttpUrlOrNullSafe(): HttpUrl? = - runCatching { toHttpUrl() }.getOrNull() - - private data class ChapterParts( - val bookId: String, - val chapterId: String, - ) - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - - private companion object { - private val DETAIL_KEYS = setOf( - "alt titles", "alternative titles", "author", "authors", - "artist", "artists", "status", "origin", "release year", - "views", "official", "purchase", "genre", "genres" - ) - private val IMAGE_ATTRS = arrayOf("src", "data-src", "data-lazy-src", "srcset") - private val IMAGE_EXTENSIONS = setOf(".jpg", ".png", ".jpeg", ".webp", ".avif", ".tiff") - private val CHAPTER_NUMBER_REGEX = Regex("""(?:chapter|ch\.?)\s*(\d+)(?:\s*[.,-]\s*(\d+))?""", RegexOption.IGNORE_CASE) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt deleted file mode 100644 index 500ae46c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt +++ /dev/null @@ -1,455 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import androidx.collection.arraySetOf -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import java.util.EnumSet - -private const val ID_SEARCH_PREFIX = "id:" - -private const val webtoons = "webtoons" -private const val canvas = "canvas" - -internal abstract class WebtoonsParser( - context: MangaLoaderContext, - source: MangaParserSource, -) : AbstractMangaParser(context, source) { - - override val configKeyDomain = ConfigKey.Domain("webtoons.com") - - private val mobileApiDomain = "m.webtoons.com" - private val staticDomain = "webtoon-phinf.pstatic.net" - - private val contentKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - webtoons to "Webtoons", - canvas to "Canvas", - ), - defaultValue = webtoons, - ) - - override val availableSortOrders: EnumSet = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.UPDATED, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36") - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(contentKey) - } - - override suspend fun getPageUrl(page: MangaPage): String { - return page.url.toAbsoluteUrl(staticDomain) - } - - private val languageCode: String - get() = when (val tag = sourceLocale.toLanguageTag()) { - "in" -> "id" - "zh" -> "zh-hant" - else -> tag - } - - private suspend fun fetchEpisodes(titleNo: Long, type: String): List { - val baseUrl = "https://$mobileApiDomain/api/v1/$type/$titleNo/episodes" - val url = if (type == "canvas") { - "$baseUrl?pageSize=99999&readingLanguageCode=$languageCode" - } else { - "$baseUrl?pageSize=99999" - } - val json = webClient.httpGet(url).parseJson() - - val episodeList = json.optJSONObject("result")?.optJSONArray("episodeList") - ?: throw ParseException("No episodes found for title $titleNo", url) - - return episodeList.mapChapters { _, jo -> - val episodeTitle = jo.getStringOrNull("episodeTitle") ?: "" - val episodeNo = jo.getInt("episodeNo") - val viewerLink = jo.getString("viewerLink") - val hasBgm = jo.optBoolean("hasBgm", false) - - MangaChapter( - id = generateUid("$titleNo-$episodeNo"), - title = buildString { - append(org.jsoup.parser.Parser.unescapeEntities(episodeTitle, false)) - append(" (ch. $episodeNo)") - if (hasBgm) append(" ♫") - }, - number = episodeNo.toFloat(), - volume = 0, - url = viewerLink, - uploadDate = jo.getLong("exposureDateMillis"), - branch = null, - scanlator = null, - source = source, - ) - }.sortedBy { it.number } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val titleNo = manga.url.toLong() - val detailsUrl = manga.publicUrl.ifBlank { - "https://$domain/$languageCode/drama/placeholder/list?title_no=$titleNo" - } - - val doc = webClient.httpGet(detailsUrl).parseHtml() - - val title = doc.select("meta[property='og:title']").attr("content") - .ifEmpty { doc.select("h1.subj, h3.subj").text().ifEmpty { manga.title } } - - val description = listOf( - doc.select("meta[property='og:description']").attr("content"), - doc.select("#_asideDetail p.summary").text(), - doc.select(".detail_header .summary").text(), - ).firstOrNull { it.isNotBlank() }.orEmpty() - - val coverUrl = doc.select("meta[property=\"og:image\"]").attr("content").let { url -> - if (url.isNotBlank()) url.toAbsoluteUrl(staticDomain) else manga.coverUrl - } - - val author = listOf( - doc.select("meta[property='com-linewebtoon:webtoon:author']").attr("content"), - doc.select(".detail_header .info .author:nth-of-type(1)").firstOrNull()?.ownText(), - doc.select(".author_area").text(), - ).firstOrNull { !it.isNullOrBlank() && it != "null" } - - val artist = listOf( - doc.select(".detail_header .info .author:nth-of-type(2)").firstOrNull()?.ownText(), - doc.select(".author_area").text().takeIf { it != author }, - ).firstOrNull { !it.isNullOrBlank() } ?: author - - val genreElements = doc.select(".detail_header .info .genre").ifEmpty { - doc.select("h2.genre") - } - val genres = genreElements.map { it.text() }.toSet() - - val dayInfo = doc.select("#_asideDetail p.day_info").text().ifEmpty { - doc.select(".day_info").text() - } - val state = when { - dayInfo.contains("UP") || dayInfo.contains("EVERY") || dayInfo.contains("NOUVEAU") -> MangaState.ONGOING - dayInfo.contains("END") || dayInfo.contains("COMPLETED") || dayInfo.contains("TERMINÉ") -> MangaState.FINISHED - else -> null - } - - val type = if ("/canvas/" in detailsUrl) "canvas" else "webtoon" - val chapters = async { fetchEpisodes(titleNo, type) }.await() - - Manga( - id = generateUid(titleNo), - title = title, - altTitles = emptySet(), - url = "$titleNo", - publicUrl = detailsUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = coverUrl, - largeCoverUrl = null, - tags = genres.map { genre -> MangaTag(title = genre, key = genre.lowercase(), source = source) }.toSet(), - authors = setOfNotNull(author, artist).filter { it != "null" }.toSet(), - description = description, - state = state, - chapters = chapters, - source = source, - ) - } - - private fun getSortOrderParam(order: SortOrder): String { - return when (order) { - SortOrder.POPULARITY -> "MANA" - SortOrder.RATING -> "LIKEIT" - SortOrder.UPDATED -> "UPDATE" - else -> "MANA" - } - } - - private fun availableTags() = arraySetOf( - MangaTag("Action", "action", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Drama", "drama", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Horror", "horror", source), - MangaTag("Romance", "romance", source), - MangaTag("Sci-Fi", "sf", source), - MangaTag("Slice of Life", "slice_of_life", source), - MangaTag("Sports", "sports", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Historical", "historical", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Superhero", "super_hero", source), - MangaTag("Heartwarming", "heartwarming", source), - MangaTag("Graphic Novel", "graphic_novel", source), - MangaTag("Informative", "tiptoon", source), - ) - - private val genreUrlMap: Map = availableTags().associate { - it.title.lowercase() to it.key - } - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query.orEmpty() - val currentContent = config[contentKey] ?: webtoons - - var directTitleNo: Long? = null - var directType: String? = null - - when { - query.startsWith("https://") -> { - val url = query.toHttpUrl() - if (!url.host.endsWith("webtoons.com")) { - throw Exception("Unsupported domain: ${url.host}. Only webtoons.com and m.webtoons.com are allowed.") - } - directTitleNo = url.queryParameter("title_no")?.toLongOrNull() - ?: throw Exception("Missing 'title_no' parameter in URL") - val path = url.pathSegments.filter(String::isNotEmpty) - if (path.size < 2) throw Exception("Invalid URL path") - if (path[0] != languageCode) { - throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") - } - directType = if (path.getOrNull(1) == "canvas") canvas else webtoons - } - - query.startsWith(ID_SEARCH_PREFIX) -> { - val idPart = query.removePrefix(ID_SEARCH_PREFIX) - val parts = idPart.split(":") - val (type, lang, titleNo) = when (parts.size) { - 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[0]}")) - 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[1]}")) - 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[2]}")) - else -> throw Exception("Invalid id search format. Use id:type:lang:titleNo, id:type:titleNo, or id:titleNo") - } - if (lang != languageCode) return emptyList() - if (type !in listOf(webtoons, canvas)) { - throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") - } - directTitleNo = titleNo - directType = type - } - - query.toLongOrNull() != null -> { - directTitleNo = query.toLong() - directType = currentContent - } - } - - if (directTitleNo != null && directType != null) { - val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" - val manga = getDetails(Manga( - id = generateUid(directTitleNo), - title = "", - altTitles = emptySet(), - url = directTitleNo.toString(), - publicUrl = detailsUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - )) - return listOf(manga) - } - - val page = offset / 20 + 1 - val inPageOffset = offset % 20 - val selectedGenre = filter.tags.firstOrNull() - - val url = buildString { - append("https://$domain/$languageCode") - when { - query.isNotBlank() -> { - append("/search") - var cleanQuery = query - var searchType: String? = null - when { - cleanQuery.startsWith("type:originals ") -> { - searchType = "originals" - cleanQuery = cleanQuery.removePrefix("type:originals ").trim() - } - cleanQuery.startsWith("type:canvas ") -> { - searchType = "canvas" - cleanQuery = cleanQuery.removePrefix("type:canvas ").trim() - } - } - if (searchType == null) { - searchType = when (currentContent) { - webtoons -> "originals" - canvas -> "canvas" - else -> null - } - } - if (searchType != null) append("/$searchType") - append("?keyword=${cleanQuery.urlEncoded()}") - if (page > 1) append("&page=$page") - } - selectedGenre != null -> { - val genreUrlPath = genreUrlMap[selectedGenre.key] ?: selectedGenre.key - append("/genres/$genreUrlPath") - append("?sortOrder=${getSortOrderParam(order)}") - if (page > 1) append("&page=$page") - } - else -> { - val rankingType = when { - currentContent == canvas -> "canvas" - else -> when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "trending" - SortOrder.UPDATED -> "originals" - else -> "popular" - } - } - append("/ranking/$rankingType") - if (page > 1) append("?page=$page") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - val items = doc.select(".webtoon_list li a, .card_wrap .card_item a") - .map { element -> createMangaFromElement(element, source as MangaParserSource, selectedGenre) } - - return items.drop(inPageOffset).take(20) - } - - private fun createMangaFromElement( - element: Element, - source: MangaParserSource, - selectedGenre: MangaTag? = null, - ): Manga { - val href = element.absUrl("href") - val titleNo = extractTitleNoFromUrl(href) - val title = element.select(".title, .card_title").text() - val thumbnailUrl = element.select("img").attr("src") - - return Manga( - id = generateUid(titleNo), - title = title, - altTitles = emptySet(), - url = titleNo.toString(), - publicUrl = href, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = thumbnailUrl.toAbsoluteUrl(staticDomain), - tags = selectedGenre?.let { setOf(it) } ?: emptySet(), - state = null, - authors = emptySet(), - largeCoverUrl = null, - description = null, - source = source, - ) - } - - private fun extractTitleNoFromUrl(url: String): Long { - return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() - ?: throw ParseException("Could not extract title_no from URL: $url", url) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = try { - val absUrl = chapter.url.toAbsoluteUrl(domain) - webClient.httpGet(absUrl).parseHtml() - } catch (e: Exception) { - throw ParseException("Failed to get pages for chapter: ${chapter.title}", chapter.url, e) - } - - val images = doc.select("div#_imageList > img").mapIndexedNotNull { i, element -> - val rawUrl = element.attr("data-url").takeIf { it.isNotBlank() } - ?: element.attr("src").takeIf { it.contains(staticDomain) } - ?: return@mapIndexedNotNull null - - MangaPage( - id = generateUid("${chapter.id}-$i"), - url = rawUrl, - preview = null, - source = source, - ) - }.toMutableList() - - if (images.isEmpty()) { - val motionPages = fetchMotionToonPages(doc) - if (motionPages.isNotEmpty()) { - images.addAll(motionPages) - } - } - - if (images.isEmpty()) { - throw ParseException("No images found in chapter.", chapter.url) - } - return images - } - - private suspend fun fetchMotionToonPages(doc: org.jsoup.nodes.Document): List { - val docString = doc.toString() - val docUrlRegex = Regex("documentURL:.*?'(.*?)'") - val motionToonPathRegex = Regex("jpg:.*?'(.*?)\\{") - - val docUrl = docUrlRegex.find(docString)?.groupValues?.get(1) ?: return emptyList() - val motionToonPath = motionToonPathRegex.find(docString)?.groupValues?.get(1) ?: return emptyList() - - return try { - val motionToonResponse = webClient.httpGet(docUrl).parseJson() - val imagesMap = motionToonResponse.optJSONObject("assets")?.optJSONObject("images") - imagesMap?.let { map -> - map.keySet().filter { it.contains("layer") }.mapIndexedNotNull { i, key -> - val relPath = map.optString(key, null) ?: return@mapIndexedNotNull null - MangaPage( - id = generateUid("motion-$i"), - url = motionToonPath + relPath, - preview = null, - source = source, - ) - } - } ?: emptyList() - } catch (_: Exception) { - emptyList() - } - } - - @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", type = ContentType.MANGA) - class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_EN) - - @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", type = ContentType.MANGA) - class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ID) - - @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es", type = ContentType.MANGA) - class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ES) - - @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", type = ContentType.MANGA) - class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_FR) - - @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", type = ContentType.MANGA) - class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_TH) - - @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", type = ContentType.MANGA) - class Chinese(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ZH) - - @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", type = ContentType.MANGA) - class German(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_DE) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt deleted file mode 100644 index 5f2ed071..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt +++ /dev/null @@ -1,44 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.removeSuffix -import java.util.* - -@MangaSourceParser("ASMHENTAI", "AsmHentai", type = ContentType.HENTAI) -internal class AsmHentai(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.ASMHENTAI, "asmhentai.com") { - - override val selectGallery = ".preview_item" - override val selectGalleryLink = ".image a" - override val selectGalleryImg = ".image img" - override val pathTagUrl = "/tags/?page=" - override val selectTags = ".tags_page ul.tags" - override val selectAuthor = "div.tags:contains(Artists:) .tag_list a span.tag" - override val idImg = "fimg" - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale.CHINESE, - Locale("tr"), - ), - ) - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".tag")?.html()?.substringBefore("<") ?: it.html().substringBefore("<") - MangaTag( - key = key, - title = name, - source = source, - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt deleted file mode 100644 index 4e555f96..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt +++ /dev/null @@ -1,18 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.GalleryParser - -@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) -internal class BuonDua(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { - - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( - "buondua.com", - "buondua.us", - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt deleted file mode 100644 index 53bdbbab..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt +++ /dev/null @@ -1,81 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.* - -@MangaSourceParser("HENTAIENVY", "HentaiEnvy", type = ContentType.HENTAI) -internal class HentaiEnvy(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAIENVY, "hentaienvy.com", pageSize = 24) { - override val selectGalleryLink = "a" - override val selectGalleryTitle = "div.title" - override val selectTags = ".tags_items" - override val selectTag = ".gt_right_tags ul:contains(Tags:)" - override val selectAuthor = ".gt_right_tags ul:contains(Artists:) a" - override val selectLanguageChapter = ".gt_right_tags ul:contains(Languages:) a" - override val idImg = "fimg" - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf( - Locale.ENGLISH, - Locale.FRENCH, - Locale.JAPANESE, - Locale.CHINESE, - Locale("es"), - Locale("ru"), - Locale("ko"), - Locale.GERMAN, - Locale("pt"), - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("/search/?s_key=") - append(query?.urlEncoded()) - append("&") - } - - else -> { - if (filter.tags.isNotEmpty()) { - if (filter.locale != null) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) - } - filter.tags.oneOrThrowIfMany()?.let { - append("/tag/") - append(it.key) - if (order == SortOrder.POPULARITY) { - append("/popular") - } - append("/?") - } - } else if (filter.locale != null) { - val locale = filter.locale - append("/language/") - append(locale?.toLanguagePath()) - append("/?") - } else { - append("/?") - } - } - } - append("page=") - append(page) - - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt deleted file mode 100644 index d0fe0b91..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt +++ /dev/null @@ -1,133 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("HENTAIFOX", "HentaiFox", type = ContentType.HENTAI) -internal class HentaiFox(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAIFOX, "hentaifox.com") { - override val selectGallery = ".lc_galleries .thumb, .related_galleries .thumb" - override val pathTagUrl = "/tags/popular/pag/" - override val selectTags = ".list_tags" - override val selectTag = "ul.tags" - override val selectLanguageChapter = "ul.languages a.tag_btn" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - ) - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("/search/?q=") - append(query?.urlEncoded()) - if (page > 1) { - append("&page=") - append(page.toString()) - } - } - - else -> { - if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { - append("/search/?q=") - append(buildQuery(filter.tags, filter.locale)) - if (page > 1) { - append("&page=") - append(page.toString()) - } - - if (order == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else if (filter.tags.isNotEmpty()) { - filter.tags.oneOrThrowIfMany()?.let { - append("/tag/") - append(it.key) - } - append("/") - if (order == SortOrder.POPULARITY) { - append("popular/") - } - - if (page > 1) { - append("/pag/") - append(page.toString()) - append("/") - } - } else if (filter.locale != null) { - val locale = filter.locale - append("/language/") - append(locale?.toLanguagePath()) - append("/") - if (order == SortOrder.POPULARITY) { - append("popular/") - } - - if (page > 1) { - append("/pag/") - append(page.toString()) - append("/") - } - } else { - if (page > 2) { - append("/pag/") - append(page.toString()) - append("/") - } else if (page > 1) { - append("/page/") - append(page.toString()) - append("/") - } - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".list_tag")?.text() ?: it.html().substringBefore("<") - MangaTag( - key = key, - title = name, - source = source, - ) - } - - private fun buildQuery(tags: Collection, language: Locale?): String { - val joiner = StringUtil.StringJoiner(" ") - tags.forEach { tag -> - joiner.add(tag.key) - } - language?.let { lc -> - joiner.add(lc.toLanguagePath()) - } - return joiner.complete() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt deleted file mode 100644 index 2943533d..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt +++ /dev/null @@ -1,796 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import androidx.collection.ArraySet -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.parseBytes -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseRaw -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.runCatchingCancellable -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.splitByWhitespace -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toCamelCase -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlDecode -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.security.MessageDigest -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.LinkedList -import java.util.Locale -import kotlin.math.min -import kotlin.text.get - -@OptIn(ExperimentalUnsignedTypes::class) -@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) -internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) { - override val configKeyDomain = ConfigKey.Domain("hitomi.la") - - private val cdnDomain = "gold-usergeneratedcontent.net" - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - private val ltnBaseUrl get() = "https://ltn.$cdnDomain" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY_TODAY, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - SortOrder.POPULARITY_YEAR, - ) - - private val localeMap: Map = mapOf( - Locale.forLanguageTag("id") to "indonesian", - Locale.forLanguageTag("jv") to "javanese", - Locale.forLanguageTag("ca") to "catalan", - Locale.forLanguageTag("ceb") to "cebuano", - Locale.forLanguageTag("cs") to "czech", - Locale.forLanguageTag("da") to "danish", - Locale.forLanguageTag("de") to "german", - Locale.forLanguageTag("et") to "estonian", - Locale.ENGLISH to "english", - Locale.forLanguageTag("es") to "spanish", - Locale.forLanguageTag("eo") to "esperanto", - Locale.forLanguageTag("fr") to "french", - Locale.forLanguageTag("it") to "italian", - Locale.forLanguageTag("hi") to "hindi", - Locale.forLanguageTag("hu") to "hungarian", - Locale.forLanguageTag("pl") to "polish", - Locale.forLanguageTag("pt") to "portuguese", - Locale.forLanguageTag("vi") to "vietnamese", - Locale.forLanguageTag("tr") to "turkish", - Locale.forLanguageTag("ru") to "russian", - Locale.forLanguageTag("uk") to "ukrainian", - Locale.forLanguageTag("ar") to "arabic", - Locale.KOREAN to "korean", - Locale.CHINESE to "chinese", - Locale.JAPANESE to "japanese", - ) - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = localeMap.keys, - ) - - private fun Locale?.getSiteLang(): String = when (this) { - null -> "all" - else -> localeMap[this] ?: "all" - } - - private suspend fun fetchAvailableTags(): Set = coroutineScope { - ('a'..'z').map { alphabet -> - async { - val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() - - doc.select(".posts > li").mapNotNull { element -> - val num = - element.ownText().let { - Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 - } - - if (num > 100) { - val url = element.selectFirst("a") - val href = - url?.attrAsRelativeUrl("href") - ?: return@mapNotNull null - - MangaTag( - title = url.ownText().toTagTitle(), - key = href.tagUrlToTag(), - source = source, - ) - } else { - null - } - } - } - }.awaitAll().flatten().toSet() - } - - private var cachedSearchIds: List = emptyList() - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List = when { - filter.query.isNullOrEmpty() -> { - if (filter.tags.isEmpty()) { - when (order) { - SortOrder.POPULARITY_TODAY -> { - getGalleryIDsFromNozomi( - "popular", - "today", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_WEEK -> { - getGalleryIDsFromNozomi( - "popular", - "week", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_MONTH -> { - getGalleryIDsFromNozomi( - "popular", - "month", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_YEAR -> { - getGalleryIDsFromNozomi( - "popular", - "year", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - else -> { - getGalleryIDsFromNozomi(null, "index", filter.locale.getSiteLang(), offset.nextOffsetRange()) - } - } - } else { - if (offset == 0) { - cachedSearchIds = - hitomiSearch( - filter.tags.joinToString(" ") { it.key }, - order, - filter.locale.getSiteLang(), - ).toList() - } - cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) - } - } - - else -> { - if (offset == 0) { - cachedSearchIds = hitomiSearch(filter.query ?: "", order).toList() - } - cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) - } - }.toMangaList() - - private fun Int.nextOffsetRange(): LongRange { - val bytes = this * 4L - return bytes.until(bytes + 100L) - } - - private suspend fun hitomiSearch( - query: String, - sortByPopularity: SortOrder = SortOrder.UPDATED, - language: String = "all", - ): Set = - coroutineScope { - val terms = query - .trim() - .replace(Regex("""^\?"""), "") - .lowercase() - .splitByWhitespace() - .map { - it.replace('_', ' ') - } - - val positiveTerms = LinkedList() - val negativeTerms = LinkedList() - - for (term in terms) { - if (term.startsWith("-")) { - negativeTerms.push(term.removePrefix("-")) - } else if (term.isNotBlank()) { - positiveTerms.push(term) - } - } - - val positiveResults = positiveTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val negativeResults = negativeTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val results = when { - sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) - sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) - sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) - sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) - sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) - positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) - else -> emptySet() - }.toMutableSet() - - fun filterPositive(newResults: Set) { - when { - results.isEmpty() -> results.addAll(newResults) - else -> results.retainAll(newResults) - } - } - - fun filterNegative(newResults: Set) { - results.removeAll(newResults) - } - - // positive results - positiveResults.forEach { - filterPositive(it.await()) - } - - // negative results - negativeResults.forEach { - filterNegative(it.await()) - } - - results - } - - // search.js - private suspend fun getGalleryIDsForQuery( - query: String, - language: String = "all", - ): Set { - query.replace("_", " ").let { - if (it.indexOf(':') > -1) { - val sides = it.split(":") - val ns = sides[0] - var tag = sides[1] - - var area: String? = ns - var lang = language - when (ns) { - "female", "male" -> { - area = "tag" - tag = it - } - - "language" -> { - area = null - lang = tag - tag = "index" - } - } - - return getGalleryIDsFromNozomi(area, tag, lang) - } - - val key = hashTerm(it) - val node = getGalleryNodeAtAddress(0) - val data = bSearch(key, node) ?: return emptySet() - - return getGalleryIDsFromData(data) - } - } - - private suspend fun getGalleryIDsFromData(data: Pair): Set { - val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" - val (offset, length) = data - require(length in 1..100000000) { - "Length $length is too long" - } - - val inbuf = getRangedResponse(url, offset.until(offset + length)) - - val galleryIDs = mutableSetOf() - - val buffer = - ByteBuffer - .wrap(inbuf) - .order(ByteOrder.BIG_ENDIAN) - - val numberOfGalleryIDs = buffer.int - - val expectedLength = numberOfGalleryIDs * 4 + 4 - - require(numberOfGalleryIDs in 1..10000000) { - "number_of_galleryids $numberOfGalleryIDs is too long" - } - require(inbuf.size == expectedLength) { - "inbuf.byteLength ${inbuf.size} != expected_length $expectedLength" - } - - for (i in 0.until(numberOfGalleryIDs)) - galleryIDs.add(buffer.int) - - return galleryIDs - } - - private suspend fun bSearch( - key: UByteArray, - node: Node, - ): Pair? { - fun compareArrayBuffers( - dv1: UByteArray, - dv2: UByteArray, - ): Int { - val top = min(dv1.size, dv2.size) - - for (i in 0.until(top)) { - if (dv1[i] < dv2[i]) { - return -1 - } else if (dv1[i] > dv2[i]) { - return 1 - } - } - - return 0 - } - - fun locateKey( - key: UByteArray, - node: Node, - ): Pair { - for (i in node.keys.indices) { - val cmpResult = compareArrayBuffers(key, node.keys[i]) - - if (cmpResult <= 0) { - return Pair(cmpResult == 0, i) - } - } - - return Pair(false, node.keys.size) - } - - fun isLeaf(node: Node): Boolean { - for (subnode in node.subNodeAddresses) - if (subnode != 0L) { - return false - } - - return true - } - - if (node.keys.isEmpty()) { - return null - } - - val (there, where) = locateKey(key, node) - if (there) { - return node.datas[where] - } else if (isLeaf(node)) { - return null - } - - val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) - return bSearch(key, nextNode) - } - - private suspend fun getGalleryIDsFromNozomi( - area: String?, - tag: String, - language: String, - range: LongRange? = null, - ): Set { - val nozomiAddress = when (area) { - null -> "$ltnBaseUrl/$tag-$language.nozomi" - else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" - } - - val bytes = getRangedResponse(nozomiAddress, range) - val nozomi = mutableSetOf() - - val arrayBuffer = ByteBuffer - .wrap(bytes) - .order(ByteOrder.BIG_ENDIAN) - - while (arrayBuffer.hasRemaining()) - nozomi.add(arrayBuffer.int) - - return nozomi - } - - private val galleriesIndexVersion = suspendLazy { - webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() - } - - private data class Node( - val keys: List, - val datas: List>, - val subNodeAddresses: List, - ) - - private fun decodeNode(data: ByteArray): Node { - val buffer = ByteBuffer - .wrap(data) - .order(ByteOrder.BIG_ENDIAN) - - val uData = data.toUByteArray() - - val numberOfKeys = buffer.int - val keys = ArrayList() - - for (i in 0.until(numberOfKeys)) { - val keySize = buffer.int - - check(keySize in 1..32) { "Invalid key size $keySize" } - - keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) - buffer.position(buffer.position() + keySize) - } - - val numberOfDatas = buffer.int - val datas = ArrayList>() - - for (i in 0.until(numberOfDatas)) { - val offset = buffer.long - val length = buffer.int - - datas.add(Pair(offset, length)) - } - - val numberOfSubNodeAddresses = 16 + 1 - val subNodeAddresses = ArrayList() - - for (i in 0.until(numberOfSubNodeAddresses)) { - val subNodeAddress = buffer.long - subNodeAddresses.add(subNodeAddress) - } - - return Node(keys, datas, subNodeAddresses) - } - - private suspend fun getGalleryNodeAtAddress(address: Long): Node { - val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index" - - val nodedata = getRangedResponse(url, address.until(address + 464)) - - return decodeNode(nodedata) - } - - private suspend fun getRangedResponse( - url: String, - range: LongRange? = null, - ): ByteArray { - val rangeHeaders = when (range) { - null -> Headers.headersOf() - else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") - } - - return webClient.httpGet(url, rangeHeaders).parseBytes() - } - - private fun hashTerm(term: String): UByteArray { - return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray() - } - - private fun sha256(data: ByteArray): ByteArray { - return MessageDigest.getInstance("SHA-256").digest(data) - } - - private suspend fun Collection.toMangaList(): List = coroutineScope { - map { id -> - async { - runCatchingCancellable { - val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { - val baseUri = it.request.url.toString() - val html = it.parseRaw() - Jsoup.parse(rewriteTnPaths(html), baseUri) - } - - Manga( - id = generateUid(id.toString()), - title = doc.selectFirstOrThrow("h1").text(), - url = id.toString(), - coverUrl = - "https:" + - doc.selectFirstOrThrow("picture > img") - .attr("data-src"), - publicUrl = - doc.selectFirstOrThrow("h1 > a") - .attrAsRelativeUrl("href") - .toAbsoluteUrl(domain), - authors = emptySet(), - tags = emptySet(), - contentRating = ContentRating.ADULT, - rating = RATING_UNKNOWN, - altTitles = emptySet(), - state = null, - source = source, - ) - }.getOrNull() - } - }.awaitAll().filterNotNull() - } - - override suspend fun getDetails(manga: Manga): Manga { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - val author = - json.optJSONArray("artists") - ?.mapJSON { it.getString("artist").toCamelCase() } - ?.joinToString() - - return manga.copy( - title = json.getString("title"), - largeCoverUrl = - json.getJSONArray("files").getJSONObject(0).let { - val hash = it.getString("hash") - val imageId = imageIdFromHash(hash) - val subDomain = 'a' + subdomainOffset(imageId) - - "https://${subDomain}tn.$cdnDomain/webpbigtn/${thumbPathFromHash(hash)}/$hash.webp" - }, - authors = setOfNotNull(author), - publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), - tags = - buildSet - { - json.optJSONArray("characters") - ?.mapToTags("character") - ?.let(::addAll) - json.optJSONArray("tags") - ?.mapToTags("tag") - ?.let(::addAll) - json.optJSONArray("artists") - ?.mapToTags("artist") - ?.let(::addAll) - json.optJSONArray("parodys") - ?.mapToTags("parody") - ?.let(::addAll) - json.optJSONArray("groups") - ?.mapToTags("group") - ?.let(::addAll) - }, - chapters = listOf( - MangaChapter( - id = generateUid(manga.url), - url = manga.url, - title = json.getStringOrNull("title"), - scanlator = json.getString("type").toTitleCase(), - number = 1f, - volume = 0, - branch = json.getString("language_localname"), - source = source, - uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), - ), - ), - ) - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) - - private fun JSONArray.mapToTags(key: String): Set { - val tags = ArraySet(length()) - mapJSON { - MangaTag( - title = - it.getString(key).toCamelCase().let { title -> - if (it.getStringOrNull("female")?.toIntOrNull() == 1) { - "$title ♀" - } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { - "$title ♂" - } else { - title - } - }, - key = it.getString("url").tagUrlToTag(), - source = source, - ).let(tags::add) - } - return tags - } - - private fun String.tagUrlToTag(): String { - val urlContent = this.split("/") - val ns = urlContent[1] - val tag = - urlContent[2] - .substringBeforeLast("-") - .urlDecode() - .replace(" ", "_") - - return if (tag.split(":")[0] in listOf("female", "male")) { - tag - } else { - "$ns:$tag" - } - } - - override suspend fun getRelatedManga(seed: Manga): List { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - - // any better way to get List from this json? - return json.getJSONArray("related").let { - 0.until(it.length()).map { i -> it.getInt(i) } - }.toMangaList() - } - - override suspend fun getPages(chapter: MangaChapter): List { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - - return json.getJSONArray("files").mapJSON { image -> - val hash = image.getString("hash") - val commonId = commonImageId() - val imageId = imageIdFromHash(hash) - val subDomain = subdomainOffset(imageId) + 1 - val thumbSubdomain = 'a' + subdomainOffset(imageId) - MangaPage( - id = generateUid(hash), - url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", - preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", - source = source, - ) - } - } - -// / ---> - - private var scriptLastRetrieval: Long = -1L - private val mutex = Mutex() - private var subdomainOffsetDefault = 0 - private val subdomainOffsetMap = mutableMapOf() - private var commonImageId = "" - - private suspend fun refreshScript() = mutex.withLock { - if (scriptLastRetrieval == -1L || (scriptLastRetrieval + 60000) < System.currentTimeMillis()) { - val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() - - subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt() - val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt() - - subdomainOffsetMap.clear() - Regex("case (\\d+):").findAll(ggScript).forEach { - val case = it.groupValues[1].toInt() - subdomainOffsetMap[case] = o - } - - commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1] - - scriptLastRetrieval = System.currentTimeMillis() - } - } - - // m <-- gg.js - private suspend fun subdomainOffset(imageId: Int): Int { - refreshScript() - return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault - } - - // b <-- gg.js - private suspend fun commonImageId(): String { - refreshScript() - return commonImageId - } - - // s <-- gg.js - private fun imageIdFromHash(hash: String): Int { - val match = Regex("(..)(.)$").find(hash) - return match!!.groupValues.let { it[2] + it[1] }.toInt(16) - } - - // real_full_path_from_hash <-- common.js - private fun thumbPathFromHash(hash: String): String { - return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") - } - - // rewrite_tn_paths <-- common.js - private suspend fun rewriteTnPaths(html: String): String { - val thumbUrlRegex = Regex( - """(?//)(?[a-z0-9.-]+\.(?:hitomi\.la|${Regex.escape(cdnDomain)}))/(?(?:avif|webp)?(?:small)?(?:big|small|medium)tn/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}\.(?:webp|avif|gif|png|jpe?g))""", - ) - - var resultHtml = html - thumbUrlRegex.findAll(html).forEach { matchResult -> - val originalUrl = matchResult.value - val groups = matchResult.groups - - val pathAfterHost = groups["pathAfterHost"]?.value ?: return@forEach - val newTnSubdomain = subdomainFromURL(originalUrl, "tn") - val correctedUrl = "${groups["protocol"]!!.value}$newTnSubdomain.$cdnDomain/$pathAfterHost" - - if (originalUrl != correctedUrl) { - resultHtml = resultHtml.replace(originalUrl, correctedUrl) - } - } - return resultHtml - } - - private suspend fun subdomainFromURL(url: String, base: String?): String { - val resultSubdomain = base ?: "b" - - // This regex extracts the last 3 hex characters from the hash in the URL - // The hash is 64 characters, so we look for the 61st character onward - val hashRegex = Regex("""/([0-9a-f]{61}[0-9a-f]{3})[./]""") - val fullHashMatch = hashRegex.find(url) - ?: // If no hash is found, default to "a" + base (typically "atn") - return "a$resultSubdomain" - - val fullHash = fullHashMatch.groupValues[1] - - val lastThreeChars = fullHash.takeLast(3) - val lastDigit = lastThreeChars.last() - val lastTwoDigits = lastThreeChars.take(2) - - val imageId = "$lastDigit$lastTwoDigits".toIntOrNull(16) - - return if (imageId != null) { - ('a' + subdomainOffset(imageId)).toString() + resultSubdomain - } else { - "a$resultSubdomain" - } - } - - private fun String.toTagTitle(): String { - return toCamelCase() - .replace("♂", "(male)") - .replace("♀", "(female)") - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt deleted file mode 100644 index 4c2f22f4..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt +++ /dev/null @@ -1,271 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import androidx.collection.ArraySet -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.flattenTo -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.ownTextOrNull -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireElementById -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.selectFirstParentOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.ArrayList -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI) -internal class ImHentai(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.IMHENTAI, 20) { - - override val availableSortOrders: Set = - EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING) - - override val configKeyDomain = ConfigKey.Domain("imhentai.xxx") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale("es"), - Locale.FRENCH, - Locale("kr"), - Locale.GERMAN, - Locale("ru"), - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.COMICS, - ContentType.IMAGE_SET, - ContentType.ARTIST_CG, - ContentType.GAME_CG, - ), - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val url = buildString { - append("https://") - append(domain) - append("/search/?page=") - append(page.toString()) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("&key=") - append(query?.urlEncoded()) - } - - else -> { - - if (filter.tags.isNotEmpty()) { - append("&key=") - filter.tags.joinTo(this, separator = ",") { it.key } - } - - var types = "&m=1&d=1&w=1&i=1&a=1&g=1" - if (filter.types.isNotEmpty()) { - types = "&m=0&d=0&w=0&i=0&a=0&g=0" - filter.types.forEach { - when (it) { - ContentType.MANGA -> types = types.replace("&m=0", "&m=1") - ContentType.DOUJINSHI -> types = types.replace("&d=0", "&d=1") - ContentType.COMICS -> types = types.replace("&w=0", "&w=1") - ContentType.IMAGE_SET -> types = types.replace("&i=0", "&i=1") - ContentType.ARTIST_CG -> types = types.replace("&a=0", "&a=1") - ContentType.GAME_CG -> types = types.replace("&g=0", "&g=1") - else -> {} - } - } - } - append(types) - - - var lang = "&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1" - filter.locale?.let { - lang = "&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0" - lang = lang.replace("${it.language}=0", "${it.language}=1") - } - append(lang) - - when (order) { - SortOrder.UPDATED -> append("<=1&pp=0") - SortOrder.POPULARITY -> append("<=0&pp=1") - SortOrder.RATING -> append("<=0&pp=0") - else -> append("<=1&pp=0") - } - } - } - - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.galleries div.thumb").map { div -> - val a = div.selectFirstOrThrow(".inner_thumb a") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = a.selectFirst("img")?.src(), - title = div.selectFirst(".caption")?.text().orEmpty(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. - private suspend fun fetchAvailableTags(): Set { - return coroutineScope { - (1..3).map { page -> - async { fetchTagsPage(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } - - private suspend fun fetchTagsPage(page: Int): Set { - val url = "https://$domain/tags/popular/?page=$page" - val root = webClient.httpGet(url).parseHtml() - return root.parseTags() - } - - private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet { - val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') - MangaTag( - key = href, - title = it.selectFirstOrThrow("h3.list_tag").text().toTitleCase(sourceLocale), - source = source, - ) - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val author = doc.selectFirst("li:contains(Artists) a.tag")?.ownTextOrNull() - manga.copy( - tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet { - val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') - MangaTag( - key = href, - title = it.ownText().toTitleCase(sourceLocale), - source = source, - ) - }, - authors = setOfNotNull(author), - chapters = listOf( - MangaChapter( - id = manga.id, - title = null, - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0, - branch = doc.selectFirst("li:contains(Language) a.tag")?.ownTextOrNull()?.toTitleCase(sourceLocale), - source = source, - ), - ), - ) - } - - override suspend fun getRelatedManga(seed: Manga): List { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("div.related") - return root.select("div.thumb").mapNotNull { div -> - val a = div.selectFirstOrThrow(".inner_thumb a") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = a.selectFirst("img")?.src(), - title = div.selectFirst(".caption")?.text().orEmpty(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() - val baseImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img") - val baseUrl = baseImg.selectFirstParentOrThrow("a").attrAsRelativeUrl("href").replace("/1/", - "/$/" - ) - val baseThumbUrl = baseImg.src()?.replace("/1t.", $$"/$t.") - val pages = ArrayList(totalPages) - repeat(totalPages) { i -> - val url = baseUrl.replace("$", (i + 1).toString()) - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = baseThumbUrl?.replace("$", (i + 1).toString()), - source = source, - ), - ) - } - return pages - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val img = doc.body().requireElementById("gimg") - return img.requireSrc() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt deleted file mode 100644 index bbe55a7a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt +++ /dev/null @@ -1,34 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.parsers.GalleryParser - -@MangaSourceParser("KIUTAKU", "Kiutaku", type = ContentType.OTHER) -internal class Kiutaku(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.KIUTAKU, "kiutaku.com") { - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val url = request.url.toString() - - val headers = if (url.contains("wp-content")) { - request.headers.newBuilder() - .removeAll(CommonHeaders.REFERER) - .build() - } else { - request.headers - } - - val newRequest = request.newBuilder() - .headers(headers) - .build() - - return chain.proceed(newRequest) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt deleted file mode 100644 index fa042ea0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import java.util.Locale - -@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) -internal class MangaCrazy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { - override val sourceLocale: Locale = Locale.ENGLISH -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt deleted file mode 100644 index 17a4be9a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt +++ /dev/null @@ -1,165 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.EnumSet - -@MangaSourceParser("MISSKON", "MissKon", type = ContentType.HENTAI) -internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MISSKON, 24) { - - override val configKeyDomain = ConfigKey.Domain("misskon.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities(isSearchSupported = true) - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - append("/page/$page/") - append("?s=") - append((filter.query?.urlEncoded() ?: "")) - } - order == SortOrder.POPULARITY -> { - append("/top3/") - } - else -> { - append("/page/$page") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select("article.item-list").map { article -> - val titleEl = article.selectFirst(".post-box-title")!! - val href = titleEl.selectFirst("a")?.attrAsRelativeUrl("href") - ?: article.parseFailed("Cannot find manga link") - - Manga( - id = generateUid(href), - title = titleEl.text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = article.selectFirst(".post-thumbnail img")?.absUrl("data-src").orEmpty(), - tags = setOf(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val postInnerEl = doc.selectFirst("article > .post-inner")!! - - return manga.copy( - tags = postInnerEl.select(".post-tag > a").mapToSet { a -> - MangaTag( - key = a.text().lowercase(), - title = a.text(), - source = source - ) - }, - chapters = listOf( - MangaChapter( - id = manga.id, - title = "Oneshot", // 1 album, idk - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source - ) - ) - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val basePageUrl = doc.selectFirst("link[rel=canonical]")?.absUrl("href") - ?: chapter.url.toAbsoluteUrl(domain) - - val pages = mutableListOf() - val pageLinks = doc.select("div.post-inner div.page-link:nth-child(1) .post-page-numbers") - - if (pageLinks.isEmpty()) { - // Single page gallery - return doc.select("div.post-inner > div.entry > p > img") - .mapNotNull { img -> img.absUrl("data-src") } - .mapIndexed { i, url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } - - // Multi-page gallery - pageLinks.forEachIndexed { index, pageEl -> - val pageDoc = when (index) { - 0 -> doc - else -> { - val url = "$basePageUrl${pageEl.text()}/" - webClient.httpGet(url).parseHtml() - } - } - - pages.addAll( - pageDoc.select("div.post-inner > div.entry > p > img") - .mapNotNull { img -> img.absUrl("data-src") } - .map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - ) - } - return pages - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt deleted file mode 100644 index 0faf0501..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt +++ /dev/null @@ -1,254 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import okhttp3.Headers -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.splitByWhitespace -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MULTPORN", "Multporn", type = ContentType.HENTAI) -internal class Multporn(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MULTPORN, 42) { - - override val configKeyDomain = ConfigKey.Domain("multporn.net") - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add(CommonHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36") - .add(CommonHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=UTF-8") - .build() - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - init { - setFirstPage(0) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableLocales = setOf( - Locale("en"), - Locale("de"), - Locale("ru"), - Locale("zh"), - Locale("es"), - ), - availableContentTypes = EnumSet.of( - ContentType.COMICS, - ContentType.HENTAI, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/search?search_api_views_fulltext=") - val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> - part.urlEncoded() - } - append(encodedQuery) - append("&undefined=Search") - append("&page=$page") - } - - filter.tags.isNotEmpty() -> { - val tag = filter.tags.first() - append("/category/") - append(tag.key) - - append("?sort_by=") - append( - when (order) { - SortOrder.NEWEST -> "created" - else -> "title" // default - } - ) - - append("&page=0,") - append(page) - } - - else -> { - append("/new") - append("?type=") - if (filter.types.isNotEmpty()) { - filter.types.oneOrThrowIfMany()?.let { - append( - when (it) { - ContentType.COMICS -> "1" - ContentType.HENTAI -> "2" - else -> "All" // all - }, - ) - } - } else append("All") - - - filter.locale?.let { - append("&language=") - append( - when (it) { - Locale("en") -> "1" - Locale("de") -> "2" - Locale("ru") -> "3" - Locale("zh") -> "4" - Locale("es") -> "5" - else -> "All" - } - ) - } - - append("&field_user_discription_value=All") - - append("&sort_by=") - append( - when (order) { - SortOrder.NEWEST -> "created&sort_order=DESC" - SortOrder.NEWEST_ASC -> "created&sort_order=ASC" - SortOrder.UPDATED -> "changed&sort_order=DESC" - SortOrder.UPDATED_ASC -> "changed&sort_order=ASC" - else -> "created&sort_order=DESC" // default - } - ) - - append("&undefined=Apply") - append("&page=$page") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".masonry-item").map { div -> - val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") - val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() - Manga( - id = generateUid(href), - title = div.select(".views-field-title").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = coverUrl, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val authors = (doc.select(".field:has(.field-label:contains(Author:)) .links a").map { it.text() } + - parseUnlabelledAuthorNames(doc)).distinct() - - val tags = listOf("Tags", "Section", "Characters") - .flatMap { type -> - doc.select(".field:has(.field-label:contains($type:)) .links a").map { it.text() } - } - .distinct() - .map { tag -> - MangaTag( - title = tag, - key = tag.lowercase().replace(" ", "_"), - source = source, - ) - }.toSet() - - val isOngoing = doc.select(".field .links a").any { it.text() == "Ongoings" } - - return manga.copy( - authors = authors.toSet(), - tags = tags, - description = buildString { - append("Pages: ") - append(doc.select(".jb-image img").size) - append("\n\n") - doc.select(".field:has(.field-label:contains(Section:)) .links a").joinTo(this, prefix = "Section: ") { it.text() } - doc.select(".field:has(.field-label:contains(Characters:)) .links a").joinTo(this, prefix = "\n\nCharacters: ") { it.text() } - }, - state = if (isOngoing) MangaState.ONGOING else MangaState.FINISHED, - chapters = listOf( - MangaChapter( - id = generateUid(manga.url), - title = "Oneshot", - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ) - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(".jb-image img").map { img -> - val url = img.attrAsAbsoluteUrl("src") - .replace("/styles/juicebox_2k/public", "") - .substringBefore("?") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun parseUnlabelledAuthorNames(document: Document): List { - val authorClasses = listOf( - "field-name-field-author", - "field-name-field-authors-gr", - "field-name-field-img-group", - "field-name-field-hentai-img-group", - "field-name-field-rule-63-section" - ) - return authorClasses.flatMap { className -> - document.select(".$className a").map { it.text().trim() } - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt deleted file mode 100644 index f23e3df5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt +++ /dev/null @@ -1,1076 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* - - -// removed first server, it has always problems -private const val TWO_IMG_SERVER = "2" -private const val THREE_IMG_SERVER = "3" -private const val FOUR_IMG_SERVER = "4" - -@MangaSourceParser("NHENTAI", "NHentai.net", type = ContentType.HENTAI) -internal class NHentaiParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.NHENTAI, 25), - MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain("nhentai.net") - - private var imageServer: String = "" - private var thumbServer: String = "" - - private var displayFullTitle: Boolean = true - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, // latest (default) - SortOrder.POPULARITY, // popular (all time) - SortOrder.POPULARITY_TODAY, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - TWO_IMG_SERVER to "First server", - THREE_IMG_SERVER to "Second server", - FOUR_IMG_SERVER to "Third server", - ), - defaultValue = TWO_IMG_SERVER, - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(preferredServerKey) - } - - private val selectedServer: String - get() = config[preferredServerKey].toString() - - companion object { - val LANG_MAP = mapOf( - "en" to "english", - "ja" to "japanese", - "zh" to "chinese", - ) - } - - private val nhTagMap: Map by lazy { - val raw = """ -2937=big breasts -35762=sole female -35763=sole male -8010=group -14283=anal -19440=lolicon -24201=stockings -10314=schoolgirl uniform -13720=nakadashi -29859=blowjob -8378=glasses -20905=full color -32341=shotacon -27553=rape -15658=bondage -23895=yaoi -27473=mosaic censorship -13989=ahegao -22942=incest -21712=males only -1207=milf -19018=dark skin -22945=double penetration -25614=paizuri -20035=x-ray -779=futanari -23237=tankoubon -21572=multi-work series -20525=defloration -14971=sex toys -8653=netorare -3735=swimsuit -19954=yuri -15348=ffm threesome -8368=full censorship -15408=femdom -29224=impregnation -29013=dilf -85295=twintails -31044=collar -85288=ponytail -24380=pantyhose -9260=cheating -28031=sister -16828=hairy -31880=bbm -30555=big penis -15782=crossdressing -31775=tentacles -27384=mind break -19175=bikini -8739=story arc -30473=muscle -24102=lactation -7752=schoolboy uniform -20617=mind control -9083=big ass -29023=tomgirl -81774=kemonomimi -1590=sweating -9162=masturbation -7256=mmf threesome -28550=teacher -190=maid -8693=uncensored -19899=exhibitionism -6343=pregnant -8050=females only -6817=unusual pupils -25871=lingerie -10988=anthology -20282=footjob -15853=mother -15785=harem -14072=huge breasts -30035=gender bender -1643=kissing -130025=anal intercourse -1033=handjob -12824=condom -31386=catgirl -10476=urination -3666=garter belt -26130=fingering -81707=beauty mark -22079=drugs -105833=gloves -4435=gag -25601=small breasts -5820=piercing -12695=prostitution -16228=demon girl -7155=cunnilingus -22950=tanlines -832=elf -31012=blindfold -17773=kimono -2820=scat -29182=blackmail -23132=bunny girl -32484=stomach deformation -2515=virginity -27063=filming -7142=bbw -21989=inflation -88846=horns -104227=tail -26953=bukkake -28800=bloomers -25050=gyaru -24676=rimjob -23632=big areolae -16533=sleeping -73750=bald -18567=monster -35972=sole dickgirl -18328=thigh high boots -5810=strap-on -29565=school swimsuit -32996=deepthroat -370=business suit -7550=monster girl -1067=inseki -50585=webtoon -12523=bestiality -27697=leotard -30645=dick growth -29631=inverted nipples -29366=tomboy -24412=bodysuit -15492=scanmark -9406=enema -35970=dickgirl on dickgirl -29399=daughter -18613=military -11941=replaced -6525=nurse -9661=cervix penetration -33129=slave -4573=corruption -5529=urethra insertion -10542=snuff -683=squirting -51399=crotch tattoo -122908=very long hair -7838=magical girl -24726=apron -23183=breast expansion -20074=latex -28426=hairy armpits -27217=guro -31285=fox girl -106119=no penetration -24764=drunk -9990=prostate massage -35968=dickgirl on male -2956=old man -32752=shibari -6900=miko -2153=wings -706=birth -10794=breast feeding -14069=ryona -25822=smell -5357=humiliation -5962=spanking -2531=transformation -21538=bike shorts -31101=incomplete -32745=chikan -16236=shemale -36957=bisexual -26952=tall girl -25663=oppai loli -7995=big nipples -32602=fisting -106733=hair buns -1088=bdsm -21283=masked face -15225=blowjob face -2633=leg lock -27378=artbook -35971=male on dickgirl -27112=tiara -107705=facial hair -24933=eyepatch -4549=torture -30206=tribadism -1037=oni -89056=hidden sex -13136=facesitting -3391=nun -25766=gokkun -5200=pegging -17531=cosplaying -28521=voyeurism -19479=nipple fuck -17349=tracksuit -22221=blood -50505=oyakodon -50486=tail plug -560=twins -23965=chloroform -15425=vore -25457=possession -129668=eye-covering bang -24984=orgasm denial -144644=extraneous ads -28589=hotpants -17752=foot licking -32282=piss drinking -19390=cousin -32589=feminization -11376=body modification -20362=gyaru-oh -28778=large insertions -27720=smegma -10811=double vaginal -3614=triple penetration -3455=chastity belt -2452=scar -31319=yandere -7354=amputee -28335=giantess -26848=waitress -28349=cbt -24967=sumata -104893=vtuber -8516=emotionless sex -26380=demon -17591=robot -17801=solo action -13640=frottage -25996=gaping -23035=aunt -23967=huge penis -31846=body writing -25744=cheerleader -24708=cowgirl -25085=swinging -18322=brother -101724=leash -10354=milking -97795=pixie cut -11089=body swap -32224=eggs -10606=pasties -3947=onahole -14573=tall man -10604=dog -14362=low lolicon -15242=lab coat -4935=farting -13468=shimapan -5620=double anal -14138=freckles -50390=josou seme -15119=dog girl -93324=fishnets -22025=prolapse -15471=asphyxiation -21774=human pet -31337=kunoichi -15712=eyemask -30126=big clit -92409=thick eyebrows -109360=cumflation -7208=catboy -31687=randoseru -24529=bride -19561=big balls -24450=chinese dress -121738=focus anal -22967=diaper -29347=miniguy -29001=parasite -25296=armpit licking -6220=orc -7546=witch -30895=sunglasses -7372=corset -28119=nose hook -8429=machine -7684=armpit sex -14516=wolf girl -15045=niece -13882=tutor -8391=public use -30811=christmas -104245=small penis -266=sundress -17501=phimosis -17800=tickling -25794=widow -7288=vomit -1215=unusual teeth -72471=dickgirls only -107503=soushuuhen -138044=exposed clothing -1352=slime -31986=age regression -23917=long tongue -24115=angel -114993=shimaidon -13722=moral degeneration -26898=age progression -27120=selfcest -7577=vampire -17676=ghost -88103=clothed female nude male -13515=coach -141098=nipple stimulation -9116=unbirth -5936=time stop -18420=all the way through -72139=clothed paizuri -27530=ball sucking -16518=coprophagia -28869=stuck in wall -2527=bandages -24621=insect -11399=metal armor -106006=large tattoo -3843=fundoshi -20120=multiple paizuri -8400=goblin -129321=mesuiki -124610=mouth mask -10693=dougi -31371=mecha girl -21450=minigirl -10685=double blowjob -118056=petplay -20789=policewoman -3031=underwater -31173=first person perspective -78262=shaved head -19064=pubic stubble -14280=bunny boy -25949=gothic lolita -23463=wrestling -16947=horse -11247=skinsuit -11073=living clothes -30786=watermarked -23073=assjob -52826=dark sclera -107478=drill hair -23225=non-h -109930=domination loss -20170=poor grammar -138200=gender change -16759=artistcg -80978=nudity only -15749=oil -30176=petrification -25848=human cattle -559=ttf threesome -14010=snake girl -11276=multiple penises -90671=original -18024=touhou project -1841=kantai collection -35605=fate grand order -20925=the idolmaster -972=granblue fantasy -78245=azur lane -17137=neon genesis evangelion -3185=love live -391=girls und panzer -11219=pokemon -15021=sailor moon -4505=mahou shoujo lyrical nanoha -128408=blue archive -10222=fate stay night -27431=to love-ru -13159=naruto -123503=genshin impact -3984=sword art online -3603=street fighter -22174=one piece -16285=puella magi madoka magica -91195=princess connect -12232=my hero academia -3163=king of fighters -26172=k-on -7259=touken ranbu -19080=code geass -37544=love live sunshine -17077=cardcaptor sakura -27547=the melancholy of haruhi suzumiya -13508=final fantasy vii -10954=shingeki no kyojin -25430=vocaloid -32687=free -4577=toheart2 -22146=dead or alive -20025=gochuumon wa usagi desu ka -8485=dragon ball z -5037=bleach -3218=bakemonogatari -12624=ore no imouto ga konna ni kawaii wake ga nai -37109=kono subarashii sekai ni syukufuku o -4369=monster hunter -127065=hololive -74788=girls frontline -24886=fate kaleid liner prisma illya -6999=toaru kagaku no railgun -22032=boku wa tomodachi ga sukunai -18350=ragnarok online -21674=dragon quest iii -14345=ojamajo doremi -7832=darkstalkers -24135=ah my goddess -32394=samurai spirits -1283=queens blade -16639=haikyuu -13924=yu-gi-oh -79467=kimetsu no yaiba -18238=danganronpa -26336=yu-gi-oh zexal -16984=persona 4 -18569=kuroko no basuke -1910=smile precure -30587=sakura taisen -16166=mahou sensei negima -12285=ranma 12 -8470=infinite stratos -32363=toaru majutsu no index -22708=saki -8708=to heart -108082=arknights -16707=detective conan -22210=guilty gear -947=gundam seed destiny -22677=tenchi muyo -23429=pretty cure -18512=strike witches -31027=lucky star -7408=league of legends -394=love hina -23201=kanon -27704=amagami -127052=nijisanji -70802=kemono friends -52098=persona 5 -22215=super robot wars -27567=hayate no gotoku -35251=osomatsu-san -7633=pripara -34823=ensemble stars -37914=re zero kara hajimeru isekai seikatsu -74918=bang dream -15041=martian successor nadesico -24783=dragon ball -120519=love live nijigasaki high school idol club -2803=love plus -5085=senki zesshou symphogear -28474=zero no tsukaima -15197=gundam build fighters -15427=dragon quest iv -1163=rozen maiden -23859=yu-gi-oh arc-v -75023=dragon quest xi -2112=dungeon ni deai o motomeru no wa machigatteiru darou ka -36418=voiceroid -28281=mitsudomoe -11624=the legend of zelda -14694=fullmetal alchemist -16847=dragon quest v -2497=urusei yatsura -5671=tengen toppa gurren lagann -22754=amagi brilliant park -20606=tsukihime -5165=gundam build fighters try -4114=macross frontier -20763=inazuma eleven -14550=sister princess -19083=jojos bizarre adventure -21052=fate hollow ataraxia -29922=teitoku -51810=gudao -16643=producer -13848=reimu hakurei -25125=asuka langley soryu -17279=sakuya izayoi -10496=patchouli knowledge -37739=shielder -3206=shinji ikari -38068=gran -4675=sanae kochiya -21779=rei ayanami -14040=fate testarossa -3870=flandre scarlet -23902=remilia scarlet -21688=atago -11373=marisa kirisame -35128=kashima -17154=sakura kinomoto -31462=satori komeiji -30080=kaga -10802=alice margatroid -17017=aya shameimaru -17862=yukari yakumo -5340=shimakaze -18935=nanoha takamachi -18896=shirou emiya -16555=rin tosaka -16130=rito yuuki -15890=reisen udongein inaba -7724=takao -27060=jeanne darc -78989=jeanne alter -7718=naruto uzumaki -5337=nami -22975=chun-li -17502=illyasviel von einzbern -20111=tifa lockhart -21131=youmu konpaku -18026=kazuto kirigaya -92923=shikikan -29856=saber -71442=minamoto no raikou -1843=asuna yuuki -51419=gudako -7488=mai shiranui -9835=koishi komeiji -16916=kasumi -30026=maki nishikino -143975=sensei -26906=izuku midoriya -37275=scathach -7696=momiji inubashiri -38039=astolfo -27794=mikoto misaka -20062=hamakaze -78285=artoria pendragon -34860=katsuki bakugou -3328=homura akemi -37687=djeeta -32200=suzuya -21108=rin shibuya -35964=nico yazawa -27494=levi ackerman -609=eren jaeger -11920=sakura haruno -20427=sailor mercury -24714=chino kafuu -31456=mikan yuuki -866=koyomi araragi -12149=kyousuke kousaka -277=haruka nanase -19926=haruna -3763=haruhi suzumiya -26427=mio akiyama -25439=hinata hyuga -17811=ran yakumo -14857=kongou -18548=kotori minami -32364=rider -15641=madoka kaname -2613=hong meiling -491=makoto tachibana -20702=koakuma -15315=tomoyo daidouji -10730=shigure -14265=touma kamijou -80311=bb -4203=mami tomoe -37706=kazuma satou -33070=umi sonoda -27172=yuyuko saigyouji -3353=yuuka kazami -2078=nagato -6311=arisu tachibana -647=belldandy -9274=maya -24889=sena kashiwazaki -15125=golden darkness -6555=sailor jupiter -25695=mika jougasaki -50929=shuten douji -33077=sailor mars -8293=minami nitta -7451=lelouch vi britannia -389=rika jougasaki -22469=prinz eugen -16108=azusa nakano -12812=tenryuu -7311=ami mizuno -6642=byakuren hijiri -7097=suwako moriya -19172=miki hoshii -9657=ayane -29433=c.c. -25220=sakura matou -14499=tsunade -10665=tenshi hinanai -16564=miku hatsune -29190=kallen stadtfeld -3312=kirino kousaka -1234=yuki nagato -26261=ranma saotome -19002=rin kaenbyou -12748=nico robin -32765=rin matsuoka -4241=fumika sagisawa -1729=tamaki kousaka -23997=ruri gokou -29684=sailor venus -19160=nitori kawashiro -27302=uzuki shimamura -23216=android 18 -8489=hibiki -7333=suguha kirigaya -1267=kodaka hasegawa -2345=morrigan aensland -9371=yamato -26087=inazuma -27532=archer -26587=miho nishizumi -12346=utsuho reiuji -37108=megumin -22407=takane shijou -15914=sasuke uchiha -2774=kyouko sakura -80930=abigail williams -81288=gudao | ritsuka fujimaru -73756=nightingale -6109=eri ayase -27492=akagi -17899=sakura kasugano -32137=cirno -11760=yui kotegawa -75029=eli ayase -11740=sailor moon -49158=narmaya -29693=ikazuchi -126586=aether -20918=iori minase -24832=misato katsuragi -2883=kasen ibara -6932=souji okita -28555=tamamo-no-mae -14428=kokoa hoto -26783=taihou -12763=rumia -401=nakoruru -72475=musashi miyamoto -23122=maho nishizumi -29188=eirin yagokoro -466=usagi tsukino -29638=kyon -15995=makoto kino -11744=amatsukaze -6175=cammy white -30331=ichika orimura -23473=mikuru asahina -28807=ruri hoshino -2572=hatate himekaidou -15291=chen -23386=fujiwara no mokou -9237=shoukaku -28763=tewi inaba -23851=gilgamesh -10672=aqua -9702=ro-500 -31074=keine kamishirasawa -32443=charlotte dunois -2196=sayaka miki -1645=zuikaku -5925=akatsuki -4196=hestia -33171=shiho nishizumi -19534=hayate yagami -79507=belfast -12433=kaede takagaki -12872=warrior -8170=len kagamine -50415=rem -14409=momoka sakurai -2211=mari illustrious makinami -99075=kokkoro -1907=rei hino -15651=miyu edelfelt -26169=musashi -8053=lum -50596=you watanabe -9883=kagami hiiragi -24509=darjeeling -11992=lala satalin deviluke -32683=hachiman hikigaya -31076=kuroko shirai -20836=red saber -12902=isuzu sento -10379=bianca whitaker -16181=nozomi toujou -27774=bismarck -28219=yui hirasawa -1271=momo velia deviluke -49852=subaru natsuki -5918=shinobu oshino -28056=link -25605=rangiku matsumoto -35313=cagliostro -18453=hero -75102=nozomi tojo -20722=mutsu -29170=yuma tsukumo -9486=nue houjuu -33049=ritsuko akizuki -23626=murakumo -20323=tsumugi kotobuki -16566=ritsu tainaka -14016=yuu narukami -11609=yoko ritona -107011=chloe von einzbern -52132=riko sakurauchi -32114=onpu segawa -11924=kagerou imaizumi - """.trimIndent() - raw.lineSequence() - .mapNotNull { line -> - val parts = line.split('=') - if (parts.size == 2 && parts[0].isNotBlank() && parts[1].isNotBlank()) { - parts[0] to parts[1] - } else null - }.toMap() - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = nhTagMap.map { (id, name) -> - MangaTag( - title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, - key = id, // numeric tag ID - source = source, - ) - }.toSet(), - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale.CHINESE, - ), - ) - - private suspend fun ensureNhConfig() { - if (imageServer.isBlank() || thumbServer.isBlank()) { - try { - val json = webClient.httpGet("https://$domain/api/v2/config").parseJson() - val imgServers = json.getJSONArray("image_servers") - .let { (0 until it.length()).map { i -> it.getString(i) } } - val thbServers = json.getJSONArray("thumb_servers") - .let { (0 until it.length()).map { i -> it.getString(i) } } - imageServer = imgServers[selectedServer.toInt() - 1] - thumbServer = thbServers[selectedServer.toInt() - 1] - } catch (_: Exception) { - imageServer = "https://i$selectedServer.nhentai.net" - thumbServer = "https://t$selectedServer.nhentai.net" - } - } - } - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - ensureNhConfig() - - // Direct ID search - val directId = extractGalleryId(filter.query) - if (directId != null) { - return try { - listOf(fetchMangaById(directId)) - } catch (_: Exception) { - emptyList() - } - } - - val selectedTag = filter.tags.firstOrNull() - if (selectedTag != null) { - val url = "https://$domain/api/v2/galleries/tagged?tag_id=${selectedTag.key}&page=$page" - val json = webClient.httpGet(url).parseJson() - val items = json.getJSONArray("result") - return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } - } - - val lang = filter.locale?.let { loc -> - LANG_MAP[loc.language] - } - - val url = when { - // Search with language - lang != null -> { - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - else -> "date" - } - "https://$domain/api/v2/search?query=language:$lang&sort=$sort&page=$page" - } - // Latest - order == SortOrder.UPDATED -> "https://$domain/api/v2/galleries?page=$page" - // Popular (all / today / week / month) - else -> { - val sort = when (order) { - SortOrder.POPULARITY_TODAY -> "popular-today" - SortOrder.POPULARITY_WEEK -> "popular-week" - SortOrder.POPULARITY_MONTH -> "popular-month" - SortOrder.POPULARITY -> "popular" - else -> "popular" - } - "https://$domain/api/v2/search?query=%22%22&sort=$sort&page=$page" - } - } - - val json = webClient.httpGet(url).parseJson() - val items = json.getJSONArray("result") - return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } - } - - private fun extractGalleryId(query: String?): Int? { - if (query.isNullOrBlank()) return null - val trimmed = query.trim() - return when { - trimmed.startsWith("id:") -> trimmed.removePrefix("id:").toIntOrNull() - trimmed.all(Char::isDigit) -> trimmed.toIntOrNull() - else -> null - } - } - - private suspend fun fetchMangaById(id: Int): Manga { - ensureNhConfig() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - return json.toMangaDetails() - } - - override suspend fun getDetails(manga: Manga): Manga { - ensureNhConfig() - val id = manga.url.removeSurrounding("/g/", "/").toInt() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - return json.toMangaDetails().copy( - coverUrl = manga.coverUrl?.ifBlank { - "${thumbServer}/${json.getJSONObject("cover").optString("path")}" - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - ensureNhConfig() - val id = chapter.url.removeSurrounding("/g/", "/").toInt() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - val pagesArray = json.getJSONArray("pages") - return (0 until pagesArray.length()).map { i -> - val pageObj = pagesArray.getJSONObject(i) - val path = pageObj.getString("path") - MangaPage( - id = generateUid(path), - url = "$imageServer/$path", - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - private fun JSONObject.toManga(): Manga { - val id = getInt("id") - val titleObj = optJSONObject("title") - val rawTitle = titleObj?.optString("english") - ?: titleObj?.optString("pretty") - ?: optString("english_title") - ?: optString("japanese_title") - ?: "Gallery $id" - - val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() - val thumbPath = optJSONObject("thumbnail")?.optString("path") - ?: optString("thumbnail").takeIf { !it.isNullOrBlank() } - ?: "galleries/$id/thumb.webp" - - return Manga( - id = generateUid("/g/$id/"), - title = title, - altTitles = emptySet(), - url = "/g/$id/", - publicUrl = "https://$domain/g/$id/", - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$thumbPath", - tags = extractTags(), - state = null, - authors = emptySet(), - source = source, - ) - } - - private fun JSONObject.toMangaDetails(): Manga { - val id = getInt("id") - val titleObj = optJSONObject("title") - val rawTitle = titleObj?.optString("english") - ?: titleObj?.optString("japanese") - ?: titleObj?.optString("pretty") - ?: "Gallery $id" - val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() - - val thumbPath = getJSONObject("thumbnail").getString("path") - val coverPath = getJSONObject("cover").optString("path", thumbPath) - val numPages = optInt("num_pages") - val uploadDate = optLong("upload_date") * 1000 - - val tags = extractTags() - val artists = getTagsOfType("artist") - val groups = getTagsOfType("group") - val author = artists.joinToString(", ").ifBlank { groups.joinToString(", ") } - - var langName: String? - var translated = false - val languageCandidates = mutableListOf() - - for (tag in tags) { - val parts = tag.key.split(":", limit = 2) - if (parts.size == 2 && parts[0] == "language") { - val name = parts[1] - when { - name.equals("translated", ignoreCase = true) -> translated = true - else -> languageCandidates.add(name) - } - } - } - - langName = languageCandidates.firstOrNull { it.equals("chinese", ignoreCase = true) } - ?: languageCandidates.firstOrNull { it.equals("japanese", ignoreCase = true) } - ?: languageCandidates.firstOrNull { it.equals("english", ignoreCase = true) } - ?: languageCandidates.firstOrNull() - - val description = buildString { - langName?.let { - val displayLang = it.replaceFirstChar { c -> c.uppercase() } - append(if (translated) "$displayLang (translated)" else displayLang) - append("\n") - } - append("Pages: $numPages") - } - - return Manga( - id = generateUid("/g/$id/"), - title = title, - altTitles = listOfNotNull(titleObj?.optString("japanese")).toSet(), - url = "/g/$id/", - publicUrl = "https://$domain/g/$id/", - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$thumbPath", - largeCoverUrl = "$imageServer/$coverPath", - tags = tags, - state = null, - authors = setOfNotNull(author.ifBlank { null }), - description = description, - chapters = listOf( - MangaChapter( - id = generateUid("/g/$id/"), - title = title, - number = 1f, - volume = 0, - url = "/g/$id/", - scanlator = groups.joinToString(", ").ifBlank { null }, - uploadDate = uploadDate, - branch = null, - source = source, - ) - ), - source = source, - ) - } - - private fun JSONObject.extractTags(): Set { - val tagsArray = optJSONArray("tags") ?: return emptySet() - return (0 until tagsArray.length()).map { i -> - val tag = tagsArray.getJSONObject(i) - val name = tag.getString("name") - MangaTag( - title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, - key = "${tag.getString("type")}:$name", - source = source, - ) - }.toSet() - } - - private fun JSONObject.getTagsOfType(type: String): List { - val tagsArray = optJSONArray("tags") ?: return emptyList() - return (0 until tagsArray.length()).mapNotNull { i -> - val tag = tagsArray.getJSONObject(i) - if (tag.getString("type") == type) tag.getString("name") else null - } - } - - private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") - private fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() - - override val authUrl: String get() = "https://$domain" - override suspend fun isAuthorized(): Boolean = true - override suspend fun getUsername(): String = "" -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt deleted file mode 100644 index 5591672b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt +++ /dev/null @@ -1,184 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.* -import java.util.* - -@MangaSourceParser("NHENTAIXXX", "NHentai.xxx", type = ContentType.HENTAI) -internal class NHentaiXXXParser(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.NHENTAIXXX, "nhentai.xxx", 25) { - - val supportedImageFormats = setOf("jpg", "webp", "jpeg", "png", "bmp") - - override val selectGallery = "div.galleries_box .gallery_item, #related-container .gallery_item" - override val selectGalleryLink = "a" - override val selectGalleryTitle = ".caption" - override val pathTagUrl = "/tags/popular?page=" - override val selectTitle = "h1" - override val selectTags = "div.tags_items" - override val selectTag = ".tags:contains(Tags)" - override val selectAuthor = ".tags:contains(Artists) span.tag_name" - override val selectLanguageChapter = - ".tags:contains(Languages) a:not([href=\"/language/translated/\"]) span.tag_name" - override val idImg = "fimg" - - override val availableSortOrders: Set = - EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf(Locale.ENGLISH, Locale.JAPANESE, Locale.CHINESE), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - val tags = filter.tags - val lang = filter.locale - if (tags.isNotEmpty() && lang != null) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) - } - - when { - lang != null -> { - append("/language/") - append(lang.toLanguagePath()) - append("/?") - } - - else -> { - append("/search/?key=") - - val joiner = StringUtil.StringJoiner("+") - tags.forEach { tag -> - joiner.add(tag.title) - } - - if (!query.isNullOrEmpty()) { - joiner.add(query.urlEncoded()) - } - append(joiner.complete()) - } - } - - when (order) { - SortOrder.POPULARITY, SortOrder.RELEVANCE -> append("&sort=popular") - SortOrder.UPDATED -> {} - else -> {} - } - - if (page > 1) { - append("&page=") - append(page.toString()) - } - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun parseMangaList(doc: Document): List { - return doc.select(selectGallery).map { div -> - val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(selectGalleryTitle).text().cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = div.selectFirstOrThrow(selectGalleryImg).src(), - tags = emptySet(), - state = null, - authors = emptySet(), - largeCoverUrl = null, - description = null, - chapters = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() - val firstPageUrl = doc.requireElementById(idImg).requireSrc() - return (1..totalPages).map { - val url = replacePageNumber(firstPageUrl, it) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - return if (hasError(page.url)) { - findAlternativeUrl(page.url) ?: page.url - } else { - page.url - } - } - - private suspend fun hasError(url: String): Boolean { - return try { - webClient.httpHead(url) - false - } catch (e: Exception) { - true - } - } - - private suspend fun findAlternativeUrl(originalUrl: String): String? { - val lastSegment = originalUrl.substringAfterLast("/") - val oldFormat = lastSegment.substringAfterLast(".", "") - - return supportedImageFormats.firstNotNullOfOrNull { format -> - val newUrl = originalUrl.replace(".$oldFormat", ".$format") - try { - webClient.httpHead(newUrl) - newUrl - } catch (e: Exception) { - null // Skip errors and continue checking other formats - } - } - } - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".tag_name")?.text() ?: it.text() - MangaTag( - key = key, - title = name.toTitleCase(sourceLocale), - source = source, - ) - } - - private fun replacePageNumber(url: String, newPageNumber: Int): String { - val lastSegment = url.substringAfterLast("/") - val extension = lastSegment.substringAfterLast(".", "") - - return if (extension.isNotEmpty()) { - url.substringBeforeLast("/") + "/$newPageNumber.$extension" - } else { - url.substringBeforeLast("/") + "/$newPageNumber" - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt deleted file mode 100644 index be1dfa16..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) -internal class Novelcrow(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { - override val tagPrefix = "comic-genre/" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt deleted file mode 100644 index 55bccbd8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt +++ /dev/null @@ -1,272 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.ar - -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("TEAMXNOVEL", "TeamXNovel", "ar") -internal class TeamXNovel(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.TEAMXNOVEL, 10) { - - override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override val configKeyDomain = ConfigKey.Domain("olympustaff.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/?search=") - append(query.urlEncoded()) - append("&page=") - append(page) - } - - else -> { - - if (order == SortOrder.UPDATED) { - if (filter.tags.isNotEmpty() || filter.demographics.isNotEmpty()) { - throw IllegalArgumentException("Updated sorting does not support other sorting filters") - } - append("/?page=") - append(page.toString()) - } else { - append("/series?page=") - append(page.toString()) - - filter.tags.oneOrThrowIfMany()?.let { - append("&genre=") - append(it.key) - } - - filter.types.forEach { - append("&type=") - append( - when (it) { - ContentType.MANGA -> "مانجا ياباني" - ContentType.MANHWA -> "مانهوا كورية" - ContentType.MANHUA -> "مانها صيني" - else -> "" - }, - ) - } - - filter.states.oneOrThrowIfMany()?.let { - append("status=") - append( - when (it) { - MangaState.ONGOING -> "مستمرة" - MangaState.FINISHED -> "مكتمل" - MangaState.ABANDONED -> "متوقف" - else -> "مستمرة" - }, - ) - } - } - } - } - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.listupd .bs .bsx").ifEmpty { - doc.select("div.post-body .box") - }.map { div -> - val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(".tt, h3").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = div.selectFirstOrThrow("img").src()?.replace("thumbnail_", ""), - tags = emptySet(), - state = when (div.selectFirst(".status")?.text()) { - "مستمرة" -> MangaState.ONGOING - "مكتمل" -> MangaState.FINISHED - "متوقف" -> MangaState.ABANDONED - else -> null - }, - authors = emptySet(), - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/series").parseHtml() - return doc.requireElementById("select_genre").select("option").mapToSet { - MangaTag( - key = it.attr("value"), - title = it.text().toTitleCase(sourceLocale), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val mangaUrl = manga.url.toAbsoluteUrl(domain) - val maxPageChapterSelect = doc.select(".pagination .page-item a") - var maxPageChapter = 1 - if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { - val i = it.attr("href").substringAfterLast("=").toInt() - if (i > maxPageChapter) { - maxPageChapter = i - } - } - } - return manga.copy( - state = when (doc.selectFirstOrThrow(".full-list-info:contains(الحالة:) a").text()) { - "مستمرة" -> MangaState.ONGOING - "مكتمل" -> MangaState.FINISHED - "متوقف" -> MangaState.ABANDONED - else -> null - }, - tags = doc.select(".review-author-info a").mapToSet { a -> - MangaTag( - key = a.attr("href").substringAfterLast("="), - title = a.text(), - source = source, - ) - }, - description = doc.selectFirstOrThrow(".review-content").html(), - chapters = run { - if (maxPageChapter == 1) { - parseChapters(doc) - } else { - coroutineScope { - val result = ArrayList(parseChapters(doc)) - result.ensureCapacity(result.size * maxPageChapter) - (2..maxPageChapter).map { i -> - async { - loadChapters(mangaUrl, i) - } - }.awaitAll() - .flattenTo(result) - result - } - } - }.reversed(), - ) - } - - private suspend fun loadChapters(baseUrl: String, page: Int): List { - return parseChapters(webClient.httpGet("$baseUrl?page=$page").parseHtml().body()) - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd hh:mm:ss", Locale.getDefault()) - - private fun parseChapters(root: Element): List { - // Try current website structure first (enhanced-chapters-grid) - val chapters = root.select("div.enhanced-chapters-grid a.chapter-link").ifEmpty { - // Fallback to potential alternative structures - root.select("div.chapter-card a").ifEmpty { - root.select("div.eplister ul a").ifEmpty { - root.select("#chapter-contact .eplister ul li a") - } - } - } - - return chapters.map { element -> - val url = element.attrAsRelativeUrl("href") - - // Extract chapter number and title from current structure - val chpNum = element.select(".chapter-number").text() - val chpTitle = element.select(".chapter-title").text() - - // Fallback to old structure if current selectors are empty - val finalChpNum = chpNum.ifEmpty { - element.select("div.chapter-info div.chapter-number").text().ifEmpty { - element.select("div.epl-num").text() - } - } - val finalChpTitle = chpTitle.ifEmpty { - element.select("div.chapter-info div.chapter-title").text().ifEmpty { - element.select("div.epl-title").text() - } - } - - val title = when { - finalChpNum.isNotBlank() -> "$finalChpNum - $finalChpTitle" - else -> finalChpTitle - } - - // Extract date from current structure, fallback to old structures - val dateText = element.select(".chapter-date").text().ifEmpty { - element.select("div.chapter-info div.chapter-date").text().ifEmpty { - element.select("div.epl-date").text() - } - }.replace(Regex("^\\s*\\S+\\s+"), "") // Remove icon text if present - - // Try to extract chapter number from data attribute or URL - val chapterNumber = element.select("div.chapter-card").attr("data-number").toFloatOrNull() - ?: finalChpNum.filter { it.isDigit() || it == '.' }.toFloatOrNull() - ?: url.substringAfterLast('/').toFloatOrNull() - ?: 0f - - MangaChapter( - id = generateUid(url), - title = title, - number = chapterNumber, - volume = 0, - url = url, - scanlator = null, - uploadDate = dateFormat.parseSafe(dateText), - branch = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select(".image_list img, .image_list canvas").map { img -> - val url = when { - img.hasAttr("src") -> img.requireSrc().toRelativeUrl(domain) - else -> img.attrAsRelativeUrl("data-src") - } - - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt deleted file mode 100644 index 079c25df..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt +++ /dev/null @@ -1,195 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("AQUAMANGA", "AquaManga", "en") -internal class AquaManga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.AQUAMANGA, "aquareader.org", 20) { - - override val withoutAjax = true - override val stylePage = "" - - override val selectChapter = ".aqua-ch-item" - - override val availableSortOrders: EnumSet = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = setOf( - MangaTag("Academy", "academy", source), - MangaTag("Action", "action", source), - MangaTag("Adaptation", "adaptation", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Crime", "crime", source), - MangaTag("Cultivation", "cultivation", source), - MangaTag("Delinquents", "delinquents", source), - MangaTag("Demons", "demons", source), - MangaTag("Drama", "drama", source), - MangaTag("Dungeons", "dungeons", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Game", "game", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Magic", "magic", source), - MangaTag("Manga", "manga", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Military", "military", source), - MangaTag("Monsters", "monsters", source), - MangaTag("Murim", "murim", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Necromancer", "necromancer", source), - MangaTag("Ninja", "ninja", source), - MangaTag("Office Workers", "office-workers", source), - MangaTag("OP-MC", "op-mc", source), - MangaTag("Overpowered", "overpowered", source), - MangaTag("Philosophical", "philosophical", source), - MangaTag("Post-Apocalyptic", "post-apocalyptic", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Rebirth", "rebirth", source), - MangaTag("Regression", "regression", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Returner", "returner", source), - MangaTag("Revenge", "revenge", source), - MangaTag("Romance", "romance", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Slice-of-Life", "slice-of-life", source), - MangaTag("Sports", "sports", source), - MangaTag("Super Power", "super-power", source), - MangaTag("Superhero", "superhero", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Survival", "survival", source), - MangaTag("System", "system", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tower", "tower", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Vampire", "vampire", source), - MangaTag("Video Games", "video-games", source), - MangaTag("Villainess", "villainess", source), - MangaTag("Virtual Reality", "virtual-reality", source), - MangaTag("Voilence", "voilence", source), - MangaTag("Webcomic", "webcomic", source), - MangaTag("Wuxia", "wuxia", source), - MangaTag("Zombies", "zombies", source), - ), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val title = doc.selectFirstOrThrow(".aqua-series-info__title").text() - val thumbnail = doc.selectFirstOrThrow(".aqua-series-cover__img").requireSrc() - val description = doc.selectFirst(".aqua-series-synopsis")?.html().orEmpty() - val status = doc.selectFirst(".aqua-series-meta__status")?.text() - val genres = doc.select(".aqua-series-genre-pill").map { it.text() }.toSet() - val author = doc.selectFirst(".aqua-series-info__creator-value a")?.ownText() - val artist = doc.selectFirst(".aqua-series-info__creator-value a")?.ownText() - - val tags = genres.mapTo(mutableSetOf()) { MangaTag(title = it, key = it, source = source) } - val statusText = status?.lowercase().orEmpty() - val state = when (statusText) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in abandoned -> MangaState.ABANDONED - in paused -> MangaState.PAUSED - else -> null - } - - val chapters = getChapters(manga, doc) - return manga.copy( - title = title, - coverUrl = thumbnail, - description = description, - tags = tags, - state = state, - authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet(), - chapters = chapters, - ) - } - - override suspend fun getChapters(manga: Manga, doc: Document): List { - return doc.select(selectChapter).mapChapters(reversed = true) { i, el -> - val a = el.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val name = el.selectFirstOrThrow(".aqua-ch-item__name").text() - val dateText = el.selectFirst(".aqua-ch-item__time")?.text()?.trim() - MangaChapter( - id = generateUid(href), - url = href + stylePage, - title = name, - number = i + 1f, - volume = 0, - uploadDate = parseChapterDate(dateText), - source = source, - scanlator = null, - branch = null, - ) - } - } - - private fun parseChapterDate(text: String?): Long { - if (text.isNullOrBlank()) return 0L - - val relativeRegex = Regex( - """(\d+)\s+(years?|months?|weeks?|days?|hours?|mins?|minutes?|sec(?:onds?)?)\s+ago""", - RegexOption.IGNORE_CASE - ) - val match = relativeRegex.matchEntire(text) - if (match != null) { - val number = match.groupValues[1].toInt() - val unit = match.groupValues[2].lowercase() - val cal = Calendar.getInstance() - when { - unit.startsWith("year") -> cal.add(Calendar.YEAR, -number) - unit.startsWith("month") -> cal.add(Calendar.MONTH, -number) - unit.startsWith("week") -> cal.add(Calendar.DAY_OF_MONTH, -number * 7) - unit.startsWith("day") -> cal.add(Calendar.DAY_OF_MONTH, -number) - unit.startsWith("hour") -> cal.add(Calendar.HOUR, -number) - unit.startsWith("min") -> cal.add(Calendar.MINUTE, -number) - unit.startsWith("sec") -> cal.add(Calendar.SECOND, -number) - } - return cal.timeInMillis - } - - val formats = listOf( - SimpleDateFormat("MMM d, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, - SimpleDateFormat("MMM dd, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, - SimpleDateFormat("MMMM d, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, - ) - for (fmt in formats) { - try { - return fmt.parse(text)?.time ?: 0L - } catch (_: Exception) {} - } - - return 0L - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt deleted file mode 100644 index c9bf0358..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt +++ /dev/null @@ -1,349 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.toJSONArrayOrNull -import org.koitharu.kotatsu.parsers.util.json.toJSONObjectOrNull -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("ASURASCANS", "AsuraComic", "en") -internal class AsuraScansParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ASURASCANS, pageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("asurascans.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.RATING, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL_DESC, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags, - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - MangaState.PAUSED, - MangaState.UPCOMING, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = "https://$domain/browse".toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - - if (!filter.query.isNullOrBlank()) { - addQueryParameter("search", filter.query) - } - - if (filter.tags.isNotEmpty()) { - addQueryParameter("genres", filter.tags.joinToString(",") { it.key }) - } - - filter.states.oneOrThrowIfMany()?.let { - addQueryParameter( - "status", - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.ABANDONED -> "dropped" - MangaState.PAUSED -> "hiatus" - else -> throw IllegalArgumentException("$it not supported") - }, - ) - } - - filter.types.oneOrThrowIfMany()?.let { - addQueryParameter( - "types", - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> throw IllegalArgumentException("$it not supported") - }, - ) - } - - addQueryParameter( - "sort", - when (order) { - SortOrder.RATING -> "rating" - SortOrder.UPDATED -> "" - SortOrder.POPULARITY -> "popular" - SortOrder.ALPHABETICAL_DESC -> "desc" - SortOrder.ALPHABETICAL -> "asc" - else -> "update" - }, - ) - }.build() - val doc = webClient.httpGet(url).parseHtml() - return doc.select("#series-grid .series-card").mapNotNull { card -> - val link = card.selectFirst("a[href*=/comics/]") ?: return@mapNotNull null - val href = link.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = card.selectFirst("img")?.src(), - title = card.selectFirst("h3")?.text()?.trim().orEmpty(), - altTitles = emptySet(), - rating = card.selectFirst("div.absolute.top-2.right-2 span")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = when (card.select("div.p-3 span").lastOrNull()?.text()?.trim()?.lowercase(Locale.ENGLISH)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "dropped" -> MangaState.ABANDONED - "coming soon" -> MangaState.UPCOMING - else -> null - }, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - private val availableTags by lazy(LazyThreadSafetyMode.NONE) { - ASURA_GENRES.mapTo(LinkedHashSet(ASURA_GENRES.size)) { title -> - MangaTag( - key = title.toAsuraGenreKey(), - title = title, - source = source, - ) - } - } - - private val tagMap by lazy(LazyThreadSafetyMode.NONE) { - availableTags.associateByTo(LinkedHashMap(availableTags.size)) { - it.title.lowercase(Locale.ENGLISH) - } - } - - private val regexDate = """(\d+)(st|nd|rd|th)""".toRegex() - private val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.US) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val selectTag = doc.select("div[class^=space] > div.flex > button.text-white") - val tags = selectTag.mapNotNullToSet { element -> - val title = element.text().trim().nullIfEmpty() ?: return@mapNotNullToSet null - tagMap[title.lowercase(Locale.ENGLISH)] ?: MangaTag( - key = title.toAsuraGenreKey(), - title = title, - source = source, - ) - } - val author = doc.selectFirst("div.grid > div:has(h3:eq(0):containsOwn(Author)) > h3:eq(1)")?.text().orEmpty() - val cutoffTime = System.currentTimeMillis() - CHAPTER_HIDE_WINDOW_MS - return manga.copy( - title = doc.selectFirst("article h1") - ?.text() - ?.trim() - ?.takeIf { it.isNotEmpty() } - ?: manga.title, - altTitles = doc.selectFirst("#alt-titles") - ?.text() - .orEmpty() - .split('•', '\n') - .mapNotNullToSet { it.trim().nullIfEmpty() }, - description = doc.selectFirst("#description-text") - ?.html() - ?.trim() - ?.takeIf { it.isNotEmpty() } - ?: doc.selectFirst("span.font-medium.text-sm")?.text().orEmpty(), - tags = tags, - authors = setOf(author), - chapters = doc.select("a.group[href*=/chapter/]").mapChapters(reversed = true) { i, a -> - val urlRelative = a.attrAsRelativeUrl("href") - val titleElement = a.selectFirst("span.font-medium") ?: a.selectFirst("span") - val chapterLabel = titleElement?.text()?.trim()?.takeIf { it.isNotEmpty() } - val chapterTitle = a.selectFirst("span.text-sm.text-white\\/50") - ?.text() - ?.trim() - ?.takeIf { it.isNotEmpty() } - val fullTitle = when { - chapterLabel != null && chapterTitle != null -> "$chapterLabel - $chapterTitle" - chapterLabel != null -> chapterLabel - else -> chapterTitle - } - val chapterNumber = chapterNumberRegex.find(chapterLabel.orEmpty()) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - ?: i + 1f - val dateText = a.selectFirst("span.text-sm.text-white\\/40") - ?.text() - ?.trim() - MangaChapter( - id = generateUid(urlRelative), - title = fullTitle, - number = chapterNumber, - volume = 0, - url = urlRelative, - scanlator = null, - uploadDate = parseChapterDate(chapterDateFormat, dateText), - branch = null, - source = source, - ) - }.filter { chapter -> - chapter.uploadDate == 0L || chapter.uploadDate <= cutoffTime - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - doc.selectFirst("astro-island[component-url*='ChapterReader']")?.attr("props")?.let { props -> - pageUrlRegex.findAll(props.replace(""", "\"")) - .map { it.groupValues[1] } - .distinct() - .toList() - .takeIf { it.isNotEmpty() } - ?.let { urls -> - return urls.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - } - - val scripts = doc.selectOrThrow("script") - val sb = StringBuilder() - for (script in scripts) { - val raw = script.data().substringBetween("self.__next_f.push(", ")", "").trim() - if (raw.isEmpty()) continue - val ja = raw.toJSONArrayOrNull() ?: continue - for (i in 0 until ja.length()) { - (ja.opt(i) as? String)?.let { sb.append(it) } - } - } - val lines = sb.toString().split('\n') - val pages = TreeMap() - for (line in lines) { - val obj = line.substringAfter(':').toJSONObjectOrNull() ?: continue - if (obj.has("order") && obj.has("url")) { - pages[obj.getInt("order")] = obj.getString("url") - } - } - return pages.values.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun String.toAsuraGenreKey(): String { - return trim() - .lowercase(Locale.ENGLISH) - .replace(asuraGenreKeyRegex, "-") - .trim('-') - } - - private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { - val value = date?.trim().orEmpty() - if (value.isEmpty()) return 0L - val lower = value.lowercase(Locale.ENGLISH) - return when { - lower == "last week" -> Calendar.getInstance().apply { - add(Calendar.WEEK_OF_YEAR, -1) - }.timeInMillis - - lower == "yesterday" -> Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) - }.timeInMillis - - lower.endsWith(" ago") -> parseRelativeDate(lower) - else -> synchronized(dateFormat) { - dateFormat.parseSafe(value.replace(regexDate, "$1")) - } - } - } - - private fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0L - val cal = Calendar.getInstance() - return when { - WordSet("second", "seconds").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("minute", "minutes", "min", "mins").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis - WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "years").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0L - } - } - - private companion object { - private const val CHAPTER_HIDE_WINDOW_MS = 6L * 60L * 60L * 1000L - private val chapterNumberRegex = Regex("""Chapter\s+(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - private val pageUrlRegex = Regex(""""url":\s*\[0,\s*"([^"]+)"""") - private val ASURA_GENRES = listOf( - "Action", - "Adventure", - "Comedy", - "Crazy MC", - "Demon", - "Dungeons", - "Fantasy", - "Game", - "Genius MC", - "Isekai", - "Magic", - "Murim", - "Mystery", - "Necromancer", - "Overpowered", - "Regression", - "Reincarnation", - "Revenge", - "Romance", - "School Life", - "Sci-fi", - "Shoujo", - "Shounen", - "System", - "Tower", - "Tragedy", - "Villain", - "Violence", - ) - private val asuraGenreKeyRegex = Regex("[^a-z0-9]+") - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt deleted file mode 100644 index a5aa24db..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt +++ /dev/null @@ -1,457 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope - -@MangaSourceParser("ATSUMARU", "Atsumaru", "en") -internal class Atsumaru(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ATSUMARU, pageSize = 40) { - - override val configKeyDomain = ConfigKey.Domain("atsu.moe") - private val baseUrl = "https://atsu.moe" - - private val apiHeaders: Headers by lazy { - Headers.Builder() - .add("Accept", "*/*") - .add("Referer", baseUrl) - .add("Content-Type", "application/json") - .build() - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, // trending (views) - SortOrder.UPDATED, // recently added (dateAdded) - SortOrder.ADDED, // recently new added mangas - SortOrder.RELEVANCE // chapters count - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - private val allGenres = listOf( - Genre("Action", "39"), Genre("Adult", "46"), Genre("Adventure", "37"), - Genre("Boys Love", "180"), Genre("Comedy", "6"), Genre("Drama", "31"), - Genre("Fantasy", "36"), Genre("Girls Love", "4"), Genre("Hentai", "10"), - Genre("Historical", "45"), Genre("Horror", "44"), Genre("Martial Arts", "29"), - Genre("Mystery", "32"), Genre("Psychological", "18"), Genre("Romance", "9"), - Genre("Sci-Fi", "1"), Genre("Slice of Life", "7"), Genre("Smut", "41"), - Genre("Supernatural", "22"), Genre("Thriller", "19"), Genre("Tragedy", "5") - ) - - private val allTags = listOf( - Tag("Blackmail", "285"), Tag("Cooking", "669"), Tag("Crimes", "288"), - Tag("Crossdressing", "167"), Tag("Murder", "250"), Tag("Prostitution", "366"), - Tag("Swordplay", "337"), Tag("Working", "248"), - Tag("Josei", "43"), Tag("Seinen", "8"), Tag("Shoujo", "40"), Tag("Shounen", "38"), - Tag("Otaku", "264"), Tag("Tsundere", "313"), Tag("Yandere", "315"), - Tag("Animal Characteristics", "274"), Tag("Beautiful Female Lead", "72"), - Tag("Big Breasts", "123"), Tag("Flat Chest", "320"), Tag("Glasses-Wearing Male Lead", "71"), - Tag("Handsome Male Lead", "68"), Tag("Kemonomimi", "279"), Tag("MILF", "339"), - Tag("Small Breasts", "124"), Tag("Young Male Lead", "787"), - Tag("Adult Cast", "159"), Tag("Bisexual", "382"), Tag("Ensemble Cast", "362"), - Tag("Female Lead", "59"), Tag("Male Lead", "58"), Tag("Non-Human Protagonist", "247"), - Tag("Primarily Adult Cast", "158"), Tag("Primarily Female Cast", "333"), - Tag("Primarily Male Cast", "335"), Tag("Primarily Teen Cast", "334"), - Tag("Strong Female Lead", "69"), Tag("Strong Male Lead", "67"), - Tag("Adapted to Anime", "166"), Tag("Based on a Light Novel", "76"), - Tag("Based on a Novel", "75"), Tag("Based on a Video Game", "77"), - Tag("Based on a Web Novel", "74"), Tag("College", "257"), Tag("Company", "1205"), - Tag("Countryside", "415"), Tag("Europe", "405"), Tag("Foreign", "336"), - Tag("High School", "162"), Tag("Hospital", "760"), Tag("Japan", "225"), - Tag("School", "107"), Tag("School Clubs", "356"), - Tag("Amnesia", "283"), Tag("Appearance Different from Personality", "651"), - Tag("Caught in the Act", "874"), Tag("Dead Family Member", "831"), - Tag("Family Drama", "848"), Tag("Flashbacks", "449"), Tag("Gender Bender", "12"), - Tag("Love Triangle", "125"), Tag("Male Lead Falls in Love First", "653"), - Tag("Misunderstandings", "647"), Tag("Past Plays a Big Role", "648"), - Tag("Reincarnation", "126"), Tag("Secret Identity", "260"), - Tag("Time Manipulation", "311"), Tag("Time Skip", "172"), Tag("Time Travel", "249"), - Tag("Tragic Past", "898"), Tag("Weak to Strong", "1064"), - Tag("Delinquents", "239"), Tag("Detectives", "240"), Tag("Idols", "281"), - Tag("Maids", "116"), Tag("Office Lady", "312"), Tag("Office Worker", "429"), - Tag("School Girl", "788"), Tag("Teachers", "175"), - Tag("Age Gap", "106"), Tag("Childhood Friends", "97"), Tag("Coworkers", "286"), - Tag("Female Harem", "163"), Tag("Friends to Lovers", "243"), Tag("Friendship", "242"), - Tag("Harem", "20"), Tag("Heterosexual", "108"), Tag("Incest", "174"), - Tag("Infidelity", "231"), Tag("Interspecies Relationship", "308"), - Tag("Love-Hate Relationship", "889"), Tag("Master-Servant Relationship", "406"), - Tag("Older Female Younger Male", "114"), Tag("Older Male Younger Female", "649"), - Tag("Older Uke Younger Seme", "880"), Tag("Siblings", "254"), - Tag("Student-Student Relationship", "573"), Tag("Student-Teacher Relationship", "177"), - Tag("Twins", "253"), - Tag("Chinese Ambience", "588"), Tag("European Ambience", "450"), - Tag("Fantasy World", "642"), Tag("Feudal Japan", "606"), Tag("Game Elements", "399"), - Tag("Game World", "641"), Tag("Isekai", "94"), Tag("Isekaied Into a Novel", "258"), - Tag("Mecha", "11"), Tag("Mythology", "259"), Tag("Urban", "338"), - Tag("Urban Fantasy", "261"), - Tag("Anal Intercourse", "100"), Tag("Bondage", "280"), Tag("Boobjob", "381"), - Tag("Borderline H", "448"), Tag("Cunnilingus", "171"), Tag("Defloration", "306"), - Tag("Dubious Consent", "985"), Tag("Ecchi", "21"), Tag("Erotica", "14"), - Tag("Exhibitionism", "287"), Tag("Group Intercourse", "373"), - Tag("Handjob", "303"), Tag("Lolicon", "28"), Tag("Masturbation", "161"), - Tag("Mature", "15"), Tag("Nakadashi", "169"), Tag("Netorare", "232"), - Tag("Nudity", "109"), Tag("Oral Intercourse", "99"), Tag("Outdoor Intercourse", "307"), - Tag("Public Intercourse", "103"), Tag("Rape", "95"), Tag("Sex Addict", "650"), - Tag("Sex Toys", "289"), Tag("Shotacon", "35"), Tag("Teens Love", "374"), - Tag("Threesome", "173"), Tag("Virginity", "369"), - Tag("Animals", "278"), Tag("Cats", "284"), Tag("Demons", "160"), - Tag("Ghosts", "229"), Tag("Gods", "176"), Tag("Monsters", "395"), - Tag("Non-human", "547"), Tag("Vampires", "252"), - Tag("21st century", "132"), Tag("Betrayal", "403"), Tag("Bullying", "235"), - Tag("Cohabitation", "228"), Tag("Coming of Age", "117"), Tag("Danmei", "305"), - Tag("Depression", "1090"), Tag("Family Life", "282"), Tag("Female Empowerment", "1816"), - Tag("Forbidden Love", "699"), Tag("Gore", "262"), Tag("Gourmet", "2"), - Tag("Harlequin", "304"), Tag("Jealousy", "881"), Tag("LGBTQ+", "326"), - Tag("Love Confession", "882"), Tag("Marriage", "360"), Tag("Mature Romance", "241"), - Tag("Medical", "350"), Tag("Military", "230"), Tag("Music", "27"), - Tag("Nobility", "127"), Tag("Obsessive Love", "893"), Tag("Orphans", "237"), - Tag("Religion", "498"), Tag("Reunion", "984"), Tag("Revenge", "227"), - Tag("Royalty", "128"), Tag("School Life", "42"), Tag("Shoujo Ai", "47"), - Tag("Shounen Ai", "23"), Tag("Special Ability", "883"), Tag("Sports", "30"), - Tag("Suicide", "309"), Tag("Super Powers", "236"), Tag("Unrequited Love", "226"), - Tag("Violence", "830"), Tag("War", "238"), Tag("Yaoi", "16"), Tag("Yuri", "33"), - Tag("4-Koma", "105"), Tag("Anthology", "113"), Tag("Chinese Novels", "1112"), - Tag("Collection of Stories", "111"), Tag("Doujinshi", "24"), Tag("Episodic", "115"), - Tag("Full color", "57"), Tag("Korean Novels", "1111"), Tag("Light Novel", "466"), - Tag("Longstrip", "93"), Tag("One Shot", "110"), Tag("Web Comic", "428"), - Tag("Web Novel", "427"), Tag("Magic", "121") - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = (allGenres.map { (name, id) -> MangaTag(name, "genre:$id", source) } + - allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List = getSearchPage(page, order, filter) - - private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query ?: "*" - val url = "$baseUrl/collections/manga/documents/search".toHttpUrl().newBuilder().apply { - addQueryParameter("q", query) - if (query != "*") { - addQueryParameter("query_by", "title,englishTitle,otherNames,authors") - addQueryParameter("query_by_weights", "4,3,2,1") - addQueryParameter("num_typos", "4,3,2,1") - } - addQueryParameter("page", page.toString()) - addQueryParameter("per_page", pageSize.toString()) - - val filterParts = mutableListOf("hidden:!=true", "views:>0") - - val ratings = filter.contentRating - when { - ratings.contains(ContentRating.ADULT) && !ratings.contains(ContentRating.SAFE) -> { - filterParts.add("isAdult:=true") - } - ratings.contains(ContentRating.SAFE) && !ratings.contains(ContentRating.ADULT) -> { - filterParts.add("isAdult:=false") - } - // both or none: no filter shows all - else -> { /* no isAdult filter */ } - } - filterParts.add("(mbContentRating:=[`Safe`,`Suggestive`,`Erotica`] || mbContentRating:!=*)") - - val types = filter.types.mapNotNull { ct -> - when (ct) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manwha" - ContentType.MANHUA -> "Manhua" - ContentType.OTHER -> "OEL" - else -> null - } - } - if (types.isNotEmpty()) { - filterParts.add("type:=[${types.joinToString(",") { "`$it`" }}]") - } - - val statusVals = filter.states.mapNotNull { st -> - when (st) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - MangaState.PAUSED -> "Hiatus" - MangaState.ABANDONED -> "Canceled" - else -> null - } - } - if (statusVals.isNotEmpty()) { - filterParts.add("status:=[${statusVals.joinToString(",") { "`$it`" }}]") - } - - val includedGenres = mutableListOf() - val excludedGenres = mutableListOf() - val includedTags = mutableListOf() - val excludedTags = mutableListOf() - - for (tag in filter.tags) { - val key = tag.key - when { - key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) - } - } - for (tag in filter.tagsExclude) { - val key = tag.key - when { - key.startsWith("genre:") -> excludedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) - } - } - - if (includedGenres.isNotEmpty()) { - filterParts.add(includedGenres.joinToString(" && ") { "genreIds:=`$it`" }) - } - if (excludedGenres.isNotEmpty()) { - filterParts.add("genreIds:!=[${excludedGenres.joinToString(",") { "`$it`" }}]") - } - if (includedTags.isNotEmpty()) { - filterParts.add(includedTags.joinToString(" && ") { "tagIds:=`$it`" }) - } - if (excludedTags.isNotEmpty()) { - filterParts.add("tagIds:!=[${excludedTags.joinToString(",") { "`$it`" }}]") - } - - addQueryParameter("filter_by", filterParts.joinToString(" && ")) - - val sort = when (order) { - SortOrder.POPULARITY -> "trending:desc" - SortOrder.UPDATED -> "released:desc" - SortOrder.ADDED -> "dateAdded:desc" - SortOrder.RELEVANCE -> "chapterCount:desc" - else -> "released:desc" - } - addQueryParameter("sort_by", sort) - addQueryParameter("sort_by", sort) - }.build() - - val json = webClient.httpGet(url, apiHeaders).parseJson() - val hits = json.optJSONArray("hits") ?: return emptyList() - return (0 until hits.length()).map { i -> - hits.getJSONObject(i).getJSONObject("document").toManga() - } - } - - // limit tags to 20 - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val result = coroutineScope { - val mangaId = manga.url.substringAfterLast("/") - - val pageDeferred = async { - webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() - } - val chaptersDeferred = async { - webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() - } - - val pageJson = pageDeferred.await() - val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga - - val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } - val description = mangaPage.optString("synopsis") ?: manga.description - - val rawRating = mangaPage.optDouble("avgRating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val posterObj = mangaPage.optJSONObject("poster") - val posterImage = posterObj?.optString("mediumImage") - val coverUrl = if (!posterImage.isNullOrEmpty()) { - "https://$domain/static/$posterImage" - } else manga.coverUrl - - val authors = mangaPage.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() - } ?: emptySet() - - val statusText = mangaPage.optString("status").orEmpty() - val state = when (statusText.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "canceled" -> MangaState.ABANDONED - else -> manga.state - } - - val tagSet = mutableSetOf() - mangaPage.optJSONArray("genres")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) - } - } - mangaPage.optJSONArray("tags")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) - } - } - - val chaptersJson = chaptersDeferred.await() - val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() - - val scanlators = mutableMapOf() - mangaPage.optJSONArray("scanlators")?.let { scanArr -> - for (i in 0 until scanArr.length()) { - val sc = scanArr.optJSONObject(i) ?: continue - val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue - val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue - scanlators[id] = name - } - } - - val chapters = (0 until chaptersArray.length()).map { i -> - val ch = chaptersArray.getJSONObject(i) - val chId = ch.getString("id") - val number = ch.optDouble("number", 0.0).toFloat() - val chTitle = ch.optString("title") - val scanId = ch.optString("scanlationMangaId").nullIfEmpty() - val scanName = scanId?.let { scanlators[it] } - val createdAt = ch.optLong("createdAt", 0L) - - MangaChapter( - id = generateUid("$mangaId/$chId"), - title = chTitle.ifBlank { "Chapter $number" }, - number = number, - volume = 0, - url = "$mangaId/$chId", - uploadDate = createdAt, - source = source, - scanlator = scanName, - branch = scanName, - ) - }.reversed() - - val groupedChapters = if (chapters.map { it.branch }.distinct().size > 1) { - chapters.map { it.copy(branch = it.branch ?: "Unknown") } - } else { - chapters.map { it.copy(branch = null, scanlator = null) } - } - - val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() - } ?: emptySet() - - manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - state = state, - tags = tagSet, - rating = rating, - chapters = groupedChapters, - altTitles = altTitles, - ) - } - - detailsCache.put(manga.url, result) - return result - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { - return size > 10 - } - } - - override suspend fun getRelatedManga(seed: Manga): List { - val mangaId = seed.url.substringAfterLast("/") - val pageJson = webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() - val mangaPage = pageJson.optJSONObject("mangaPage") ?: return emptyList() - val similar = mangaPage.optJSONArray("similarManga") ?: return emptyList() - return (0 until similar.length()).map { i -> - similar.getJSONObject(i).toManga() - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val (mangaId, chapterId) = chapter.url.split("/") - val url = "$baseUrl/api/read/chapter".toHttpUrl().newBuilder() - .addQueryParameter("mangaId", mangaId) - .addQueryParameter("chapterId", chapterId) - .build() - val json = webClient.httpGet(url, apiHeaders).parseJson() - val pagesArray = json.getJSONObject("readChapter").getJSONArray("pages") - return (0 until pagesArray.length()).map { i -> - val page = pagesArray.getJSONObject(i) - val imagePath = page.getString("image") - val fullUrl = when { - imagePath.startsWith("http") -> imagePath - imagePath.startsWith("//") -> "https:$imagePath" - else -> "https://$domain/static/${imagePath.removePrefix("/").removePrefix("static/")}" - } - MangaPage(id = generateUid(fullUrl), url = fullUrl, preview = null, source = source) - } - } - - private fun JSONObject.toManga(): Manga { - val id = getString("id") - val title = optString("title").ifEmpty { optString("englishTitle", "Unknown") } - - val imageRaw: Any? = when { - has("poster") -> get("poster") - has("image") -> getString("image") - else -> null - } - - val imagePath: String? = when (imageRaw) { - is String -> imageRaw.takeIf { it.isNotBlank() } - is JSONObject -> { - imageRaw.optString("image").nullIfEmpty() - ?: imageRaw.optString("mediumImage").nullIfEmpty() - ?: imageRaw.optString("smallImage").nullIfEmpty() - } - else -> null - }?.removePrefix("/")?.removePrefix("static/") - - val coverUrl = if (!imagePath.isNullOrBlank()) { - when { - imagePath.startsWith("http") -> imagePath - imagePath.startsWith("//") -> "https:$imagePath" - else -> "https://$domain/static/$imagePath" - } - } else null - - return Manga( - id = generateUid(id), - title = title, - altTitles = emptySet(), - url = "/manga/$id", - publicUrl = "$baseUrl/manga/$id", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = coverUrl, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - private data class Genre(val name: String, val id: String) - private data class Tag(val name: String, val id: String) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt deleted file mode 100644 index 7699d15d..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt +++ /dev/null @@ -1,251 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.FormBody -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody -import okhttp3.Response -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) -internal class Batcave(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { - - override val configKeyDomain = ConfigKey.Domain("batcave.biz") - - private var publishers: List> = emptyList() - private var genres: List> = emptyList() - private var filterParseFailed = false - - private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - - private val apiClient: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor(::refererInterceptor) - .addInterceptor(::dleGuardInterceptor) - .build() - OkHttpWebClient(httpClient, source) - } - - private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } - - private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { - val request = chain.request() - val url = request.url.toString() - val referer = when { - url.contains("readcomicsonline.ru") -> "https://readcomicsonline.ru/" - else -> "https://$domain/" - } - return chain.proceed(request.newBuilder().header("Referer", referer).build()) - } - - private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { - val originalRequest = chain.request() - val response = chain.proceed(originalRequest) - if (response.request.url.pathSegments.firstOrNull() != "_c") { - return response - } - response.close() - val url = if (originalRequest.method == "GET") { - originalRequest.url.toString() - } else { - "https://$domain/" - } - context.requestBrowserAction(this, url) - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = emptySet(), - ) - } - - private fun parseFilterValues(filterItems: JSONObject, key: String): List> { - val obj = filterItems.optJSONObject(key) ?: return emptyList() - val values = obj.optJSONArray("values") ?: return emptyList() - val result = mutableListOf>() - for (i in 0 until values.length()) { - val item = values.getJSONObject(i) - result.add(item.getString("value") to item.getInt("id")) - } - return result - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - if (query != null) return searchManga(query, page) - - val urlBuilder = StringBuilder().apply { - append("https://$domain/ComicList/") - if (filter.yearFrom != YEAR_UNKNOWN) append("y[from]=${filter.yearFrom}/") - if (filter.yearTo != YEAR_UNKNOWN) append("y[to]=${filter.yearTo}/") - val pIds = filter.tags.filter { it.key.startsWith("p_") }.map { it.key.removePrefix("p_") } - val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } - if (pIds.isNotEmpty()) append("p=${pIds.joinToString(",")}/") - if (gIds.isNotEmpty()) append("g=${gIds.joinToString(",")}/") - append("sort") - if (page > 1) append("/page/$page/") - } - val url = urlBuilder.toString() - - val sortPair = when (order) { - SortOrder.POPULARITY -> "rating" to "desc" - SortOrder.UPDATED -> "date" to "desc" - else -> "" to "" - } - - return if (sortPair.first.isEmpty()) { - parseMangaList(apiClient.httpGet(url).parseHtml()) - } else { - val formBody = FormBody.Builder() - .add("dlenewssortby", sortPair.first) - .add("dledirection", sortPair.second) - .add("set_new_sort", "dle_sort_xfilter") - .add("set_direction_sort", "dle_direction_xfilter") - .build() - val request = Request.Builder().url(url).post(formBody).build() - val response = rawHttpClient.newCall(request).execute() - parseMangaList(response.parseHtml()) - } - } - - private suspend fun searchManga(query: String, page: Int): List { - val encoded = query.urlEncoded() - val url = if (page == 1) "https://$domain/search/$encoded/" - else "https://$domain/search/$encoded/page/$page/" - return parseMangaList(apiClient.httpGet(url).parseHtml()) - } - - private fun parseMangaList(doc: org.jsoup.nodes.Document): List { - return doc.select("#dle-content > .readed").map { element -> - val a = element.selectFirst(".readed__title > a") ?: return@map null - val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") - Manga( - id = generateUid(a.attrAsRelativeUrl("href")), - url = a.attrAsRelativeUrl("href"), - publicUrl = a.absUrl("href"), - title = a.ownText(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - }.filterNotNull() - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title - val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") - val description = doc.selectFirst("div.page__text")?.text() - val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() - val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { - "Ongoing" -> MangaState.ONGOING - else -> MangaState.FINISHED - } - - val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() - ?: throw ParseException("Chapter data script not found", manga.url) - val json = JSONObject( - script.substringAfter("window.__DATA__ = ").substringBeforeLast(";").trim() - ) - val newsId = json.getLong("news_id") - val chaptersArray = json.getJSONArray("chapters") - val chapters = (0 until chaptersArray.length()).map { i -> - val ch = chaptersArray.getJSONObject(i) - MangaChapter( - id = generateUid("/reader/$newsId/${ch.getInt("id")}"), - url = "/reader/$newsId/${ch.getInt("id")}", - title = ch.optString("title"), - number = ch.optDouble("posi", 0.0).toFloat(), - volume = 0, - scanlator = null, - uploadDate = dateFormat.parseSafe(ch.optString("date")), - branch = null, - source = source, - ) - }.reversed() - - val result = manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - description = description, - authors = setOfNotNull(author), - state = state, - tags = emptySet(), - chapters = chapters, - ) - detailsCache[manga.url] = result - return result - } - - override suspend fun getPages(chapter: MangaChapter): List { - val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) - val id = Regex("^\\d+").find(rawId)?.value ?: rawId - val jsonBody = JSONObject().apply { - put("news_id", newsId) - put("chapter_id", id) - }.toString() - - val request = Request.Builder() - .url("https://$domain/engine/ajax/controller.php?mod=api&action=reader/getChapterData") - .header("Referer", "https://$domain/") - .header("Content-Type", "application/json") - .post( - jsonBody - .toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) - ) - .build() - - val response = rawHttpClient.newCall(request).execute() - val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) - val data = json.getJSONObject("data") - val images = data.getJSONArray("images") - - return (0 until images.length()).map { i -> - var img = images.getString(i).trim() - if (!img.startsWith("http")) img = "https://$domain$img" - MangaPage(id = generateUid(img), url = img, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - - private fun SimpleDateFormat.parseSafe(date: String?): Long = - if (date.isNullOrBlank()) 0L else runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt deleted file mode 100644 index 86488f68..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("COFFEEMANGA", "CoffeeManga", "en") -internal class CoffeeManga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.COFFEEMANGA, "coffeemanga.ink") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt deleted file mode 100644 index 2b252806..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt +++ /dev/null @@ -1,244 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import androidx.collection.arraySetOf -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("DEMONICSCANS", "DemonicScans", "en") -internal class DemonicScans(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.DEMONICSCANS, 25) { - - override val configKeyDomain = ConfigKey.Domain("demonicscans.org") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchTags(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val hasQuery = !filter.query.isNullOrEmpty() - val isNewest = order == SortOrder.NEWEST && !hasQuery && filter.tags.isEmpty() - val isAlpha = order == SortOrder.ALPHABETICAL && !hasQuery && filter.tags.isEmpty() - val isAlphaDesc = order == SortOrder.ALPHABETICAL_DESC && !hasQuery && filter.tags.isEmpty() - val url = when { - hasQuery -> "https://$domain/search.php?manga=" + (filter.query ?: "") - isNewest -> "https://$domain/lastupdates.php?list=$page" - isAlpha -> buildString { - append("https://$domain/advanced.php") - append("?list=$page") - append("&status=all") - append("&orderby=NAME ASC") - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { tag -> - append("&genre[]=") - append(tag.key) - } - } - } - isAlphaDesc -> buildString { - append("https://$domain/advanced.php") - append("?list=$page") - append("&status=all") - append("&orderby=NAME DESC") - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { tag -> - append("&genre[]=") - append(tag.key) - } - } - } - else -> buildString { - append("https://$domain/advanced.php") - append("?list=$page") - append("&status=all") - append("&orderby=VIEWS DESC") - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { tag -> - append("&genre[]=") - append(tag.key) - } - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - val selector = when { - hasQuery -> "body > a[href]" - isNewest -> "div#updates-container > div.updates-element" - else -> "div#advanced-content > div.advanced-element" - } - - return doc.select(selector).map { element -> - when { - isNewest -> parseNewestManga(element) - else -> parseNormalManga(element, hasQuery) - } - } - } - - private fun parseNewestManga(element: Element): Manga { - val info = element.selectFirst("div.updates-element-info")!! - val anchor = info.selectFirst("a")!! - val href = anchor.attr("href").let { - if (it.startsWith("http", ignoreCase = true)) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" - } - return Manga( - id = generateUid(href), - title = anchor.ownText(), - altTitles = emptySet(), - url = href, - publicUrl = href, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = element.selectFirst("div.thumb img")?.attr("src")?.let { url -> - if (url.startsWith("http", ignoreCase = true)) url else "https://$domain$url" - }, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source - ) - } - - private fun parseNormalManga(element: Element, hasQuery: Boolean): Manga { - val anchor = if (hasQuery) element else element.selectFirst("a")!! - val href = anchor.attr("href").let { - if (it.startsWith("http", ignoreCase = true)) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" - } - return Manga( - id = generateUid(href), - title = if (hasQuery) - element.selectFirst("div.seach-right > div")?.text().orEmpty() - else - element.selectFirst("h1")?.ownText().orEmpty(), - altTitles = emptySet(), - url = href, - publicUrl = href, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = anchor.selectFirst("img")?.attr("src")?.let { url -> - if (url.startsWith("http", ignoreCase = true)) url else "https://$domain$url" - }, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source - ) - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url).parseHtml() - val info = doc.selectFirst("div#manga-info-container") - val title = info?.selectFirst("h1.big-fat-titles")?.ownText().orEmpty() - val thumbnail = info?.selectFirst("div#manga-page img")?.attrAsAbsoluteUrlOrNull("src") - val genre = info?.select("div.genres-list > li")?.joinToString { it.text() } ?: "" - val description = info?.selectFirst("div#manga-info-rightColumn > div > div.white-font")?.textOrNull() - val author = info?.select("div#manga-info-stats > div:has(> li:eq(0):contains(Author)) > li:eq(1)")?.textOrNull() - val statusText = info?.select("div#manga-info-stats > div:has(> li:eq(0):contains(Status)) > li:eq(1)")?.text() - val state = when (statusText) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - } - - val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { i, el -> - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) - val href = el.attr("href").let { - if (it.startsWith("http")) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" - } - val date = el.selectFirst("span")?.text() - MangaChapter( - id = generateUid(href), - title = el.ownText(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - uploadDate = dateFormat.parseSafe(date), - branch = null, - source = source - ) - } - - return manga.copy( - title = title, - coverUrl = thumbnail, - tags = genre.split(", ").filter { it.isNotBlank() }.mapToSet { - MangaTag(title = it.lowercase().replace(" ", "-").toTitleCase(sourceLocale), key = it, source) - }, - description = description, - state = state, - authors = if (author.isNullOrBlank()) emptySet() else setOf(author), - chapters = chapters - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url).parseHtml() - return doc.select("div > img.imgholder").map { - val url = it.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } - - private fun fetchTags() = arraySetOf( - MangaTag("Action", "1", source), - MangaTag("Adventure", "2", source), - MangaTag("Comedy", "3", source), - MangaTag("Cooking", "34", source), - MangaTag("Doujinshi", "25", source), - MangaTag("Drama", "4", source), - MangaTag("Ecchi", "19", source), - MangaTag("Fantasy", "5", source), - MangaTag("Gender Bender", "30", source), - MangaTag("Harem", "10", source), - MangaTag("Historical", "28", source), - MangaTag("Horror", "8", source), - MangaTag("Isekai", "33", source), - MangaTag("Josei", "31", source), - MangaTag("Martial Arts", "6", source), - MangaTag("Mature", "22", source), - MangaTag("Mecha", "32", source), - MangaTag("Mystery", "15", source), - MangaTag("One Shot", "26", source), - MangaTag("Psychological", "11", source), - MangaTag("Romance", "12", source), - MangaTag("School Life", "13", source), - MangaTag("Sci-fi", "16", source), - MangaTag("Seinen", "17", source), - MangaTag("Shoujo", "14", source), - MangaTag("Shoujo Ai", "23", source), - MangaTag("Shounen", "7", source), - MangaTag("Shounen Ai", "29", source), - MangaTag("Slice of Life", "21", source), - MangaTag("Smut", "27", source), - MangaTag("Sports", "20", source), - MangaTag("Supernatural", "9", source), - MangaTag("Tragedy", "18", source), - MangaTag("Webtoons", "24", source), - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt deleted file mode 100644 index 03153da3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt +++ /dev/null @@ -1,24 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MangaReaderParser - -@MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") -internal class EnThunderScans(context: MangaLoaderContext) : - MangaReaderParser( - context, - MangaParserSource.ENTHUNDERSCANS, - "en-thunderscans.com", - pageSize = 30, - searchPageSize = 10, - ) { - override val listUrl = "/comics" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isTagsExclusionSupported = false, - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt deleted file mode 100644 index 4f0520b2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt +++ /dev/null @@ -1,263 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import androidx.collection.ArraySet -import okhttp3.HttpUrl -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.SinglePageMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy -import java.util.EnumSet -import java.util.concurrent.TimeUnit - -@MangaSourceParser("FLAMECOMICS", "FlameComics", "en") -internal class FlameComics(context: MangaLoaderContext) : - SinglePageMangaParser(context, MangaParserSource.FLAMECOMICS) { - - private val commonPrefix = suspendLazy(initializer = ::fetchCommonPrefix) - private val removeSpecialCharsRegex = Regex("[^A-Za-z0-9 ]") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override val configKeyDomain = ConfigKey.Domain("flamecomics.xyz") - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("browse.json") - .build() - - val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") - - val allManga = json.mapJSONNotNull { jo -> - parseManga(jo).takeIf { it.tags.matches(filter) } - } - - // Filter by search if provided - val query = filter.query - val filteredManga = if (!query.isNullOrEmpty()) { - val normalizedQuery = removeSpecialCharsRegex.replace(query.lowercase(), "") - allManga.filter { manga -> - val titles = mutableListOf(manga.title) - titles.addAll(manga.altTitles) - - titles.any { title -> - normalizedQuery in removeSpecialCharsRegex.replace(title.lowercase(), "") - } - } - } else { - allManga - } - - return filteredManga - .let { list -> - when (order) { - SortOrder.ALPHABETICAL -> list.sortedBy { it.title } - else -> list - } - } - } - - override suspend fun getDetails(manga: Manga): Manga = getDetailsImpl(manga.url.toLong()) - - override suspend fun getPages(chapter: MangaChapter): List { - val (seriesId, token) = chapter.url.split('?') - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("series") - .addPathSegment(seriesId) - .addPathSegment("$token.json") - .addQueryParameter("id", seriesId) - .addQueryParameter("token", token) - .build() - val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") - .getJSONObject("chapter") - .getJSONObject("images") - .entries() - return json.map { (i, jo) -> - MangaPage( - id = generateUid("$i|$token"), - url = imageUrl(seriesId, token + "/" + jo.getString("name"), 1920), - preview = imageUrl(seriesId, token + "/" + jo.getString("name"), 128), - source = source, - ) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val seriesId = link.pathSegments.lastOrNull()?.toLongOrNull() ?: return null - return getDetailsImpl(seriesId) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) - - private suspend fun fetchCommonPrefix(): String { - val raw = webClient.httpGet(urlBuilder().build()).parseRaw() - val regex = Regex("/_next/static/([^/]+)/_buildManifest\\.js") - val prefix = raw.findGroupValue(regex) - return checkNotNull(prefix) { - "Unable to find common prefix in page. Looking for pattern: /_next/static/([^/]+)/_buildManifest\\.js\n" + - "Page content sample: ${raw.take(1000)}" - } - } - - private fun imageUrl(seriesId: Any, url: String, width: Int) = urlBuilder() - .addPathSegment("_next") - .addPathSegment("image") - .addQueryParameter( - "url", - urlBuilder("cdn") - .addPathSegment("uploads") - .addPathSegment("images") - .addPathSegment("series") - .addPathSegment(seriesId.toString()) - .addPathSegments(url) - .build().toString(), - ) - .addQueryParameter("w", width.toString()) - .addQueryParameter("q", "100") - .build() - .toString() - - private suspend fun fetchAvailableTags(): Set { - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("browse.json") - .build() - return webClient.httpGet(url).parseJson() - .getJSONObject("pageProps") - .getJSONArray("series") - .mapJSONNotNull { it.getStringOrNull("categories") } - .flatMapTo(ArraySet()) { - JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } - } - } - - private fun parseManga(jo: JSONObject): Manga { - // Try different possible field names for series ID, including string conversion - val seriesId = jo.getLongOrDefault("series_id", -1L).takeIf { it != -1L } - ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } // Handle novels - ?: jo.getStringOrNull("series_id")?.toLongOrNull() - ?: jo.getStringOrNull("novel_id")?.toLongOrNull() // Handle novels as string - ?: jo.getLongOrDefault("id", -1L).takeIf { it != -1L } - ?: jo.getStringOrNull("id")?.toLongOrNull() - ?: jo.getLongOrDefault("seriesId", -1L).takeIf { it != -1L } - ?: jo.getStringOrNull("seriesId")?.toLongOrNull() - ?: jo.getLongOrDefault("series", -1L).takeIf { it != -1L } - ?: jo.getStringOrNull("series")?.toLongOrNull() - ?: throw IllegalArgumentException( - "No valid series ID found. Available fields: ${jo.keys().asSequence().joinToString()}\n" + - "series_id value: ${jo.optString("series_id", "MISSING")}\n" + - "novel_id value: ${jo.optString("novel_id", "MISSING")}\n" + - "JSON sample: ${jo.toString().take(500)}" - ) - val cover = jo.getStringOrNull("cover") - val author = jo.getStringOrNull("author") - return Manga( - id = generateUid(seriesId), - title = jo.getString("title"), - altTitles = jo.getStringOrNull("altTitles")?.let { - JSONArray(it).toStringSet() - }.orEmpty(), - url = seriesId.toString(), - publicUrl = "https://${domain}/series/$seriesId", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = if (cover != null) { - imageUrl(seriesId, cover, 384) - } else { - null - }, - tags = jo.getStringOrNull("categories")?.let { - JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } - }.orEmpty(), - state = when (jo.getStringOrNull("status")) { - "Dropped" -> MangaState.ABANDONED - "Completed" -> MangaState.FINISHED - "Hiatus" -> MangaState.PAUSED - "Ongoing" -> MangaState.ONGOING - else -> null - }, - authors = setOfNotNull(author), - largeCoverUrl = if (cover != null) { - imageUrl(seriesId, cover, 720) - } else { - null - }, - description = jo.getStringOrNull("description"), - source = source, - ) - } - - private suspend fun getDetailsImpl(seriesId: Long): Manga { - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("series") - .addPathSegment("$seriesId.json") - .addQueryParameter("id", seriesId.toString()) - .build() - val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") - val series = json.getJSONObject("series") - val chapters = json.getJSONArray("chapters") - return parseManga(series).copy( - chapters = chapters.mapJSON { jo -> - val chapterId = jo.getLong("chapter_id") - val number = jo.getFloatOrDefault("chapter", 0f) - MangaChapter( - id = generateUid(longOf(seriesId.toInt(), chapterId.toInt())), - title = jo.getStringOrNull("name"), - number = number, - volume = 0, - url = seriesId.toString() + "?" + jo.getStringOrNull("token").orEmpty(), - scanlator = null, - uploadDate = TimeUnit.SECONDS.toMillis(jo.getLongOrDefault("release_date", 0L)), - branch = jo.getStringOrNull("language"), - source = source, - ) - }.reversed(), - ) - } - - private fun Set.matches(filter: MangaListFilter): Boolean { - if (filter.tags.isNotEmpty() && !containsAll(filter.tags)) { - return false - } - for (tag in filter.tagsExclude) { - if (contains(tag)) { - return false - } - } - return true - } - - private fun String.toMangaTag() = MangaTag( - title = this.toTitleCase(sourceLocale), - key = this.lowercase().trim(), - source = source, - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt deleted file mode 100644 index b11da65b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt +++ /dev/null @@ -1,230 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadthemeParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.text.SimpleDateFormat -import java.util.EnumSet - -@MangaSourceParser("KALISCAN", "kaliscan.io", "en") -internal class KaliScan(context: MangaLoaderContext) : - MadthemeParser(context, MangaParserSource.KALISCAN, "kaliscan.io", pageSize=24) { - - override val selectDesc = ".summary .content, .summary .content ~ p" - override val selectState = ".detail .meta > p > strong:contains(Status) ~ a" - override val selectAlt = ".detail h2" - override val selectTag = ".detail .meta > p > strong:contains(Genres) ~ a" - override val selectDate = ".chapter-update" - override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isAuthorSearchSupported = true, - isTagsExclusionSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - append("/search") - - append("?page=") - append(page.toString()) - - filter.query?.let { - append("&q=") - append(it.urlEncoded()) - } - - append("&sort=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("updated_at") - SortOrder.ALPHABETICAL -> append("name") - SortOrder.NEWEST -> append("created_at") - SortOrder.RATING -> append("rating") - else -> append("updated_at") - } - - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { tag -> - append("&include[]=") - append(tag.key) - } - } - - if (filter.tagsExclude.isNotEmpty()) { - filter.tagsExclude.forEach { tag -> - append("&exclude[]=") - append(tag.key) - } - } - - append("&include_mode=and") - append("&bookmark=off") - - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> "all" - }, - ) - } ?: append("&status=all") - - filter.author?.takeIf { it.isNotBlank() }?.let { author -> - append("&author=") - append(author.urlEncoded()) - } - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List { - return doc.select(".book-detailed-item").map { div -> - val link = div.selectFirstOrThrow("a") - val href = link.attrAsRelativeUrl("href") - val title = link.attr("title").ifEmpty { - div.selectFirst(".title")?.text() ?: "" - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = div.selectFirst("img")?.attr("data-src")?.ifEmpty { - div.selectFirst("img")?.attr("src") - }, - title = title, - altTitles = emptySet(), - rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst(".detail h1")?.text() ?: manga.title - - val authors = doc.select(".detail .meta > p > strong:contains(Authors) ~ a") - .map { it.text().trim(',', ' ') } - .toSet() - - val tags = doc.select(selectTag).mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().trim(',', ' ').toTitleCase(), - source = source, - ) - } - - val altNames = doc.selectFirst(selectAlt)?.text() - ?.split(',', ';') - ?.mapNotNull { it -> it.trim().takeIf { it != title } } - ?.toSet() ?: emptySet() - - val description = doc.select(selectDesc).text() - val statusText = doc.selectFirst(selectState)?.text()?.lowercase() ?: "" - val state = when (statusText) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - val nsfw = doc.selectFirst("#adt-warning") != null - - val coverUrl = doc.selectFirst("#cover img")?.attr("data-src") - - val chapters = doc.select(selectChapter).mapChapters(reversed = true) - { i, element -> - val link = element.selectFirst("a") ?: return@mapChapters null - val href = link.attrAsRelativeUrl("href") - val chapterTitle = element.selectFirst(".chapter-title")?.text()?.trim() ?: return@mapChapters null - val dateText = element.selectFirst(selectDate)?.text()?.trim() - - MangaChapter( - id = generateUid(href), - url = href, - title = chapterTitle, - uploadDate = parseChapterDate( - SimpleDateFormat(datePattern, sourceLocale), - dateText, - ), - source = source, - number = i + 1f, - volume = 0, - scanlator = null, - branch = null, - ) - } - - return manga.copy( - title = title, - altTitles = altNames, - authors = authors, - tags = tags, - description = description, - state = state, - largeCoverUrl = coverUrl, - chapters = chapters, - contentRating = if (nsfw || isNsfwSource) ContentRating.ADULT else ContentRating.SAFE, - ) - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/search").parseHtml() - - return doc.selectFirst(".checkbox-group.genres")?.select(".checkbox-wrapper")?.mapNotNullToSet { element -> - val input = element.selectFirst("input") ?: return@mapNotNullToSet null - val key = input.attr("value").takeIf { it.isNotEmpty() } ?: return@mapNotNullToSet null - val label = element.selectFirst(".radio__label")?.text() ?: key - - MangaTag( - key = key, - title = label, - source = source, - ) - } ?: emptySet() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt deleted file mode 100644 index 7e31556c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MangaThemesia - -@MangaSourceParser("KINGOFSHOJO", "King of Shojo", "en") -internal class KingofShojo(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.KINGOFSHOJO, "kingofshojo.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt deleted file mode 100644 index fc19cc98..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt +++ /dev/null @@ -1,27 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.LikeMangaParser -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.network.CommonHeaders - -private const val USER_AGENT = "Usagi/1 (Android)" - -@MangaSourceParser("LIKEMANGA", "LikeManga", "en") -internal class LikeManga(context: MangaLoaderContext) : - LikeMangaParser(context, MangaParserSource.LIKEMANGA, "likemanga.ink") { - override val userAgentKey = ConfigKey.UserAgent(USER_AGENT,) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = request.newBuilder() - .header(CommonHeaders.USER_AGENT, USER_AGENT) - .build() - - return chain.proceed(newRequest) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt deleted file mode 100644 index 8af146a5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt +++ /dev/null @@ -1,236 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGAFOX", "MangaFox", "en") -internal class MangaFox(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAFOX, pageSize = 24) { - - private val baseUrl = "fanfox.net" - private val mobileUrl = "https://m.fanfox.net" - private val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH) - - override val configKeyDomain = ConfigKey.Domain(baseUrl) - - private val apiClient: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request().newBuilder() - .header("Referer", "https://$baseUrl/") - .build() - chain.proceed(request) - } - .build() - OkHttpWebClient(httpClient, source) - } - - override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true - ) - - @Volatile - private var tagsCache: Set? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = getOrFetchTags() - ) - } - - private suspend fun getOrFetchTags(): Set { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTags().also { tagsCache = it } - } - } - - private fun fetchTags(): Set { - return listOf( - "action", "adventure", "comedy", "drama", "fantasy", - "harem", "romance", "school-life", "shounen", "shoujo", - "seinen", "sci-fi", "slice-of-life", "supernatural", - "ecchi", "mystery", "psychological", "historical", - "martial-arts", "mecha", "sports" - ).map { - MangaTag(key = it, title = it.replace("-", " "), source = source) - }.toSet() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildUrl(page, filter, order) - val doc = apiClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun buildUrl(page: Int, filter: MangaListFilter, order: SortOrder): String { - val query = filter.query?.trim().orEmpty() - val tag = filter.tags.firstOrNull()?.key - return when { - !tag.isNullOrBlank() && query.isEmpty() -> - "https://m.fanfox.net/search/cate/$tag" - query.isNotEmpty() -> - "https://m.fanfox.net/search?k=${query.urlEncoded()}" - else -> - buildString { - append("https://$baseUrl/directory/") - if (page > 1) append("$page.html") - if (order == SortOrder.UPDATED) append("?latest") - } - } - } - - private fun parseMangaList(doc: Document): List { - val searchItems = doc.select(".post-list li") - if (searchItems.isNotEmpty()) { - return searchItems.mapNotNull { el -> - val a = el.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = a.attr("title").ifBlank { a.selectFirst(".cover-info .title")?.text() ?: a.text() }, - url = href, - publicUrl = a.absUrl("href"), - coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - source = source - ) - } - } - - val dirSelectors = listOf( - "ul.manga-list-1-list li", - "ul.manga-list-4-list li", - "ul.manga-list-2-list li" - ) - for (sel in dirSelectors) { - val items = doc.select(sel) - if (items.isNotEmpty()) { - return items.mapNotNull { el -> - val a = el.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = a.attr("title").ifBlank { a.text() }, - url = href, - publicUrl = a.absUrl("href"), - coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - source = source - ) - } - } - } - return emptyList() - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() - val info = doc.selectFirst(".detail-info-right")!! - - val chapters = doc.select("ul.detail-main-list li").mapIndexedNotNull { i, el -> - val a = el.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("p")?.text() ?: a.ownText() - val rawDate = el.text() - .let { text -> - Regex("""(Today|Yesterday|\w{3}\s\d{1,2},\s\d{4}|\d{4}-\d{2}-\d{2})""") - .find(text)?.value - } - val parsedDate = parseChapterDate(rawDate) - MangaChapter( - id = generateUid(href), - title = name, - number = extractChapterNumber(name), - volume = 0, - url = href, - uploadDate = parsedDate.takeIf { it > 0L } - ?: (System.currentTimeMillis() - (i * 86_400_000L)), - scanlator = null, - branch = null, - source = source - ) - }.sortedBy { it.number } - - return manga.copy( - title = doc.selectFirst("h1")?.text() ?: manga.title, - authors = info.select(".detail-info-right-say a").mapToSet { it.text() }, - tags = info.select(".detail-info-right-tag-list a").mapToSet { - MangaTag(key = it.text().lowercase(), title = it.text(), source = source) - }, - description = info.selectFirst("p.fullcontent")?.text(), - state = parseStatus(info.selectFirst(".detail-info-right-title-tip")?.text().orEmpty()), - coverUrl = doc.selectFirst(".detail-info-cover-img")?.attr("abs:src").orEmpty(), - chapters = chapters - ) - } - - private fun parseStatus(status: String): MangaState? = when { - status.contains("Ongoing", true) -> MangaState.ONGOING - status.contains("Completed", true) -> MangaState.FINISHED - else -> null - } - - private fun extractChapterNumber(name: String): Float { - return Regex("""(?i)(chapter|ch\.)\s*(\d+(\.\d+)?)""") - .find(name) - ?.groupValues - ?.getOrNull(2) - ?.toFloatOrNull() - ?: 0f - } - - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrBlank()) return 0L - val clean = date.trim() - return when { - "Today" in clean -> Calendar.getInstance().timeInMillis - "Yesterday" in clean -> Calendar.getInstance().apply { add(Calendar.DATE, -1) }.timeInMillis - clean.matches(Regex("""\d{4}-\d{2}-\d{2}""")) -> runCatching { - val parts = clean.split("-") - Calendar.getInstance().apply { set(parts[0].toInt(), parts[1].toInt() - 1, parts[2].toInt()) }.timeInMillis - }.getOrDefault(0L) - else -> runCatching { dateFormat.parse(clean)?.time ?: 0L }.getOrDefault(0L) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val mobilePath = chapter.url.replace("/manga/", "/roll_manga/") - val url = "$mobileUrl$mobilePath" - val doc = apiClient.httpGet(url).parseHtml() - return doc.select("#viewer img").map { - val imgUrl = it.attr("abs:data-original") - MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - override suspend fun getRelatedManga(seed: Manga): List = emptyList() -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt deleted file mode 100644 index 92bab010..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt +++ /dev/null @@ -1,156 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import java.text.SimpleDateFormat -import kotlinx.coroutines.awaitAll - -@MangaSourceParser("MANGAGG", "MangaGG", "en") -internal class MangaGG(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAGG, "mangagg.com") { - - override val tagPrefix = "genre/" - override val datePattern = "MM/dd/yyyy" - - override suspend fun loadChapters( - mangaUrl: String, - document: Document, - ): List = coroutineScope { - - val baseUrl = mangaUrl - .toAbsoluteUrl(domain) - .removeSuffix('/') - - val chapters = mutableListOf() - - var page = 1 - var finished = false - - while (!finished) { - - val jobs = (page until page + 5).map { currentPage -> - async { - fetchChapterPage( - baseUrl, - currentPage, - ) - } - } - - val results = jobs.awaitAll() - - finished = results.all { - it.isEmpty() - } - - results.forEach { - chapters.addAll(it) - } - - page += 5 - } - - chapters - .distinctBy { - it.id - } - .reversed() - .mapIndexed { index, chapter -> - chapter.copy( - number = index + 1f - ) - } - } - - - private suspend fun fetchChapterPage( - mangaUrl: String, - page: Int, - ): List { - - val url = - "$mangaUrl/ajax/chapters/?t=$page" - - return runCatching { - - webClient - .httpPost( - url, - emptyMap(), - ) - .parseHtml() - .let(::parseChapterPage) - - }.getOrDefault(emptyList()) - } - - - private fun parseChapterPage( - doc: Document, - ): List { - - val dateFormat = - SimpleDateFormat( - datePattern, - sourceLocale, - ) - - return doc - .select(selectChapter) - .map { li -> - parseChapterItem( - li, - dateFormat, - ) - } - } - - - private fun parseChapterItem( - li: org.jsoup.nodes.Element, - dateFormat: SimpleDateFormat, - ): MangaChapter { - - val a = li.selectFirstOrThrow("a") - - val href = a.attrAsRelativeUrl("href") - - val dateText = - li.selectFirst("a.c-new-tag") - ?.attr("title") - ?: li.selectFirst(selectDate) - ?.text() - - val name = - a.selectFirst("p") - ?.text() - ?: a.ownText() - - return MangaChapter( - id = generateUid(href), - title = name, - number = 0f, - volume = 0, - url = href + stylePage, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), - source = source, - scanlator = null, - branch = null, - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt deleted file mode 100644 index c99da478..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt +++ /dev/null @@ -1,242 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGAKEKO", "MangaGeko", "en") -internal class MangaGeko(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAKEKO, 30) { - - override val configKeyDomain = ConfigKey.Domain("www.mgeko.cc") - private val baseUrl = "https://$domain" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.NEWEST, // recently_added - SortOrder.UPDATED, // latest - SortOrder.POPULARITY, // popular_all_time - SortOrder.RATING, // rating - SortOrder.ALPHABETICAL, // az - SortOrder.ALPHABETICAL_DESC, // za - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - private val dateFormats = listOf( - SimpleDateFormat("MMMM dd, yyyy, h:mm a", Locale.ENGLISH), - SimpleDateFormat("MMMM dd, yyyy, h a", Locale.ENGLISH), - ) - - private val allGenres = setOf( - "Action", "Adventure", "Comedy", "Cooking", "Manga", "Drama", "Fantasy", - "Gender bender", "Harem", "Historical", "Horror", "Isekai", "Josei", - "Manhua", "Manhwa", "Martial arts", "Mature", "Mecha", "Medical", - "Mystery", "One shot", "Psychological", "Romance", "School life", - "Sci fi", "Seinen", "Shoujo", "Shounen", "Slice of life", "Sports", - "Supernatural", "Tragedy", "Webtoons", "Ladies", - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = allGenres.map { MangaTag(it, it.lowercase().replace(" ", "_"), source) }.toSet() - return MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - if (!filter.query.isNullOrEmpty()) { - return searchManga(page, filter.query!!) - } - return browseManga(page, order, filter) - } - - private suspend fun searchManga(page: Int, query: String): List { - val url = "$baseUrl/search/".toHttpUrl().newBuilder() - .addQueryParameter("search", query) - .addQueryParameter("results", page.toString()) - .build() - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".novel-item").mapNotNull { it.toManga() } - } - - private suspend fun browseManga(page: Int, order: SortOrder, filter: MangaListFilter): List { - val sort = when (order) { - SortOrder.NEWEST -> "recently_added" - SortOrder.UPDATED -> "latest" - SortOrder.POPULARITY -> "popular_all_time" - SortOrder.RATING -> "rating" - SortOrder.ALPHABETICAL -> "az" - SortOrder.ALPHABETICAL_DESC -> "za" - else -> "latest" - } - - val url = "$baseUrl/browse-comics/data/".toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("sort", sort) - - val safeMode = if (ContentRating.ADULT !in filter.contentRating) "1" else "0" - addQueryParameter("safe_mode", safeMode) - - if (filter.tags.isNotEmpty()) { - addQueryParameter("include_genres", filter.tags.joinToString(",") { it.key.replace("_", " ") }) - } - if (filter.tagsExclude.isNotEmpty()) { - addQueryParameter("exclude_genres", filter.tagsExclude.joinToString(",") { it.key.replace("_", " ") }) - } - - if (filter.states.isNotEmpty()) { - val status = when (filter.states.first()) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - MangaState.PAUSED -> "Hiatus" - else -> null - } - status?.let { addQueryParameter("status", it) } - } - - if (filter.types.isNotEmpty()) { - val type = when (filter.types.first()) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Webtoon" - else -> null - } - type?.let { addQueryParameter("type", it) } - } - }.build() - - val response = webClient.httpGet(url) - val json = JSONObject(response.body.string()) - val htmlFragment = json.getString("results_html") - val doc = Jsoup.parseBodyFragment(htmlFragment, baseUrl) - return doc.select(".comic-card").map { it.toManga() } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val header = doc.selectFirstOrThrow(".novel-header") - - val title = header.selectFirst(".novel-title")?.text()?.trim() ?: manga.title - val cover = header.selectFirst(".cover img")?.let { - it.attr("data-src").ifBlank { it.attr("src") } - } ?: manga.coverUrl - - val author = header.selectFirst(".author a")?.attr("title")?.trim()?.takeIf { - it.lowercase() != "updating" - } - - val state = when { - header.selectFirst("div.header-stats strong.completed") != null -> MangaState.FINISHED - header.selectFirst("div.header-stats strong.ongoing") != null -> MangaState.ONGOING - else -> null - } - - val tags = doc.select(".categories a[href*=genre]").mapToSet { a -> - val name = a.ownText().trim().split(" ").joinToString(" ") { word -> - word.lowercase().replaceFirstChar { it.uppercase() } - } - MangaTag(name, name.lowercase().replace(" ", "_"), source) - } - - val chapters = fetchChapters(manga.url) - - return manga.copy( - title = title, - altTitles = emptySet(), - coverUrl = cover, - largeCoverUrl = cover, - state = state, - authors = setOfNotNull(author), - description = null, - tags = tags, - chapters = chapters, - ) - } - - private suspend fun fetchChapters(mangaUrl: String): List { - val doc = webClient.httpGet(mangaUrl.toAbsoluteUrl(domain) + "all-chapters/").parseHtml() - return doc.select("ul.chapter-list > li").mapNotNull { li -> - val a = li.selectFirst("a") ?: return@mapNotNull null - val url = a.attrAsRelativeUrl("href") - val titleEl = li.selectFirst(".chapter-title, .chapter-number") ?: return@mapNotNull null - val chapterName = titleEl.ownText() - val dateStr = li.selectFirst(".chapter-update")?.attr("datetime") - ?.replace(".", "")?.replace("Sept", "Sep") ?: "" - val date = dateFormats.firstNotNullOfOrNull { fmt -> fmt.parseSafe(dateStr) } ?: 0L - MangaChapter( - id = generateUid(url), - title = "Chapter $chapterName", - number = 0f, - volume = 0, - url = url, - uploadDate = date, - source = source, - scanlator = null, - branch = null, - ) - }.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-reader img").mapNotNull { img -> - val src = img.absUrl("src") - if (src.isEmpty()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun Element.toManga(): Manga { - val titleEl = selectFirst(".comic-card__title a") ?: selectFirst(".novel-title") - val title = titleEl?.text()?.trim() ?: "" - val href = selectFirst("a")?.attrAsRelativeUrl("href") ?: "" - val cover = selectFirst(".comic-card__cover img, .novel-cover img")?.let { - it.attr("data-src").ifBlank { it.attr("src") } - } ?: "" - return Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt deleted file mode 100644 index 28783ee2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt +++ /dev/null @@ -1,219 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadthemeParser -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.EnumSet - -@MangaSourceParser("MANGAJINX", "MangaJinx", "en") -internal class MangaJinx(context: MangaLoaderContext) : - MadthemeParser(context, MangaParserSource.MANGAJINX, "mgjinx.com", pageSize=24) { - - override val selectDesc = ".summary .content, .summary .content ~ p" - override val selectState = ".detail .meta > p > strong:contains(Status) ~ a" - override val selectAlt = ".detail h2" - override val selectTag = ".detail .meta > p > strong:contains(Genres) ~ a" - override val selectDate = ".chapter-update" - override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isAuthorSearchSupported = true, - isTagsExclusionSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - append("/search") - - append("?page=") - append(page.toString()) - - filter.query?.let { - append("&q=") - append(it.urlEncoded()) - } - - append("&sort=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("updated_at") - SortOrder.ALPHABETICAL -> append("name") - SortOrder.NEWEST -> append("created_at") - SortOrder.RATING -> append("rating") - else -> append("updated_at") - } - - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { tag -> - append("&include[]=") - append(tag.key) - } - } - - if (filter.tagsExclude.isNotEmpty()) { - filter.tagsExclude.forEach { tag -> - append("&exclude[]=") - append(tag.key) - } - } - - append("&include_mode=and") - append("&bookmark=off") - - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> "all" - }, - ) - } ?: append("&status=all") - - filter.author?.takeIf { it.isNotBlank() }?.let { author -> - append("&author=") - append(author.urlEncoded()) - } - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List { - return doc.select(".book-detailed-item").map { div -> - val link = div.selectFirstOrThrow("a") - val href = link.attrAsRelativeUrl("href") - val title = link.attr("title").ifEmpty { - div.selectFirst(".title")?.text() ?: "" - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = div.selectFirst("img")?.attr("data-src")?.ifEmpty { - div.selectFirst("img")?.attr("src") - }, - title = title, - altTitles = emptySet(), - rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst(".detail h1")?.text() ?: manga.title - - val authors = doc.select(".detail .meta > p > strong:contains(Authors) ~ a") - .map { it.text().trim(',', ' ') } - .toSet() - - val tags = doc.select(selectTag).mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().trim(',', ' ').toTitleCase(), - source = source, - ) - } - - val altNames = doc.selectFirst(selectAlt)?.text() - ?.split(',', ';') - ?.mapNotNull { it -> it.trim().takeIf { it != title } } - ?.toSet() ?: emptySet() - - val description = doc.select(selectDesc).text() - val statusText = doc.selectFirst(selectState)?.text()?.lowercase() ?: "" - val state = when (statusText) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - val nsfw = doc.selectFirst("#adt-warning") != null - - val coverUrl = doc.selectFirst("#cover img")?.attr("data-src") - - val chapters = doc.select(selectChapter).mapChapters(reversed = true) - { i, element -> - val link = element.selectFirst("a") ?: return@mapChapters null - val href = link.attrAsRelativeUrl("href") - val chapterTitle = element.selectFirst(".chapter-title")?.text()?.trim() ?: return@mapChapters null - val dateText = element.selectFirst(selectDate)?.text()?.trim() - - MangaChapter( - id = generateUid(href), - url = href, - title = chapterTitle, - uploadDate = parseChapterDate( - SimpleDateFormat(datePattern, sourceLocale), - dateText, - ), - source = source, - number = i + 1f, - volume = 0, - scanlator = null, - branch = null, - ) - } - - return manga.copy( - title = title, - altTitles = altNames, - authors = authors, - tags = tags, - description = description, - state = state, - largeCoverUrl = coverUrl, - chapters = chapters, - contentRating = if (nsfw || isNsfwSource) ContentRating.ADULT else ContentRating.SAFE, - ) - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/search").parseHtml() - - return doc.selectFirst(".checkbox-group.genres")?.select(".checkbox-wrapper")?.mapNotNullToSet { element -> - val input = element.selectFirst("input") ?: return@mapNotNullToSet null - val key = input.attr("value").takeIf { it.isNotEmpty() } ?: return@mapNotNullToSet null - val label = element.selectFirst(".radio__label")?.text() ?: key - - MangaTag( - key = key, - title = label, - source = source, - ) - } ?: emptySet() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt deleted file mode 100644 index 508a8099..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt +++ /dev/null @@ -1,340 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNullToSet -import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale -import okhttp3.Interceptor -import okhttp3.Response - -@MangaSourceParser("MANGAK", "MangaK", "en", ContentType.MANGA) -internal class MangaK(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAK, pageSize = 24) { - - override val configKeyDomain = ConfigKey.Domain("mangak.io") - - private val apiUrl = "https://api.mangak.io" - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - if (!response.isSuccessful && request.url.host.matches(IMAGE_FALLBACK_REGEX)) { - response.close() - val newUrl = request.url.newBuilder().host("rx.rzyn.net").build() - return chain.proceed(request.newBuilder().url(newUrl).build()) - } - return response - } - - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, -// MangaState.PAUSED, -// MangaState.ABANDONED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ), - availableDemographics = EnumSet.of( - Demographic.SHOUJO, - Demographic.SEINEN, - Demographic.SHOUNEN, - Demographic.JOSEI, - ), - availableContentRating = EnumSet.of( - ContentRating.SAFE, - ContentRating.SUGGESTIVE, - ContentRating.ADULT, - ), - ) - - private suspend fun fetchTags(): Set { - val json = webClient.httpGet("$apiUrl/genres").parseJson() - return json.getJSONObject("data").getJSONArray("items").mapJSONToSet { item -> - MangaTag( - title = item.getString("name").toTitleCase(sourceLocale), - key = item.getString("slug"), - source = source, - ) - } - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val allExcludedTags = filter.tagsExclude.map { it.key }.toMutableSet() - val query = filter.query - val url = buildString { - append(apiUrl) - append("/titles/search?page=") - append(page) - append("&limit=") - append(pageSize) - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.NEWEST -> "newest" - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "rating" - else -> "latest" - }, - ) - - if (!query.isNullOrEmpty()) { - append("&q=") - // api does not like commans and character limits to 60-70 - append(query.replace(",", "").urlEncoded()) - } - - if (filter.tags.isNotEmpty()) { - append("&genres=") - append(filter.tags.joinToString(",") { it.key }) - } - - if (allExcludedTags.isNotEmpty()) { - append("&exclude=") - append(allExcludedTags.joinToString(",")) - } - - filter.types.oneOrThrowIfMany()?.let { - append("&type=") - append( - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> null - } ?: return@let - ) - } - - filter.contentRating.oneOrThrowIfMany()?.let { - append("&content_rating=") - append( - when (it) { - ContentRating.SAFE -> "safe" - ContentRating.SUGGESTIVE -> "suggestive" - ContentRating.ADULT -> "pornographic" - } - ) - } - - filter.demographics.oneOrThrowIfMany()?.let { - append("&demographic=") - append( - when (it) { - Demographic.SHOUJO -> "shoujo" - Demographic.SEINEN -> "seinen" - Demographic.SHOUNEN -> "shounen" - Demographic.JOSEI -> "josei" - else -> null - } ?: return@let - ) - } - - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" -// MangaState.PAUSED -> "hiatus" -// MangaState.ABANDONED -> "cancelled" - else -> null - } ?: return@let - ) - } - } - - val json = webClient.httpGet(url).parseJson() - return json.getJSONObject("data").getJSONArray("items").mapJSON { item -> - item.toManga() - } - } - - private fun JSONObject.toManga(): Manga { - val relativeUrl = getString("url") - return Manga( - id = generateUid(getString("id")), - title = getString("name"), - altTitles = parseAltTitles(), - url = getString("id"), - publicUrl = "https://$domain$relativeUrl", - rating = optDouble("rating", 0.0).toRating(), - contentRating = optString("content_rating").nullIfEmpty().toContentRating(), - coverUrl = optString("cover").nullIfEmpty(), - tags = parseTags(), - state = optString("status").toMangaState(), - authors = emptySet(), - description = optString("summary").nullIfEmpty(), - source = source, - ) - } - - - private fun String?.toContentRating() = when (this) { - "safe" -> ContentRating.SAFE - "suggestive" -> ContentRating.SUGGESTIVE - "erotica", "pornographic" -> ContentRating.ADULT - else -> null - } - - - private fun JSONObject.parseAltTitles(): Set { - val result = mutableSetOf() - optJSONArray("alt_names")?.let { arr -> - for (i in 0 until arr.length()) { - arr.optJSONObject(i)?.optString("name")?.nullIfEmpty()?.let { result.add(it) } - } - } - optString("alt_name").nullIfEmpty()?.let { result.add(it) } - return result - } - - - private fun JSONObject.parseTags(): Set = - optJSONArray("genres")?.mapJSONNotNullToSet { genre -> - val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null - MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) - }.orEmpty() - - - override suspend fun getDetails(manga: Manga): Manga { - val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() - .getJSONObject("data").getJSONObject("title") - val cv = json.optLong("cv") - val authors = json.optJSONArray("authors")?.mapJSONNotNullToSet { - it.optString("name").nullIfEmpty() - }.orEmpty() - val artists = json.optJSONArray("artists")?.mapJSONNotNullToSet { - it.optString("name").nullIfEmpty() - }.orEmpty() - return manga.copy( - title = json.optString("name").nullIfEmpty() ?: manga.title, - altTitles = json.parseAltTitles().ifEmpty { manga.altTitles }, - description = json.optString("summary").nullIfEmpty() ?: manga.description, - tags = json.parseTags().ifEmpty { manga.tags }, - authors = authors + artists, - state = json.optString("status").toMangaState() ?: manga.state, - rating = json.optDouble("rating", 0.0).toRating(), - chapters = fetchChapters(manga.url, cv), - ) - } - - - private suspend fun fetchChapters(id: String, cv: Long): List { - val json = webClient.httpGet("$apiUrl/titles/$id/chapters?cv=$cv").parseJson() - val chapters = json.getJSONObject("data").getJSONArray("chapters").mapJSON { it } - val total = chapters.size - return chapters.mapIndexed { index, chapter -> - MangaChapter( - id = generateUid(chapter.getString("id")), - title = chapter.optString("name").nullIfEmpty(), - number = chapter.optDouble("chapter_number", Double.NaN).takeUnless { it.isNaN() }?.toFloat() - ?: (total - index).toFloat(), - volume = 0, - url = chapter.getString("url"), - scanlator = null, - uploadDate = dateFormat.parseSafe(chapter.optString("updated_at").substringBefore('.')), - branch = null, - source = source, - ) - }.reversed() - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet("https://$domain${chapter.url}").parseHtml() - val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() - ?: throw ParseException("Cannot find chapter data", chapter.url) - val images = JSONObject(raw) - .getJSONObject("props") - .getJSONObject("pageProps") - .getJSONObject("initialChapter") - .getJSONArray("images") - return (0 until images.length()).map { i -> - val imageUrl = images.getString(i) - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - - private fun Double.toRating() = if (this > 0.0) (this / 5.0).toFloat() else RATING_UNKNOWN - - private fun String?.toMangaState() = when (this?.lowercase(Locale.US)) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled" -> MangaState.ABANDONED - else -> null - } - - companion object { - private val IMAGE_FALLBACK_REGEX = Regex("rx\\.qvzr[a-z]\\.org") - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt deleted file mode 100644 index a4b9072b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt +++ /dev/null @@ -1,396 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -private const val two = "?sv=mk" -private const val three = "?sv=3" - -@MangaSourceParser("MANGAKATANA", "MangaKatana", "en") -internal class MangaKatana(context: MangaLoaderContext): - PagedMangaParser(context, MangaParserSource.MANGAKATANA, pageSize = 24) { - - private val baseUrl = "https://mangakatana.com" - - override val configKeyDomain = ConfigKey.Domain("mangakatana.com") - - override val availableSortOrders = EnumSet.of( - SortOrder.UPDATED, - SortOrder.NEWEST, - )!! - - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - two to "First server", - three to "Second server", - ), - defaultValue = two, - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(preferredServerKey) - } - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = availableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - ), - ) - } - - private fun MangaState.toMKStatus(): Int { - return when (this) { - MangaState.ONGOING -> 1 - MangaState.FINISHED -> 2 - MangaState.ABANDONED -> 0 - else -> 1 - } - } - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true - ) - - private val dateFormat = SimpleDateFormat("MMM-dd-yyyy", Locale.US) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - - val url = buildUrl(page, order, filter) - val doc = webClient.httpGet(url).parseHtml() - - return doc.select("div#book_list > div.item").map { element -> - - val link = element.selectFirst("div.text > h3 > a")!! - val thumb = element.selectFirst("img")!! - - Manga( - id = generateUid(link.absUrl("href")), - title = link.ownText().ifEmpty { link.text() }, - url = link.absUrl("href"), - publicUrl = link.absUrl("href"), - - coverUrl = thumb.absUrl("src"), - - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - - source = source - ) - } - } - - private fun buildUrl( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): String { - val query = filter.query?.trim().orEmpty() - val tags = filter.tags.map { it.key }.filter { it.isNotBlank() } - val status = filter.states.firstOrNull() - - return when { - query.isNotEmpty() -> { - if (page == 1) { - "$baseUrl/?search=${query.urlEncoded()}&search_by=m_name" - } else { - "$baseUrl/page/$page?search=${query.urlEncoded()}&search_by=m_name" - } - } - - status != null -> { - "$baseUrl/genres/?" + - "filter=1" + - "&include_mode=and" + - "&bookmark_opts=off" + - "&chapters=1" + - "&order=latest" + - "&status=${status.toMKStatus()}" + - "&page=$page" - } - - tags.isNotEmpty() -> - "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" - - else -> when (order) { - SortOrder.NEWEST -> "$baseUrl/new-manga?page=$page" - SortOrder.UPDATED -> "$baseUrl/latest/page/$page" - else -> "$baseUrl/latest/page/$page" - } - } - } - - override suspend fun getDetails(manga: Manga): Manga { - - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() - - val chapters = doc.select("tr:has(.chapter)") - .mapNotNull { tr -> - val a = tr.selectFirst("a") ?: return@mapNotNull null - val href = a.absUrl("href") - - if (href.isBlank() || href == "#" || !href.startsWith("http")) return@mapNotNull null - - MangaChapter( - id = generateUid(href), - title = a.text(), - number = extractChapterNumber(href), - volume = 0, - url = href, - uploadDate = parseDate(tr.selectFirst(".update_time")?.text()), - scanlator = null, - branch = null, - source = source - ) - } - .sortedBy { it.number } - - return manga.copy( - title = doc.selectFirst("h1")?.text() - ?: doc.selectFirst("h1.heading")?.text() - ?: manga.title, - - authors = doc.select(".author").eachText().toSet(), - - tags = doc.select(".genres a").map { - MangaTag( - key = it.text().lowercase(), - title = it.text(), - source = source - ) - }.toSet(), - - description = buildString { - append(doc.select(".summary > p").text()) - val alt = doc.select(".alt_name").text() - if (alt.isNotBlank()) { - append("\n\nAlt name(s): $alt") - } - }.trim(), - - state = parseStatus(doc.select(".value.status").text()), - - coverUrl = doc.select("div.media div.cover img").attr("abs:src"), - chapters = chapters - ) - } - - private fun parseStatus(status: String): MangaState? { - return when { - status.contains("Ongoing", true) -> MangaState.ONGOING - status.contains("Completed", true) -> MangaState.FINISHED - else -> null - } - } - - private fun extractChapterNumber(url: String): Float { - return Regex("""/c(\d+(\.\d+)?)""") - .find(url) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - ?: 0f - } - - private fun parseDate(text: String?): Long { - if (text.isNullOrBlank()) return 0L - return runCatching { - dateFormat.parse(text)?.time ?: 0L - }.getOrDefault(0L) - } - - - private fun availableTags() = setOf( - MangaTag("4 koma", "4-koma", source), - MangaTag("Action", "action", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Artbook", "artbook", source), - MangaTag("Award winning", "award-winning", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Doujinshi", "doujinshi", source), - MangaTag("Drama", "drama", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Erotica", "erotica", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Loli", "loli", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("One shot", "one-shot", source), - MangaTag("Overpowered MC", "overpowered-mc", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Romance", "romance", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Sexual violence", "sexual-violence", source), - MangaTag("Shota", "shota", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujo-ai", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Shounen Ai", "shounen-ai", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Sports", "sports", source), - MangaTag("Super power", "super-power", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Survival", "survival", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Yaoi", "yaoi", source), - MangaTag("Yuri", "yuri", source), - ) - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - override suspend fun getPages(chapter: MangaChapter): List { - - val url = applyServer(chapter.url) - val doc = webClient.httpGet(url).parseHtml() - - parseThzq(doc)?.let { - if (it.isNotEmpty()) return it - } - parseHtmlImages(doc)?.let { - if (it.isNotEmpty()) return it - } - val fallbackDoc = webClient.httpGet(chapter.url).parseHtml() - parseThzq(fallbackDoc)?.let { - if (it.isNotEmpty()) return it - } - - return emptyList() - } - - private fun applyServer(url: String): String { - val server = config[preferredServerKey] - return url + server - } - - private fun parseThzq(doc: Document): List? { - - val script = doc.select("script") - .asSequence() - .map { it.data() } - .firstOrNull { it.contains("thzq") } - ?: return null - - val start = script.indexOf("var thzq") - if (start == -1) return null - - val arrStart = script.indexOf('[', start) - if (arrStart == -1) return null - - var depth = 0 - var end = -1 - - for (i in arrStart until script.length) { - when (script[i]) { - '[' -> depth++ - ']' -> { - if (--depth == 0) { - end = i - break - } - } - } - } - - if (end == -1) return null - - val raw = script.substring(arrStart, end) - - val urls = ArrayList(32) - - var i = 0 - while (i < raw.length) { - if (raw[i] == '\'' || raw[i] == '"') { - val quote = raw[i] - val startIdx = ++i - - while (i < raw.length && raw[i] != quote) { - i++ - } - - if (i > startIdx) { - val url = raw.substring(startIdx, i) - if (url.startsWith("http")) { - urls.add(url) - } - } - } - i++ - } - - if (urls.isEmpty()) return null - - return urls.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } - - private fun parseHtmlImages(doc: Document): List? { - - val pages = doc.select("div.wrap_img img") - .asSequence() - .mapNotNull { img -> - - val url = img.attr("data-src") - .ifEmpty { img.attr("src") } - - if (url.isEmpty() || !url.startsWith("http")) return@mapNotNull null - - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - .toList() - - return pages.ifEmpty { null } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt deleted file mode 100644 index 2ecac3ad..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt +++ /dev/null @@ -1,375 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGATOWN", "MangaTown", "en") -internal class MangaTown(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGATOWN, 30) { - - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isYearSupported = false, - ) - - - @Volatile - private var allTagsCache: List? = null - - private suspend fun getAllTags(): List { - if (allTagsCache == null) { - allTagsCache = fetchAvailableTags().toList() - } - return allTagsCache!! - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getAllTags().toSet(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - ), - availableDemographics = EnumSet.of( - Demographic.SHOUJO, - Demographic.SEINEN, - Demographic.SHOUNEN, - Demographic.JOSEI, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHUA, - ContentType.MANHWA, - ), - ) - - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim() - val hasFilters = filter.types.isNotEmpty() || filter.demographics.isNotEmpty() || - filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || - filter.states.isNotEmpty() - - val url = when { - hasFilters -> { - val builder = "https://$domain/search".toHttpUrl().newBuilder() - .addQueryParameter("name_method", "cw") - .addQueryParameter("name", query ?: "") - .addQueryParameter("author_method", "cw") - .addQueryParameter("author", "") - .addQueryParameter("artist_method", "cw") - .addQueryParameter("artist", "") - - builder.addQueryParameter("type", - if (filter.types.size == 1) when (filter.types.first()) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> "" - } else "" - ) - - builder.addQueryParameter("demographic", - if (filter.demographics.size == 1) when (filter.demographics.first()) { - Demographic.SEINEN -> "SEINEN" - Demographic.SHOUJO -> "SHOUJO" - Demographic.SHOUNEN -> "SHOUNEN" - Demographic.JOSEI -> "JOSEI" - else -> "" - } else "" - ) - - if (filter.states.size == 1) { - when (filter.states.first()) { - MangaState.FINISHED -> builder.addQueryParameter("is_completed", "1") - MangaState.ONGOING -> builder.addQueryParameter("is_completed", "0") - else -> builder.addQueryParameter("is_completed", "") - } - } else { - builder.addQueryParameter("is_completed", "") - } - - val allTags = getAllTags() - val includedKeys = filter.tags.map { it.key }.toSet() - val excludedKeys = filter.tagsExclude.map { it.key }.toSet() - for (tag in allTags) { - val value = when (tag.key) { - in includedKeys -> "1" - in excludedKeys -> "2" - else -> "0" - } - builder.addQueryParameter("genres[${tag.title}]", value) - } - - builder.addQueryParameter("released_method", "eq") - builder.addQueryParameter("released", "") - builder.addQueryParameter("rating_method", "eq") - builder.addQueryParameter("rating", "") - builder.addQueryParameter("advopts", "1") - builder.addQueryParameter("page", page.toString()) - - builder.build().toString() - } - !query.isNullOrEmpty() -> { - // Simple search - "https://$domain/search?page=$page&name=${query.urlEncoded()}" - } - order == SortOrder.UPDATED -> { - "https://$domain/latest/$page.htm" - } - else -> { - // Directory browse - buildString { - append("https://$domain/directory/") - append("0-") - append( - if (filter.tags.isNotEmpty()) filter.tags.oneOrThrowIfMany()?.key ?: "0" - else "0" - ) - append("-0-") - append( - if (filter.states.isNotEmpty()) { - filter.states.oneOrThrowIfMany()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> "0" - } - } ?: "0" - } else "0" - ) - append("-0-0/") - append(page).append(".htm") - append( - when (order) { - SortOrder.POPULARITY -> "" - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - else -> "?last_chapter_time.za" - } - ) - } - } - } - - val doc = webClient.httpGet(url, getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("ul.manga_pic_list") ?: return emptyList() - val manga = root.select("li") - if (manga.isEmpty()) return emptyList() - - return manga.mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - val author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim() - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), - source = source, - altTitles = emptySet(), - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - authors = setOfNotNull(author), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), - source = source, - ) - }.orEmpty(), - url = href, - contentRating = null, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - ) - } - } - - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("section.main") - ?.selectFirst("div.article_content") ?: return manga - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - - if (root.select("div.chapter_content:contains(has been licensed)").isNotEmpty()) { - return manga.copy(state = MangaState.ABANDONED, chapters = bypassLicensedChapters(manga)) - } - - return manga.copy( - tags = manga.tags + (info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), - source = source, - ) - }.orEmpty()), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val linkEl = li.selectFirst("a") ?: return@mapChapters null - val href = linkEl.attrAsRelativeUrl("href") - val name = buildString { - append(linkEl.text()) - li.select("span").filter { span -> - !span.hasClass("time") && !span.hasClass("new") - }.forEach { span -> - if (isNotEmpty()) append(" ") - append(span.text()) - } - } - MangaChapter( - id = generateUid(href), - url = href, - source = source, - number = i + 1f, - volume = 0, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - title = name.nullIfEmpty(), - scanlator = null, - branch = null, - ) - } ?: emptyList(), - ) - } - - - private suspend fun bypassLicensedChapters(manga: Manga): List { - val subdomain = "m." + domain.removePrefix("www.") - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(subdomain), getRequestHeaders()).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { a.ownText() } - MangaChapter( - id = generateUid(href), - url = href, - source = source, - number = i + 1f, - volume = 0, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - title = name.nullIfEmpty(), - scanlator = null, - branch = null, - ) - } - } - - - private fun parseChapterDate(dateFormat: SimpleDateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.parseSafe(date) - } - } - - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-") - MangaTag( - source = source, - key = key, - title = a.text().toTitleCase(), - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl, getRequestHeaders()).parseHtml() - val pageSelect = doc.body().selectFirst("div.page_select select") ?: return emptyList() - val options = pageSelect.select("option").filterNot { it.attr("value").endsWith("featured.html") } - if (options.isNotEmpty()) { - return options.mapIndexed { _, option -> - val href = option.attrAsRelativeUrl("value") - MangaPage( - id = generateUid(href), - url = href, - preview = null, - source = source, - ) - } - } - val imgElements = doc.select("div#viewer img") - return imgElements.map { img -> - val src = img.attrAsAbsoluteUrl("src") - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - - override suspend fun getPageUrl(page: MangaPage): String { - if (page.url.startsWith("http")) return page.url - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - return doc.selectFirst("div#viewer img")?.attrAsAbsoluteUrl("src") - ?: throw Exception("Could not find image") - } - - - private fun String.nullIfEmpty(): String? = ifEmpty { null } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt deleted file mode 100644 index 1e74f249..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt +++ /dev/null @@ -1,470 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGADOTNET", "Mangadot.net", "en", ContentType.MANGA) -internal class Mangadotnet(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGADOTNET, 20) { - - override val configKeyDomain = ConfigKey.Domain("mangadot.net") - private val baseUrl = "https://mangadot.net" - private val apiBase = "$baseUrl/api" - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) - - private val genreNames = setOf( - "Josei", "Seinen", "Shoujo", "Shounen", - "Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Historical", "Horror", "Mecha", "Mystery", "Psychological", - "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", - "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", - "Medical", "Military", "Music", "School Life", "Webtoon", - "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", - "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", - "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", - "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", - "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", - "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", - "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", - "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", - "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", - "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", - "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", - "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", - "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", - "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", - "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", - "Wuxia", "Yaoi", "Yuri", "Zombies" - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(showAdultKey) - } - - private fun decodeRsc(flat: JSONArray): Any? { - val cache = arrayOfNulls(flat.length()) - val nil = Any() - fun resolve(i: Int): Any? { - if (i < 0 || i >= flat.length()) return null - val cached = cache[i] - if (cached != null) return if (cached === nil) null else cached - val result: Any? = when (val el = flat.opt(i)) { - JSONObject.NULL -> null - is String, is Number, is Boolean -> el - is JSONArray -> (0 until el.length()).mapTo(mutableListOf()) { j -> resolve(el.optInt(j, -1)) } - is JSONObject -> { - val map = mutableMapOf() - for (key in el.keys()) { - val actualKey = if (key.startsWith("_")) flat.optString(key.removePrefix("_").toInt(), key) else key - map[actualKey] = resolve(el.optInt(key, -1)) - } - map - } - else -> null - } - cache[i] = result ?: nil - return result - } - return resolve(0) - } - - @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscData(url: String, route: String): Map? { - val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) ?: return null - val routeValue = (decoded as? Map)?.get(route) as? Map ?: return null - return routeValue.asMap("data") - } - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchWithFiltersSupported = true, - ) - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, // latest - SortOrder.POPULARITY, // most viewed - SortOrder.RATING, // top rated - SortOrder.ALPHABETICAL, // a-z - SortOrder.RELEVANCE, // most tracked - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = genreNames.map { name -> MangaTag(name, name, source) }.toSet() - return MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableDemographics = EnumSet.of(Demographic.JOSEI, Demographic.SEINEN, Demographic.SHOUJO, Demographic.SHOUNEN), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || - filter.states.isNotEmpty() || filter.types.isNotEmpty() || filter.demographics.isNotEmpty()) { - return getSearchPage(page, order, filter) - } - return getBrowsePage(page, order) - } - - private suspend fun getBrowsePage(page: Int, order: SortOrder): List { - val sortParam = when (order) { - SortOrder.POPULARITY -> "views" - SortOrder.RATING -> "rating" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "tracked" - else -> null - } - val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/view-all/latest-updates.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) - if (page > 1) addQueryParameter("page", page.toString()) - sortParam?.let { addQueryParameter("sortBy", it) } - addQueryParameter("_routes", "pages/ViewAllPage") - }.build().toString() - return parseViewAllPage(url) - } - - private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/search.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) - if (!filter.query.isNullOrBlank()) addQueryParameter("search", filter.query!!) - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "56") - - val (sortBy, sortOrder) = when (order) { - SortOrder.UPDATED -> "latest" to "desc" - SortOrder.ALPHABETICAL -> "alphabetical" to "asc" - SortOrder.POPULARITY -> "views" to "desc" - SortOrder.RATING -> "rating" to "desc" - else -> "relevance" to "desc" - } - addQueryParameter("sortBy", sortBy) - addQueryParameter("sortOrder", sortOrder) - - if (filter.states.size == 1) { - addQueryParameter("status", filter.states.first().toApiStatus()) - } - - filter.types.forEach { ct -> - when (ct) { - ContentType.MANGA -> addQueryParameter("origin", "JP") - ContentType.MANHWA -> addQueryParameter("origin", "KR") - ContentType.MANHUA -> addQueryParameter("origin", "CN") - ContentType.OTHER -> addQueryParameter("origin", "ONESHOT") - else -> {} - } - } - - filter.demographics.forEach { demographic -> - addQueryParameter("genre", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) - } - - filter.tags.forEach { tag -> addQueryParameter("genre", tag.key) } - filter.tagsExclude.forEach { tag -> addQueryParameter("genre", "-${tag.key}") } - - addQueryParameter("_routes", "pages/SearchPage") - }.build().toString() - - val dataMap = fetchRscData(url, "pages/SearchPage") ?: return emptyList() - return parseMangaList(dataMap) - } - - private suspend fun parseViewAllPage(url: String): List { - val viewAllData = fetchRscData(url, "pages/ViewAllPage") ?: return emptyList() - val mangaListMap = viewAllData.asMap("data") ?: return emptyList() - return parseMangaList(mangaListMap) - } - - private fun parseMangaList(data: Map): List { - val list = data["manga_list"] as? List<*> ?: data["results"] as? List<*> ?: return emptyList() - return list.filterIsInstance>().map { parseMangaFromList(it) } - } - - private fun parseMangaFromList(data: Map): Manga { - val id = (data["id"] as? Number)?.toInt()?.toString() ?: "" - val title = data["title"] as? String ?: "" - val photo = data["photo"] as? String - val coverUrl = photo?.let { - when { - it.startsWith("/") -> "$baseUrl$it" - it.startsWith("http") -> it - else -> null - } - } - return Manga( - id = generateUid(id), - url = id, - publicUrl = "$baseUrl/manga/$id", - coverUrl = coverUrl, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - @Suppress("UNCHECKED_CAST") - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return manga - - val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return manga - val title = mangaInfo["title"] as? String ?: manga.title - val description = mangaInfo["description"] as? String - val photo = mangaInfo["photo"] as? String - val coverUrl = photo?.let { - when { - it.startsWith("/") -> "$baseUrl$it" - it.startsWith("http") -> it - else -> null - } - } ?: manga.coverUrl - val bannerImage = mangaInfo["banner_image"] as? String - val largeCoverUrl = bannerImage?.let { - when { - it.startsWith("/") -> "$baseUrl$it" - it.startsWith("http") -> it - else -> null - } - } - - val genres = (mangaInfo["genres"] as? List<*>)?.filterIsInstance()?.mapNotNull { it.trim().ifBlank { null } }.orEmpty() - val altTitles = (mangaInfo["alt_titles"] as? List<*>)?.filterIsInstance()?.mapNotNull { it.trim().ifBlank { null } }?.toSet().orEmpty() - val origin = mangaInfo["country_of_origin"] as? String - val authorRaw = mangaInfo["authors"] as? String - val author = parseJsonArrayString(authorRaw)?.joinToString(", ") ?: authorRaw - - val statusText = mangaInfo["status_text"] as? String - val status = mangaInfo["status"] as? String - val state = when { - !statusText.isNullOrBlank() -> when (statusText.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - else -> null - } - !status.isNullOrBlank() -> when (status.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - else -> null - } - else -> null - } - val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { MangaTag(key = it, title = it, source = source) } - when (origin) { - "JP" -> tagSet.add(MangaTag("Manga", "Manga", source)) - "KR" -> tagSet.add(MangaTag("Manhwa", "Manhwa", source)) - "CN" -> tagSet.add(MangaTag("Manhua", "Manhua", source)) - } - - val tagsArray = mangaInfo["tags"] as? List> - tagsArray?.flatMap { category -> - (category["tags"] as? List> ?: emptyList()) - .mapNotNull { it["name"] as? String } - }?.distinct()?.take(20)?.forEach { tagSet.add(MangaTag(it, it, source)) } - - val rawRating = (mangaInfo["avg_rating"] as? Number)?.toDouble() ?: -1.0 - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val chapters = fetchChapters(manga.url) - - val result = manga.copy( - title = title, - coverUrl = coverUrl, - largeCoverUrl = largeCoverUrl, - rating = rating, - description = description, - altTitles = altTitles, - tags = tagSet, - state = state, - authors = setOfNotNull(author).filterTo(mutableSetOf()) { it.isNotBlank() }, - chapters = chapters, - ) - - detailsCache[manga.url] = result - return result - } - - private fun parseJsonArrayString(raw: String?): List? { - if (raw.isNullOrBlank()) return null - return try { - val arr = JSONArray(raw) - (0 until arr.length()).mapNotNull { arr.optString(it) } - } catch (_: Exception) { null } - } - - private suspend fun fetchChapters(mangaId: String): List { - val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=en").parseJsonArray() - val allChapters = (0 until response.length()).map { response.getJSONObject(it) } - - val chaptersByTeam = mutableMapOf>() - for (ch in allChapters) { - val team = listOf( - ch.optString("group_name", ""), - ch.optString("scanlator_name", "") - ).firstOrNull { it.isNotBlank() && !it.equals("null", ignoreCase = true) } - ?: "Unknown" - chaptersByTeam.getOrPut(team) { mutableListOf() }.add(ch) - } - - val allNumbers = allChapters.map { it.optDouble("chapter_number", 0.0).toFloat() }.distinct().sorted() - val builder = ChaptersListBuilder(allChapters.size * chaptersByTeam.size) - - for ((team, teamChapters) in chaptersByTeam) { - val mapByNum = teamChapters.associateBy { it.optDouble("chapter_number", 0.0).toFloat() } - for (num in allNumbers) { - val chapter = mapByNum[num] ?: continue - val chId = chapter.getString("id") - val src = chapter.optString("source", "user") - val name = chapter.optString("chapter_title", "").nullIfEmpty() - val date = chapter.optString("date_added", "").nullIfEmpty() - val title = buildString { - val numStr = num.toString().removeSuffix(".0") - append("Chapter $numStr") - val cleanName = name?.trim()?.takeIf { it.isNotEmpty() && !it.equals("null", ignoreCase = true) } - if (cleanName != null && !cleanName.contains(numStr)) { - append(": ").append(cleanName) - } - } - val chapterUrl = JSONObject().apply { - put("id", chId) - put("source", src) - }.toString() - builder.add( - MangaChapter( - id = generateUid("$team-$chId"), - title = title, - number = num, - volume = 0, - url = chapterUrl, - uploadDate = date?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = team, - branch = team, - ) - ) - } - } - - val volumesResponse = runCatching { - webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=en").parseJsonArray() - }.getOrNull() - - if (volumesResponse != null) { - val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } - .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } - - for (vol in volumes) { - if (vol.optString("language", "en") != "en") continue - val volId = vol.getInt("id") - val volNum = vol.optDouble("volume_number", 0.0).toFloat() - val volDate = vol.optString("date_added", "").nullIfEmpty() - val volSource = vol.optString("source", "user") - val volGroup = vol.optString("group_name", "").nullIfEmpty() - ?: vol.optString("scanlator_name", "").nullIfEmpty() - ?: "Unknown" - val volumeUrl = JSONObject().apply { - put("id", volId) - put("source", volSource) - }.toString() - builder.add( - MangaChapter( - id = generateUid("vol-$volId"), - title = "Volume ${volNum.toString().removeSuffix(".0")}", - number = volNum, - volume = 0, - url = volumeUrl, - uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = volGroup, - branch = "Volume", - ) - ) - } - } - - return builder.toList() - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterData = JSONObject(chapter.url) - val chapterId = chapterData.getString("id") - val source = chapterData.optString("source", "user") - val segment = if (source == "user") "uploads" else "chapters" - val response = webClient.httpGet("$baseUrl/api/$segment/$chapterId/images").parseJson() - val data = response.optJSONObject("data") - val imagesArray = if (data != null && data.has("images")) data.getJSONArray("images") else response.getJSONArray("images") - return (0 until imagesArray.length()).mapNotNull { i -> - val img = imagesArray.getJSONObject(i) - val imgUrl = img.optString("url", "").nullIfEmpty() ?: return@mapNotNull null - val fullUrl = when { - imgUrl.startsWith("/") -> "$baseUrl$imgUrl" - imgUrl.startsWith("http") -> imgUrl - else -> return@mapNotNull null - } - MangaPage( - id = generateUid(fullUrl), - url = fullUrl, - preview = null, - source = this.source, - ) - } - } - - - @Suppress("UNCHECKED_CAST") - override suspend fun getRelatedManga(seed: Manga): List { - val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" - val data = fetchRscData(url, "pages/MangaDetailPage") ?: return emptyList() - - val related = mutableListOf() - (data["suggestions"] as? List<*>)?.filterIsInstance>()?.mapTo(related) { parseMangaFromList(it) } - val relationsData = data.asMap("relationsData") - val relations = relationsData?.asMap("relations") as? Map>> - relations?.values?.forEach { list -> list.mapTo(related) { parseMangaFromList(it) } } - - return related - } - - private fun MangaState.toApiStatus() = when (this) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - MangaState.PAUSED -> "Hiatus" - else -> "" - } - - @Suppress("UNCHECKED_CAST") - private fun Map.asMap(key: String): Map? = this[key] as? Map - private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt deleted file mode 100644 index 0d9fbed8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt +++ /dev/null @@ -1,685 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.Interceptor -import okhttp3.Response -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.bitmap.Bitmap -import org.koitharu.kotatsu.parsers.bitmap.Rect -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* -import javax.crypto.Cipher -import javax.crypto.spec.IvParameterSpec -import javax.crypto.spec.SecretKeySpec -import kotlin.math.min - -@MangaSourceParser("MANGAGO", "Mangago", "en") -internal class MangagoParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAGO, pageSize = 20), Interceptor { - - override val configKeyDomain = ConfigKey.Domain("www.mangago.me") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getAvailableTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - private fun getAvailableTags(): Set = setOf( - MangaTag("Yaoi", "Yaoi", source), - MangaTag("Doujinshi", "Doujinshi", source), - MangaTag("Shounen Ai", "Shounen Ai", source), - MangaTag("Shoujo", "Shoujo", source), - MangaTag("Yuri", "Yuri", source), - MangaTag("Romance", "Romance", source), - MangaTag("Fantasy", "Fantasy", source), - MangaTag("Comedy", "Comedy", source), - MangaTag("Smut", "Smut", source), - MangaTag("Adult", "Adult", source), - MangaTag("School Life", "School Life", source), - MangaTag("Mystery", "Mystery", source), - MangaTag("One Shot", "One Shot", source), - MangaTag("Ecchi", "Ecchi", source), - MangaTag("Shounen", "Shounen", source), - MangaTag("Martial Arts", "Martial Arts", source), - MangaTag("Shoujo Ai", "Shoujo Ai", source), - MangaTag("Supernatural", "Supernatural", source), - MangaTag("Drama", "Drama", source), - MangaTag("Action", "Action", source), - MangaTag("Adventure", "Adventure", source), - MangaTag("Harem", "Harem", source), - MangaTag("Historical", "Historical", source), - MangaTag("Horror", "Horror", source), - MangaTag("Josei", "Josei", source), - MangaTag("Mature", "Mature", source), - MangaTag("Mecha", "Mecha", source), - MangaTag("Psychological", "Psychological", source), - MangaTag("Sci-fi", "Sci-fi", source), - MangaTag("Seinen", "Seinen", source), - MangaTag("Slice Of Life", "Slice Of Life", source), - MangaTag("Sports", "Sports", source), - MangaTag("Gender Bender", "Gender Bender", source), - MangaTag("Tragedy", "Tragedy", source), - MangaTag("Bara", "Bara", source), - MangaTag("Shotacon", "Shotacon", source), - MangaTag("Webtoons", "Webtoons", source), - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query.orEmpty() - if (!query.isNullOrEmpty()) { - val url = buildString { - append("https://") - append(domain) - append("/r/l_search?name=") - append(query.urlEncoded()) - append("&page=") - append(page) - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - val url = when (order) { - SortOrder.NEWEST -> { - buildString { - append("https://") - append(domain) - append("/list/new/") - if (filter.tags.isNotEmpty()) { - filter.tags.joinTo(this, ",") { it.key } - } else { - append("all") - } - append("/") - append(page) - append("/") - } - } - else -> { - buildString { - append("https://") - append(domain) - append("/genre/") - if (filter.tags.isNotEmpty()) { - filter.tags.joinTo(this, ",") { it.key } - } else { - append("all") - } - append("/") - append(page) - append("/?") - val states = filter.states - val showFinished = states.isEmpty() || states.contains(MangaState.FINISHED) - val showOngoing = states.isEmpty() || states.contains(MangaState.ONGOING) - append("f=").append(if (showFinished) "1" else "0") - append("&o=").append(if (showOngoing) "1" else "0") - append("&sortby=") - when (order) { - SortOrder.POPULARITY -> append("view") - SortOrder.UPDATED -> append("update_date") - else -> append("update_date") - } - append("&e=") - if (filter.tagsExclude.isNotEmpty()) { - filter.tagsExclude.joinTo(this, ",") { it.key } - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - private fun parseMangaList(doc: Document): List { - return doc.select(".box, .updatesli, .pic_list > li").mapNotNull { element -> - val linkElement = element.selectFirst(".thm-effect") ?: return@mapNotNull null - val href = linkElement.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null - val title = linkElement.attr("title").ifEmpty { - linkElement.selectFirst("h2 a")?.text() ?: return@mapNotNull null - } - val thumbnailElem = linkElement.selectFirst("img") ?: return@mapNotNull null - val thumbnailUrl = thumbnailElem.attr("abs:data-src").ifEmpty { - thumbnailElem.attr("abs:src") - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = title, - altTitles = emptySet(), - coverUrl = thumbnailUrl, - largeCoverUrl = null, - description = null, - tags = emptySet(), - authors = emptySet(), - state = null, - rating = RATING_UNKNOWN, - contentRating = ContentRating.SAFE, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val infoBlock = doc.requireElementById("information") - - val title = doc.selectFirstOrThrow(".w-title h1").text() - val thumbnail = infoBlock.selectFirstOrThrow("img").absUrl("src") - - var description = infoBlock.selectFirst(".manga_summary")?.let { summary -> - summary.selectFirst("font")?.remove() - summary.text() - }.orEmpty() - - var author: String? = null - var genres = emptySet() - var state: MangaState? = null - - infoBlock.select(".manga_info li, .manga_right tr").forEach { el -> - when (el.selectFirst("b, label")?.text()?.lowercase()) { - "alternative:" -> description += "\n\n${el.text()}" - "status:" -> state = when (el.selectFirstOrThrow("span").text().lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - "author(s):", "author:" -> author = el.select("a").joinToString { it.text() } - "genre(s):" -> genres = el.select("a").mapToSet { - MangaTag( - key = it.text(), - title = it.text(), - source = source, - ) - } - } - } - - val chapters = parseChapterList(doc) - - return manga.copy( - title = title, - coverUrl = thumbnail, - description = description, - authors = author?.let { setOf(it) } ?: emptySet(), - tags = genres, - state = state, - chapters = chapters, - ) - } - - private data class ChapterParseData( - val name: String, - val url: String, - val dateUpload: Long, - val scanlator: String?, - ) - - private fun parseChapterList(doc: Document): List { - val rawChapters = doc.select("table#chapter_table > tbody > tr, table.uk-table > tbody > tr") - .mapNotNull { element -> - val link = element.selectFirst("a.chico") ?: return@mapNotNull null - val url = link.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null - val name = link.text().trim() - - val dateText = element.select("td:last-child").text().trim() - val dateUpload = runCatching { - dateFormat.parse(dateText)?.time - }.getOrNull() ?: 0L - - val scanlator = element.selectFirst("td.no a, td.uk-table-shrink a")?.text()?.trim() - ?.ifEmpty { null } - - ChapterParseData(name, url, dateUpload, scanlator) - } - .reversed() - - return buildChapterList(rawChapters) - } - - private fun buildChapterList(chapters: List): List { - val scanlatorCounts = mutableMapOf() - for (chapter in chapters) { - val scanlator = chapter.scanlator ?: extractTitleSuffix(chapter.name) ?: continue - scanlatorCounts[scanlator] = (scanlatorCounts[scanlator] ?: 0) + 1 - } - val preferredScanlator = scanlatorCounts.maxByOrNull { it.value }?.key - - val regularChapters = mutableMapOf() - val specialChapters = mutableListOf() - - for (chapter in chapters) { - val chapterNum = extractChapterNumber(chapter.name) - if (chapterNum != null) { - val existing = regularChapters[chapterNum] - if (existing == null) { - regularChapters[chapterNum] = chapter - } else { - val existingSource = existing.scanlator ?: extractTitleSuffix(existing.name) - val newSource = chapter.scanlator ?: extractTitleSuffix(chapter.name) - - val existingIsPreferred = existingSource == preferredScanlator - val newIsPreferred = newSource == preferredScanlator - - if (newIsPreferred && !existingIsPreferred) { - regularChapters[chapterNum] = chapter - } else if (!newIsPreferred && existingIsPreferred) { - } else { - if (chapter.name.length > existing.name.length) { - regularChapters[chapterNum] = chapter - } - } - } - } else { - specialChapters.add(chapter) - } - } - - val result = mutableListOf() - - val sortedRegular = regularChapters.entries.sortedBy { it.key } - for ((chapterNum, chapter) in sortedRegular) { - result.add( - MangaChapter( - id = generateUid(chapter.url), - url = chapter.url, - title = chapter.name, - number = chapterNum, - volume = 0, - uploadDate = chapter.dateUpload, - scanlator = chapter.scanlator, - branch = null, - source = source, - ), - ) - } - - val baseSpecialNumber = (regularChapters.keys.maxOrNull() ?: 0f) + 10000f - for ((index, chapter) in specialChapters.withIndex()) { - result.add( - MangaChapter( - id = generateUid(chapter.url), - url = chapter.url, - title = chapter.name, - number = baseSpecialNumber + index, - volume = 0, - uploadDate = chapter.dateUpload, - scanlator = chapter.scanlator, - branch = null, - source = source, - ), - ) - } - - return result - } - - private fun extractTitleSuffix(title: String): String? { - val regex = Regex("""(?:ch\.?|chapter)\s*\d+(?:\.\d+)?\s*[:\-]\s*(.+)""", RegexOption.IGNORE_CASE) - return regex.find(title)?.groupValues?.get(1)?.trim()?.takeIf { it.isNotEmpty() } - } - - private fun extractChapterNumber(title: String): Float? { - return try { - val regex = Regex("""(?:ch\.?|chapter|vol\.?\s*\d+\s+ch\.?)\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - val match = regex.find(title) - match?.groupValues?.get(1)?.toFloat() - } catch (e: Exception) { - null - } - } - - private val jsCache = mutableMapOf>() - private val maxCacheTime = 1000 * 60 * 5 // 5 minutes - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val pageDropdown = doc.select("div.controls ul#dropdown-menu-page") - if (pageDropdown.isNotEmpty()) { - val pagesCount = pageDropdown.select("li").size - - val cleanUrl = fullUrl.removeSuffix("/") - val lastSegment = cleanUrl.substringAfterLast("/") - - val isPgFormat = lastSegment.startsWith("pg-") - val pageNumber = lastSegment.toIntOrNull() - val isPageNumber = pageNumber != null && pageNumber < 1000 - - val baseUrl: String - val buildBatchUrl: (Int) -> String - - if (isPgFormat) { - baseUrl = cleanUrl.substringBeforeLast("/") - buildBatchUrl = { batchNum -> "$baseUrl/pg-$batchNum/" } - } else if (isPageNumber) { - baseUrl = cleanUrl.substringBeforeLast("/") - buildBatchUrl = { batchNum -> "$baseUrl/$batchNum/" } - } else { - baseUrl = cleanUrl - buildBatchUrl = { batchNum -> "$baseUrl/$batchNum/" } - } - - val batchSize = 5 - val allImages = mutableListOf() - var batchStart = 1 - - val js = getDeobfuscatedJS(doc) - val cols = getColsFromDoc(doc) ?: "" - - while (allImages.size < pagesCount) { - val batchUrl = buildBatchUrl(batchStart) - val batchDoc = if (batchStart == 1) doc else webClient.httpGet(batchUrl).parseHtml() - val batchImages = decryptImageList(batchDoc, batchUrl) - - if (batchImages.isEmpty()) { - break - } - - allImages.addAll(batchImages) - batchStart += batchSize - - if (batchStart > pagesCount + batchSize) { - break - } - } - - return allImages.take(pagesCount).mapIndexed { index, imageUrl -> - val url = if (imageUrl.contains("cspiclink") && js != null) { - val descramblingKey = getDescramblingKey(js, imageUrl) - "$imageUrl#desckey=$descramblingKey&cols=$cols" - } else { - imageUrl - } - - MangaPage( - id = generateUid("$fullUrl-$index"), - url = url, - preview = null, - source = source, - ) - } - } - - val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() - - if (imgsrcsScript != null) { - val images = decryptImageList(doc, fullUrl) - val cols = getColsFromDoc(doc) ?: "" - val js = getDeobfuscatedJS(doc) ?: throw Exception("Could not get JS") - - return images.mapIndexed { index, imageUrl -> - val url = if (imageUrl.contains("cspiclink")) { - val descramblingKey = getDescramblingKey(js, imageUrl) - "$imageUrl#desckey=$descramblingKey&cols=$cols" - } else { - imageUrl - } - - if (url.isBlank()) { - throw Exception("Final image URL at index $index is blank after processing") - } - - MangaPage( - id = generateUid("$fullUrl-$index"), - url = url, - preview = null, - source = source, - ) - } - } - - throw Exception("Could not find pages") - } - - override suspend fun getPageUrl(page: MangaPage): String { - if (page.url.isBlank()) { - throw Exception("Page URL is blank in getPageUrl") - } - return page.url - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - private suspend fun decryptImageList(doc: Document, sourceUrl: String): List { - val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() - ?: throw Exception("Could not find imgsrcs") - - val imgsrcRaw = IMG_SRCS_REGEX.find(imgsrcsScript)?.groupValues?.get(1) - ?: throw Exception("Could not extract imgsrcs") - - val imgsrcs = context.decodeBase64(imgsrcRaw) - - val deobfChapterJs = getDeobfuscatedJS(doc) ?: throw Exception("Could not deobfuscate JS") - - val key = findHexEncodedVariable(deobfChapterJs, "key").decodeHex() - val iv = findHexEncodedVariable(deobfChapterJs, "iv").decodeHex() - - val cipher = Cipher.getInstance("AES/CBC/NoPadding") - val keySpec = SecretKeySpec(key, "AES") - cipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) - val decryptedBytes = cipher.doFinal(imgsrcs) - - var imageList = String(decryptedBytes, Charsets.UTF_8).trimEnd('\u0000') - imageList = unscrambleImageList(imageList, deobfChapterJs) - - return imageList.split(",") - .map { it.trim() } - .filter { it.isNotBlank() } - .map { url -> - if (url.startsWith("https://") && url.contains("/_") || url.contains("https://iweb_")) { - url.replaceFirst("https://", "http://") - } else { - url - } - } - } - - private suspend fun getDeobfuscatedJS(doc: Document): String? { - val chapterJsUrl = doc.select("script[src*=chapter.js]").firstOrNull()?.absUrl("src") ?: return null - - val now = System.currentTimeMillis() - val cached = jsCache[chapterJsUrl] - if (cached != null && now - cached.second < maxCacheTime) { - return cached.first - } - - val obfuscatedChapterJs = webClient.httpGet(chapterJsUrl).parseRaw() - val deobf = deobfuscateSoJsonV4(obfuscatedChapterJs) - - jsCache[chapterJsUrl] = Pair(deobf, now) - return deobf - } - - private fun getColsFromDoc(doc: Document): String? { - val chapterJsUrl = doc.select("script[src*=chapter.js]").firstOrNull()?.absUrl("src") - val cached = if (chapterJsUrl != null) jsCache[chapterJsUrl] else null - val js = cached?.first ?: return null - return COLS_REGEX.find(js)?.groupValues?.get(1) - } - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - val fragment = response.request.url.fragment - - if (fragment == null || !fragment.contains("desckey=")) { - return response - } - - return context.redrawImageResponse(response) { bitmap -> - val key = fragment.substringAfter("desckey=").substringBefore("&") - val cols = fragment.substringAfter("cols=").toIntOrNull() ?: return@redrawImageResponse bitmap - unscrambleImage(bitmap, key, cols) - } - } - - private fun unscrambleImage(bitmap: Bitmap, key: String, cols: Int): Bitmap { - val width = bitmap.width - val height = bitmap.height - - val result = context.createBitmap(width, height) - - val unitWidth = width / cols - val unitHeight = height / cols - - val keyArray = key.split("a") - - for (idx in 0 until cols * cols) { - val keyval = keyArray.getOrNull(idx)?.takeIf { it.isNotEmpty() }?.toIntOrNull() ?: 0 - - val heightY = keyval / cols - val dy = heightY * unitHeight - val dx = (keyval - heightY * cols) * unitWidth - - val widthY = idx / cols - val sy = widthY * unitHeight - val sx = (idx - widthY * cols) * unitWidth - - val w = min(unitWidth, width - dx) - val h = min(unitHeight, height - dy) - - val srcRect = Rect(sx, sy, sx + w, sy + h) - val dstRect = Rect(dx, dy, dx + w, dy + h) - - result.drawBitmap(bitmap, srcRect, dstRect) - } - - return result - } - - private fun deobfuscateSoJsonV4(jsf: String): String { - if (!jsf.startsWith("['sojson.v4']")) { - throw Exception("Obfuscated code header mismatch. Expected sojson.v4") - } - - val splitRegex = Regex("[a-zA-Z]+") - val args = jsf.substring(240, jsf.length - 59).split(splitRegex) - - return args.map { it.toInt().toChar() }.joinToString("") - } - - private fun findHexEncodedVariable(input: String, variable: String): String { - val regex = Regex("""var $variable\s*=\s*CryptoJS\.enc\.Hex\.parse\("([0-9a-zA-Z]+)"\)""") - return regex.find(input)?.groupValues?.get(1) - ?: throw Exception("Could not find variable: $variable") - } - - private fun String.decodeHex(): ByteArray { - check(length % 2 == 0) { "Must have an even length" } - return chunked(2).map { it.toInt(16).toByte() }.toByteArray() - } - - private fun unscrambleImageList(imageList: String, js: String): String { - var imgList = imageList - - val keyLocations = KEY_LOCATION_REGEX.findAll(js) - .map { it.groupValues[1].toInt() } - .distinct() - .sorted() - .toList() - - if (keyLocations.isEmpty()) { - return imgList - } - - val unscrambleKey = try { - keyLocations.map { loc -> - if (loc >= imgList.length) { - throw NumberFormatException("Position $loc is beyond string length") - } - imgList[loc].toString().toInt() - } - } catch (e: NumberFormatException) { - return imgList - } - - keyLocations.forEachIndexed { idx, loc -> - imgList = imgList.removeRange((loc - idx)..(loc - idx)) - } - - imgList = imgList.unscramble(unscrambleKey) - - return imgList - } - - private fun String.unscramble(keys: List): String { - var s = this - keys.reversed().forEach { key -> - for (i in s.length - 1 downTo key) { - if (i % 2 != 0) { - val sourceIdx = i - key - if (sourceIdx >= 0) { - val temp = s[sourceIdx] - s = s.substring(0, sourceIdx) + s[i] + s.substring(sourceIdx + 1) - s = s.substring(0, i) + temp + s.substring(i + 1) - } - } - } - } - return s - } - - private suspend fun getDescramblingKey(deobfChapterJs: String, imageUrl: String): String { - val imgkeys = deobfChapterJs - .substringAfter("var renImg = function(img,width,height,id){", "") - .substringBefore("key = key.split(", "") - .split("\n") - .filter { line -> JS_FILTERS.none { filter -> line.contains(filter) } } - .joinToString("\n") - .replace("img.src", "url") - - if (imgkeys.isEmpty()) { - throw Exception("Failed to extract image key extraction code from chapter.js") - } - - val js = """ - function replacePos(strObj, pos, replacetext) { - var str = strObj.substr(0, pos) + replacetext + strObj.substring(pos + 1, strObj.length); - return str; - } - function getDescramblingKey(url) { $imgkeys; return key; } - getDescramblingKey("$imageUrl"); - """.trimIndent() - - val result = context.evaluateJs("https://$domain/", js) - - if (result.isNullOrEmpty()) { - throw Exception("Failed to evaluate JavaScript to get descrambling key. The image format or site structure may have changed.") - } - - return result - } - - private companion object { - private val IMG_SRCS_REGEX = Regex("""var imgsrcs\s*=\s*['"]([a-zA-Z0-9+=/]+)['"]""") - private val COLS_REGEX = Regex("""var\s*widthnum\s*=\s*heightnum\s*=\s*(\d+);""") - private val KEY_LOCATION_REGEX = Regex("""str\.charAt\(\s*(\d+)\s*\)""") - private val JS_FILTERS = listOf("jQuery", "document", "getContext", "toDataURL", "getImageData", "width", "height") - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt deleted file mode 100644 index 7b3f82d0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANGAREAD", "MangaRead", "en") -internal class MangaRead(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt deleted file mode 100644 index 205ae6ea..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt +++ /dev/null @@ -1,861 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.Headers -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseRaw -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import okhttp3.HttpUrl -import java.security.MessageDigest -import java.time.ZoneOffset -import java.time.ZonedDateTime -import java.time.format.DateTimeFormatter -import java.util.ArrayList -import java.util.Calendar -import java.util.EnumSet -import java.util.HashSet -import java.util.LinkedHashSet -import java.util.Locale - -@MangaSourceParser("MANGATAROORG", "Mangataro", "en", ContentType.MANGA) -internal class Mangataro(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGATAROORG, 48) { - - override val configKeyDomain = ConfigKey.Domain("mangataro.org") - - private val chapterNumberRegex = Regex("""(\d+(?:\.\d+)?)""") - - private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - isYearSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = genreTags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - ) - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val payload = JSONObject().apply { - put("page", page) - put("search", filter.query.orEmpty()) - - // Year - val yearArray = JSONArray() - if (filter.year > 0) yearArray.put(filter.year.toString()) - put("years", yearArray.toString()) - - // Genres (tags) - put("genres", filter.tags.toGenrePayload()) - - // Types - val typeArray = JSONArray() - filter.types.forEach { ct -> - val apiType = when (ct) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.OTHER -> "Novel" - else -> null - } - if (apiType != null) typeArray.put(apiType) - } - put("types", typeArray.toString()) - - // Statuses - val statusArray = JSONArray() - filter.states.forEach { state -> - val apiStatus = when (state) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - else -> null - } - if (apiStatus != null) statusArray.put(apiStatus) - } - put("statuses", statusArray.toString()) - - put("sort", order.toApiSort()) - put("genreMatchMode", "any") - } - val json = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) - return List(json.length()) { index -> - val item = json.getJSONObject(index) - item.toManga() - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val detailUrl = manga.url.urlWithoutFragment().toAbsoluteUrl(domain) - val document = fetchDocument(detailUrl) - val jsonLd = document.findSeriesJsonLd() - val description = document.extractDescription(jsonLd) - val altTitles = document.extractAltTitles(jsonLd) - val tags = document.extractTags(jsonLd) - val authors = document.extractAuthors(jsonLd) - val state = document.extractStatus() - val rating = document.extractRating(jsonLd) - val mangaId = extractMangaId(manga.url) - ?: document.selectFirst("[data-manga-id]")?.attr("data-manga-id") - ?: jsonLd?.optString("identifier")?.nullIfEmpty() - ?: throw ParseException("Unable to determine manga id", detailUrl) - val chapters = loadChapters(mangaId, detailUrl) - return manga.copy( - description = description, - tags = tags.ifEmpty { manga.tags }, - authors = authors.ifEmpty { manga.authors }, - altTitles = altTitles.ifEmpty { manga.altTitles }, - state = state, - rating = rating, - chapters = chapters, - ) - } - - private suspend fun loadChapters(mangaId: String, referer: String): List { - val headers = Headers.headersOf("Referer", referer) - val result = ArrayList() - val seenIds = HashSet() - var offset = 0 - val limit = 500 - var expectedTotal = Int.MAX_VALUE - while (offset < expectedTotal) { - val (token, timestamp) = generateChapterToken() - val requestUrl = buildChapterUrl(mangaId, offset, limit, token, timestamp) - val raw = webClient.httpGet(requestUrl, headers).parseRaw().trim() - if (raw.isEmpty()) { - break - } - val json = runCatching { JSONObject(raw) }.getOrNull() ?: break - if (!json.optBoolean("success", false)) { - break - } - expectedTotal = json.optInt("total", expectedTotal) - val chaptersArray = json.optJSONArray("chapters") ?: break - if (chaptersArray.length() == 0) { - break - } - for (i in 0 until chaptersArray.length()) { - val item = chaptersArray.getJSONObject(i) - val key = item.optString("id").ifEmpty { item.optString("url") } - if (!seenIds.add(key)) { - continue - } - val fallbackNumber = (offset + i + 1).toFloat() - result.add(item.toMangaChapter(fallbackNumber)) - } - offset += chaptersArray.length() - if (!json.optBoolean("has_more", false)) { - break - } - } - return result - } - - private fun buildChapterUrl( - mangaId: String, - offset: Int, - limit: Int, - token: String, - timestamp: String, - ): String { - return HttpUrl.Builder() - .scheme("https") - .host(domain) - .addPathSegment("auth") - .addPathSegment("manga-chapters") - .addQueryParameter("manga_id", mangaId) - .addQueryParameter("offset", offset.toString()) - .addQueryParameter("limit", limit.toString()) - .addQueryParameter("order", "ASC") - .addQueryParameter("_t", token) - .addQueryParameter("_ts", timestamp) - .build() - .toString() - } - - private fun generateChapterToken(): Pair { - val timestamp = (System.currentTimeMillis() / 1000L).toString() - val hourKey = ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter) - val secret = "mng_ch_$hourKey" - val hashInput = timestamp + secret - val token = MessageDigest - .getInstance("MD5") - .digest(hashInput.toByteArray()) - .toHex() - .substring(0, 16) - return token to timestamp - } - - private fun ByteArray.toHex(): String { - if (isEmpty()) { - return "" - } - val chars = CharArray(size * 2) - var index = 0 - for (element in this) { - val unsigned = element.toInt() and 0xFF - chars[index++] = Character.forDigit((unsigned ushr 4) and 0xF, 16) - chars[index++] = Character.forDigit(unsigned and 0xF, 16) - } - return String(chars) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - val chapterId = chapterUrl.extractChapterId() - ?: fetchDocument(chapterUrl).body().attr("data-chapter-id").nullIfEmpty() - if (chapterId != null) { - val pages = loadChapterPages(chapterId, chapterUrl) - if (pages.isNotEmpty()) { - return pages - } - } - val document = fetchDocument(chapterUrl) - val containers = document.select("div.comic-image-container img") - val seen = HashSet(containers.size) - return containers.mapNotNull { element -> - val imageUrl = element.resolveImageUrl() - ?.takeUnless { it.startsWith("data:") } - ?: return@mapNotNull null - if (!seen.add(imageUrl)) { - return@mapNotNull null - } - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun loadChapterPages(chapterId: String, referer: String): List { - val headers = Headers.headersOf( - "Accept", - "application/json", - "Referer", - referer, - ) - val raw = webClient.httpGet("https://$domain/auth/chapter-content?chapter_id=$chapterId", headers) - .parseRaw() - .trim() - if (raw.isEmpty()) { - return emptyList() - } - val json = raw.toJsonObjectOrNull() ?: return emptyList() - if (!json.optBoolean("success", false) || json.optString("chapter_type") == "text") { - return emptyList() - } - val images = json.optJSONArray("images") ?: return emptyList() - return List(images.length()) { index -> - val imageUrl = images.getString(index) - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - private fun JSONObject.toManga(): Manga { - val id = optString("id") - val rawUrl = optString("url") - val relativeUrl = if (rawUrl.isBlank()) "/" else rawUrl.toRelativeUrl(domain) - val internalUrl = relativeUrl.withMangaId(id) - val publicUrl = rawUrl.nullIfEmpty() ?: relativeUrl.toAbsoluteUrl(domain) - return Manga( - id = generateUid(internalUrl), - title = optString("title"), - altTitles = emptySet(), - url = internalUrl, - publicUrl = publicUrl, - rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, - contentRating = null, - coverUrl = optString("cover").nullIfEmpty(), - tags = emptySet(), - state = optString("status").toMangaState(), - authors = emptySet(), - description = optString("description").nullIfEmpty(), - source = source, - ) - } - - private fun JSONObject.toMangaChapter(fallbackNumber: Float): MangaChapter { - val chapterUrl = optString("url") - val rawChapter = optString("chapter") - val chapterLabel = rawChapter.nullIfEmpty() ?: fallbackNumber.toChapterLabel() - val rawTitle = optString("title").trim() - val title = when { - rawTitle.equals("N/A", ignoreCase = true) -> "Chapter $chapterLabel" - rawTitle.isEmpty() -> null - else -> rawTitle - } - val number = chapterNumberRegex.find(rawChapter)?.value?.toFloatOrNull() - return MangaChapter( - id = generateUid(chapterUrl), - title = title, - number = number ?: fallbackNumber, - volume = 0, - url = chapterUrl.toRelativeUrl(domain), - scanlator = null, - uploadDate = parseChapterDate(optString("date")), - branch = null, - source = source, - ) - } - - private fun Float.toChapterLabel(): String = if (this % 1f == 0f) { - toInt().toString() - } else { - replaceTrailingZeros() - } - - private fun Float.replaceTrailingZeros(): String { - val text = toString() - val trimmed = text.trimEnd('0') - return if (trimmed.endsWith('.')) { - trimmed.dropLast(1) - } else { - trimmed - } - } - - private fun String?.toMangaState(): MangaState? { - val normalized = this?.lowercase(Locale.US)?.trim() ?: return null - return when { - normalized.contains("ongoing") -> MangaState.ONGOING - normalized.contains("complete") || normalized.contains("completed") -> MangaState.FINISHED - normalized.contains("hiatus") -> MangaState.PAUSED - normalized.contains("canceled") || normalized.contains("cancelled") -> MangaState.ABANDONED - normalized.contains("upcoming") || normalized.contains("tba") -> MangaState.UPCOMING - else -> null - } - } - - private fun parseChapterDate(raw: String?): Long { - val value = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L - if (!value.endsWith("ago")) { - return 0L - } - val amount = chapterNumberRegex.find(value)?.value?.toFloatOrNull() ?: return 0L - val calendar = Calendar.getInstance() - when { - value.contains("second") -> calendar.add(Calendar.SECOND, -amount.toInt()) - value.contains("minute") -> calendar.add(Calendar.MINUTE, -amount.toInt()) - value.contains("hour") -> calendar.add(Calendar.HOUR_OF_DAY, -amount.toInt()) - value.contains("day") -> calendar.add(Calendar.DAY_OF_YEAR, -amount.toInt()) - value.contains("week") -> calendar.add(Calendar.WEEK_OF_YEAR, -amount.toInt()) - value.contains("month") -> calendar.add(Calendar.MONTH, -amount.toInt()) - value.contains("year") -> calendar.add(Calendar.YEAR, -amount.toInt()) - else -> return 0L - } - return calendar.timeInMillis - } - - private fun SortOrder.toApiSort(): String = when (this) { - SortOrder.POPULARITY -> "popular_desc" - SortOrder.UPDATED -> "post_desc" - SortOrder.NEWEST -> "release_desc" - SortOrder.ALPHABETICAL -> "title_desc" - else -> "post_desc" - } - - private fun Document.extractDescription(jsonLd: JSONObject?): String? { - val fromJson = jsonLd?.optString("description")?.nullIfEmpty() - if (fromJson != null) { - return fromJson - } - return selectFirst("div[data-description], div#description, div#synopsis, div:matchesOwn((?i)description)") - ?.nextElementSibling() - ?.text() - ?.nullIfEmpty() - ?: selectFirst("meta[name='description']")?.attr("content")?.nullIfEmpty() - ?: selectFirst("meta[property='og:description']")?.attr("content")?.nullIfEmpty() - } - - private fun Document.extractAltTitles(jsonLd: JSONObject?): Set { - val alt = mutableSetOf() - jsonLd?.opt("alternateName")?.let { value -> - when (value) { - is JSONArray -> { - repeat(value.length()) { index -> - value.optString(index).nullIfEmpty()?.let(alt::add) - } - } - is String -> value.nullIfEmpty()?.let(alt::add) - } - } - if (alt.isEmpty()) { - selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { alt.addAll(it) } - } - return alt - } - - private fun Document.extractTags(jsonLd: JSONObject?): Set { - val tags = LinkedHashSet() - jsonLd?.opt("genre")?.let { genre -> - when (genre) { - is JSONArray -> repeat(genre.length()) { index -> - genre.optString(index).nullIfEmpty()?.let { addTag(tags, it) } - } - is String -> genre.nullIfEmpty()?.let { addTag(tags, it) } - } - } - if (tags.isEmpty()) { - select("a[href*='/genre/']").mapNotNullToSet { anchor -> - anchor.text().nullIfEmpty()?.let { title -> - val href = anchor.attr("href").trimEnd('/') - val slug = href.substringAfterLast('/').substringBefore('?').nullIfEmpty() - ?: return@let null - MangaTag( - key = slug, - title = title, - source = source, - ) - } - } - .takeIf { it.isNotEmpty() } - ?.let(tags::addAll) - } - return tags - } - - private fun addTag(target: MutableSet, value: String) { - val normalized = value.trim().nullIfEmpty() ?: return - target.add( - MangaTag( - key = normalized.lowercase(Locale.US).replace(' ', '-'), - title = normalized, - source = source, - ), - ) - } - - private fun Document.extractAuthors(jsonLd: JSONObject?): Set { - val authors = LinkedHashSet() - fun addName(value: Any?) { - when (value) { - is JSONArray -> repeat(value.length()) { index -> addName(value.opt(index)) } - is JSONObject -> addName(value.opt("name")) - is String -> value.nullIfEmpty()?.let(authors::add) - } - } - addName(jsonLd?.opt("author")) - addName(jsonLd?.opt("creator")) - if (authors.isEmpty()) { - select("a[href*='/author/'], a[href*='/artist/']").forEach { anchor -> - anchor.text().nullIfEmpty()?.let(authors::add) - } - } - return authors - } - - private fun Document.extractStatus(): MangaState? { - val label = select("*").firstOrNull { element -> - element.ownText().trim().equals("Status", ignoreCase = true) - } - val statusText = label?.let { element -> - element.findSiblingText()?.nullIfEmpty() - } - ?: selectFirst("[data-status]")?.attr("data-status")?.nullIfEmpty() - ?: selectFirst("span.status, div.status")?.text()?.nullIfEmpty() - return statusText.toMangaState() - } - - private fun Document.extractRating(jsonLd: JSONObject?): Float { - jsonLd?.optJSONObject("aggregateRating")?.optDouble("ratingValue")?.takeIf { !it.isNaN() }?.let { - return it.toFloat() - } - return selectFirst("svg ~ span.font-semibold")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN - } - - private fun Document.findSeriesJsonLd(): JSONObject? { - for (script in select("script[type=application/ld+json]")) { - val raw = script.data().trim() - if (raw.isEmpty()) { - continue - } - val obj = raw.toJsonObjectOrNull() - if (obj != null) { - if (obj.matchesSeriesType()) { - return obj - } - continue - } - val array = raw.toJsonArrayOrNull() ?: continue - repeat(array.length()) { index -> - val candidate = array.optJSONObject(index) ?: return@repeat - if (candidate.matchesSeriesType()) { - return candidate - } - } - } - return null - } - - private fun String.toJsonObjectOrNull(): JSONObject? = runCatching { JSONObject(this) }.getOrNull() - private fun String.toJsonArrayOrNull(): JSONArray? = runCatching { JSONArray(this) }.getOrNull() - private fun JSONObject.matchesSeriesType(): Boolean { - val type = optString("@type") - if (type.isNullOrEmpty()) { - return false - } - return type.equals("ComicSeries", true) || - type.equals("ComicStory", true) || - type.equals("Book", true) || - type.equals("CreativeWorkSeries", true) - } - - private fun String?.withMangaId(id: String?): String { - val value = this?.nullIfEmpty() ?: return "/" - val cleanId = id?.nullIfEmpty() ?: return value - return if (value.contains('#')) { - if (value.contains("mid=")) value else "$value&mid=$cleanId" - } else { - "$value#mid=$cleanId" - } - } - - private fun extractMangaId(rawUrl: String): String? { - val fragment = rawUrl.substringAfter('#', "") - if (fragment.isNotEmpty()) { - fragment.split('&').firstOrNull { it.startsWith("mid=") }?.let { part -> - return part.substringAfter("mid=").nullIfEmpty() - } - } - val query = rawUrl.substringAfter('?', "") - if (query.isNotEmpty()) { - query.split('&').firstOrNull { it.startsWith("mid=") || it.startsWith("mangaId=") }?.let { part -> - return part.substringAfter('=').nullIfEmpty() - } - } - return null - } - - private fun String.urlWithoutFragment(): String = substringBefore('#') - - private fun String.extractChapterId(): String? = substringBefore('#') - .substringBefore('?') - .substringAfterLast('-') - .takeIf { it.all(Char::isDigit) } - ?.nullIfEmpty() - - private suspend fun postJsonArray(url: String, payload: JSONObject): JSONArray { - val raw = webClient.httpPost(url, payload).parseRaw().trim() - if (raw.isEmpty()) { - return JSONArray() - } - return raw.toJsonArrayOrNull() ?: JSONArray() - } - - private suspend fun fetchDocument(url: String): Document = webClient.httpGet(url).parseHtml() - - private fun Collection.toGenrePayload(): String { - if (isEmpty()) { - return "[]" - } - val ids = LinkedHashSet(size) - for (tag in this) { - val numericKey = tag.key.toIntOrNull()?.toString() - ?: genreTags.firstOrNull { ref -> - ref.title.equals(tag.title, ignoreCase = true) - }?.key?.toIntOrNull()?.toString() - if (numericKey != null) { - ids.add(numericKey) - } - } - if (ids.isEmpty()) { - return "[]" - } - return buildString(ids.size * 4) { - append('[') - ids.forEachIndexed { index, value -> - if (index > 0) { - append(',') - } - append(value) - } - append(']') - } - } - - private fun tag(id: String, title: String) = MangaTag(title = title, key = id, source = source) - - private fun Element.findSiblingText(): String? { - nextElementSibling()?.text()?.nullIfEmpty()?.let { return it } - val parent = parent() ?: return null - val children = parent.children() - val index = children.indexOf(this) - if (index == -1) { - return null - } - for (i in index + 1 until children.size) { - val text = children[i].text().nullIfEmpty() - if (text != null) { - return text - } - } - return null - } - - private fun Element.resolveImageUrl(): String? { - return attrAsAbsoluteUrlOrNull("data-src") - ?: attrAsAbsoluteUrlOrNull("data-lazy-src") - ?: attrAsAbsoluteUrlOrNull("src") - } - - private val genreTags by lazy { - setOf( - tag("7", "Action"), - tag("12", "Adventure"), - tag("17", "Award Winning"), - tag("80", "Boys Love"), - tag("20", "Comedy"), - tag("8", "Drama"), - tag("40", "Ecchi"), - tag("71", "Erotica"), - tag("2", "Fantasy"), - tag("34", "Full Color"), - tag("49", "Girls Love"), - tag("64", "Harem"), - tag("18", "Historical"), - tag("44", "Horror"), - tag("3", "Isekai"), - tag("43", "Josei"), - tag("1172", "Long Strip"), - tag("14", "Manga"), - tag("33", "Manhua"), - tag("6", "Manhwa"), - tag("4", "Martial Arts"), - tag("24", "Mystery"), - tag("48", "One-shot"), - tag("26", "Psychological"), - tag("5", "Reincarnation"), - tag("29", "Romance"), - tag("10", "School"), - tag("15", "Sci-Fi"), - tag("19", "Seinen"), - tag("42", "Shoujo"), - tag("13", "Shounen"), - tag("47", "Slice of Life"), - tag("25", "Sports"), - tag("9", "Supernatural"), - tag("32", "Time Travel"), - tag("39", "Webtoon"), - tag("2094", "4-Koma"), - tag("1050", "Abandoned Children"), - tag("2386", "Ability Steal"), - tag("2397", "Absent Parents"), - tag("4012", "Academy"), - tag("999", "Accelerated Growth"), - tag("1351", "Adaptation"), - tag("1000", "Adapted to Anime"), - tag("5072", "Adapted to Drama CD"), - tag("1001", "Adapted to Game"), - tag("5069", "Adapted to Manga"), - tag("963", "Adapted to Manhua"), - tag("1002", "Adapted to Manhwa"), - tag("5070", "Adapted to Visual Novel"), - tag("1058", "Adopted Children"), - tag("88", "Adult"), - tag("16", "Adult Cast"), - tag("2861", "Age Regression"), - tag("964", "Alchemy"), - tag("965", "Aliens"), - tag("966", "Alternate World"), - tag("2862", "Ancient Times"), - tag("1135", "Animals"), - tag("2096", "Anthology"), - tag("93", "Anthropomorphic"), - tag("5071", "Appearance Different from Actual Age"), - tag("1037", "Aristocracy"), - tag("2387", "Army"), - tag("1005", "Army Building"), - tag("1047", "Arranged Marriage"), - tag("2401", "Arrogant Characters"), - tag("4551", "Artbook"), - tag("4004", "Artifacts"), - tag("2863", "Assassins"), - tag("98", "Avant Garde"), - tag("5279", "Battle Academy"), - tag("2388", "Battle Competition"), - tag("1137", "BD"), - tag("968", "Beast Companions"), - tag("969", "Beautiful Female Lead"), - tag("2864", "Betrayal"), - tag("971", "Blacksmith"), - tag("4237", "Boys' Love"), - tag("1038", "Brotherhood"), - tag("2398", "Business Management"), - tag("1006", "Calm Protagonist"), - tag("2392", "Cautious Protagonist"), - tag("94", "CGDCT"), - tag("4929", "Character growth"), - tag("51", "Childcare"), - tag("4818", "Children's"), - tag("2393", "Cold Love Interests"), - tag("2394", "Cold Protagonist"), - tag("92", "Combat Sports"), - tag("2118", "Cooking"), - tag("1753", "Crime"), - tag("81", "Crossdressing"), - tag("2395", "Cruel Characters"), - tag("1039", "Cunning Protagonist"), - tag("4817", "Cute Stuffs"), - tag("4930", "Dark"), - tag("1044", "Death of Loved Ones"), - tag("52", "Delinquents"), - tag("1376", "Demons"), - tag("975", "Dense Protagonist"), - tag("58", "Detective"), - tag("4008", "Devoted love interests"), - tag("79", "Doujinshi"), - tag("1040", "Dragons"), - tag("1048", "Drugs"), - tag("4795", "Dungeon"), - tag("1011", "Dungeons"), - tag("66", "Educational"), - tag("976", "Elemental Magic"), - tag("4010", "Elves"), - tag("4005", "Family"), - tag("59", "Gag Humor"), - tag("36", "Game"), - tag("979", "Game Elements"), - tag("106", "Gender Bender"), - tag("2093", "Genderswap"), - tag("1134", "Ghosts"), - tag("4739", "Gyaru"), - tag("76", "High Stakes Game"), - tag("82", "Hentai"), - tag("4932", "Human nonhuman relationship"), - tag("95", "Idols (Female)"), - tag("101", "Idols (Male)"), - tag("109", "Indonesian"), - tag("91", "Iyashikei"), - tag("107", "Kids"), - tag("4006", "Kingdoms"), - tag("1020", "Level System"), - tag("55", "Light Novel"), - tag("4207", "Loli"), - tag("89", "Lolicon"), - tag("1173", "Web Comic"), - tag("41", "Love Polygon"), - tag("100", "Love Status Quo"), - tag("2061", "Mafia"), - tag("35", "Magic"), - tag("2095", "Magical Girls"), - tag("68", "Magical Sex Shift"), - tag("97", "Mahou Shoujo"), - tag("4933", "Male"), - tag("4792", "Mangataro Exclusive"), - tag("1414", "Monster Girls"), - tag("37", "Monsters"), - tag("1024", "Multiple POV"), - tag("38", "Murim"), - tag("83", "Music"), - tag("53", "Mythology"), - tag("4601", "Ninja"), - tag("103", "OEL"), - tag("2062", "Office Workers"), - tag("4834", "Official Colored"), - tag("4210", "Oneshot"), - tag("50", "Organized Crime"), - tag("70", "Otaku Culture"), - tag("21", "Parody"), - tag("72", "Performing Arts"), - tag("105", "Pets"), - tag("63", "Philosophical"), - tag("1136", "Police"), - tag("1557", "Post-Apocalyptic"), - tag("102", "Racing"), - tag("4014", "Reincarnated in another world"), - tag("61", "Reverse Harem"), - tag("28", "Romantic Subtext"), - tag("60", "Samurai"), - tag("73", "School Life"), - tag("1413", "Self-Published"), - tag("4116", "Sexual Violence"), - tag("90", "Shoujo Ai"), - tag("104", "Shounen Ai"), - tag("54", "Showbiz"), - tag("4009", "Skill books"), - tag("85", "Smut"), - tag("84", "Space"), - tag("77", "Strategy Game"), - tag("22", "Super Power"), - tag("4098", "Superhero"), - tag("56", "Survival"), - tag("75", "Suspense"), - tag("995", "Sword And Magic"), - tag("27", "Team Sports"), - tag("591", "Thai"), - tag("1341", "Thriller"), - tag("4742", "Traditional Games"), - tag("65", "Tragedy"), - tag("4011", "Transported to another world"), - tag("110", "Urban Fantasy"), - tag("45", "Vampire"), - tag("2102", "Vampires"), - tag("31", "Video Game"), - tag("1366", "Video Games"), - tag("62", "Villainess"), - tag("4485", "Virtual Reality"), - tag("57", "Visual Arts"), - tag("69", "Workplace"), - tag("1350", "Wuxia"), - tag("2391", "Xianxia"), - tag("962", "Xuanhuan"), - tag("86", "Yaoi"), - tag("99", "Yuri"), - tag("1342", "Zombies"), - ) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt deleted file mode 100644 index ffb883a6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt +++ /dev/null @@ -1,78 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import androidx.collection.arraySetOf -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHUAUS", "Manhuaus", "en") -internal class Manhuaus(context: MangaLoaderContext): - MadaraParser(context, MangaParserSource.MANHUAUS, "manhuaus.com") { - - override val postReq = false - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isAuthorSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - private fun availableTags() = arraySetOf( - MangaTag("Action", "action", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Anime", "anime", source), - MangaTag("Cartoon", "cartoon", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Comic", "comic", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Detective", "detective", source), - MangaTag("Doujinshi", "doujinshi", source), - MangaTag("Drama", "drama", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Josei", "josei", source), - MangaTag("Live Action", "live-action", source), - MangaTag("Manga", "manga", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mature", "mature", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Mystery", "mystery", source), - MangaTag("One Shot", "one-shot", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Romance", "romance", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujo-ai", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Shounen Ai", "shounen-ai", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Smut", "smut", source), - MangaTag("Soft Yaoi", "soft-yaoi", source), - MangaTag("Soft Yuri", "soft-yuri", source), - MangaTag("Sports", "sports", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Yaoi", "yaoi", source), - MangaTag("Yuri", "yuri", source), - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt deleted file mode 100644 index ffc91246..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHWANEX", "ManhwaNex", "en") -internal class ManhwaNex(context: MangaLoaderContext): - MadaraParser(context, MangaParserSource.MANHWANEX, "manhwanex.com") { - - override val postReq = false - -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt deleted file mode 100644 index b264ee08..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHWATOP", "ManhwaTop", "en") -internal class ManhwaTop(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANHWATOP, "manhwatop.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt deleted file mode 100644 index f00bc585..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt +++ /dev/null @@ -1,103 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* - -@MangaSourceParser("MANHWAZ", "ManhwaZ", "en", ContentType.MANGA) -internal class Manhwaz(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANHWAZ, "manhwaz.com", 40) { - - override val listUrl = "genre/manhwa" - override val tagPrefix = "genre/" - override val withoutAjax = true - override val selectTestAsync = "div.list-chapter" - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableStates = emptySet(), - availableContentRating = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/search?s=") - append(query.urlEncoded()) - append("&page=") - append(page.toString()) - } - - else -> { - - val tag = filter.tags.oneOrThrowIfMany() - if (filter.tags.isNotEmpty()) { - append("/$tagPrefix") - append(tag?.key.orEmpty()) - append("?page=") - append(page.toString()) - append("&") - } else { - append("/$listUrl") - append("?page=") - append(page.toString()) - append("&") - } - - append("m_orderby=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.NEWEST -> append("new") - SortOrder.RATING -> append("rating") - else -> append("latest") - } - } - } - } - val doc = webClient.httpGet(url).parseHtml() - - return doc.select("div.row.c-tabs-item__content").ifEmpty { - doc.select("div.page-item-detail") - }.map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") - val author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText() - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src(), - title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), - altTitles = emptySet(), - rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, - tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), - source = source, - ) - }.orEmpty(), - authors = setOfNotNull(author), - state = when (summary?.selectFirst(".mg_status")?.selectFirst(".summary-content")?.ownText() - ?.lowercase().orEmpty()) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - else -> null - }, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt deleted file mode 100644 index 3a7f8f4c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt +++ /dev/null @@ -1,295 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.network.UserAgents -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* -import okhttp3.Interceptor -import okhttp3.Response - -// TODO -// add filter options - -@MangaSourceParser("MGREADIO", "Mgread.io", "en") -internal class MgreadIo(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MGREADIO, 24) { - - override val configKeyDomain = ConfigKey.Domain("mgread.io") - override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request().newBuilder() - .header("Referer", "https://$domain/") - .header("Origin", "https://$domain") - .build() - return chain.proceed(request) - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US) - private val restChapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US).apply { - timeZone = TimeZone.getTimeZone("GMT+7") - } - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - - if (query != null) { - val url = "https://$domain/wp-json/initlise/v1/search".toHttpUrl().newBuilder() - .addQueryParameter("term", query) - .addQueryParameter("page", page.toString()) - .build() - val jsonArray = JSONArray(webClient.httpGet(url).parseRaw()) - return (0 until jsonArray.length()).mapNotNull { i -> - val obj = jsonArray.getJSONObject(i) - val title = obj.getString("title").let { org.jsoup.Jsoup.parse(it).text() } - val mangaUrl = obj.getString("url").trim().takeIf(String::isNotEmpty) ?: return@mapNotNull null - val thumb = obj.optString("thumb", null) - Manga( - id = generateUid(mangaUrl), - title = title, - url = mangaUrl, - publicUrl = mangaUrl.toAbsoluteUrl(domain), - coverUrl = thumb, - source = source, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - ) - }.filterNot { it.isAnime() } - } - - val slug = if (order == SortOrder.POPULARITY) "manga-ranking" else "recently-updated" - val url = if (page == 1) "https://$domain/$slug/" else "https://$domain/$slug/page/$page/" - val doc = webClient.httpGet(url).parseHtml() - return parseMangaGrid(doc).filterNot { it.isAnime() } - } - - private fun parseMangaGrid(doc: Document): List { - return doc.select(".manga-item-grid").map { element -> - val titleElement = element.selectFirst("h2 a[href*='/manga/']") - ?: element.selectFirst("a[href*='/manga/']:not([href*='/chapter-'])") - ?: return@map null - val href = titleElement.absUrl("href") - Manga( - id = generateUid(href), - title = titleElement.text(), - url = href.toHttpUrl().encodedPath, - publicUrl = href, - coverUrl = element.selectFirst("img")?.imageUrl(), - source = source, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - ) - }.filterNotNull() - } - - private fun Manga.isAnime(): Boolean { - val t = title.lowercase() - return t.startsWith("anime -") || t.startsWith("anime –") || - url.substringAfter("/manga/", "").startsWith("anime-") - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst("#manga-title")?.ownText() - ?: doc.selectFirst("meta[property=og:title]")?.attr("content")?.substringBefore(" [Ch.")?.trim() - ?: manga.title - - val cover = doc.selectFirst(".story-cover img")?.imageUrl() - ?: doc.selectFirst("meta[property=og:image]")?.attr("content") - ?: manga.coverUrl - - val descElement = doc.selectFirst("#manga-description") - val descriptionText = descElement?.wholeText()?.trim() - ?: doc.selectFirst("meta[name=description]")?.attr("content")?.trim() - - val genre = doc.select("#genre-tags a[href*='/genre/']") - .joinToString { it.ownText().ifEmpty { it.text() } } - - val statusText = doc.selectFirst("#manga-status")?.text()?.lowercase(Locale.US)?.trim() - val state = when (statusText) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "season end", "source hiatus", "caught up" -> MangaState.PAUSED - "dropped" -> MangaState.ABANDONED - else -> null - } - - val metaRow = doc.selectFirst("#manga-title + div") - val metadata = buildList { - metaRow?.ownText()?.substringBefore("Chapters")?.trim()?.takeIf(String::isNotEmpty)?.let { - add("Chapters: $it") - } - doc.selectFirst("#comic-othername")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Alternative title: $it") - } - doc.selectFirst(".init-review-info")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Rating: $it") - } - metaRow?.selectFirst(".init-plugin-suite-view-count-number")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Views: $it") - } - doc.selectFirst("#last-updated")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Last updated: $it") - } - } - - val description = buildString { - if (descriptionText != null) append(descriptionText) - if (metadata.isNotEmpty()) { - if (isNotEmpty()) append("\n\n") - metadata.joinTo(this, separator = "\n") - } - }.trim().takeIf(String::isNotEmpty) - - val tags = genre.split(",").mapNotNull { it.trim().takeIf(String::isNotEmpty) }.map { name -> - MangaTag(name, name.lowercase(), source) - }.toSet() - - val mangaId = doc.selectFirst("#manga-title[data-id], #chapter-search-input[data-manga-id]") - ?.attr("data-id")?.ifEmpty { doc.selectFirst("#chapter-search-input[data-manga-id]")?.attr("data-manga-id") } - ?.toIntOrNull() - - val chapters = if (mangaId != null) { - fetchChaptersApi(mangaId, manga.url) - } else { - doc.select(".chapter-list .chapter-item").mapNotNull { chapterFromElement(it) } - } - - return manga.copy( - title = title, - coverUrl = cover, - description = description, - tags = tags, - state = state, - chapters = chapters, - ) - } - - private suspend fun fetchChaptersApi(mangaId: Int, mangaPath: String): List { - val result = mutableListOf() - var page = 1 - do { - val url = "https://$domain/wp-json/initmanga/v1/chapters".toHttpUrl().newBuilder() - .addQueryParameter("manga_id", mangaId.toString()) - .addQueryParameter("paged", page.toString()) - .addQueryParameter("per_page", "50") - .build() - val json = webClient.httpGet(url).parseJson() - val items = json.optJSONArray("items") ?: break - val totalPages = json.optInt("total_pages", 1) - for (i in 0 until items.length()) { - val item = items.getJSONObject(i) - val number = item.optDouble("number", -1.0).toFloat() - val title = item.optString("title", "") - val slug = item.optString("slug", "") - val createdAt = item.optString("created_at", "") - val chapterName = if (title.isNotEmpty()) "Chapter ${number.toInt()} - $title" else "Chapter ${number.toInt()}" - val cleanPath = mangaPath.substringBefore("/chapter/").trimEnd('/') - result.add( - MangaChapter( - id = generateUid("$cleanPath/$slug/"), - title = chapterName, - number = number, - url = "$cleanPath/$slug/", - uploadDate = restChapterDateFormat.parseSafe(createdAt), - source = source, - volume = 0, - scanlator = null, - branch = null, - ) - ) - } - page++ - } while (page <= totalPages) - return result.sortedBy { it.number } - } - - private fun chapterFromElement(element: Element): MangaChapter? { - val urlElement = element.selectFirst("a[href*='/chapter-']") ?: return null - val chapterUrl = urlElement.absUrl("href") - val name = element.selectFirst("h3")?.text() - ?: chapterUrl.substringBeforeLast('/').substringAfterLast('/').replace('-', ' ').replaceFirstChar { it.uppercase() } - val number = Regex("""/chapter-(\d+(?:\.\d+)?)/""").find(chapterUrl)?.groupValues?.get(1)?.toFloatOrNull() ?: -1f - val dateStr = element.selectFirst("time[datetime]")?.attr("datetime") - val date = dateStr?.parseChapterDate() ?: 0L - return MangaChapter( - id = generateUid(chapterUrl.toHttpUrl().encodedPath), - title = name, - number = number, - url = chapterUrl.toHttpUrl().encodedPath, - uploadDate = date, - source = source, - volume = 0, - scanlator = null, - branch = null, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-content img[src]").mapIndexed { _, img -> - val src = img.absUrl("src") - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun String?.parseChapterDate(): Long { - if (isNullOrBlank()) return 0L - val normalized = this - .replace(Regex("""([+-]\d{2}):(\d{2})$"""), "$1$2") - .replace(Regex("""Z$"""), "+0000") - return chapterDateFormat.parseSafe(normalized) - } - - private fun Element.imageUrl(): String? { - return when (normalName()) { - "meta" -> attr("content") - else -> attr("abs:data-src").ifEmpty { - attr("abs:data-lazy-src").ifEmpty { attr("abs:src") } - } - }.takeIf(String::isNotEmpty) - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - private fun SimpleDateFormat.parseSafe(date: String): Long = - runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt deleted file mode 100644 index 6a3530ea..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource - -@MangaSourceParser("TWENTYFOURHNOVEL", "24HNovel", "en") -internal class Novel24hParser(context: MangaLoaderContext) : - MadaraParser( - context = context, - source = MangaParserSource.TWENTYFOURHNOVEL, - domain = "24hnovel.com" - ) { - override val withoutAjax = true -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt deleted file mode 100644 index ea35e04a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("TOPMANHUA", "ManhuaTop", "en") -internal class TopManhua(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.TOPMANHUA, "manhuatop.org") { - override val tagPrefix = "manhua-genre/" - override val listUrl = "manhua/" - override val datePattern = "MM/dd/yyyy" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt deleted file mode 100644 index 145e51ba..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt +++ /dev/null @@ -1,378 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.LinkResolver -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.getCookies -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("WEEBCENTRAL", "Weeb Central", "en") -internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.WEEBCENTRAL), - MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain("weebcentral.com") - - override val authUrl: String - get() = "https://$domain" - - override suspend fun isAuthorized(): Boolean = - context.cookieJar.getCookies(domain).any { it.name == "access_token" } - - override suspend fun getUsername(): String { - return webClient.httpGet("https://$domain/users/me/profiles") - .parseHtml() - .selectFirstOrThrow("div:has(section > .avatar) .text-4xl") - .text() - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.POPULARITY, - SortOrder.POPULARITY_ASC, - SortOrder.RATING, - SortOrder.RATING_ASC, - SortOrder.ADDED, - SortOrder.ADDED_ASC, - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - ) - - override val filterCapabilities: MangaListFilterCapabilities = - MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val document = webClient.httpGet("https://$domain/search") - .parseHtml() - - val tags = document.select("section[x-show=show_filter] div:contains(tags) fieldset label").mapToSet { - MangaTag( - title = it.selectFirstOrThrow("span").text(), - key = it.selectFirstOrThrow("input[id$=value]").attr("value"), - source = source, - ) - } - - val states = EnumSet.of( - MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED, - ) - - val types = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS, - ) - - val rating = EnumSet.of( - ContentRating.SAFE, ContentRating.SUGGESTIVE, - ) - - return MangaListFilterOptions( - availableTags = tags, - availableStates = states, - availableContentTypes = types, - availableContentRating = rating, - ) - } - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { - val url = "https://$domain/search/data".toHttpUrl().newBuilder().apply { - addQueryParameter("limit", "32") - addQueryParameter("offset", offset.toString()) - filter.query?.let { - val query = it - .replace(Regex("""[^a-zA-Z0-9\s]"""), " ") - .replace(Regex("""\s+"""), " ") - .trim() - addQueryParameter("text", query) - } - addQueryParameter( - name = "sort", - value = when (order) { - SortOrder.RELEVANCE -> "Best Match" - SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "Alphabet" - SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "Popularity" - SortOrder.RATING, SortOrder.RATING_ASC -> "Subscribers" - SortOrder.ADDED, SortOrder.ADDED_ASC -> "Recently Added" - SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "Latest Updates" - else -> throw UnsupportedOperationException("unsupported order: $order") - }, - ) - addQueryParameter( - name = "order", - value = when (order) { - SortOrder.RELEVANCE, SortOrder.ALPHABETICAL, SortOrder.POPULARITY_ASC, SortOrder.RATING_ASC, SortOrder.ADDED_ASC, SortOrder.UPDATED_ASC -> "Ascending" - SortOrder.ALPHABETICAL_DESC, SortOrder.POPULARITY, SortOrder.RATING, SortOrder.ADDED, SortOrder.UPDATED -> "Descending" - else -> throw UnsupportedOperationException("unsupported order: $order") - }, - ) - addQueryParameter("official", "Any") - addQueryParameter("anime", "Any") - with(filter.contentRating) { - addQueryParameter( - name = "adult", - value = when { - isEmpty() -> "Any" - ContentRating.SAFE in this && ContentRating.SUGGESTIVE in this -> "Any" - ContentRating.SAFE in this -> "False" - ContentRating.SUGGESTIVE in this -> "True" - else -> throw UnsupportedOperationException("unsupported content rating: $this") - }, - ) - } - filter.states.forEach { state -> - addQueryParameter( - name = "included_status", - value = when (state) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Complete" - MangaState.ABANDONED -> "Canceled" - MangaState.PAUSED -> "Hiatus" - else -> throw UnsupportedOperationException("unsupported state: $state") - }, - ) - } - filter.types.forEach { type -> - addQueryParameter( - name = "included_type", - value = when (type) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "OEL" - else -> throw UnsupportedOperationException("unsupported type: $type") - }, - ) - } - filter.tags.forEach { tag -> - addQueryParameter("included_tag", tag.key) - } - filter.tagsExclude.forEach { tag -> - addQueryParameter("excluded_tag", tag.key) - } - addQueryParameter("display_mode", "Full Display") - }.build() - - val document = webClient.httpGet(url).parseHtml() - - return document.select("article:has(section)").map { element -> - val mangaId = element.selectFirstOrThrow("a") - .attrAsAbsoluteUrl("href") - .toHttpUrl() - .pathSegments[1] - val author = document.select("div:contains(author) a").eachText().joinToString().nullIfEmpty() - val title = - element.selectFirst("div.text-ellipsis.truncate.text-white.text-center.text-lg.z-20.w-\\[90\\%\\]") - ?.text() - ?: "No name" - Manga( - id = generateUid(mangaId), - url = mangaId, - publicUrl = "https://$domain/series/$mangaId", - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = if (element.selectFirst("svg:has(style:containsData(ff0000))") == null) { - ContentRating.SAFE - } else { - ContentRating.SUGGESTIVE - }, - coverUrl = element.selectFirst("picture img")?.attrAsAbsoluteUrlOrNull("src"), - tags = element.selectFirst("div:contains(Tag(s): )")?.text() - ?.substringAfter("Tag(s): ") - ?.split(", ") - ?.mapToSet { - MangaTag( - title = it, - key = it, - source = source, - ) - } - .orEmpty(), - state = when (document.selectFirst("div:contains(status) span")?.text()) { - "Ongoing" -> MangaState.ONGOING - "Complete" -> MangaState.FINISHED - "Canceled" -> MangaState.ABANDONED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = setOfNotNull(author), - largeCoverUrl = null, - chapters = null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val document = webClient.httpGet("https://$domain/series/${manga.url}") - .parseHtml() - - val chapters = async { getChapters(manga.url, document) } - - val sectionLeft = document.select("section[x-data] > section")[0] - val sectionRight = document.select("section[x-data] > section")[1] - val author = sectionLeft.select("ul > li:has(strong:contains(Author)) > span > a") - .eachText().joinToString() - - manga.copy( - title = sectionRight.selectFirstOrThrow("h1").text(), - altTitles = sectionRight.select("li:has(strong:contains(Associated Name)) li") - .eachText().toSet(), - publicUrl = "https://$domain/series/${manga.url}", - rating = RATING_UNKNOWN, - contentRating = if (sectionLeft.selectFirst("ul > li > strong:contains(Official Translation) + a:contains(Yes)") != null) { - ContentRating.SUGGESTIVE - } else { - ContentRating.SAFE - }, - coverUrl = sectionLeft.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), - tags = sectionLeft.select("ul > li:has(strong:contains(Tag)) a").mapToSet { - MangaTag( - title = it.text(), - key = it.text(), - source = source, - ) - }, - state = when (sectionLeft.selectFirst("ul > li:has(strong:contains(Status)) > a")?.text()) { - "Ongoing" -> MangaState.ONGOING - "Complete" -> MangaState.FINISHED - "Canceled" -> MangaState.ABANDONED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = setOf(author), - description = Element("div").also { desc -> - sectionRight.selectFirst("li:has(strong:contains(Description)) > p")?.let { - desc.appendChild(it) - } - - val ul = Element("ul") - sectionLeft.select("ul > li:has(strong:contains(Track)) abbr").stream().forEach { abbr -> - abbr.selectFirst("a")?.attr("href")?.let { url -> - val a = Element("a") - .text( - abbr.attr("title"), - ) - .attr("href", url) - - ul.appendChild( - Element("li").appendChild(a), - ) - } - } - - if (ul.children().isNotEmpty()) { - desc.append("
Links:") - desc.appendChild(ul) - } - }.outerHtml(), - chapters = chapters.await(), - source = source, - ) - } - - private suspend fun getChapters(mangaId: String, mangaDocument: Document): List { - val document = if (mangaDocument.selectFirst("#chapter-list > button[hx-get*=full-chapter-list]") != null) { - webClient.httpGet("https://$domain/series/$mangaId/full-chapter-list").parseHtml() - } else { - mangaDocument - } - - return document.select("div[x-data] > a").mapChapters(reversed = true) { i, element -> - val chapterId = element.attrAsAbsoluteUrl("href") - .toHttpUrl() - .pathSegments[1] - val name = element.selectFirstOrThrow("span.flex > span").text() - - MangaChapter( - id = generateUid(chapterId), - url = chapterId, - title = name, - number = Regex("""(? "Official" - else -> null - }, - uploadDate = dateFormat.parseSafe( - element.selectFirst("time[datetime]")?.attr("datetime"), - ), - branch = null, - source = source, - ) - } - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) - - override suspend fun getPages(chapter: MangaChapter): List { - val url = "https://$domain".toHttpUrl().newBuilder().apply { - addPathSegment("chapters") - addPathSegment(chapter.url) - addPathSegment("images") - addQueryParameter("is_prev", "False") - addQueryParameter("reading_style", "long_strip") - }.build() - - val document = webClient.httpGet(url).parseHtml() - - return document.select("section[x-data~=scroll] > img").map { element -> - val pageUrl = element.attrAsAbsoluteUrl("src") - - MangaPage( - id = generateUid(pageUrl), - url = pageUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val mangaId = link.pathSegments[1] - - return resolver.resolveManga(this, mangaId) - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt deleted file mode 100644 index 98e8a4e3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt +++ /dev/null @@ -1,59 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.text.SimpleDateFormat - -@MangaSourceParser("ZINMANGA", "ZinManga.net", "en") -internal class Zinmanga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.ZINMANGA, "zinmanga.net") { - override val datePattern = "yyyy-MM-dd" - override val withoutAjax = true - - override val selectRequiredLogin = ".content-blocked" - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://www.zinmanga.net/") - .build() - - override suspend fun getChapters(manga: Manga, doc: Document): List { - val slug = manga.url.removeSuffix("/").substringAfterLast('/') - val mangaUrl = manga.url.removeSuffix("/") - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val collected = ArrayList() - var page = 1 - while (true) { - val url = "https://$domain/api/comics/$slug/chapters?page=$page" - val data = webClient.httpGet(url).parseJson().getJSONObject("data") - data.getJSONArray("chapters").mapJSON { jo -> - val chapterSlug = jo.getString("chapter_slug") - val href = "$mangaUrl/$chapterSlug" + stylePage - collected += MangaChapter( - id = generateUid("$mangaUrl/$chapterSlug"), - title = jo.getStringOrNull("chapter_name"), - number = jo.getFloatOrDefault("chapter_num", 0f), - volume = 0, - url = href, - uploadDate = parseChapterDate( - dateFormat, - jo.getStringOrNull("updated_at")?.substringBefore('T'), - ), - source = source, - scanlator = null, - branch = null, - ) - } - val lastPage = data.getIntOrDefault("last_page", page) - if (page >= lastPage) break - page++ - } - return collected.sortedBy { it.number } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt deleted file mode 100644 index 280d6b47..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("ALLPORNCOMIC", "AllPornComic", "en", ContentType.HENTAI) -internal class AllPornComic(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.ALLPORNCOMIC, "allporncomic.com", pageSize = 24) { - override val withoutAjax = true - override val listUrl = "porncomic/" - override val tagPrefix = "porncomic-cat/" - override val datePattern = "MMMM dd, yyyy" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt deleted file mode 100644 index 19cc6732..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt +++ /dev/null @@ -1,286 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.urlEncoded - -@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) -internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( - context, MangaParserSource.COMICLAND, 20) { - - override val configKeyDomain = - ConfigKey.Domain("comicland.org") - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - - val newRequest = request.newBuilder() - .header( - "Origin", - "https://comicland.org" - ) - .header( - "Referer", - "https://comicland.org/" - ) - .build() - - return chain.proceed(newRequest) - } - - override val availableSortOrders = - setOf( - SortOrder.UPDATED, // Recommended - SortOrder.POPULARITY, // Popular - SortOrder.NEWEST // Official/Ongoing equivalent - ) - - override val filterCapabilities = - MangaListFilterCapabilities( - isMultipleTagsSupported = false, - isTagsExclusionSupported = false, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - isAuthorSearchSupported = false - ) - - private val api = - "https://api.comicland.org/api" - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter - ): List { - - - val offset = page * 20 - val endpoint = - if (!filter.query.isNullOrBlank()) { - "$api/comic/search?q=${filter.query!!.urlEncoded()}" - } else { - val category = - filter.tags - .firstOrNull() - ?.key - when(category) { - "official" -> - "$api/comics/official" - "ongoing" -> - "$api/comics" - "popular" -> - "$api/comics/popular" - else -> - "$api/comics" - } - } - - val separator = - if (endpoint.contains("?")) "&" else "?" - - val response = - webClient.httpGet( - "$endpoint${separator}offset=$offset&limit=20" - ) - - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") - ?: return emptyList() - - val array = - data.optJSONArray("list") - ?: data.optJSONArray("items") - ?: return emptyList() - - val result = ArrayList() - for (i in 0 until array.length()) { - val comic = - array.getJSONObject(i) - val slug = - comic.optString("slug") - result.add( - Manga( - id = generateUid(slug), - title = comic.optString("title"), - url = slug, - publicUrl = "https://comicland.org/comic/$slug", - coverUrl = comic.optString("cover_url"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - contentRating = ContentRating.ADULT, - source = source - ) - ) - } - return result - } - - override suspend fun getDetails( - manga: Manga - ): Manga { - val response = - webClient.httpGet( - "$api/comic/detail?slug=${manga.url}" - ) - - val root = - JSONObject(response.body.string()) - - val comic = - root.optJSONObject("data") - ?: return manga - - val chapters = - ArrayList() - - val chapterArray = - comic.optJSONArray("chapters") - - if (chapterArray != null) { - for (i in 0 until chapterArray.length()) { - val chapter = - chapterArray.getJSONObject(i) - - val index = - chapter.optDouble( - "chapter_index", - 0.0 - ) - chapters.add( - MangaChapter( - id = generateUid("${comic.optString("slug")}/$index"), - title = chapter.optString("title"), - number = index.toFloat(), - volume = 0, - url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", - scanlator = null, - uploadDate = 0, - branch = null, - source = source - ) - ) - } - } - return manga.copy( - title = - comic.optString( - "title", - manga.title - ), - description = - comic.optString( - "description", - "" - ), - coverUrl = - comic.optString( - "cover_url", - manga.coverUrl - ), - authors = - parseNames( - comic.optJSONArray("authors") - ), - tags = emptySet(), - chapters = chapters - ) - } - - - private fun parseNames( - array: org.json.JSONArray? - ): Set { - if (array == null) - return emptySet() - return buildSet { - for (i in 0 until array.length()) { - add( - array.getJSONObject(i) - .optString("name") - ) - } - } - } - - - override suspend fun getPages( - chapter: MangaChapter - ): List { - - val slug = - chapter.url - .substringAfter("/comic/") - .substringBefore("/chapter/") - - val index = - chapter.url - .substringAfter("/chapter/") - - val response = - webClient.httpGet( - "$api/chapter/pages_by_index?slug=$slug&index=$index" - ) - - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") - ?: return emptyList() - - val pages = - data.optJSONArray("pages") - ?: return emptyList() - - return buildList { - for (i in 0 until pages.length()) { - val image = - pages.getString(i) - add( - MangaPage( - id = generateUid(image), - url = image, - preview = null, - source = source - ) - ) - } - } - } - - private val categories = mapOf( - "recommended" to "/comics", - "official" to "/comics/official", - "ongoing" to "/comics?status=ongoing", - "popular" to "/comics/popular" - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = categories.map { (key, _) -> - MangaTag( - title = key.replaceFirstChar { it.uppercase() }, - key = key, - source = source - ) - }.toSet(), - - availableContentRating = emptySet() - ) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt deleted file mode 100644 index 1366a933..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt +++ /dev/null @@ -1,397 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.urlDecode -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.text.SimpleDateFormat -import java.util.Base64 -import org.koitharu.kotatsu.parsers.config.ConfigKey - -private const val SERVER_PNG = "png" // old method uses now image_fallback -private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format - -@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) -internal class HentaiNexus(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { - override val selectGallery = "div.container div.columns div.column" - override val selectGalleryLink = "a" - override val selectGalleryTitle = ".card-header" - override val selectTitle = ".title" - override val selectTag = "tr:contains(Tags) td:nth-child(2) span.tag a" - override val selectAuthor = "tr:contains(Artist) td:nth-child(2) a" - override val selectLanguageChapter = "" - override val selectUrlChapter = "" - override val selectTotalPage = "div.column.is-2-fullhd" - - val selectReadUrl = "a:contains(Read Online)" - val selectPublisher = "tr:contains(Publisher) td:nth-child(2)" - val selectPublishedDate = "tr:contains(Published) td:nth-child(2)" - val selectDescription = "tr:contains(Description) td:nth-child(2)" - - var mangaInternalId: String = "" /* use as a flag for reloading data */ - var mangaPages: List = listOf() - - private val pageCache = object : LinkedHashMap>(10, 0.75f, true) { - override fun removeEldestEntry( - eldest: MutableMap.MutableEntry> - ): Boolean { - return size > 5 - } - } - - private val preferredImageModeKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - SERVER_PNG to "Old method (png)", - SERVER_WEBP to "New method (webp/avif)", - ), - defaultValue = SERVER_PNG, - ) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(preferredImageModeKey) - } - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - isAuthorSearchSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val document = webClient.httpGet("https://$domain/explore/categories/tag").parseHtml() - val tags = document.select("div.container div.columns div.column").mapToSet {div -> - val tag = div.selectFirstOrThrow("a").attr("href") - .substring(8) // href="/?q=tag:value" - .urlDecode() - .trim(' ', '\"') - MangaTag( - title = tag.replace(Regex("\\b[a-z]")) { it.value.uppercase(sourceLocale) }, - key = tag, - source = source, - ) - } - - return MangaListFilterOptions( - availableTags = tags - ) - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val url = buildString { - append("https://$domain/page/$page") - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("?q=") - append(query?.urlEncoded()) - } - - else -> { - val queries = mutableListOf() - - if (!filter.author.isNullOrEmpty()) { - queries.add("artist:${filter.author}") - } - - filter.tags.map { - val key = it.key - when { - key.split(" ").count() > 1 -> { - "\"${key.replace(Regex("\\s+")) { "+" }}\"" - } - else -> { - key - } - } - }.also { - if (it.count() > 0) { - queries.add("tag:${ it.joinToString("+") }") - } - } - - if (queries.count() > 0) { - append("?q=${queries.joinToString("+")}") - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun parseMangaList(doc: Document): List { - return doc.select(selectGallery).map { div -> - val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(selectGalleryTitle).text().cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = div.selectFirst(selectGalleryImg)?.src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val tags = doc.select(selectTag).mapToSet { - val value = it.attr("href").substring(8).urlDecode().trim('\"') /* /?q=tag:"blow job" */ - MangaTag( - title = value.replace(Regex("\\b[a-z]")) { x -> x.value.uppercase(sourceLocale) }, - key = value, - source = source - ) - } - val authors = doc.select(selectAuthor).mapToSet { - it.attr("href").substring(11).urlDecode() /* /?q=artist:Danchino */ - } - - mangaPages = getPagesInternal(manga.url, doc) - mangaInternalId = manga.url.split("/").last() - - val format = SimpleDateFormat("dd MMMM yyyy") - - return manga.copy( - title = doc.select(selectTitle).text().cleanupTitle(), - tags = tags, - authors = authors, - chapters = listOf( - MangaChapter( - id = manga.id, - title = manga.title, - number = 0f, - volume = 0, - url = manga.url, - scanlator = doc.select(selectPublisher).text().replace(Regex(" \\([\\d,]+\\)")) { "" }, - uploadDate = format.parseSafe(doc.select(selectPublishedDate).text()), - branch = "English", - source = source, - ), - ), - description = doc.selectFirst(selectDescription)?.text() ?: "" - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { - mangaPages = getPagesInternal(chapter.url) - mangaInternalId = chapter.url.split("/").last() - - // preload image urls - val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") - if (reader != null && !pageCache.containsKey(reader)) { - pageCache[reader] = getPageUrlInternal(reader) - } - } - - return mangaPages - } - - override suspend fun getPageUrl(page: MangaPage): String { - val readerUrl = page.url.substringBefore("#") - - val pages = pageCache[readerUrl] - ?: getPageUrlInternal(readerUrl).also { - pageCache[readerUrl] = it - } - - val index = page.url.substringAfter("#").toInt() - 1 - - return pages[index] - } - - private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List { - val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() - val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") - - return document.select(selectTotalPage).mapIndexed { index, element -> - val url = "$readUrl#${index + 1}" - MangaPage( - id = generateUid(url), - url = url, - preview = element.select("img").attr("src"), - source = source, - ) - } - } - - private suspend fun getPageUrlInternal(pageUrl: String): List { - val doc = webClient - .httpGet(pageUrl.toAbsoluteUrl(domain)) - .parseHtml() - val script = doc.select("script") - .firstOrNull { it.data().contains("initReader") } - ?: throw Exception("Reader script not found") - val encryptedPagesData = script.data() - .substringAfter("initReader(\"") - .substringBefore("\",") - .replace("\\n", "") - .replace("\\r", "") - .trim() - val decryptedString = decrypt(encryptedPagesData) - .replace("\\/", "/") - val jsonArray = JSONArray(decryptedString) - val mode = config[preferredImageModeKey] ?: SERVER_WEBP - return buildList { - for (i in 0 until jsonArray.length()) { - val item = jsonArray.getJSONObject(i) - val url = when (mode) { - SERVER_PNG -> { - val image = item.optString("image") - .takeIf { it.isNotBlank() } - val fallback = item.optString("image_fallback") - .takeIf { it.isNotBlank() } - fallback ?: image - } - SERVER_WEBP -> { - var found: String? = null - val keys = item.keys() - while (keys.hasNext()) { - val value = item.optString(keys.next()) - if ( - value.startsWith("http") && - ( - value.contains(".webp") || - value.contains(".avif") || - value.contains(".jpg") || - value.contains(".jpeg") || - value.contains(".png") - ) - ) { - found = value - break - } - } - found - } - else -> null - } - if (!url.isNullOrBlank()) { - add(normalizeImageUrl(url)) - } - } - } - } - - private fun normalizeImageUrl(url: String): String { - return when { - url.startsWith("http") -> url - - url.startsWith("//") -> - "https:$url" - - url.startsWith("/") -> - "https://$domain$url" - - else -> - "https://$domain/$url" - } - } - - private fun decrypt(encodedData: String): String { - val xorKey = listOf('h','e','n','t','a','i','n','e','x','u','s','.','c','o','m') - val keyLength = minOf(xorKey.size, 64) - - // Decode base64 string into characters (1 byte per char, unsigned) - val decodedBytes = Base64.getDecoder().decode(encodedData) - val decodedChars = decodedBytes.map { (it.toInt() and 0xFF).toChar() }.toMutableList() - - // XOR first 64 characters with the key - for (i in 0 until keyLength) { - decodedChars[i] = (decodedChars[i].code xor xorKey[i].code).toChar() - } - - val decodedString = decodedChars.joinToString("") - - // Prime sieve: first 16 primes - val sieve = BooleanArray(257) - val primeIndexes = mutableListOf() - var j = 2 - while (primeIndexes.size < 16) { - if (!sieve[j]) { - primeIndexes.add(j) - for (k in j * 2..256 step j) sieve[k] = true - } - j++ - } - - // Hash from first 64 chars - var hash = 0 - for (k in 0 until 64) { - hash = hash xor decodedString[k].code - repeat(8) { - hash = if ((hash and 1) == 1) { - (hash ushr 1) xor 12 - } else { - hash ushr 1 - } - } - } - hash = hash and 7 - - // RC4 key scheduling - val s = IntArray(256) { it } - j = 0 - for (i in 0 until 256) { - j = (j + s[i] + decodedString[i % 64].code) % 256 - s[i] = s[j].also { s[j] = s[i] } - } - - // Decrypt rest of the chars - val result = StringBuilder() - var i = 0 - j = 0 - var keyStream = 0 - var rnd = 0 - val step = primeIndexes[hash] - - for (n in 0 until decodedString.length - 64) { - i = (i + step) % 256 - j = (keyStream + s[(j + s[i]) % 256]) % 256 - keyStream = (keyStream + i + s[i]) % 256 - s[i] = s[j].also { s[j] = s[i] } - - rnd = s[(j + s[(i + s[(rnd + keyStream) % 256]) % 256]) % 256] - val decryptedChar = decodedString[n + 64].code xor rnd - result.append(decryptedChar.toChar()) - } - - return result.toString() - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt deleted file mode 100644 index 891e56eb..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt +++ /dev/null @@ -1,28 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* - -@MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) -internal class HentaixComic(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.HENTAIXCOMIC, "hentaixcomic.com", 16) { - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/?s=&post_type=wp-manga").parseHtml() - val set = mutableSetOf() - val titles = mutableSetOf() - doc.select("div.checkbox-group input[type=checkbox]").forEach { input -> - val key = input.attr("value") - val title = input.nextElementSibling()?.text()?.toTitleCase() - if (key.isNotEmpty() && !title.isNullOrEmpty() && titles.add(title.lowercase())) { - set.add(MangaTag(key = key, title = title, source = source)) - } - } - return set - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt deleted file mode 100644 index 730a153e..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt +++ /dev/null @@ -1,293 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.Headers -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJsonArray -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.Calendar -import java.util.EnumSet - -@MangaSourceParser("HEYTOON", "HeyToon", "en", ContentType.HENTAI) -internal class HeyToon(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.HEYTOON, pageSize = 54, searchPageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("heytoon.net") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getAvailableGenres(), - ) - - private val headers = Headers.headersOf("X-Requested-With", "XMLHttpRequest") - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - return when { - !query.isNullOrEmpty() -> { - if (page > 1) return emptyList() - - val url = "https://$domain/api/complete-search?keyword=${query.urlEncoded()}" - val response = webClient.httpGet(url, headers).parseJsonArray() - - response.mapJSON { comic -> - val linkComic = comic.getString("linkComic") - val id = linkComic.substringAfterLast("/").substringBefore(".html") - Manga( - id = generateUid(id), - url = linkComic.toRelativeUrl(domain), - publicUrl = linkComic.toAbsoluteUrl(domain), - title = comic.getString("title"), - coverUrl = comic.getStringOrNull("raw_thumb"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - contentRating = null, - largeCoverUrl = null, - description = null, - chapters = null, - ) - } - } - - else -> { - val url = buildString { - append("https://") - append(domain) - append("/en/genres") - - filter.tags.firstOrNull()?.let { - append("/") - append(it.key) - } - - append("?orderBy=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - else -> append("latest") - } - - if (page > 1) { - append("&page=") - append(page) - } - } - - val doc = webClient.httpGet(url).parseHtml() - parseMangaList(doc) - } - } - } - - private fun parseMangaList(doc: Document): List { - return doc.select("div.comicItemCon a").map { element -> - val href = element.attrAsRelativeUrl("href") - val id = href.substringAfterLast("/").substringBefore(".html").substringBefore("-") - val img = element.selectFirstOrThrow("img[alt!=badge]") - Manga( - id = generateUid(id), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = img.attr("title").ifEmpty { img.attr("alt") }, - coverUrl = img.attr("data-src").toAbsoluteUrl(domain), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - contentRating = null, - largeCoverUrl = null, - description = null, - chapters = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirstOrThrow("#titleSubWrapper h1.titCon").text() - val description = doc.selectFirst("#modal_detail .cont_area p")?.text() - val genres = parseGenres(doc) - val status = parseStatus(doc) - val chapters = parseChapters(doc) - - return manga.copy( - title = title, - description = description, - tags = genres, - state = status, - coverUrl = doc.selectFirst("meta[property=og:image]")?.attr("content") ?: manga.coverUrl, - chapters = chapters, - ) - } - - private fun parseGenres(doc: Document): Set { - return doc.select("div.previewItemCon a[href*=/genres/]").mapNotNullToSet { a -> - val key = a.attr("href").substringAfterLast("/genres/").substringBefore("?") - if (key.isEmpty()) return@mapNotNullToSet null - MangaTag( - key = key, - title = key, - source = source, - ) - } - } - - private fun parseStatus(doc: Document): MangaState? { - return doc.select(".badgeArea span").eachText().let { badges -> - when { - badges.any { it.contains("Up") } -> MangaState.ONGOING - badges.any { it.contains("Completed") } -> MangaState.FINISHED - else -> null - } - } - } - - private fun parseChapters(doc: Document): List { - return doc.select(".episodeListConPC a#episodeItemCon").mapChapters(reversed = false) { i, element -> - val href = element.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - title = element.selectFirstOrThrow(".comicInfo p.episodeStitle").text(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - uploadDate = parseDateOrNull(element.selectFirst(".comicInfo .episodeDate")?.text()), - branch = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - - return doc.select("#comicContent img").map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun getAvailableGenres(): Set { - val genres = listOf( - "Detective", - "Spin-Off", - "Mommy", - "Uncensored", - "New", - "In-Law", - "Cheating", - "MILF", - "Harem", - "College", - "Business", - "Supernatural", - "Thriller", - "Adventure", - "Romance", - "Drama", - ) - - return genres.mapToSet { genre -> - MangaTag( - key = genre, - title = genre, - source = source, - ) - } - } - - private fun parseDateOrNull(dateStr: String?): Long { - if (dateStr.isNullOrEmpty()) return 0 - return try { - val parts = dateStr.split(" ") - if (parts.size != 3) return 0 - - val month = when (parts[0].lowercase()) { - "jan" -> Calendar.JANUARY - "feb" -> Calendar.FEBRUARY - "mar" -> Calendar.MARCH - "apr" -> Calendar.APRIL - "may" -> Calendar.MAY - "jun" -> Calendar.JUNE - "jul" -> Calendar.JULY - "aug" -> Calendar.AUGUST - "sep" -> Calendar.SEPTEMBER - "oct" -> Calendar.OCTOBER - "nov" -> Calendar.NOVEMBER - "dec" -> Calendar.DECEMBER - else -> return 0 - } - - val day = parts[1].removeSuffix(",").toIntOrNull() ?: return 0 - val year = parts[2].toIntOrNull() ?: return 0 - - Calendar.getInstance().apply { - set(Calendar.YEAR, year) - set(Calendar.MONTH, month) - set(Calendar.DAY_OF_MONTH, day) - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } catch (e: Exception) { - 0 - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt deleted file mode 100644 index 620224f9..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.Broken -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser - -@Broken -@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) -internal class Hiperdex(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("x-svc-gate", "f5pabmx7sdek") - .build() -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt deleted file mode 100644 index db897ad0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt +++ /dev/null @@ -1,176 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.network.UserAgents -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* - -private const val F_URL = "fullUrl=" - -@MangaSourceParser("MADARADEX", "MadaraDex", "en", ContentType.HENTAI) -internal class MadaraDex(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MADARADEX, "madaradex.org") { - - init { - context.cookieJar.insertCookies(domain, "wpmanga-adault=1") - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.remove(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders() - .newBuilder() - .set(CommonHeaders.USER_AGENT, UserAgents.CHROME_DESKTOP) - .set(CommonHeaders.REFERER, "https://$domain/") - .build() - - override val authUrl: String - get() = "https://$domain" - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name.contains("cm_uaid") - } - } - - override val listUrl = "title/" - override val tagPrefix = "genre/" - override val postReq = true - override val stylePage = "" - - - override suspend fun getPages(chapter: MangaChapter): List { - - val fullUrl = chapter.url.toAbsoluteUrl(domain) - - val doc = webClient - .httpGet(fullUrl) - .parseHtml() - - - val images = extractImages(doc) - - - if (images.isEmpty()) { - throw ParseException( - "No images found, try to log in", - fullUrl - ) - } - - - return images.map { image -> - - val imageUrl = image - .toRelativeUrl(domain) - .toHttpUrl() - .newBuilder() - .fragment(F_URL + fullUrl) - .build() - .toString() - - - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - - private fun extractImages(document: Document): List { - - val pageBreakImages = - document - .select("div.page-break img") - .mapNotNull { - it.attr("data-src") - .ifBlank { - it.attr("src") - } - .nullIfEmpty() - } - - - if (pageBreakImages.isNotEmpty()) { - return pageBreakImages - } - - - return document - .body() - .select(selectBodyPage) - .flatMap { body -> - body.select(selectPage) - .flatMap { page -> - page.select("img") - .mapNotNull { - it.attr("src") - .nullIfEmpty() - } - } - } - } - - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val url = request.url - - val fullUrl = url.fragment - ?.substringAfter(F_URL, "") - - if (!fullUrl.isNullOrEmpty()) { - - copyCookies() - - val cleanUrl = url.newBuilder() - .fragment(null) - .build() - - val newRequest = request.newBuilder() - .header( - CommonHeaders.REFERER, - fullUrl - ) - .header( - CommonHeaders.USER_AGENT, - UserAgents.CHROME_DESKTOP - ) - .url(cleanUrl) - .build() - - return chain.proceed(newRequest) - } - - return super.intercept(chain) - } - - - private fun copyCookies() { - val cookies = context.cookieJar.getCookies(domain) - - cookies.forEach { - context.cookieJar.insertCookies( - "cdn.$domain", - "${it.name}=${it.value}" - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt deleted file mode 100644 index b6ae5367..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt +++ /dev/null @@ -1,11 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.Manga18Parser - -@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) -internal class Manga18(context: MangaLoaderContext) : - Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt deleted file mode 100644 index 20995789..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt +++ /dev/null @@ -1,106 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.EnumSet - -@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) -internal class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { - - override val tagPrefix = "publication-genre/" - override val withoutAjax: Boolean = true - override val datePattern: String = "MMMM d, yyyy" - override val stylePage: String = "?style=list" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) - - override suspend fun getChapters(manga: Manga, doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - - val chapters = doc.body() - .select("li.wp-manga-chapter") - .mapNotNull { li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - - val name = a.selectFirst("p")?.text() - ?: a.ownText() - - val dateText = - li.selectFirst("a.c-new-tag")?.attr("title") - ?: li.selectFirst("span.chapter-release-date i")?.text() - - MangaChapter( - id = generateUid(href), - title = name, - number = extractChapterNumber(name) ?: 0f, - volume = 0, - url = link, - uploadDate = parseChapterDate(dateFormat, dateText), - scanlator = null, - branch = null, - source = source, - ) - } - .sortedWith( - compareBy { it.number } - .thenBy { it.title } - ) - - return chapters - } - - private fun extractChapterNumber(name: String): Float? { - val match = Regex( - """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", - RegexOption.IGNORE_CASE - ).find(name) - - return match - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/series/").parseHtml() - val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") - - return elements.mapNotNullToSet { a -> - val href = a.attr("href") - .removeSuffix("/") - .substringAfterLast(tagPrefix, "") - - if (href.isBlank()) return@mapNotNullToSet null - - MangaTag( - key = href, - title = a.text() - .replace(Regex("""\s*\(\d+\)"""), "") - .replace(Regex("""[^\x20-\x7E]"""), "") - .trim() - .toTitleCase(), - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt deleted file mode 100644 index c2b4e8ae..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt +++ /dev/null @@ -1,162 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.host -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.HashSet -import java.util.Locale - -@MangaSourceParser("MANHWA18CC", "Manhwa18.cc", "en", ContentType.HENTAI) -internal class Manhwa18CC(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANHWA18CC, "manhwa18.cc", 24) { - - override val datePattern = "dd MMM yyyy" - override val sourceLocale: Locale = Locale.ENGLISH - override val listUrl = "webtoons/" - override val tagPrefix = "webtoon-genre/" - override val withoutAjax = true - override val selectTestAsync = "ul.row-content-chapter" - override val selectDate = "span.chapter-time" - override val selectChapter = "li.a-h" - override val selectBodyPage = "div.read-content" - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - @Volatile - private var tagsCache: Set? = null - private val tagsMutex = Mutex() - - override suspend fun fetchAvailableTags(): Set { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTagsFromSite().also { tagsCache = it } - } - } - - private suspend fun fetchTagsFromSite(): Set { - val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() - val list = doc.body().selectFirstOrThrow("div.sub-menu").select("ul li") - val keySet = HashSet(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().ifEmpty { - a.selectFirst(".menu-image-title")?.text() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableStates = emptySet(), - availableContentRating = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - when { - !query.isNullOrEmpty() -> { - append("/search?q=") - append(query.urlEncoded()) - append("&page=") - append(page.toString()) - } - else -> { - val tag = filter.tags.oneOrThrowIfMany() - if (filter.tags.isNotEmpty()) { - append("/$tagPrefix") - append(tag?.key.orEmpty()) - } else { - append("/$listUrl") - } - if (page > 1) { - append(page.toString()) - } - append("?orderby=") - when (order) { - SortOrder.POPULARITY -> append("trending") - SortOrder.UPDATED -> append("latest") - SortOrder.ALPHABETICAL -> append("alphabet") - SortOrder.RATING -> append("rating") - else -> append("latest") - } - } - } - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.manga-lists div.manga-item").map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src(), - title = div.selectFirstOrThrow("h3").text(), - altTitles = emptySet(), - rating = div.selectFirst(".item-rate span")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow(selectBodyPage) - return root.select("img").map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt deleted file mode 100644 index c8ac5f66..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt +++ /dev/null @@ -1,240 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) -internal class Manhwa18Com(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { - - override val configKeyDomain = ConfigKey.Domain("manhwa18.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, - SortOrder.NEWEST, SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) - - private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - private val numericRegex = Regex("""\d+""") - - @Volatile - private var tagsCache: Set? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - ) - - private suspend fun getOrFetchTags(): Set { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTags().also { tagsCache = it } - } - } - - private suspend fun fetchTags(): Set { - val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() - val list = doc.select("[data-genre-id]") - if (list.isEmpty()) return emptySet() - return list.mapNotNull { item -> - val id = item.attr("data-genre-id") - val name = item.text().trim() - if (id.isNotEmpty() && name.isNotEmpty()) { - MangaTag(name.toTitleCase(Locale.ENGLISH), id, source) - } else null - }.toSet() - } - - private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { - doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } - doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } - val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") - ?: doc.selectFirst(".au-cover-art")?.attr("style") - if (styleUrl != null) { - val url = styleUrl.substringAfter("url('").substringBefore("')") - .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } - if (url.isNotEmpty()) { - return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) - } - } - return fallback - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - val url = buildString { - append("https://$domain/tim-kiem?page=$page") - if (query != null) { - append("&q=") - append(query.urlEncoded()) - } - append("&accept_genres=") - if (filter.tags.isNotEmpty()) { - append(filter.tags.joinToString(",") { it.key }) - } - append("&reject_genres=") - if (filter.tagsExclude.isNotEmpty()) { - append(filter.tagsExclude.joinToString(",") { it.key }) - } - append("&sort=") - append( - when (order) { - SortOrder.ALPHABETICAL -> "az" - SortOrder.POPULARITY -> "top" - SortOrder.UPDATED -> "update" - SortOrder.NEWEST -> "new" - SortOrder.RATING -> "like" - else -> "update" - } - ) - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "1" - MangaState.FINISHED -> "3" - MangaState.PAUSED -> "2" - else -> "" - } - ) - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".thumb-item-flow").map { element -> - val a = element.selectFirst("a") ?: return@map null - val absUrl = a.attrAsAbsoluteUrl("href") - val title = element.selectFirst(".series-title a")?.text() ?: a.text() - val cover = element.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg") ?: "" - Manga( - id = generateUid(absUrl.toRelativeUrl(domain)), - title = title, - altTitles = emptySet(), - url = absUrl.toRelativeUrl(domain), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - }.filterNotNull() - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title - val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) - val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() - - val statusText = doc.select(".au-stat") - .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } - ?.selectFirst(".v")?.text()?.trim() - val state = when (statusText?.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "on hold" -> MangaState.PAUSED - else -> null - } - - val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> - val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } - ?: return@mapNotNull null - MangaTag(a.text().trim(), slug, source) - }.toSet() - - val chapters = parseChapters(doc) - - val fullManga = manga.copy( - title = title, - coverUrl = coverUrl, - description = description, - state = state, - tags = tags, - authors = emptySet(), - chapters = chapters, - ) - - detailsCache[manga.url] = fullManga - return fullManga - } - - private fun parseChapters(doc: org.jsoup.nodes.Document): List { - return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> - val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) - val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null - val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() - val uploadDate = dateStr?.let { - runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) - } ?: 0L - val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() - ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() - ?: 0f - MangaChapter( - id = generateUid(chUrl), - title = chTitle, - number = number, - volume = 0, - url = chUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ) - }.sortedBy { it.number } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-content img.lazy").mapNotNull { img -> - val src = img.attrAsAbsoluteUrl("data-src") - .takeIf { it.isNotEmpty() } - ?: img.attrAsAbsoluteUrl("src").takeIf { it.isNotEmpty() } - ?: return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt deleted file mode 100644 index 89aaa5e3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt +++ /dev/null @@ -1,259 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.attrOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectLast -import org.koitharu.kotatsu.parsers.util.textOrNull -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) -internal class Manhwa210(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { - - override val configKeyDomain = ConfigKey.Domain("manhwa210.com") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - @Volatile - private var tagsCache: Set? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - private suspend fun getOrFetchTags(): Set { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTagsFromSite().also { tagsCache = it } - } - } - - private suspend fun fetchTagsFromSite(): Set { - val doc = webClient.httpGet("https://$domain").parseHtml() - return doc.select("ul.grid.grid-cols-2 a").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text(), - source = source, - ) - } - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() - - val details = manga.copy( - altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), - state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - }, - tags = root.select("div.mt-2:contains(Genres) a.bg-gray-500").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text(), - source = source, - ) - }, - authors = setOfNotNull(author), - description = root.selectFirst("meta[name=description]")?.attrOrNull("content"), - chapters = root.select("div.justify-between ul.overflow-y-auto.overflow-x-hidden a") - .mapNotNull { a -> - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() - val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() - val number = extractChapterNumber(name) - MangaChapter( - id = generateUid(href), - title = name, - number = number, - volume = 0, - url = href, - scanlator = null, - uploadDate = runCatching { chapterDateFormat.parse(dateText)?.time ?: 0L }.getOrDefault(0L), - branch = null, - source = source, - ) - } - .sortedBy { it.number } - ) - - detailsCache[manga.url] = details - return details - } - - private fun extractChapterNumber(title: String): Float { - val regex = Regex("""(?i)(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""") - regex.find(title)?.let { match -> - return match.groupValues[1].toFloatOrNull() ?: 0f - } - Regex("""(\d+(?:\.\d+)?)""").find(title)?.let { - return it.value.toFloatOrNull() ?: 0f - } - return 0f - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append("https://") - append(domain) - val query = filter.query - when { - !query.isNullOrEmpty() -> { - append("/search") - append("?filter[name]=") - append(query.urlEncoded()) - if (page > 1) { - append("&page=") - append(page) - } - append("&sort=") - append( - when (order) { - SortOrder.POPULARITY -> "-views" - SortOrder.UPDATED -> "-updated_at" - SortOrder.NEWEST -> "-created_at" - SortOrder.ALPHABETICAL -> "name" - SortOrder.ALPHABETICAL_DESC -> "-name" - else -> "-updated_at" - }, - ) - } - filter.tags.isNotEmpty() -> { - val tag = filter.tags.first() - append("/genre/") - append(tag.key) - append("?page=") - append(page) - } - else -> { - append("/list") - append("?sort=") - append( - when (order) { - SortOrder.POPULARITY -> "-views" - SortOrder.UPDATED -> "-updated_at" - SortOrder.NEWEST -> "-created_at" - SortOrder.ALPHABETICAL -> "name" - SortOrder.ALPHABETICAL_DESC -> "-name" - else -> "-updated_at" - }, - ) - append("&page=") - append(page) - } - } - if (filter.states.isNotEmpty()) { - append("&filter[status]=") - filter.states.forEach { - append( - when (it) { - MangaState.ONGOING -> "2," - MangaState.FINISHED -> "1," - else -> "1,2" - }, - ) - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.grid div.relative").map { div -> - val href = div.selectFirst("a[href^=/manga/]")?.attrOrNull("href") - ?: div.parseFailed("Cant find manga image!") - val coverUrl = div.selectFirst("div.cover")?.attr("style") - ?.substringAfter("url('")?.substringBefore("')") - Manga( - id = generateUid(href), - title = div.select("div.p-2 a.text-ellipsis").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = coverUrl.orEmpty(), - tags = setOf(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.text-center img.lazy").mapNotNull { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt deleted file mode 100644 index bff2c95b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.HeanCmsParser - -@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) -internal class OmegaScans(context: MangaLoaderContext) : - HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt deleted file mode 100644 index 0236ac51..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt +++ /dev/null @@ -1,392 +0,0 @@ -@file:Suppress("DEPRECATION") - -package org.koitharu.kotatsu.parsers.site.kotatsu.en.mangabox - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.parsers.MangaboxParser -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toTitleCase -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.LinkedHashSet -import java.util.Locale -import java.util.TimeZone -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.mapToSet - -@MangaSourceParser("MANGABAT", "MangaBats", "en") -internal class Mangabats(context: MangaLoaderContext) : - MangaboxParser(context, MangaParserSource.MANGABAT) { - - override val configKeyDomain = ConfigKey.Domain("mangabats.com") - - override val webClient = OkHttpWebClient( - context.httpClient.newBuilder() - .build(), - source, - ) - - - override val listUrl = "/genre/all" - override val searchUrl = "/search/story" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - ) - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val chaptersDeferred = async { getChapters(doc) } - - val stateText = doc.findInfo("Status")?.lowercase() - - val state = when { - stateText?.contains("ongoing") == true -> MangaState.ONGOING - stateText?.contains("completed") == true || - stateText?.contains("finish") == true -> MangaState.FINISHED - else -> null - } - - val alt = doc.findInfo("Alternative") - - manga.copy( - tags = doc.body() - .select(selectTag) - .mapToSet { a -> - val href = a.attr("href") - - MangaTag( - key = href.substringAfterLast("category=") - .substringBefore("&") - .takeIf { it != href } - ?: href.removeSuffix("/") - .substringAfterLast("/"), - - title = a.text().toTitleCase(), - source = source, - ) - }, - - description = "If empty Chapters, reload Manga!", - altTitles = setOfNotNull(alt), - state = state, - chapters = chaptersDeferred.await() - ) - } - - private fun Document.findInfo(label: String): String? { - return select("ul.manga-info-text li") - .firstOrNull { it.text().contains(label, ignoreCase = true) } - ?.text() - ?.substringAfter(":") - ?.trim() - ?.nullIfEmpty() - } - - @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) - override val searchQueryCapabilities: MangaSearchQueryCapabilities - get() = MangaSearchQueryCapabilities( - SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), - isMultiple = false, - isExclusive = true, - ), - SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - ) - - override suspend fun getListPage( - query: MangaSearchQuery, - page: Int - ): List { - - val titleQuery = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TITLE_NAME } - ?.value as? String - - val tagCriterion = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TAG } - - val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key - ?: tagCriterion?.values?.firstOrNull()?.toString() - - val state = query.criteria.filterIsInstance>() - .firstOrNull { it.field == STATE } - ?.values - ?.firstOrNull() as? MangaState - - val url = if (!titleQuery.isNullOrBlank()) { - - "https://$domain/search/story/${ - normalizeSearchQuery(titleQuery).replace('_', '-') - }?page=$page" - - } else { - - val base = if (!tagKey.isNullOrBlank()) { - "https://$domain/genre/$tagKey" - } else { - "https://$domain/genre/all" - } - - val filter = when { - state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 - state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 - - state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 - state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 - - query.order == SortOrder.POPULARITY -> 7 - query.order == SortOrder.NEWEST -> 1 - - else -> 4 // all + latest - } - - "$base?filter=$filter&page=$page" - } - - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain").parseHtml() - val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") - .drop(1) - - val result = LinkedHashSet() - - for (a in tags) { - val key = a.attr("href") - .removeSuffix("/") - .substringAfter("/genre/") - .substringBefore("?") - .nullIfEmpty() - ?: continue - - if (result.any { it.key == key }) { - continue - } - - val title = a.text() - .replace(" Manga", "") - .toTitleCase(sourceLocale) - - result.add( - MangaTag( - key = key, - title = title, - source = source, - ), - ) - } - - return result - } - - override suspend fun getChapters(doc: Document): List { - val slug = doc.location() - .substringAfter("/manga/", "") - .substringBefore("/") - .trim() - .ifEmpty { return super.getChapters(doc) } - - val apiChapters = runCatching { - fetchChaptersApi(slug) - }.getOrDefault(emptyList()) - - return apiChapters.ifEmpty { - super.getChapters(doc) - } - } - - private suspend fun fetchChaptersApi(slug: String): List { - - suspend fun request(offset: Int): JSONObject { - val apiUrl = - "https://$domain/api/manga/$slug/chapters" + - "?limit=$CHAPTER_LIST_TAKE&offset=$offset" - - return webClient.httpGet(apiUrl).parseJson() - } - - fun extractChapters(json: JSONObject): List { - val array = json - .optJSONObject("data") - ?.optJSONArray("chapters") - ?: return emptyList() - - return buildList { - for (i in 0 until array.length()) { - array.optJSONObject(i)?.let(::add) - } - } - } - // first request - val first = request(0) - - val chapters = mutableListOf() - chapters += extractChapters(first) - - val pagination = - first.optJSONObject("data") - ?.optJSONObject("pagination") - - val hasMore = - pagination?.optBoolean("has_more", false) ?: false - - if (!hasMore) { - return mapChapters(slug, chapters) - } - - val total = - pagination.optInt("total", 0) - - val offsets = - (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) - .toList() - // parallel loading - coroutineScope { - val results = offsets.map { offset -> - async { - request(offset) - } - }.awaitAll() - results.forEach { - chapters += extractChapters(it) - } - } - return mapChapters(slug, chapters) - } - - private fun mapChapters( - slug: String, - chapters: List - ): List { - - return chapters.mapNotNull { chapter -> - val chapterSlug = - chapter.optString("chapter_slug") - .nullIfEmpty() - ?: return@mapNotNull null - - val chapterName = - chapter.optString("chapter_name") - .nullIfEmpty() - ?: "Chapter" - - val number = - chapter.optString("chapter_num") - .toFloatOrNull() - ?: 0f - - val url = "/manga/$slug/$chapterSlug" - - MangaChapter( - id = generateUid(url), - title = chapterName, - number = number, - volume = 0, - url = url, - uploadDate = parseApiDate( - chapter.optString("updated_at") - ), - source = source, - scanlator = null, - branch = null, - ) - }.sortedBy { - it.number - } - } - - - private fun parseApiDate(date: String?): Long { - if (date.isNullOrBlank()) { - return 0L - } - - val formats = listOf( - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), - ).onEach { - it.timeZone = TimeZone.getTimeZone("UTC") - } - - for (format in formats) { - val parsed = format.parseSafe(date) - if (parsed != 0L) { - return parsed - } - } - - return 0L - } - - @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://www.mangabats.com/") - .build() - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - - if (request.method == "GET" || request.method == "POST") { - val newRequest = request.newBuilder() - .removeHeader("Content-Encoding") - .removeHeader("Accept-Encoding") - .build() - - return chain.proceed(newRequest) - } - - return chain.proceed(request) - } - - companion object { - - private const val CHAPTER_LIST_TAKE = 1000 - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt deleted file mode 100644 index 186cd712..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt +++ /dev/null @@ -1,392 +0,0 @@ -@file:Suppress("DEPRECATION") - -package org.koitharu.kotatsu.parsers.site.kotatsu.en.mangabox - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.parsers.MangaboxParser -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toTitleCase -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.LinkedHashSet -import java.util.Locale -import java.util.TimeZone -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.mapToSet - -@MangaSourceParser("MANGANATOGG", "Manganato.gg", "en") -internal class ManganatoGG(context: MangaLoaderContext) : - MangaboxParser(context, MangaParserSource.MANGANATOGG) { - - override val configKeyDomain = ConfigKey.Domain("manganato.gg") - - override val webClient = OkHttpWebClient( - context.httpClient.newBuilder() - .build(), - source, - ) - - - override val listUrl = "/genre/all" - override val searchUrl = "/search/story" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - ) - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val chaptersDeferred = async { getChapters(doc) } - - val stateText = doc.findInfo("Status")?.lowercase() - - val state = when { - stateText?.contains("ongoing") == true -> MangaState.ONGOING - stateText?.contains("completed") == true || - stateText?.contains("finish") == true -> MangaState.FINISHED - else -> null - } - - val alt = doc.findInfo("Alternative") - - manga.copy( - tags = doc.body() - .select(selectTag) - .mapToSet { a -> - val href = a.attr("href") - - MangaTag( - key = href.substringAfterLast("category=") - .substringBefore("&") - .takeIf { it != href } - ?: href.removeSuffix("/") - .substringAfterLast("/"), - - title = a.text().toTitleCase(), - source = source, - ) - }, - - description = "If empty Chapters, reload Manga!", - altTitles = setOfNotNull(alt), - state = state, - chapters = chaptersDeferred.await() - ) - } - - private fun Document.findInfo(label: String): String? { - return select("ul.manga-info-text li") - .firstOrNull { it.text().contains(label, ignoreCase = true) } - ?.text() - ?.substringAfter(":") - ?.trim() - ?.nullIfEmpty() - } - - @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) - override val searchQueryCapabilities: MangaSearchQueryCapabilities - get() = MangaSearchQueryCapabilities( - SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), - isMultiple = false, - isExclusive = true, - ), - SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - ) - - override suspend fun getListPage( - query: MangaSearchQuery, - page: Int - ): List { - - val titleQuery = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TITLE_NAME } - ?.value as? String - - val tagCriterion = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TAG } - - val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key - ?: tagCriterion?.values?.firstOrNull()?.toString() - - val state = query.criteria.filterIsInstance>() - .firstOrNull { it.field == STATE } - ?.values - ?.firstOrNull() as? MangaState - - val url = if (!titleQuery.isNullOrBlank()) { - - "https://$domain/search/story/${ - normalizeSearchQuery(titleQuery).replace('_', '-') - }?page=$page" - - } else { - - val base = if (!tagKey.isNullOrBlank()) { - "https://$domain/genre/$tagKey" - } else { - "https://$domain/genre/all" - } - - val filter = when { - state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 - state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 - - state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 - state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 - - query.order == SortOrder.POPULARITY -> 7 - query.order == SortOrder.NEWEST -> 1 - - else -> 4 // all + latest - } - - "$base?filter=$filter&page=$page" - } - - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain").parseHtml() - val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") - .drop(1) - - val result = LinkedHashSet() - - for (a in tags) { - val key = a.attr("href") - .removeSuffix("/") - .substringAfter("/genre/") - .substringBefore("?") - .nullIfEmpty() - ?: continue - - if (result.any { it.key == key }) { - continue - } - - val title = a.text() - .replace(" Manga", "") - .toTitleCase(sourceLocale) - - result.add( - MangaTag( - key = key, - title = title, - source = source, - ), - ) - } - - return result - } - - override suspend fun getChapters(doc: Document): List { - val slug = doc.location() - .substringAfter("/manga/", "") - .substringBefore("/") - .trim() - .ifEmpty { return super.getChapters(doc) } - - val apiChapters = runCatching { - fetchChaptersApi(slug) - }.getOrDefault(emptyList()) - - return apiChapters.ifEmpty { - super.getChapters(doc) - } - } - - private suspend fun fetchChaptersApi(slug: String): List { - - suspend fun request(offset: Int): JSONObject { - val apiUrl = - "https://$domain/api/manga/$slug/chapters" + - "?limit=$CHAPTER_LIST_TAKE&offset=$offset" - - return webClient.httpGet(apiUrl).parseJson() - } - - fun extractChapters(json: JSONObject): List { - val array = json - .optJSONObject("data") - ?.optJSONArray("chapters") - ?: return emptyList() - - return buildList { - for (i in 0 until array.length()) { - array.optJSONObject(i)?.let(::add) - } - } - } - // first request - val first = request(0) - - val chapters = mutableListOf() - chapters += extractChapters(first) - - val pagination = - first.optJSONObject("data") - ?.optJSONObject("pagination") - - val hasMore = - pagination?.optBoolean("has_more", false) ?: false - - if (!hasMore) { - return mapChapters(slug, chapters) - } - - val total = - pagination.optInt("total", 0) - - val offsets = - (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) - .toList() - // parallel loading - coroutineScope { - val results = offsets.map { offset -> - async { - request(offset) - } - }.awaitAll() - results.forEach { - chapters += extractChapters(it) - } - } - return mapChapters(slug, chapters) - } - - private fun mapChapters( - slug: String, - chapters: List - ): List { - - return chapters.mapNotNull { chapter -> - val chapterSlug = - chapter.optString("chapter_slug") - .nullIfEmpty() - ?: return@mapNotNull null - - val chapterName = - chapter.optString("chapter_name") - .nullIfEmpty() - ?: "Chapter" - - val number = - chapter.optString("chapter_num") - .toFloatOrNull() - ?: 0f - - val url = "/manga/$slug/$chapterSlug" - - MangaChapter( - id = generateUid(url), - title = chapterName, - number = number, - volume = 0, - url = url, - uploadDate = parseApiDate( - chapter.optString("updated_at") - ), - source = source, - scanlator = null, - branch = null, - ) - }.sortedBy { - it.number - } - } - - - private fun parseApiDate(date: String?): Long { - if (date.isNullOrBlank()) { - return 0L - } - - val formats = listOf( - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), - ).onEach { - it.timeZone = TimeZone.getTimeZone("UTC") - } - - for (format in formats) { - val parsed = format.parseSafe(date) - if (parsed != 0L) { - return parsed - } - } - - return 0L - } - - @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://www.mangabats.com/") - .build() - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - - if (request.method == "GET" || request.method == "POST") { - val newRequest = request.newBuilder() - .removeHeader("Content-Encoding") - .removeHeader("Accept-Encoding") - .build() - - return chain.proceed(newRequest) - } - - return chain.proceed(request) - } - - companion object { - - private const val CHAPTER_LIST_TAKE = 1000 - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt deleted file mode 100644 index cf97ba11..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt +++ /dev/null @@ -1,301 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.EnumSet -import java.util.TimeZone -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock - -@MangaSourceParser("AINZSCANS", "AinzScans", "id") -internal class AinzScans(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.AINZSCANS, pageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("v2.ainzscans01.com") - - private val apiBase = "https://api.ainzscans01.com/api" - - override val sourceLocale: Locale = Locale.US - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres().values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), - ) - - @Volatile - private var genresCache: Map? = null - private val genresMutex = Mutex() - - private suspend fun getOrFetchGenres(): Map { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - val jsonStr = webClient.httpGet("$apiBase/genres").body?.string().orEmpty() - val map = mutableMapOf() - try { - val array = JSONArray(jsonStr) - for (i in 0 until array.length()) { - val obj = array.getJSONObject(i) - val name = obj.getString("name") - val slug = obj.getString("slug") - map[slug] = MangaTag(title = name, key = slug, source = source) - } - } catch (_: Exception) { } - genresCache = map - map - } - } - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .build() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = buildString { - append(apiBase) - append("/search?type=COMIC") - append("&limit=").append(pageSize) - append("&page=").append(page) - - filter.query?.takeIf { it.isNotBlank() }?.let { - append("&q=").append(it.urlEncoded()) - } - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "new" - SortOrder.RATING -> "rate" - SortOrder.ALPHABETICAL -> "az" - else -> "latest" - } - ) - filter.states.oneOrThrowIfMany()?.let { state -> - append("&status=") - append( - when (state) { - MangaState.ONGOING -> "ONGOING" - MangaState.FINISHED -> "COMPLETED" - MangaState.PAUSED -> "HIATUS" - else -> "" - } - ) - } - filter.tags.firstOrNull()?.let { tag -> - append("&genre=").append(tag.key) - } - filter.types.oneOrThrowIfMany()?.let { type -> - append("&comic_type=") - append( - when (type) { - ContentType.MANGA -> "MANGA" - ContentType.MANHWA -> "MANHWA" - ContentType.MANHUA -> "MANHUA" - else -> "" - } - ) - } - } - - val json = webClient.httpGet(url).body?.string().orEmpty() - return parseListJson(json) - } - - private fun parseListJson(json: String): List { - val result = mutableListOf() - try { - val obj = JSONObject(json) - val dataArray = obj.optJSONArray("data") ?: return emptyList() - for (i in 0 until dataArray.length()) { - val item = dataArray.getJSONObject(i) - val slug = item.optString("slug").ifEmpty { continue } - val relativeUrl = "/comic/$slug" - val title = item.optString("title").ifEmpty { slug } - val cover = item.optString("poster_image_url").nullIfEmpty() - result.add( - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - publicUrl = "https://$domain$relativeUrl", - title = title, - coverUrl = cover, - rating = RATING_UNKNOWN, - contentRating = ContentRating.SAFE, - tags = emptySet(), - state = null, - authors = emptySet(), - altTitles = emptySet(), - source = source, - ) - ) - } - } catch (_: Exception) { - } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val slug = manga.url.substringAfter("/comic/") - val json = webClient.httpGet("$apiBase/series/comic/$slug").body?.string().orEmpty() - val result = parseDetailsJson(json, manga) - - detailsCache[manga.url] = result - return result - } - - private fun parseDetailsJson(json: String, manga: Manga): Manga { - try { - val obj = JSONObject(json) - val slug = obj.optString("slug") - val relativeUrl = "/comic/$slug" - - val title = obj.optString("title").ifEmpty { manga.title } - val desc = obj.optString("synopsis").nullIfEmpty()?.let { org.jsoup.Jsoup.parse(it).text() } - val rating = obj.optString("rating_average").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN - val state = when (obj.optString("comic_status").uppercase()) { - "ONGOING" -> MangaState.ONGOING - "COMPLETED" -> MangaState.FINISHED - "HIATUS" -> MangaState.PAUSED - else -> null - } - val author = obj.optString("author_name").takeIf { it.isNotEmpty() } - val artist = obj.optString("artist_name").takeIf { it.isNotEmpty() } - val genres = obj.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).mapNotNull { i -> - val g = arr.getJSONObject(i) - val gSlug = g.optString("slug") - val gName = g.optString("name") - if (gSlug.isNotEmpty() && gName.isNotEmpty()) MangaTag(gName, gSlug, source) else null - }.toSet() - } ?: emptySet() - - val chapters = parseUnitsJson(obj.optJSONArray("units"), slug) - - return manga.copy( - title = title, - url = relativeUrl, - publicUrl = "https://$domain$relativeUrl", - description = desc, - rating = rating, - state = state, - authors = setOfNotNull(author, artist), - tags = genres, - chapters = chapters, - ) - } catch (_: Exception) { - return manga - } - } - - private fun parseUnitsJson(array: JSONArray?, seriesSlug: String): List { - if (array == null) return emptyList() - val chapters = mutableListOf() - for (i in 0 until array.length()) { - val obj = array.getJSONObject(i) - val chSlug = obj.optString("slug").ifEmpty { continue } - val number = obj.optString("number") - val date = obj.optString("created_at") - val uploadDate = parseDate(date) - chapters.add( - MangaChapter( - id = generateUid("/comic/$seriesSlug/chapter/$chSlug"), - url = "/comic/$seriesSlug/chapter/$chSlug", - title = "Chapter ${number.removeSuffix(".00")}", - number = number.toFloatOrNull() ?: (chapters.size + 1).toFloat(), - volume = 0, - uploadDate = uploadDate, - scanlator = null, - branch = null, - source = source, - ) - ) - } - return chapters.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List { - val slug = chapter.url.substringAfter("/comic/").substringBefore("/chapter/") - val chSlug = chapter.url.substringAfterLast("/") - val json = webClient.httpGet("$apiBase/series/comic/$slug/chapter/$chSlug").body?.string().orEmpty() - try { - val obj = JSONObject(json) - val chapterObj = obj.optJSONObject("chapter") ?: return emptyList() - val pagesArray = chapterObj.optJSONArray("pages") ?: return emptyList() - return (0 until pagesArray.length()).mapNotNull { i -> - val pageObj = pagesArray.getJSONObject(i) - val rawUrl = pageObj.optString("image_url").nullIfEmpty() ?: return@mapNotNull null - val fixedUrl = fixImageUrl(rawUrl) - MangaPage(id = generateUid(fixedUrl), url = fixedUrl, preview = null, source = source) - } - } catch (_: Exception) { - return emptyList() - } - } - - private fun fixImageUrl(url: String): String { - var fixed = if (url.startsWith("http")) url else "https://api.ainzscans01.com$url" - - if (fixed.contains("googleusercontent.com") || fixed.contains("bp.blogspot.com")) { - fixed = fixed.replace(Regex("""=[swh]\d+[^/?]*($|\?)""", RegexOption.IGNORE_CASE), "=s0$1") - .replace(Regex("""/[swh]\d+[^/]*/""", RegexOption.IGNORE_CASE), "/s0/") - } - - val httpUrl = fixed.toHttpUrlOrNull() - if (httpUrl != null && httpUrl.queryParameterNames.any { it == "w" || it == "width" || it == "resize" }) { - fixed = httpUrl.newBuilder() - .removeAllQueryParameters("w") - .removeAllQueryParameters("width") - .removeAllQueryParameters("resize") - .build() - .toString() - } - return fixed - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - private fun parseDate(dateStr: String?): Long { - if (dateStr.isNullOrEmpty()) return 0L - return try { - dateFormat.parse(dateStr)?.time ?: 0L - } catch (_: Exception) { 0L } - } - - private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt deleted file mode 100644 index 225bb7a3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt +++ /dev/null @@ -1,178 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import okhttp3.Interceptor -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import androidx.collection.arraySetOf - -@MangaSourceParser("MGKOMIK", "MGKomik", "id") -internal class MGKomik(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MGKOMIK, "id.mgkomik.cc"), Interceptor, MangaParserAuthProvider { - - override val datePattern = "dd MMM yy" - override val tagPrefix = "genres/" - override val selectDesc = "div.description-summary div.summary__content p" - - override val withoutAjax = true - - override fun intercept(chain: Interceptor.Chain): okhttp3.Response { - val request = chain.request() - val path = request.url.encodedPath - val isAjax = path.contains("admin-ajax.php") || - path.contains("wp-json") || - path.endsWith("/ajax/chapters") - - return if (isAjax) { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .header("X-Requested-With", "XMLHttpRequest") - .header("Sec-Fetch-Dest", "empty") - .header("Sec-Fetch-Mode", "cors") - .header("Sec-Fetch-Site", "same-origin") - .header("Origin", "https://$domain") - .header("Priority", "u=1, i") - .removeHeader("Sec-Fetch-User") - .removeHeader("Upgrade-Insecure-Requests") - .build() - ) - } else { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .build() - ) - } - } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - if (order == SortOrder.POPULARITY) { - val p = page + 1 - val url = buildString { - append("https://") - append(domain) - append("/komik") - if (p > 1) { - append("/page/") - append(p) - } - append("/?m_orderby=trending") - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - return super.getListPage(page, order, filter) - } - - override suspend fun fetchAvailableTags(): Set { - return super.fetchAvailableTags() - } - - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - - private fun availableTags() = arraySetOf( - MangaTag("2 hours ago", "2-hours-ago", source), - MangaTag("7 hours ago", "7-hours-ago", source), - MangaTag("8 hours ago", "8-hours-ago", source), - MangaTag("11 hours ago", "11-hours-ago", source), - MangaTag("13 hours ago", "13-hours-ago", source), - MangaTag("15 hours ago", "15-hours-ago", source), - MangaTag("1 day ago", "1-day-ago", source), - MangaTag("2 days ago", "2-days-ago", source), - MangaTag("Action", "action", source), - MangaTag("Adaptation", "adaptation", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Animals", "animals", source), - MangaTag("Apocalypse", "apocalypse", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Crime", "crime", source), - MangaTag("Demon", "demon", source), - MangaTag("Demons", "demons", source), - MangaTag("Drama", "drama", source), - MangaTag("Dungeons", "dungeons", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Fight", "fight", source), - MangaTag("Fighting", "fighting", source), - MangaTag("Full Color", "full-color", source), - MangaTag("Game", "game", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Josei(W)", "joseiw", source), - MangaTag("Kids", "kids", source), - MangaTag("Magic", "magic", source), - MangaTag("Manga", "manga", source), - MangaTag("Mangatoon", "mangatoon", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mature", "mature", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Military", "military", source), - MangaTag("Modern", "modern", source), - MangaTag("Monsters", "monsters", source), - MangaTag("Murim", "murim", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Office Workers", "office-workers", source), - MangaTag("One-Shot", "one-shot", source), - MangaTag("Overpowered", "overpowered", source), - MangaTag("Police", "police", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Regression", "regression", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Revenge", "revenge", source), - MangaTag("Reverse Harem", "reverse-harem", source), - MangaTag("Rofan", "rofan", source), - MangaTag("Romance", "romance", source), - MangaTag("School", "school", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujoai", source), - MangaTag("Shoujo AI", "shoujo-ai", source), - MangaTag("Shoujo(G)", "shoujog", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Smut", "smut", source), - MangaTag("Sports", "sports", source), - MangaTag("Super Power", "super-power", source), - MangaTag("Superhero", "superhero", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Supranatural", "supranatural", source), - MangaTag("Survival", "survival", source), - MangaTag("System", "system", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Transmigration", "transmigration", source), - MangaTag("Vampire", "vampire", source), - MangaTag("Villainess", "villainess", source), - MangaTag("Violence", "violence", source), - MangaTag("War", "war", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Webtoons", "webtoons", source), - MangaTag("Wuxia", "wuxia", source), - MangaTag("Yuri", "yuri", source), - ) - -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt deleted file mode 100644 index 1a684f3c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt +++ /dev/null @@ -1,177 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.ZeistMangaParser -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList -import java.text.SimpleDateFormat - -@MangaSourceParser("REYUME", "ReYume", "id") -internal class ReYume(context: MangaLoaderContext) : - ZeistMangaParser(context, MangaParserSource.REYUME, "www.re-yume.my.id") { - - override val selectTags = "a[rel=tag]" - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query - val url = buildString { - append("https://") - append(domain) - append("/feeds/posts/default/-/") - when { - !query.isNullOrEmpty() -> { - append("Series") - append("?alt=json&orderby=published&max-results=") - append((maxMangaResults + 1).toString()) - append("&start-index=") - append(startIndex.toString()) - append("&q=label:Series+") - append(query.urlEncoded()) - } - else -> { - if (filter.tags.isNotEmpty()) { - append(filter.tags.joinToString("/") { it.key.urlEncoded() }) - } else { - append("Series") - } - append("?alt=json&orderby=published&max-results=") - append((maxMangaResults + 1).toString()) - append("&start-index=") - append(startIndex.toString()) - } - } - } - - val json = webClient.httpGet(url).parseJson().getJSONObject("feed") - val list = if (json.toString().contains("\"entry\":")) { - parseMangaList(json.getJSONArray("entry")) - } else { - emptyList() - } - return if (list.size > maxMangaResults) list.dropLast(1) else list - } - - override suspend fun getDetails(manga: Manga):Manga { - val baseDetails = super.getDetails(manga) - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = if (baseDetails.title == "Unknown manga" || baseDetails.title.isEmpty()) { - doc.selectFirst("h1#post-title")?.text() ?: baseDetails.title - } else { - baseDetails.title - } - - var desc = baseDetails.description - if (desc.isNullOrEmpty()) { - desc = doc.getElementById("syn_bod")?.text() ?: "" - } - - var author = baseDetails.authors - if (author.isEmpty()) { - val authorText = doc.getElementById("tauther")?.text() ?: doc.getElementById("tauthers")?.text() - if (!authorText.isNullOrEmpty()) { - author = setOf(authorText) - } - } - - return baseDetails.copy( - title = title, - description = desc, - authors = author, - ) - } - - override fun parseMangaList(json: JSONArray): List { - return super.parseMangaList(json).map { - val cleanUrl = it.url.substringBefore("?m=1").substringBefore("&m=1") - val cleanPublic = it.publicUrl.substringBefore("?m=1").substringBefore("&m=1") - it.copy( - url = cleanUrl, - publicUrl = cleanPublic, - id = generateUid(cleanUrl) - ) - } - } - - override suspend fun loadChapters(mangaUrl: String, doc: Document): List { - val label = doc.selectFirst(".chapter_get")?.attr("data-labelchapter") - ?: doc.selectFirst(".libraryAdd")?.attr("data-title") - ?: return super.loadChapters(mangaUrl, doc) - - val url = buildString { - append("https://") - append(domain) - append("/feeds/posts/default/-/") - append(label.urlEncoded()) - append("?alt=json&orderby=published&max-results=9999") - } - - val json = webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList().reversed() - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val mangaTitle = doc.selectFirst("h1#post-title")?.text().orEmpty() - - return json.mapIndexedNotNull { i, j -> - val name = j.getJSONObject("title").getString("\$t") - val chapterName = if (mangaTitle.isNotEmpty() && name.contains(mangaTitle, ignoreCase = true)) { - name.replace(mangaTitle, "", ignoreCase = true).trim().trim('-').trim().takeIf { it.isNotEmpty() } ?: name - } else { - name - } - val href = j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" }.getString("href") - val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") - val slug = mangaUrl.substringAfterLast('/') - val slugChapter = href.substringAfterLast('/') - if (slug == slugChapter) return@mapIndexedNotNull null - MangaChapter( - id = generateUid(href), - url = href, - title = chapterName, - number = i + 1f, - volume = 0, - branch = null, - uploadDate = dateFormat.parseSafe(dateText), - scanlator = null, - source = source, - ) - } - } - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain").parseHtml() - val script = doc.selectFirst("script:containsData(filterGenre =)")?.data() - if (script != null) { - val genres = script.substringAfter("filterGenre = [").substringBefore("]") - .replace("'", "").replace("\"", "").split(",") - return genres.mapToSet { - val tag = it.trim() - MangaTag(key = tag, title = tag, source = source) - } - } - return super.fetchAvailableTags() - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(selectPage) - .filter { el -> el.parents().any { it.className().contains("separator") } } - .map { img -> - MangaPage( - id = generateUid(img.requireSrc()), - url = img.requireSrc(), - preview = null, - source = source, - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt deleted file mode 100644 index 93aa0fae..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt +++ /dev/null @@ -1,257 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.util.* -import javax.crypto.Mac -import javax.crypto.spec.SecretKeySpec - -@MangaSourceParser("WESTMANGA", "WestManga", "id") -internal class WestmangaParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.WESTMANGA, pageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("v1.westmanga.cc") - - private val apiDomain: String - get() = "data.mantweh.online" - - private val accessKey = "WM_WEB_FRONT_END" - private val secretKey = "xxxoidj" - - private var genresCache: Set? = null - private val mutex = Mutex() - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.NEWEST, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchGenres(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = "https://$apiDomain/api/contents".toHttpUrl().newBuilder().apply { - if (!filter.query.isNullOrEmpty()) { - addQueryParameter("q", filter.query) - } - addQueryParameter("page", page.toString()) - addQueryParameter("per_page", "20") - addQueryParameter("type", "Comic") - - when (order) { - SortOrder.UPDATED -> addQueryParameter("sort", "updated_at") - SortOrder.POPULARITY -> addQueryParameter("sort", "views") - SortOrder.ALPHABETICAL -> addQueryParameter("sort", "title") - SortOrder.NEWEST -> addQueryParameter("sort", "created_at") - else -> {} - } - - filter.tags.forEach { tag -> - addQueryParameter("genre[]", tag.key) - } - - filter.states.oneOrThrowIfMany()?.let { - addQueryParameter("status", when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - else -> return@let - }) - } - - filter.types.oneOrThrowIfMany()?.let { - addQueryParameter("country", when (it) { - ContentType.MANGA -> "JP" - ContentType.MANHWA -> "KR" - ContentType.MANHUA -> "CN" - else -> return@let - }) - } - }.build() - - val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() - val mangaArray = response.optJSONArray("data") ?: return emptyList() - - return mangaArray.mapJSON { jo -> - val slug = jo.getString("slug") - Manga( - id = generateUid(slug), - title = jo.getString("title"), - altTitles = emptySet(), - url = "/manga/$slug/", - publicUrl = "https://$domain/comic/$slug", - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = jo.getString("cover"), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.removeSurrounding("/manga/", "/") - val url = "https://$apiDomain/api/comic/$slug".toHttpUrl() - - val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() - val data = response.getJSONObject("data") - - val tags = buildSet { - when (data.getStringOrNull("country_id") ?: data.getStringOrNull("country")) { - "JP" -> add(MangaTag("Manga", "manga", source)) - "CN" -> add(MangaTag("Manhua", "manhua", source)) - "KR" -> add(MangaTag("Manhwa", "manhwa", source)) - } - if (data.optBoolean("color", false)) { - add(MangaTag("Colored", "colored", source)) - } - data.optJSONArray("genres")?.let { genres -> - for (i in 0 until genres.length()) { - val genre = genres.getJSONObject(i) - add(MangaTag( - title = genre.getString("name"), - key = genre.getInt("id").toString(), - source = source, - )) - } - } - } - - val description = buildString { - data.getStringOrNull("sinopsis")?.let { synopsis -> - append(org.jsoup.Jsoup.parseBodyFragment(synopsis).wholeText().trim()) - } - data.getStringOrNull("alternative_name")?.let { alt -> - append("\n\nAlternative Name: ") - append(alt.trim()) - } - } - - val state = when (data.getStringOrNull("status")) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - else -> null - } - - val chapters = data.optJSONArray("chapters")?.mapJSON { chapterObj -> - val chapterSlug = chapterObj.getString("slug") - val updatedAt = chapterObj.optJSONObject("updated_at")?.getLongOrDefault("time", 0L) - ?: chapterObj.getLongOrDefault("updated_at", 0L) - MangaChapter( - id = generateUid(chapterSlug), - url = "/$chapterSlug/", - title = "Chapter ${chapterObj.getString("number")}", - number = chapterObj.getString("number").toFloatOrNull() ?: 0f, - volume = 0, - branch = null, - uploadDate = updatedAt * 1000, - scanlator = null, - source = source, - ) - } ?: emptyList() - - return manga.copy( - title = data.getString("title"), - coverUrl = data.getString("cover"), - tags = tags, - authors = setOfNotNull(data.getStringOrNull("author")), - description = description.takeIf { it.isNotEmpty() }, - state = state, - chapters = chapters.reversed(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val slug = chapter.url.removeSurrounding("/", "/") - val url = "https://$apiDomain/api/v/$slug".toHttpUrl() - - val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() - val images = response.optJSONObject("data")?.getJSONArray("images") - ?: response.getJSONArray("images") - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - val imageUrl = images.getString(i) - pages.add( - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - ) - } - return pages - } - - private suspend fun fetchGenres(): Set = mutex.withLock { - genresCache?.let { return@withLock it } - - val url = "https://$apiDomain/api/contents/genres".toHttpUrl() - val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() - val genresArray = response.optJSONArray("data") ?: return@withLock emptySet() - - val genres = genresArray.mapJSONToSet { genreObj -> - MangaTag( - title = genreObj.getString("name"), - key = genreObj.getInt("id").toString(), - source = source, - ) - } - - genresCache = genres - return@withLock genres - } - - private fun createApiHeaders(url: okhttp3.HttpUrl): Headers { - val timestamp = (System.currentTimeMillis() / 1000).toString() - val message = "wm-api-request" - val key = timestamp + "GET" + url.encodedPath + accessKey + secretKey - - val mac = Mac.getInstance("HmacSHA256") - val secretKeySpec = SecretKeySpec(key.toByteArray(Charsets.UTF_8), "HmacSHA256") - mac.init(secretKeySpec) - val hash = mac.doFinal(message.toByteArray(Charsets.UTF_8)) - val signature = hash.joinToString("") { "%02x".format(it) } - - return Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .add("Referer", "https://$domain/") - .add("x-wm-request-time", timestamp) - .add("x-wm-accses-key", accessKey) - .add("x-wm-request-signature", signature) - .build() - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt deleted file mode 100644 index 625f0b21..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt +++ /dev/null @@ -1,184 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id.hentais - -import okhttp3.Headers -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* - -@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) -internal class BacaKomikpics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance - - override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") - .build() - - override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = false, // weird handling of website - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val url = when (order) { - SortOrder.POPULARITY -> "https://$domain/komik-populer/" - SortOrder.UPDATED -> { - if (page == 0) "https://$domain/?s=&post_type=komik_series" - else "https://$domain/page/${page + 1}/?s&post_type=komik_series" - } - else -> "https://$domain/komik-populer/" - } - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List { - return doc.select("a.card").mapNotNull { card -> - val href = card.attrAsAbsoluteUrl("href") - val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null - val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() - val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() - val status = card.selectFirst(".status-label")?.text()?.trim() - Manga( - id = generateUid(href), - url = href.toRelativeUrl(domain), - publicUrl = href, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = cover ?: "", - tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), - state = when (status) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst(".post-title h1")?.text()?.trim() - ?: doc.selectFirst("h1.entry-title")?.text()?.trim() - ?: manga.title - - val description = doc.selectFirst(".sinopsis")?.text()?.trim() - ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" - - val cover = doc.selectFirst(".series-thumb img")?.attr("src") - ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl - - val tags = doc.select("a.genre-link").mapNotNull { a -> - val genre = a.text().trim() - val key = a.attr("href").substringAfter("/genre/").substringBefore("/") - if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null - }.toSet() - - val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() - ?.substringAfter("Status :")?.trim() - val state = when { - statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING - statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED - statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED - else -> null - } - - val chapters = extractChapters(doc) - - return manga.copy( - title = title, - description = description, - coverUrl = cover, - tags = tags, - state = state, - authors = emptySet(), - chapters = chapters, - ) - } - - private fun extractChapters(doc: Document): List { - val script = doc.select("script").find { it.data().contains("chapterData") } - ?: return emptyList() - val data = script.data() - - val prefixIndex = data.indexOf("chapterData") - if (prefixIndex == -1) return emptyList() - - val windowStart = (prefixIndex - 500).coerceAtLeast(0) - val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) - val window = data.substring(windowStart, windowEnd) - - val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) - val match = arrayRegex.find(window) ?: return emptyList() - val jsonStr = match.groupValues[1].replace("\\/", "/") - - return try { - val jsonArray = JSONArray(jsonStr) - (0 until jsonArray.length()).mapNotNull { i -> - val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null - val chTitle = obj.optString("title", "") - val chUrl = obj.optString("url", "") - val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) - .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) - MangaChapter( - id = generateUid(chUrl), - title = chTitle, - number = number, - volume = 0, - url = chUrl.toRelativeUrl(domain), - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ) - } - } catch (e: Exception) { - e.printStackTrace() - emptyList() - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> - var src = img.attr("src") - if (src.isBlank()) src = img.attr("data-src") - if (src.isBlank()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun String.nullIfEmpty() = ifBlank { null } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt deleted file mode 100644 index b6c01ea9..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt +++ /dev/null @@ -1,21 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.DoujinDesuParser -import java.util.EnumSet - -@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) -internal class DoujinDesu(context: MangaLoaderContext) : - DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { - - override val defaultTypes: String = "doujinshi,manga,manhwa" - - override val availableContentTypes: Set = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.MANHWA - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt deleted file mode 100644 index b5dbcac1..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser - -@MangaSourceParser("HIPERTOON", "Hipertoon", "pt") -internal class HiperToon(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.HIPERTOON, "hipertoon.com") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt deleted file mode 100644 index e2737907..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt +++ /dev/null @@ -1,218 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("ROXINHA", "Roxinha", "pt") -internal class Roxinha(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ROXINHA, pageSize = 24) { - - override val configKeyDomain = ConfigKey.Domain("roxinha.online") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - - ) - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .build() - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( - availableTags = emptySet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS // Webtoon - ), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val offset = (page - 1) * pageSize - val url = "https://$domain/api/manga/search/advanced".toHttpUrl().newBuilder().apply { - addQueryParameter("limit", pageSize.toString()) - addQueryParameter("offset", offset.toString()) - addQueryParameter("mode", "default") - - if (!filter.query.isNullOrBlank()) { - addQueryParameter("q", filter.query!!) - } - - val (sortField, sortOrder) = when (order) { - SortOrder.ALPHABETICAL -> "title" to "ASC" - SortOrder.UPDATED -> "updatedAt" to "DESC" - SortOrder.POPULARITY -> "views" to "DESC" - SortOrder.RATING -> "avgRating" to "DESC" - else -> "title" to "ASC" - } - addQueryParameter("sort", sortField) - addQueryParameter("order", sortOrder) - - if (filter.states.isNotEmpty()) { - val status = when (filter.states.first()) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> null - } - status?.let { addQueryParameter("status", it) } - } - - if (filter.types.isNotEmpty()) { - val type = when (filter.types.first()) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Webtoon" - else -> null - } - type?.let { addQueryParameter("type", it) } - } - }.build() - - val json = webClient.httpGet(url).parseJson() - val mangasArray = json.optJSONArray("mangas") ?: return emptyList() - - val list = mutableListOf() - for (i in 0 until mangasArray.length()) { - val item = mangasArray.getJSONObject(i) - list.add(item.toManga()) - } - return list - } - - private fun JSONObject.toManga(): Manga { - val id = getInt("id") - return Manga( - id = generateUid(id.toString()), - url = id.toString(), - publicUrl = "https://$domain/manga/$id", - title = getString("title"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } ?: "", - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - override suspend fun getDetails(manga: Manga): Manga { - val id = manga.url - val json = webClient.httpGet("https://$domain/api/manga/$id").parseJson() - - val title = json.getString("title") - val cover = json.optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } - val author = json.optString("author", null).nullIfEmpty() - val description = json.optString("description", null).nullIfEmpty() - val status = json.optString("status", null) - val genresRaw = json.optString("genres", null) - val tags = genresRaw?.split(",")?.mapNotNull { it -> it.trim().takeIf { it.isNotEmpty() } }?.map { - MangaTag(it, it.lowercase().replace(" ", "_"), source) - }.orEmpty().toSet() - - val state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - - val chaptersArray = json.optJSONArray("chapters") - val chapters = chaptersArray?.let { arr -> - (0 until arr.length()).map { i -> arr.getJSONObject(i).toChapter() } - } ?: emptyList() - - return manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - largeCoverUrl = cover ?: manga.largeCoverUrl, - authors = setOfNotNull(author), - description = description, - state = state, - tags = tags, - chapters = chapters, - ) - } - - private fun JSONObject.toChapter(): MangaChapter { - val id = getInt("id") - val number = optDouble("chapterNumber", Double.NaN).let { if (it.isNaN()) -1f else it.toFloat() } - val rawTitle = optString("title", null) - val createdAt = optString("createdAt", null) - - val title = if (!rawTitle.isNullOrBlank() && rawTitle.toDoubleOrNull() == null) { - rawTitle - } else { - "Capítulo ${number.toString().removeSuffix(".0")}" - } - - return MangaChapter( - id = generateUid(id.toString()), - title = title, - number = number, - volume = 0, - url = id.toString(), // chapter ID - uploadDate = createdAt?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = null, - branch = null, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterId = chapter.url - // access ticket - val ticketJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId/access").parseJson() - val ticket = ticketJson.getString("ticket") - - // fetch chapter pages with ticket header - val pagesHeaders = getRequestHeaders().newBuilder() - .add("x-chapter-access", ticket) - .build() - val pagesJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId", pagesHeaders).parseJson() - val pagesArray = pagesJson.optJSONArray("pages") ?: return emptyList() - - return (0 until pagesArray.length()).map { i -> - val url = pagesArray.getString(i) - val fullUrl = if (url.startsWith("http")) url else "https://$domain$url" - MangaPage( - id = generateUid(fullUrl), - url = fullUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - private fun String.nullIfEmpty() = ifBlank { null } - private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt deleted file mode 100644 index 492eea9b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt +++ /dev/null @@ -1,17 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt.hentais - -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser - -@MangaSourceParser("LERHENTAIS", "LerHentais", "pt", ContentType.HENTAI) -internal class LerHentais(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.LERHENTAIS, "lerhentais.com") { - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("x-flux-node", "G2ZsDdWhUwdU82Vw") - .build() -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt deleted file mode 100644 index 4b8eeed8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.vi - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("TRUYENTUOITHO", "Truyện Tuổi Thơ", "vi") -internal class TruyenTuoiTho(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.TRUYENTUOITHO, "truyentuoitho.com") { - override val datePattern = "dd/MM/yyyy" - override val withoutAjax = true -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt deleted file mode 100644 index 04cc78c5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt +++ /dev/null @@ -1,43 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.vi.hentai - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml - -@MangaSourceParser("TRUYENTRANHDAMMYY", "Truyện Tranh Đam Mỹ", "vi", ContentType.HENTAI) -internal class TruyenTranhDamMyy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.TRUYENTRANHDAMMYY, "truyentranhdammyy.site") { - override val postReq = true - override suspend fun loadChapters(mangaUrl: String, document: Document): List { - val mangaId = document.select("div#manga-chapters-holder").attr("data-id") - val url = "https://$domain/wp-admin/admin-ajax.php" - val postData = "action=manga_get_chapters&manga=$mangaId" - val doc = webClient.httpPost(url, postData).parseHtml() - return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") - val link = href + stylePage - val name = a.selectFirst("p")?.text() ?: a.ownText() - MangaChapter( - id = generateUid(href), - url = link, - title = name, - number = i + 1f, - volume = 0, - branch = null, - uploadDate = 0, // Correct datePattern not found. - scanlator = null, - source = source, - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt deleted file mode 100644 index 17c4fc08..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.zh.hentai - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.parsers.GalleryParser - -@MangaSourceParser("XIUTAKU", "Xiutaku", "zh", type = ContentType.OTHER) -internal class Xiutaku(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.XIUTAKU, "xiutaku.com") { - - override suspend fun getFilterOptions(): MangaListFilterOptions = - MangaListFilterOptions(availableTags = fetchTags()) -} \ No newline at end of file From cb4dec89eb06758ba675d90bf6698f53540638aa Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 14:12:39 +0200 Subject: [PATCH 008/434] Add files via upload --- .../kotlin/tsuki/parsers/DoujinDesuParser.kt | 397 ++++++ .../tsuki/parsers/GalleryAdultsParser.kt | 252 ++++ .../kotlin/tsuki/parsers/GalleryParser.kt | 149 +++ .../kotlin/tsuki/parsers/HeanCmsParser.kt | 241 ++++ src/main/kotlin/tsuki/parsers/HiperParser.kt | 398 ++++++ .../kotlin/tsuki/parsers/LikeMangaParser.kt | 309 +++++ .../kotlin/tsuki/parsers/LilianaParser.kt | 266 ++++ src/main/kotlin/tsuki/parsers/MadaraParser.kt | 936 ++++++++++++++ .../kotlin/tsuki/parsers/MadthemeParser.kt | 449 +++++++ .../kotlin/tsuki/parsers/Manga18Parser.kt | 276 +++++ .../kotlin/tsuki/parsers/MangaReaderParser.kt | 461 +++++++ .../kotlin/tsuki/parsers/MangaThemesia.kt | 414 +++++++ .../kotlin/tsuki/parsers/MangaboxParser.kt | 459 +++++++ .../kotlin/tsuki/parsers/ZeistMangaParser.kt | 384 ++++++ .../kotlin/tsuki/site/all/MangaDexParser.kt | 540 +++++++++ src/main/kotlin/tsuki/site/all/MangaFire.kt | 494 ++++++++ src/main/kotlin/tsuki/site/all/MangaPlus.kt | 297 +++++ src/main/kotlin/tsuki/site/all/Mangaball.kt | 891 ++++++++++++++ src/main/kotlin/tsuki/site/all/ProjectSuki.kt | 490 ++++++++ src/main/kotlin/tsuki/site/all/Webtoons.kt | 459 +++++++ .../tsuki/site/all/hentais/AsmHentai.kt | 46 + .../kotlin/tsuki/site/all/hentais/BuonDua.kt | 19 + .../tsuki/site/all/hentais/HentaiEnvy.kt | 90 ++ .../tsuki/site/all/hentais/HentaiFox.kt | 136 +++ .../tsuki/site/all/hentais/HitomiLaParser.kt | 798 ++++++++++++ .../kotlin/tsuki/site/all/hentais/IMHentai.kt | 248 ++++ .../kotlin/tsuki/site/all/hentais/Kiutaku.kt | 35 + .../tsuki/site/all/hentais/MangaCrazy.kt | 15 + .../kotlin/tsuki/site/all/hentais/Misskon.kt | 152 +++ .../kotlin/tsuki/site/all/hentais/Multporn.kt | 235 ++++ .../tsuki/site/all/hentais/NHentaiParser.kt | 1079 +++++++++++++++++ .../site/all/hentais/NHentaiXXXParser.kt | 187 +++ .../tsuki/site/all/hentais/Novelcrow.kt | 14 + src/main/kotlin/tsuki/site/ar/TeamXNovel.kt | 275 +++++ src/main/kotlin/tsuki/site/en/AquaManga.kt | 200 +++ src/main/kotlin/tsuki/site/en/AsuraScan.kt | 354 ++++++ src/main/kotlin/tsuki/site/en/Atsumaru.kt | 459 +++++++ src/main/kotlin/tsuki/site/en/Batcave.kt | 254 ++++ src/main/kotlin/tsuki/site/en/CoffeeManga.kt | 11 + src/main/kotlin/tsuki/site/en/DemonicScan.kt | 247 ++++ .../kotlin/tsuki/site/en/EnThunderScans.kt | 25 + src/main/kotlin/tsuki/site/en/FlameComics.kt | 272 +++++ src/main/kotlin/tsuki/site/en/KaliScan.kt | 211 ++++ src/main/kotlin/tsuki/site/en/KingofShojo.kt | 11 + src/main/kotlin/tsuki/site/en/LikeManga.kt | 29 + src/main/kotlin/tsuki/site/en/MangaFox.kt | 240 ++++ src/main/kotlin/tsuki/site/en/MangaGG.kt | 153 +++ src/main/kotlin/tsuki/site/en/MangaGeko.kt | 245 ++++ src/main/kotlin/tsuki/site/en/MangaJinx.kt | 211 ++++ src/main/kotlin/tsuki/site/en/MangaK.kt | 322 +++++ src/main/kotlin/tsuki/site/en/MangaKatana.kt | 398 ++++++ src/main/kotlin/tsuki/site/en/MangaTown.kt | 377 ++++++ src/main/kotlin/tsuki/site/en/Mangadotnet.kt | 474 ++++++++ src/main/kotlin/tsuki/site/en/Mangago.kt | 688 +++++++++++ src/main/kotlin/tsuki/site/en/Mangaread.kt | 14 + src/main/kotlin/tsuki/site/en/Mangataro.kt | 845 +++++++++++++ src/main/kotlin/tsuki/site/en/Manhuaus.kt | 80 ++ src/main/kotlin/tsuki/site/en/ManhwaNex.kt | 15 + src/main/kotlin/tsuki/site/en/ManhwaTop.kt | 11 + src/main/kotlin/tsuki/site/en/ManhwaZ.kt | 123 ++ src/main/kotlin/tsuki/site/en/Mgread.kt | 299 +++++ src/main/kotlin/tsuki/site/en/Novel24h.kt | 17 + src/main/kotlin/tsuki/site/en/RoliaScan.kt | 797 ++++++++++++ src/main/kotlin/tsuki/site/en/TopManhua.kt | 15 + src/main/kotlin/tsuki/site/en/WeebCentral.kt | 358 ++++++ src/main/kotlin/tsuki/site/en/ZinManga.kt | 62 + .../tsuki/site/en/hentais/AllPornComic.kt | 18 + .../kotlin/tsuki/site/en/hentais/ComicLand.kt | 287 +++++ .../tsuki/site/en/hentais/HentaiNexus.kt | 379 ++++++ .../tsuki/site/en/hentais/HentaiXComic.kt | 34 + .../kotlin/tsuki/site/en/hentais/HeyToon.kt | 273 +++++ .../kotlin/tsuki/site/en/hentais/Hiperdex.kt | 21 + .../kotlin/tsuki/site/en/hentais/MadaraDex.kt | 185 +++ .../kotlin/tsuki/site/en/hentais/Manga18.kt | 12 + .../tsuki/site/en/hentais/MangaDistrict.kt | 103 ++ .../tsuki/site/en/hentais/Manhwa18CC.kt | 144 +++ .../tsuki/site/en/hentais/Manhwa18Com.kt | 243 ++++ .../kotlin/tsuki/site/en/hentais/Manhwa210.kt | 238 ++++ .../kotlin/tsuki/site/en/hentais/OmegaScan.kt | 12 + .../tsuki/site/en/mangabox/MangaBats.kt | 392 ++++++ .../tsuki/site/en/mangabox/ManganatoGG.kt | 392 ++++++ src/main/kotlin/tsuki/site/id/AinzScans.kt | 304 +++++ src/main/kotlin/tsuki/site/id/MGKomik.kt | 182 +++ src/main/kotlin/tsuki/site/id/ReYume.kt | 179 +++ src/main/kotlin/tsuki/site/id/Westmanga.kt | 264 ++++ .../tsuki/site/id/hentais/BacaKomikpics.kt | 186 +++ .../tsuki/site/id/hentais/DoujinDesu.kt | 23 + src/main/kotlin/tsuki/site/pt/HiperToon.kt | 11 + src/main/kotlin/tsuki/site/pt/Roxinha.kt | 222 ++++ .../tsuki/site/pt/hentais/LerHentais.kt | 19 + .../kotlin/tsuki/site/vi/TruyenTuoiTho.kt | 14 + .../tsuki/site/vi/hentai/TruyenTranhDamMyy.kt | 46 + .../kotlin/tsuki/site/zh/hentai/Xiutaku.kt | 17 + 93 files changed, 23908 insertions(+) create mode 100644 src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/GalleryParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/HeanCmsParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/HiperParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/LikeMangaParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/LilianaParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/MadaraParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/MadthemeParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/Manga18Parser.kt create mode 100644 src/main/kotlin/tsuki/parsers/MangaReaderParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/MangaThemesia.kt create mode 100644 src/main/kotlin/tsuki/parsers/MangaboxParser.kt create mode 100644 src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/MangaDexParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/MangaFire.kt create mode 100644 src/main/kotlin/tsuki/site/all/MangaPlus.kt create mode 100644 src/main/kotlin/tsuki/site/all/Mangaball.kt create mode 100644 src/main/kotlin/tsuki/site/all/ProjectSuki.kt create mode 100644 src/main/kotlin/tsuki/site/all/Webtoons.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/Misskon.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/Multporn.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt create mode 100644 src/main/kotlin/tsuki/site/ar/TeamXNovel.kt create mode 100644 src/main/kotlin/tsuki/site/en/AquaManga.kt create mode 100644 src/main/kotlin/tsuki/site/en/AsuraScan.kt create mode 100644 src/main/kotlin/tsuki/site/en/Atsumaru.kt create mode 100644 src/main/kotlin/tsuki/site/en/Batcave.kt create mode 100644 src/main/kotlin/tsuki/site/en/CoffeeManga.kt create mode 100644 src/main/kotlin/tsuki/site/en/DemonicScan.kt create mode 100644 src/main/kotlin/tsuki/site/en/EnThunderScans.kt create mode 100644 src/main/kotlin/tsuki/site/en/FlameComics.kt create mode 100644 src/main/kotlin/tsuki/site/en/KaliScan.kt create mode 100644 src/main/kotlin/tsuki/site/en/KingofShojo.kt create mode 100644 src/main/kotlin/tsuki/site/en/LikeManga.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaFox.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaGG.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaGeko.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaJinx.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaK.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaKatana.kt create mode 100644 src/main/kotlin/tsuki/site/en/MangaTown.kt create mode 100644 src/main/kotlin/tsuki/site/en/Mangadotnet.kt create mode 100644 src/main/kotlin/tsuki/site/en/Mangago.kt create mode 100644 src/main/kotlin/tsuki/site/en/Mangaread.kt create mode 100644 src/main/kotlin/tsuki/site/en/Mangataro.kt create mode 100644 src/main/kotlin/tsuki/site/en/Manhuaus.kt create mode 100644 src/main/kotlin/tsuki/site/en/ManhwaNex.kt create mode 100644 src/main/kotlin/tsuki/site/en/ManhwaTop.kt create mode 100644 src/main/kotlin/tsuki/site/en/ManhwaZ.kt create mode 100644 src/main/kotlin/tsuki/site/en/Mgread.kt create mode 100644 src/main/kotlin/tsuki/site/en/Novel24h.kt create mode 100644 src/main/kotlin/tsuki/site/en/RoliaScan.kt create mode 100644 src/main/kotlin/tsuki/site/en/TopManhua.kt create mode 100644 src/main/kotlin/tsuki/site/en/WeebCentral.kt create mode 100644 src/main/kotlin/tsuki/site/en/ZinManga.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manga18.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt create mode 100644 src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt create mode 100644 src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt create mode 100644 src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt create mode 100644 src/main/kotlin/tsuki/site/id/AinzScans.kt create mode 100644 src/main/kotlin/tsuki/site/id/MGKomik.kt create mode 100644 src/main/kotlin/tsuki/site/id/ReYume.kt create mode 100644 src/main/kotlin/tsuki/site/id/Westmanga.kt create mode 100644 src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt create mode 100644 src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt create mode 100644 src/main/kotlin/tsuki/site/pt/HiperToon.kt create mode 100644 src/main/kotlin/tsuki/site/pt/Roxinha.kt create mode 100644 src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt create mode 100644 src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt create mode 100644 src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt create mode 100644 src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt diff --git a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt new file mode 100644 index 00000000..2bbdb13e --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -0,0 +1,397 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlBuilder + +import okhttp3.Headers +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.net.URLDecoder +import java.time.Instant +import java.util.EnumSet +import kotlin.math.abs +import kotlin.text.iterator +import kotlin.time.Duration.Companion.milliseconds +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + + +internal abstract class DoujinDesuParser( + context: MangaLoaderContext, + source: MangaParserSource +): PagedMangaParser(context, source, pageSize = 18) { + + protected abstract val defaultTypes: String + protected abstract val availableContentTypes: Set + + override val configKeyDomain = ConfigKey.Domain("doujin.desu.xxx") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + @Volatile + private var genresCache: Set? = null + private val genresMutex = Mutex() + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override val defaultSortOrder: SortOrder = SortOrder.UPDATED + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = availableContentTypes, + ) + + private suspend fun getOrFetchGenres(): Set { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + genresCache = fetchAvailableTags() + genresCache!! + } + } + } + + private suspend fun fetchAvailableTags(): Set { + val url = "/api/terms?taxonomy=genre".toAbsoluteUrl(domain) + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val array = JSONArray(decrypted) + return (0 until array.length()).mapTo(mutableSetOf()) { i -> + val obj = array.getJSONObject(i) + MangaTag(key = obj.getString("slug"), title = obj.getString("name"), source = source) + } + } + + override fun getRequestHeaders() = Headers.Builder() + .add("X-Requested-With", "XMLHttpRequest") + .add("Referer", "https://$domain/") + .add("X-App-Secret", "dfdf72051dbfdc7d76889ebd31324e74") + .build() + + private suspend fun executeWithCloudflareRetry(url: String, headers: Headers): Response { + var response = webClient.httpGet(url, headers) + if (response.code == 403) { + val bodyString = response.body.string() + if ("Unauthorized" in bodyString || "Access denied" in bodyString) { + response.close() + refreshCloudflare() + response = webClient.httpGet(url, headers) + } + } + if (response.code in 500..599) { + response.close() + delay(1000L.milliseconds) + response = webClient.httpGet(url, headers) + } + return response + } + + private suspend fun refreshCloudflare() { + webClient.httpGet("https://$domain/", getRequestHeaders()).close() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val limit = pageSize + val offset = (page - 1) * limit + + val url = urlBuilder().apply { + addPathSegment("api") + addPathSegment("manga") + addQueryParameter("limit", limit.toString()) + addQueryParameter("offset", offset.toString()) + + filter.query?.let { addQueryParameter("search", it) } + + val sortParam = when (order) { + SortOrder.POPULARITY -> "rating" + SortOrder.ALPHABETICAL -> "title_asc" + SortOrder.NEWEST -> "newest" + SortOrder.UPDATED -> "latest_chapter" + else -> "latest_chapter" + } + addQueryParameter("sort", sortParam) + + filter.types.oneOrThrowIfMany()?.let { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.DOUJINSHI -> "doujinshi" + else -> null + }?.let { type -> addQueryParameter("type", type) } + } + + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> null + }?.let { status -> addQueryParameter("status", status) } + } + + if (filter.tags.isNotEmpty()) { + addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) + } + }.build() + + val jsonResponse = executeWithCloudflareRetry(url.toString(), getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val array = JSONArray(decrypted) + + return (0 until array.length()).map { i -> + val obj = array.getJSONObject(i) + val slug = obj.getString("slug") + val href = "/manga/$slug" + Manga( + id = generateUid(href), + title = obj.getString("title"), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = obj.optDouble("rating", 0.0).toFloat() / 10f, + contentRating = ContentRating.ADULT, + coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> + if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) + }, + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val slug = manga.url.removePrefix("/manga/").removeSuffix("/") + val url = "/api/manga/$slug".toAbsoluteUrl(domain) + + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val obj = JSONObject(decrypted) + + val state = when (obj.optString("status")) { + "completed" -> MangaState.FINISHED + "ongoing" -> MangaState.ONGOING + else -> null + } + + val author = obj.optString("author").takeIf { it.isNotEmpty() && it != "null" } + + val tags = mutableSetOf() + val genresArray = obj.optJSONArray("manga_genres") + if (genresArray != null) { + for (i in 0 until genresArray.length()) { + val genreObj = genresArray.getJSONObject(i).getJSONObject("genres") + tags.add( + MangaTag( + key = genreObj.getString("slug"), + title = genreObj.getString("name"), + source = source, + ) + ) + } + } + + val chaptersArray = obj.getJSONArray("chapters") + val chapters = ArrayList(chaptersArray.length()) + for (i in 0 until chaptersArray.length()) { + val chapObj = chaptersArray.getJSONObject(i) + val chNum = chapObj.optDouble("chapter_number", 0.0).toFloat() + val chTitle = if (chNum == chNum.toLong().toFloat()) { + "Chapter ${chNum.toLong()}" + } else { + "Chapter $chNum" + } + chapters.add( + MangaChapter( + id = generateUid("/reader/${chapObj.getString("id")}"), + title = chTitle, + number = chNum, + volume = 0, + url = "/reader/${chapObj.getString("id")}", + scanlator = null, + uploadDate = runCatching { Instant.parse(chapObj.optString("created_at")).toEpochMilli() }.getOrDefault(0L), + branch = null, + source = source, + ) + ) + } + + val rawDesc = obj.optString("description").takeIf { it.isNotEmpty() && it != "null" } + val cleanDesc = rawDesc?.let { html -> + val document = Jsoup.parseBodyFragment(html) + val fullText = document.text() + val cutoffRegex = Regex("""download\s+(batch|volume)""", RegexOption.IGNORE_CASE) + val cutoffIndex = cutoffRegex.find(fullText)?.range?.first + val textBeforeDownload = if (cutoffIndex != null) fullText.substring(0, cutoffIndex) else fullText + textBeforeDownload + .replace(Regex("""^Sinopsis:\s*""", RegexOption.IGNORE_CASE), "") + .replace(Regex("""https?://\S+"""), "") + .replace(Regex("""\[url=.*?]|\[/url]"""), "") + .trim() + } + + val coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> + if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) + } + + val fullManga = manga.copy( + authors = setOfNotNull(author), + description = cleanDesc, + state = state, + rating = obj.optDouble("rating", 0.0).toFloat() / 10f, + tags = tags, + coverUrl = coverUrl ?: manga.coverUrl, + chapters = chapters.reversed() + ) + + detailsCache[manga.url] = fullManga + return fullManga + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chId = chapter.url.removePrefix("/reader/").removeSuffix("/") + val url = "/api/chapters/$chId".toAbsoluteUrl(domain) + + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val obj = JSONObject(decrypted) + + val pagesList = mutableListOf() + val contentUrls = obj.optJSONArray("content_urls") + if (contentUrls != null && contentUrls.length() > 0) { + for (i in 0 until contentUrls.length()) { + pagesList.add( + MangaPage( + id = generateUid(contentUrls.getString(i)), + url = transformPageUrl(contentUrls.getString(i)), + preview = null, + source = source, + ) + ) + } + } else { + val signedUrlsStr = obj.optString("signed_content_urls") + if (signedUrlsStr.isNotEmpty()) { + val signedUrls = JSONArray(signedUrlsStr) + for (i in 0 until signedUrls.length()) { + pagesList.add( + MangaPage( + id = generateUid(signedUrls.getString(i)), + url = transformPageUrl(signedUrls.getString(i)), + preview = null, + source = source, + ) + ) + } + } + } + + return pagesList + } + + private fun toSigned32(x: Long): Int { + var y = x and 0xFFFFFFFFL + if (y >= 0x80000000L) y -= 0x100000000L + return y.toInt() + } + + private fun wH(e: Int): String { + val t = "doujindesu-scrapers-cannot-read-this-super-secret-salt-2026-v2_$e" + var a = 0 + for (ch in t) { + a = (a shl 5) - a + ch.code + a = toSigned32(a.toLong()) + } + var l = if (a != 0) abs(a).toLong() else 123456789L + return buildString { + repeat(32) { + l = (l * 1664525L + 1013904223L) % 4294967296L + append((33 + (l % 93)).toInt().toChar()) + } + } + } + + private fun lU(): List { + val now = System.currentTimeMillis() + val t = now / 3600000L + return listOf(wH(t.toInt()), wH((t - 1).toInt()), wH((t + 1).toInt())) + } + + private fun yre(encryptedHex: String, key: String): String { + val bytes = encryptedHex.chunked(2).mapNotNull { it.toIntOrNull(16) } + var d = 42 + val keyLen = key.length + return buildString { + for ((idx, byteVal) in bytes.withIndex()) { + val keyChar = key[idx % keyLen].code + val k = byteVal xor keyChar xor (idx * 13) xor d + append((k and 0xFF).toChar()) + d = (d + byteVal) % 256 + } + } + } + + private fun decrypt(encryptedHex: String): String { + for (key in lU()) { + try { + val decoded = yre(encryptedHex, key) + return URLDecoder.decode(decoded, "UTF-8") + } catch (_: Exception) { + } + } + throw RuntimeException("Decryption failed") + } + + private fun transformPageUrl(page: String): String = when { + page.contains("/uploads/") && !page.contains("/storage/uploads/") -> + page.replace("/uploads/", "/storage/uploads/") + page.contains("/upload/") && !page.contains("/storage/upload/") -> + page.replace("/upload/", "/storage/upload/") + else -> page + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt new file mode 100644 index 00000000..fab5860a --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt @@ -0,0 +1,252 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.ErrorMessages + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +internal abstract class GalleryAdultsParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.JAPANESE, + Locale.CHINESE, + Locale("es"), + Locale("ru"), + Locale("ko"), + Locale.GERMAN, + Locale("id"), + Locale.ITALIAN, + Locale("pt"), + Locale("tr"), + Locale("th"), + Locale("vi"), + ), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("/search/?q=") + append(query?.urlEncoded()) + append("&") + } + + else -> { + val tag = filter.tags.oneOrThrowIfMany() + val lang = filter.locale + if (tag != null && lang != null) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) + } + if (tag != null) { + append("/tag/") + append(tag.key) + append("/?") + } else if (filter.locale != null) { + val locale = filter.locale + append("/language/") + append(locale?.toLanguagePath()) + append("/?") + } else { + append("/?") + } + } + } + append("page=") + append(page) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + protected open val selectGallery = ".thumb" + protected open val selectGalleryLink = ".inner_thumb a" + protected open val selectGalleryImg = "img" + protected open val selectGalleryTitle = "h2" + private val regexBrackets = Regex("\\[[^]]+]|\\([^)]+\\)") + private val regexSpaces = Regex("\\s+") + + protected open fun parseMangaList(doc: Document): List { + return doc.select(selectGallery).map { div -> + val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(selectGalleryTitle).text().cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = div.selectFirst(selectGalleryImg)?.src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + //Tags are deliberately reduced because there are too many and this slows down the application. + //only the most popular ones are taken. + private suspend fun fetchAvailableTags(): Set { + return coroutineScope { + (1..3).map { page -> + async { getTags(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } + + protected open val pathTagUrl = "/tags/popular/?page=" + protected open val selectTags = ".tags_page ul.tags li" + + private suspend fun getTags(page: Int): Set { + val url = "https://$domain$pathTagUrl$page" + val root = webClient.httpGet(url).parseHtml().selectFirstOrThrow(selectTags) + return root.parseTags() + } + + protected open fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.html().substringBefore("<") + MangaTag( + key = key, + title = name.toTitleCase(), + source = source, + ) + } + + protected open val selectTitle = "h1.title" + protected open val selectTag = "div.tags:contains(Tags:) .tag_list" + protected open val selectAuthor = "ul.artists a.tag_btn" + protected open val selectLanguageChapter = "div.tags:contains(Languages:) .tag_list a span.tag" + protected open val selectUrlChapter = "#cover a, .cover a, .left_cover a, .g_thumb a, .gallery_left a, .gt_left a" + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val urlChapters = doc.selectFirstOrThrow(selectUrlChapter).attr("href") + val tag = doc.selectFirst(selectTag)?.parseTags() + val branch = doc.select(selectLanguageChapter).joinToString(separator = " / ") { + it.html().substringBefore("<") + } + val author = doc.selectFirst(selectAuthor)?.html()?.substringBefore(" { + return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) + } + + protected open val selectTotalPage = ".total_pages, .num-pages, .tp" + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() + val rawUrl = chapter.url.removeSuffix("/").substringBeforeLast("/") + "/" + return (1..totalPages).map { + val url = "$rawUrl$it/" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + protected open val idImg = "gimg" + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.requireElementById(idImg).requireSrc() + } + + protected fun String.cleanupTitle() = replace(regexBrackets, "") + .replace(regexSpaces, " ") + .trim() + + protected open fun Locale.toLanguagePath() = getDisplayLanguage(Locale.ENGLISH).lowercase() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/GalleryParser.kt b/src/main/kotlin/tsuki/parsers/GalleryParser.kt new file mode 100644 index 00000000..ce61f0a8 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/GalleryParser.kt @@ -0,0 +1,149 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.attrOrNull +import tsuki.util.attrOrThrow +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectFirstOrThrow +import tsuki.util.urlBuilder + +import java.text.SimpleDateFormat +import java.util.EnumSet + +internal abstract class GalleryParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set + get() = EnumSet.of(SortOrder.NEWEST, SortOrder.POPULARITY) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = urlBuilder().apply { + when { + !query.isNullOrEmpty() -> addQueryParameter("search", query) + filter.tags.isNotEmpty() -> addPathSegments(filter.tags.first().key) + order == SortOrder.POPULARITY -> addPathSegment("hot") + } + + addQueryParameter("start", (page*pageSize).toString()) + }.build() + + val content = webClient.httpGet(url).parseHtml() + return content.select("div.items-row").map { el -> + val titleEl = el.selectFirstOrThrow("div.page-header a.item-link") + val relUrl = titleEl.attrOrThrow("href") + Manga( + id = generateUid(relUrl), + url = relUrl, + title = titleEl.text(), + altTitles = emptySet(), + publicUrl = relUrl.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = el.selectFirst("div.item-thumb img")?.attr("src"), + tags = el.select("div.item-tags > a.tag").mapNotNullToSet { tagEl -> + MangaTag( + title = tagEl.text(), + key = tagEl.attrAsRelativeUrlOrNull("href") + ?.removePrefix("/") ?: return@mapNotNullToSet null, + source = source, + ) + }, + state = MangaState.FINISHED, + authors = emptySet(), + largeCoverUrl = null, + description = null, + chapters = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val content = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val description = content.selectFirst("div.article-info")?.text()?.trim() ?: "" + val df = SimpleDateFormat("HH:mm dd-MM-yyyy") + val time = content.selectFirst("div.article-info > small")?.text()?.trim() + val chapters = content.selectFirst("nav.pagination")?.select("a.pagination-link") + ?.mapChapters { index, element -> + val relUrl = element.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(relUrl), + title = null, + number = index + 1f, + volume = 0, + url = relUrl, + scanlator = null, + uploadDate = df.parseSafe(time), + branch = null, + source = source, + ) + }.orEmpty() + return manga.copy(chapters = chapters, description = description) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return content.selectFirstOrThrow("div.article-fulltext").select("p > img").mapNotNull { el -> + val url = el.attrOrNull("src") ?: return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + protected suspend fun fetchTags(): Set { + val root = webClient.httpGet("https://$domain").parseHtml() + return root.select("div#navbar-main a.navbar-item").map { a -> + MangaTag( + title = a.text(), + key = a.attr("href").removePrefix("/"), + source = source, + ) + }.toSet() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt new file mode 100644 index 00000000..af40847d --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt @@ -0,0 +1,241 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.json.unescapeJson +import tsuki.util.mapChapters +import tsuki.util.parseJson +import tsuki.util.parseSafe + +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +internal abstract class HeanCmsParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), + ) + + protected open val pathManga = "series" + protected open val apiPath + get() = "api.$domain" + + protected open val paramsUpdated = "latest" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(apiPath) + append("/query?query_string=") + + query?.let { + append(query.urlEncoded()) + } + + append("&series_type=Comic&perPage=$pageSize") + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.ABANDONED -> "Dropped" + MangaState.PAUSED -> "Hiatus" + else -> "" + }, + ) + } + + append("&orderBy=") + when (order) { + SortOrder.POPULARITY -> append("total_views&order=desc") + SortOrder.POPULARITY_ASC -> append("total_views&order=asc") + SortOrder.UPDATED -> append("$paramsUpdated&order=desc") + SortOrder.UPDATED_ASC -> append("$paramsUpdated&order=asc") + SortOrder.NEWEST -> append("created_at&order=desc") + SortOrder.NEWEST_ASC -> append("created_at&order=asc") + SortOrder.ALPHABETICAL -> append("title&order=asc") + SortOrder.ALPHABETICAL_DESC -> append("title&order=desc") + else -> append("latest&order=desc") + } + append("&tags_ids=") + append("[".urlEncoded()) + append(filter.tags.joinToString(",") { it.key }) + append("]".urlEncoded()) + + append("&page=") + append(page.toString()) + } + + + return parseMangaList(webClient.httpGet(url).parseJson()) + } + + protected open val cdn = "api.$domain/" + + private fun parseMangaList(response: JSONObject): List { + return response.getJSONArray("data").mapJSON { it -> + val id = it.getLong("id") + val publicUrl = "/series/${it.getString("series_slug")}" + val title = it.getString("title") + val cover = if (it.getString("thumbnail").startsWith("https://")) { + it.getString("thumbnail") + } else { + "https://$cdn${it.getString("thumbnail")}" + } + + Manga( + id = generateUid(id), + url = id.toString(), + title = title, + altTitles = setOfNotNull(it.getString("alternative_names").takeIf { it.isNotBlank() }), + publicUrl = publicUrl.toAbsoluteUrl(domain), + description = it.getString("description"), + rating = it.getFloatOrDefault("rating", RATING_UNKNOWN) / 5f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = cover, + tags = emptySet(), + state = when (it.getString("status")) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Dropped" -> MangaState.ABANDONED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + protected open val datePattern = "yyyy-MM-dd" + + protected open fun reqUrl(seriesId: Long): String { + return "https://$apiPath/chapter/query?page=1&perPage=9999&series_id=$seriesId" + } + + override suspend fun getDetails(manga: Manga): Manga { + val seriesId = manga.url.toLongOrNull() ?: manga.id // backward compatibility + val url = reqUrl(seriesId) // keep old API url, can replace it + val response = webClient.httpGet(url).parseJson() + val data = response.getJSONArray("data").asTypedList() + val dateFormat = SimpleDateFormat(datePattern, Locale.ENGLISH) + return manga.copy( + chapters = data.mapChapters(reversed = true) { i, it -> + val chapterUrl = + "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" + MangaChapter( + id = generateUid(it.getLong("id")), + title = it.getString("chapter_name"), + number = i + 1f, + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = dateFormat.parseSafe(it.getString("created_at").substringBefore("T")), + branch = null, + source = source, + ) + }, + ) + } + + protected open val selectPages = ".flex > img:not([alt])" + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select(selectPages).map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/comics").parseHtml() + val regex = Regex("\"tags\\\\?\":\\s*\\[(.+?)]\\s*[},]") + val tags = doc.select("script").joinToString("") { it.html() } + .let { fullHtml -> + regex.find(fullHtml)?.groupValues?.getOrNull(1) + } + ?.unescapeJson() + ?.replace(Regex(""""]\)\s*self\.__next_f\.push\(\[\d+,""""), "") + ?.let { "[$it]" } + ?: return emptySet() + + return JSONArray(tags).mapJSONToSet { + MangaTag( + key = it.getInt("id").toString(), + title = it.getString("name").toTitleCase(sourceLocale), + source = source, + ) + } + } + +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/HiperParser.kt b/src/main/kotlin/tsuki/parsers/HiperParser.kt new file mode 100644 index 00000000..756c5fa9 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/HiperParser.kt @@ -0,0 +1,398 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.ContentType +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import okhttp3.Headers +import okhttp3.Request +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +abstract class HiperParser( + context: MangaLoaderContext, + source: MangaParserSource, + protected val domainName: String, +) : PagedMangaParser(context, source, pageSize = 30) { + + protected open val availableContentTypes: Set = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA) + + protected open val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED) + + protected open val availableContentRating: Set = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + + protected open val mangaPath = "manga" + + override val configKeyDomain = ConfigKey.Domain(domainName) + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.RELEVANCE + ) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + val client: OkHttpWebClient by lazy { + val httpClient = context.httpClient.newBuilder() + .addInterceptor { chain -> + val request = chain.request() + val newRequest = request.newBuilder() + .header("Referer", "https://$domainName/") + .header("Origin", "https://$domainName") + .build() + var response = chain.proceed(newRequest) + if (response.code == 401) { + response.close() + val acceptHeaders = Headers.Builder() + .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + .build() + val homeRequest = Request.Builder() + .url("https://$domainName/") + .headers(acceptHeaders) + .build() + chain.proceed(homeRequest).close() + response = chain.proceed(request) + } + response + } + .build() + OkHttpWebClient(httpClient, source) + } + + @Volatile + private var genresListCache: List? = null + + override suspend fun getFilterOptions(): MangaListFilterOptions { + if (genresListCache == null) { + try { + val input = "%7B%220%22%3A%7B%22json%22%3Anull%2C%22meta%22%3A%7B%22values%22%3A%5B%22undefined%22%5D%7D%7D%7D" + val res = client.httpGet(apiUrl("search.genres", input), getRequestHeaders()) + val body = res.body?.string() + if (body != null) { + val root = JSONArray(body) + val jsonArray = root.optJSONObject(0) + ?.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") + if (jsonArray != null) { + val tags = mutableListOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.optJSONObject(i) ?: continue + val name = obj.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name, name, source)) + } + if (tags.isNotEmpty()) genresListCache = tags + } + } + } catch (_: Exception) {} + } + return MangaListFilterOptions( + availableTags = genresListCache?.toSet() ?: emptySet(), + availableContentTypes = availableContentTypes, + availableStates = availableStates, + availableContentRating = availableContentRating, + ) + } + + private fun apiUrl(trpcPath: String, input: String) = "https://$domainName/api/trpc/$trpcPath?batch=1&input=${input.urlEncoded()}" + + private fun searchPayload(query: String, sort: String, limit: Int, offset: Int, filter: MangaListFilter): String { + val genres = filter.tags.map { it.key } + val types = filter.types.map { it.name.lowercase() } + val statusStr = filter.states.firstOrNull()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> null + } + } + val contentRatingStr = filter.contentRating.firstOrNull()?.let { + when (it) { + ContentRating.SAFE -> "safe" + ContentRating.SUGGESTIVE -> "suggestive" + ContentRating.ADULT -> "pornographic" + } + } + return JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject().apply { + put("q", query) + put("sort", sort) + put("filters", JSONObject().apply { + if (genres.isNotEmpty()) put("genres", JSONArray(genres)) else put("genres", JSONObject.NULL) + if (types.isNotEmpty()) put("type", types.first()) else put("type", JSONObject.NULL) + if (statusStr != null) put("status", statusStr) else put("status", JSONObject.NULL) + if (contentRatingStr != null) put("contentRating", contentRatingStr) else put("contentRating", JSONObject.NULL) + put("author", JSONObject.NULL) + put("artist", JSONObject.NULL) + put("year", JSONObject.NULL) + }) + put("limit", limit) + put("offset", offset) + put("maxRating", "pornographic") + }) + put("meta", JSONObject().apply { + put("values", JSONObject().apply { + if (genres.isEmpty()) put("filters.genres", JSONArray().put("undefined")) + if (types.isEmpty()) put("filters.type", JSONArray().put("undefined")) + if (statusStr == null) put("filters.status", JSONArray().put("undefined")) + if (contentRatingStr == null) put("filters.contentRating", JSONArray().put("undefined")) + put("filters.author", JSONArray().put("undefined")) + put("filters.artist", JSONArray().put("undefined")) + put("filters.year", JSONArray().put("undefined")) + }) + }) + }) + }.toString() + } + + private fun detailsPayload(slug: String) = JSONObject().apply { + put("0", JSONObject().apply { + put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { put("slug", slug) }) + }) + }.toString() + + private fun chaptersPayload(mangaId: Long) = JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { + put("seriesId", mangaId) + put("sort", "best") + put("page", 1) + put("limit", 20) + }) + put("meta", JSONObject().apply { + put("values", JSONObject().apply { put("chapterId", JSONArray().put("undefined")) }) + }) + }) + put("2", JSONObject().apply { + put("json", JSONObject().apply { put("seriesId", mangaId) }) + }) + }.toString() + + private fun pagesPayload(slug: String, number: Float) = JSONObject().apply { + put("0", JSONObject().apply { + put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { put("slug", slug) }) + }) + put("2", JSONObject().apply { + put("json", JSONObject().apply { + put("seriesSlug", slug) + put("chapterNumber", number.toDouble()) + }) + }) + put("3", JSONObject().apply { + put("json", JSONObject().apply { put("position", "footer_bottom") }) + }) + }.toString() + + private fun parseLastItemJsonArray(body: String): JSONArray? = try { + val root = JSONArray(body) + root.optJSONObject(root.length() - 1) + ?.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") + } catch (_: Exception) { null } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val q = filter.query?.trim().orEmpty() + val sort = when (order) { + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "recent" + SortOrder.NEWEST -> "newest" + SortOrder.NEWEST_ASC -> "oldest" + SortOrder.RATING -> "score" + SortOrder.ALPHABETICAL -> "alphabetical" + SortOrder.RELEVANCE -> "relevance" + else -> "newest" + } + val input = searchPayload(q, sort, pageSize, (page - 1) * pageSize, filter) + val res = client.httpGet(apiUrl("search.query", input), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val root = JSONArray(body) + val first = root.optJSONObject(0) ?: return emptyList() + val hits = first.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONObject("json") + ?.optJSONArray("hits") ?: return emptyList() + return (0 until hits.length()).mapNotNull { i -> + hits.optJSONObject(i)?.let { obj -> + val id = obj.optLong("id", -1).takeIf { it > 0 } ?: return@mapNotNull null + val slug = obj.optString("slug", "").takeIf { it.isNotEmpty() } ?: return@mapNotNull null + val title = obj.optString("title", "") + val cover = obj.optString("coverUrl", "") + Manga( + id = generateUid("/$mangaPath/$slug#$id"), + url = "/$mangaPath/$slug#$id", + publicUrl = "https://$domainName/$mangaPath/$slug#$id", + title = title, coverUrl = cover, + altTitles = emptySet(), authors = emptySet(), tags = emptySet(), + rating = RATING_UNKNOWN, state = null, + contentRating = ContentRating.ADULT, source = source + ) + } + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#") + .takeIf { it.isNotBlank() } ?: return manga + val res = client.httpGet(apiUrl("auth.me,series.bySlugWithGenres", detailsPayload(slug)), getRequestHeaders()) + val body = res.body?.string() ?: return manga + val detailJson = try { + val arr = JSONArray(body) + arr.getJSONObject(arr.length() - 1) + .optJSONObject("result")?.optJSONObject("data")?.optJSONObject("json") + } catch (_: Exception) { null } ?: return manga + + val title = detailJson.optString("title", manga.title) + val desc = detailJson.optString("synopsis", manga.description) + val cover = detailJson.optString("coverUrl", manga.coverUrl) + val authors = detailJson.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val genres = detailJson.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val status = detailJson.optString("status", null) + val chapters = loadChapters(manga) + + return manga.copy( + title = title, description = desc, coverUrl = cover, + authors = authors.toSet(), + tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = when (status?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + }, + chapters = chapters, + ) + } + + private suspend fun loadChapters(manga: Manga): List { + val mangaId = manga.url.substringAfterLast("#").toLongOrNull() ?: return emptyList() + val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#").ifBlank { return emptyList() } + val res = client.httpGet(apiUrl("auth.me,comments.list,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val chaptersArray = parseLastItemJsonArray(body) ?: return emptyList() + return (0 until chaptersArray.length()).mapNotNull { i -> + chaptersArray.optJSONObject(i)?.let { obj -> + val number = obj.optDouble("number", 0.0).toFloat() + val createdAt = obj.optString("createdAt", null) + val uploadDate = createdAt?.let { dateFormat.parse(it)?.time } ?: 0L + MangaChapter( + id = generateUid("/$mangaPath/$slug/$number"), + url = "/$mangaPath/$slug/$number", + title = "Chapter ${number.formatForTitle()}", + number = number, volume = 0, uploadDate = uploadDate, + scanlator = null, branch = null, source = source + ) + } + }.sortedBy { it.number } + } + + // triyng first tRPC API if it fails then chapter html scraping + override suspend fun getPages(chapter: MangaChapter): List { + val path = chapter.url.removePrefix("/$mangaPath/") + val slug = path.substringBefore("/") + val num = path.substringAfter("/").toFloatOrNull() ?: return emptyList() + + val input = pagesPayload(slug, num) + val res = client.httpGet(apiUrl("auth.me,series.bySlug,reader.chapterPages", input), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val apiPages = parsePages(body) + if (!apiPages.isNullOrEmpty()) return apiPages + + val chapterUrl = "https://$domainName/$mangaPath/$slug/$num" + val doc = client.httpGet(chapterUrl).parseHtml() + + // search in json results image URLs + val script = doc.select("script").find { it.data().contains("\"webpUrl\"") } + ?: doc.select("script").find { it.data().contains("\"avifUrl\"") } + if (script != null) { + val arrayRegex = Regex("""\[\s*\{.*?\"(?:webp|avif)Url\".*?\}\s*\]""", RegexOption.DOT_MATCHES_ALL) + val match = arrayRegex.find(script.data())?.value + if (match != null) { + return try { + val jsonArray = JSONArray(match) + (0 until jsonArray.length()).mapNotNull { i -> + val page = jsonArray.optJSONObject(i) ?: return@mapNotNull null + val url = page.optString("webpUrl", "").ifEmpty { + page.optString("avifUrl", "") + }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } catch (_: Exception) { emptyList() } + } + } + + return emptyList() + } + + private fun parsePages(body: String): List? { + val elements = try { JSONArray(body) } catch (_: Exception) { return null } + for (i in 0 until elements.length()) { + if (elements.optJSONObject(i)?.has("error") == true) return null + } + val last = elements.optJSONObject(elements.length() - 1) ?: return null + val items = last.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") ?: return null + return (0 until items.length()).mapNotNull { i -> + val page = items.optJSONObject(i) ?: return@mapNotNull null + val url = page.optString("webpUrl", "").ifEmpty { + page.optString("avifUrl", "") + }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getPageUrl(page: MangaPage) = page.url + + private fun Float.formatForTitle() = if (this == toLong().toFloat()) toLong().toString() else toString() + + companion object { + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt new file mode 100644 index 00000000..6dafdb8b --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt @@ -0,0 +1,309 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.concatUrl +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.selectLast + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.Calendar + +internal abstract class LikeMangaParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 36, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = + EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + append("/?act=search") + + query?.let { + append("&f") + append("[keyword]".urlEncoded()) + append("=") + append(query.urlEncoded()) + } + + append("&f") + append("[sortby]".urlEncoded()) + append("=") + when (order) { + SortOrder.UPDATED -> append("lastest-chap") + SortOrder.NEWEST -> append("lastest-manga") + SortOrder.POPULARITY -> append("top-manga") + SortOrder.POPULARITY_TODAY -> append("top-day") + SortOrder.POPULARITY_WEEK -> append("top-week") + SortOrder.POPULARITY_MONTH -> append("top-month") + else -> append("lastest-chap") + } + + if (filter.tags.isNotEmpty()) { + append("&f") + append("[genres]".urlEncoded()) + append("=") + filter.tags.oneOrThrowIfMany()?.let { + append(it.key) + } + } + + filter.states.oneOrThrowIfMany()?.let { + append("&f") + append("[status]".urlEncoded()) + append("=") + append( + when (it) { + MangaState.ONGOING -> "in-process" + MangaState.FINISHED -> "complete" + MangaState.PAUSED -> "pause" + else -> "all" + }, + ) + } + + if (page > 1) { + append("&pageNum=") + append(page) + } + + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.card-body div.video").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.selectFirstOrThrow("p.title-manga").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = div.selectFirstOrThrow("img").src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/genres/").parseHtml() + return doc.select("ul.nav-genres li:not(.text-center) a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val mangaId = manga.url.toAbsoluteUrl(domain).removeSuffix("/").substringAfterLast("-").toInt() + val maxPageChapterSelect = doc.getElementById("nav_list_chapter_id_detail")?.select("a:not(.next)") + var maxPageChapter = 1 + if (!maxPageChapterSelect.isNullOrEmpty()) { + maxPageChapterSelect.map { + val i = it.text().toInt() + if (i > maxPageChapter) { + maxPageChapter = i + } + } + } + val author = doc.selectLast("li.author p")?.textOrNull() + return manga.copy( + altTitles = setOfNotNull(doc.selectFirst(".list-info li.othername h2")?.textOrNull()), + tags = doc.select("li.kind a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + }, + authors = setOfNotNull(author), + description = doc.requireElementById("summary_shortened").html(), + chapters = run { + if (maxPageChapter == 1) { + parseChapters(doc) + } else { + coroutineScope { + val result = ArrayList(parseChapters(doc)) + result.ensureCapacity(result.size * maxPageChapter) + (2..maxPageChapter).map { i -> + async { + loadChapters(mangaId, i) + } + }.awaitAll() + .flattenTo(result) + result + } + } + }.reversed(), + ) + } + + private suspend fun loadChapters(mangaId: Int, page: Int): List { + val json = + webClient.httpGet( + "https://$domain/?act=ajax&code=load_list_chapter&manga_id=$mangaId&page_num=$page&chap_id=0&keyword=", + ) + .parseJson().getString("list_chap") + val doc = Jsoup.parse(json) + return parseChapters(doc) + } + + private val dateFormat = SimpleDateFormat("MMMM d, yyyy", Locale.ENGLISH) + private fun parseChapters(root: Element): List { + return root.select("li.wp-manga-chapter") + .map { li -> + val url = li.selectFirstOrThrow("a").attrAsRelativeUrl("href") + val dateText = if (li.selectFirstOrThrow(".chapter-release-date").text() == "New") { + "today" + } else { + li.selectFirstOrThrow(".chapter-release-date").text() + } + val chapNum = url.substringAfter("chapter-").substringBefore("-") + + MangaChapter( + id = generateUid(url), + title = li.selectFirstOrThrow("a").text(), + number = chapNum.toFloatOrNull() ?: 0f, + volume = 0, + url = url, + scanlator = null, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val testJson = doc.selectFirst("div.reading input#next_img_token") + if (testJson != null) { + val jsonRaw = testJson.attr("value").split(".")[1] + val jsonData = JSONObject(context.decodeBase64(jsonRaw).toString(Charsets.UTF_8)) + val jsonImg = context.decodeBase64(jsonData.getString("data")).toString(Charsets.UTF_8) + val images = jsonImg.replace("\\", "").replace("[", "").replace("]", "").replace("\"", "").split(",") + val baseUrl = doc.selectFirstOrThrow(".reading-detail img").src() + val cdn = baseUrl?.substringBefore("manga/", "")?.ifEmpty { + baseUrl.toHttpUrl().resolve("/").toString() + } + return images.map { img -> + val url = concatUrl(cdn.orEmpty(), img) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + } else { + return doc.select(".reading-detail img").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + } + + } + + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + d.startsWith("today") -> Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + + else -> dateFormat.parseSafe(date) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/LilianaParser.kt b/src/main/kotlin/tsuki/parsers/LilianaParser.kt new file mode 100644 index 00000000..44f5411d --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/LilianaParser.kt @@ -0,0 +1,266 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.mapChapters +import tsuki.util.parseJson +import tsuki.util.selectOrThrow + +import androidx.collection.scatterSetOf +import kotlinx.coroutines.coroutineScope +import org.jsoup.Jsoup +import org.jsoup.nodes.Element + +import java.util.EnumSet + +internal abstract class LilianaParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 24, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.POPULARITY_MONTH, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_TODAY, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.RATING, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = getAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED + ), + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/search/") + append(page) + append("/?keyword=") + append((filter.query?.urlEncoded() ?: "")) + } + + else -> { + append("/filter/") + append(page) + + append("/?sort=") + when (order) { + SortOrder.UPDATED -> append("latest-updated") + SortOrder.POPULARITY -> append("views") + SortOrder.POPULARITY_MONTH -> append("views_month") + SortOrder.POPULARITY_WEEK -> append("views_week") + SortOrder.POPULARITY_TODAY -> append("views_day") + SortOrder.ALPHABETICAL -> append("az") + SortOrder.ALPHABETICAL_DESC -> append("za") + SortOrder.NEWEST -> append("new") + SortOrder.NEWEST_ASC -> append("old") + SortOrder.RATING -> append("score") + else -> append("latest-updated") + } + + append("&genres=") + filter.tags.joinTo(this, ",") { it.key } + + append("¬Genres=") + filter.tagsExclude.joinTo(this, ",") { it.key } + + if (filter.states.isNotEmpty()) { + append("&status=") + append( + when (filter.states.first()) { + MangaState.ONGOING -> "on-going" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on-hold" + MangaState.ABANDONED -> "canceled" + else -> "all" + }, + ) + } + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div#main div.grid > div").map { parseSearchManga(it) } + } + + private fun parseSearchManga(element: Element): Manga { + val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = element.selectFirst("img")?.src(), + title = element.selectFirst(".text-center a")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + + @JvmField + protected val ongoing = scatterSetOf( + "on-going", "đang tiến hành", "進行中", + ) + + @JvmField + protected val finished = scatterSetOf( + "completed", "hoàn thành", "完了", + ) + + @JvmField + protected val abandoned = scatterSetOf( + "canceled", "đã huỷ bỏ", "キャンセル", + ) + + @JvmField + protected val paused = scatterSetOf( + "on-hold", "tạm dừng", "一時停止", + ) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val author = doc.selectFirst("div.y6x11p i.fas.fa-user + span.dt")?.textOrNull()?.takeUnless { + it.equals("updating", true) + } + return manga.copy( + description = doc.selectFirst("div#syn-target")?.html(), + largeCoverUrl = doc.selectFirst(".a1 > figure img")?.src(), + tags = doc.select(".a2 div > a[rel='tag'].label").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('/'), + title = a.text().toTitleCase(sourceLocale), + source = source, + ) + }, + authors = setOfNotNull(author), + state = when (doc.selectFirst("div.y6x11p i.fas.fa-rss + span.dt")?.text()?.lowercase().orEmpty()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in paused -> MangaState.PAUSED + in abandoned -> MangaState.ABANDONED + else -> null + }, + chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = true) { i, element -> + val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = element.selectFirst("a")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + uploadDate = element.selectFirst("time[datetime]")?.attr("datetime")?.toLongOrNull()?.times(1000) + ?: 0L, + branch = null, + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val script = doc.selectFirstOrThrow("script:containsData(const CHAPTER_ID)").data() + + val chapterId = script.substringAfter("const CHAPTER_ID = ", "").substringBefore(';', "") + check(chapterId.isNotEmpty()) { "Failed to get chapter id" } + + val ajaxUrl = buildString { + append("https://") + append(domain) + append("/ajax/image/list/chap/") + append(chapterId) + } + + val responseJson = webClient.httpGet(ajaxUrl).parseJson() + + check(responseJson.getBooleanOrDefault("status", false)) { responseJson.getString("msg") } + + val pageListDoc = Jsoup.parse(responseJson.getString("html")) + + return pageListDoc.selectOrThrow("div").map { + val url = it.selectFirstOrThrow("img").attr("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + protected open suspend fun getAvailableTags(): Set = coroutineScope { + val doc = webClient.httpGet("https://$domain/filter").parseHtml() + doc.select("div.advanced-genres > div > .advance-item").mapToSet { element -> + MangaTag( + key = element.selectFirstOrThrow("span[data-genre]").attr("data-genre"), + title = element.text().toTitleCase(sourceLocale), + source = source, + ) + } + } +} + diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt new file mode 100644 index 00000000..0af51490 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -0,0 +1,936 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.MangaParserAuthProvider +import tsuki.exception.AuthRequiredException +import tsuki.exception.ParseException + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapChapters +import tsuki.util.selectOrThrow +import tsuki.util.CryptoAES +import tsuki.util.getCookies +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.selectLast +import tsuki.util.toMutableMap +import tsuki.util.toRelativeUrl + +import androidx.collection.scatterSetOf +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import org.jsoup.HttpStatusException +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.net.HttpURLConnection +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Base64 +import java.util.Calendar +import java.util.EnumSet + +internal abstract class MadaraParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 12, +) : PagedMangaParser(context, source, pageSize), MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + // Change these values only if the site does not support manga listings via ajax + protected open val withoutAjax = false + protected open val authorSearchSupported = false + + override val availableSortOrders: Set = setupAvailableSortOrders() + + private fun setupAvailableSortOrders(): Set { + return if (!withoutAjax) { + EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.RELEVANCE, + ) + } else { + EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.RELEVANCE, + ) + } + } + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = !withoutAjax, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = true, + isAuthorSearchSupported = authorSearchSupported, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + MangaState.UPCOMING, + ), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + ) + + override val authUrl: String + get() = "https://${domain}" + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name.contains("wordpress_logged_in") + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml().body() + return body.selectFirst(".c-user_name")?.text() + ?: run { + throw if (body.selectFirst("#loginform") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } + + protected open val tagPrefix = "manga-genre/" + protected open val datePattern = "MMMM d, yyyy" + protected open val stylePage = "?style=list" + protected open val postReq = false + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + protected fun Element.tableValue(): Element { + for (p in parents()) { + val children = p.children() + if (children.size == 2) { + return children[1] + } + } + parseFailed("Cannot find tableValue for node ${text()}") + } + + @JvmField + protected val ongoing = scatterSetOf( + "مستمرة", + "en curso", + "ongoing", + "on going", + "OnGoing", + "ativo", + "en cours", + "en cours \uD83D\uDFE2", + "en cours de publication", + "activo", + "đang tiến hành", + "em lançamento", + "онгоінг", + "publishing", + "devam ediyor", + "em andamento", + "in corso", + "güncel", + "berjalan", + "продолжается", + "updating", + "lançando", + "in arrivo", + "emision", + "en emision", + "مستمر", + "curso", + "en marcha", + "publicandose", + "publicando", + "连载中", + "đang làm", + "em postagem", + "devam eden", + "em progresso", + "atualizações semanais", + "em lançamento", + "devam ediyo", + ) + + @JvmField + protected val finished = scatterSetOf( + "completed", + "complete", + "completo", + "complété", + "fini", + "achevé", + "terminé", + "terminé ⚫", + "tamamlandı", + "đã hoàn thành", + "hoàn thành", + "مكتملة", + "завершено", + "завершен", + "finished", + "finalizado", + "completata", + "one-shot", + "bitti", + "tamat", + "completado", + "concluído", + "concluido", + "已完结", + "bitmiş", + "end", + "منتهية", + "tamamlanan", + "مكتمل", + ) + + @JvmField + protected val abandoned = scatterSetOf( + "canceled", + "cancelled", + "cancelado", + "cancellato", + "cancelados", + "dropped", + "discontinued", + "abandonné", + "iptal edildi", + "đã hủy", + "ملغي", + "заброшено", + "annulé", + ) + + @JvmField + protected val paused = scatterSetOf( + "hiatus", + "on hold", + "pausado", + "en espera", + "en pause", + "en attente", + "durduruldu", + "beklemede", + "đang chờ", + "متوقف", + "заморожено", + ) + + @JvmField + protected val upcoming = scatterSetOf( + "upcoming", + "لم تُنشَر بعد", + "prochainement", + "à venir", + "in arrivo", + ) + + // can be changed to retrieve tags see getTags + protected open val listUrl = "manga/" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + if (withoutAjax) { + val pages = page + 1 + + val url = buildString { + append("https://") + append(domain) + + if (pages > 1) { + append("/page/") + append(pages.toString()) + } + append("/?s=") + append(filter.query?.urlEncoded() ?: "") + + append("&post_type=wp-manga") + + // Known bug: in some cases, if there are no manga with the associated tags, the source returns the full list of manga + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { + append("&genre[]=") + append(it.key) + } + } + + filter.states.forEach { + append("&status[]=") + when (it) { + MangaState.ONGOING -> append("on-going") + MangaState.FINISHED -> append("end") + MangaState.ABANDONED -> append("canceled") + MangaState.PAUSED -> append("on-hold") + MangaState.UPCOMING -> append("upcoming") + else -> throw IllegalArgumentException("$it not supported") + } + } + + filter.contentRating.oneOrThrowIfMany()?.let { + append("&adult=") + append( + when (it) { + ContentRating.SAFE -> "0" + ContentRating.ADULT -> "1" + else -> "" + }, + ) + } + + if (filter.year != 0) { + append("&release=") + append(filter.year.toString()) + } + + filter.author?.takeIf { it.isNotEmpty() }?.let { + append("&author=") + // should be like "minamida-usuke" + append(it.lowercase().replace(" ", "-")) + } + + // Support artist + //filter.artist?.let { + // append("&artist=") + // append(filter.artist) + //} + + + append("&m_orderby=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + SortOrder.NEWEST -> append("new-manga") + SortOrder.ALPHABETICAL -> append("alphabet") + SortOrder.RATING -> append("rating") + SortOrder.RELEVANCE -> {} + else -> {} + } + } + val html = try { + webClient.httpGet(url).parseHtml() + } catch (e: HttpStatusException) { + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { + return emptyList() + } else { + throw ParseException("Can't fetch data from source!", url) + } + } + return parseMangaList(html) + } else { + + val payload = createRequestTemplate() + + payload["page"] = page.toString() + + filter.query?.takeIf { it.isNotEmpty() }?.let { + payload["vars[s]"] = it.urlEncoded() + } + + if (filter.tags.isNotEmpty()) { + payload["vars[tax_query][0][taxonomy]"] = "wp-manga-genre" + payload["vars[tax_query][0][field]"] = "slug" + filter.tags.forEachIndexed { i, it -> + payload["vars[tax_query][0][terms][$i]"] = it.key + } + payload["vars[tax_query][0][operator]"] = "IN" + } + + if (filter.tagsExclude.isNotEmpty()) { + payload["vars[tax_query][1][taxonomy]"] = "wp-manga-genre" + payload["vars[tax_query][1][field]"] = "slug" + filter.tagsExclude.forEachIndexed { i, it -> + payload["vars[tax_query][1][terms][$i]"] = it.key + } + payload["vars[tax_query][1][operator]"] = "NOT IN" + } + + if (filter.year != 0) { + payload["vars[tax_query][2][taxonomy]"] = "wp-manga-release" + payload["vars[tax_query][2][field]"] = "slug" + payload["vars[tax_query][2][terms][]"] = filter.year.toString() + } + + filter.author?.takeIf { it.isNotEmpty() }?.let { + payload["vars[tax_query][3][taxonomy]"] = "wp-manga-author" + payload["vars[tax_query][3][field]"] = "name" + payload["vars[tax_query][3][terms][0]"] = it + payload["vars[tax_query][3][operator]"] = "IN" + } + + // Support artist + // filter.artist.let { + // payload["vars[tax_query][4][taxonomy]"] = "wp-manga-artist" + // payload["vars[tax_query][4][field]"] = "name" + // payload["vars[tax_query][4][terms][0]"] = filter.artist + // payload["vars[tax_query][4][operator]"] = "IN" + //} + + if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.year != 0) { + payload["vars[tax_query][relation]"] = "AND" + } + + when (order) { + SortOrder.POPULARITY -> { + payload["vars[meta_key]"] = "_wp_manga_views" + payload["vars[orderby]"] = "meta_value_num" + payload["vars[order]"] = "desc" + } + + SortOrder.POPULARITY_ASC -> { + payload["vars[meta_key]"] = "_wp_manga_views" + payload["vars[orderby]"] = "meta_value_num" + payload["vars[order]"] = "asc" + } + + SortOrder.UPDATED -> { + payload["vars[meta_key]"] = "_latest_update" + payload["vars[orderby]"] = "meta_value_num" + payload["vars[order]"] = "desc" + } + + SortOrder.UPDATED_ASC -> { + payload["vars[meta_key]"] = "_latest_update" + payload["vars[orderby]"] = "meta_value_num" + payload["vars[order]"] = "asc" + } + + SortOrder.NEWEST -> { + payload["vars[orderby]"] = "date" + payload["vars[order]"] = "desc" + } + + SortOrder.NEWEST_ASC -> { + payload["vars[orderby]"] = "date" + payload["vars[order]"] = "asc" + } + + SortOrder.ALPHABETICAL -> { + payload["vars[orderby]"] = "post_title" + payload["vars[order]"] = "asc" + } + + SortOrder.ALPHABETICAL_DESC -> { + payload["vars[orderby]"] = "post_title" + payload["vars[order]"] = "desc" + } + + SortOrder.RATING -> { + payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" + payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" + + payload["vars[orderby][query_avarage_reviews]"] = "DESC" + payload["vars[orderby][query_total_reviews]"] = "DESC" + } + + SortOrder.RATING_ASC -> { + payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" + payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" + + payload["vars[orderby][query_avarage_reviews]"] = "ASC" + payload["vars[orderby][query_total_reviews]"] = "ASC" + } + + SortOrder.RELEVANCE -> { + payload["vars[orderby]"] = "" + } + + else -> payload["vars[orderby]"] = "" + } + + filter.states.forEach { + payload["vars[meta_query][0][0][key]"] = "_wp_manga_status" + payload["vars[meta_query][0][0][compare]"] = "IN" + payload["vars[meta_query][0][0][value][]"] = + when (it) { + MangaState.ONGOING -> "on-going" + MangaState.FINISHED -> "end" + MangaState.ABANDONED -> "canceled" + MangaState.PAUSED -> "on-hold" + MangaState.UPCOMING -> "upcoming" + else -> throw IllegalArgumentException("$it not supported") + } + } + + filter.contentRating.oneOrThrowIfMany()?.let { + payload["vars[meta_query][0][1][key]"] = "manga_adult_content" + payload["vars[meta_query][0][1][value]"] = + when (it) { + ContentRating.SAFE -> "" + ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" + else -> "" + } + } + + val html = try { + webClient.httpPost( + "https://$domain/wp-admin/admin-ajax.php", + payload, + ).parseHtml() + } catch (e: HttpStatusException) { + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { + return emptyList() + } else { + throw ParseException("Can't fetch data from source!", domain) + } + } + return parseMangaList(html) + } + } + + protected open fun parseMangaList(doc: Document): List { + val elements = doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail, div.manga__item") + } + + // Avoid "Content not found or removed" errors + if (elements.isEmpty()) { + return emptyList() + } + + return elements.map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") + val author = summary?.selectFirst(".mg_author, .mg_artists")?.selectFirst("a")?.ownText() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src(), + title = (summary?.selectFirst("h3, h4") + ?: div.selectFirst(".manga-name, .post-title h2 a, .post-title"))?.text() + .orEmpty(), + altTitles = emptySet(), + rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), + source = source, + ) + }.orEmpty(), + authors = setOfNotNull(author), + state = when ( + summary?.selectFirst(".mg_status") + ?.selectFirst(".summary-content") + ?.ownText()?.lowercase() + .orEmpty() + ) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in abandoned -> MangaState.ABANDONED + in paused -> MangaState.PAUSED + in upcoming -> MangaState.UPCOMING + else -> null + }, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + protected open suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix('/').substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().ifEmpty { + a.selectFirst(".menu-image-title")?.textOrNull() + }?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null, + source = source, + ) + } + } + + protected open val selectDesc = + "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" + protected open val selectGenre = "div.genres-content a" + protected open val selectTestAsync = "div.listing-chapters_wrap" + protected open val selectState = + "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + + "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + + "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(Статус)," + + "div.post-content_item:contains(状态), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" + protected open val selectAlt = + ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" + + protected open suspend fun createMangaTag(a: Element): MangaTag? { + return MangaTag( + key = a.attr("href").removeSuffix("/").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val href = doc.selectFirst("head meta[property='og:url']")?.attr("content")?.toRelativeUrl(domain) ?: manga.url + val testCheckAsync = doc.select(selectTestAsync) + val chaptersDeferred = if (testCheckAsync.isEmpty()) { + async { loadChapters(href, doc) } + } else { + async { getChapters(manga, doc) } + } + + val desc = doc.select(selectDesc).html() + + val stateDiv = doc.selectFirst(selectState)?.selectLast("div.summary-content") + + val state = stateDiv?.let { + when (it.text().lowercase()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in abandoned -> MangaState.ABANDONED + in paused -> MangaState.PAUSED + else -> null + } + } + + val alt = doc.body().select(selectAlt).firstOrNull()?.tableValue()?.textOrNull() + + manga.copy( + title = doc.selectFirst("h1")?.textOrNull() ?: manga.title, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + tags = doc.body().select(selectGenre).mapToSet { a -> createMangaTag(a) }.filterNotNull().toSet(), + description = desc, + altTitles = setOfNotNull(alt), + state = state, + chapters = chaptersDeferred.await(), + contentRating = if (doc.selectFirst(".adult-confirm") != null || isNsfwSource) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + ) + } + + + protected open val selectDate = "span.chapter-release-date i" + protected open val selectChapter = "li.wp-manga-chapter" + + protected open suspend fun getChapters(manga: Manga, doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + title = name, + number = i + 1f, + volume = 0, + url = link, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + protected open val postDataReq = "action=manga_get_chapters&manga=" + + protected open suspend fun loadChapters(mangaUrl: String, document: Document): List { + val doc = if (postReq) { + val mangaId = document.select("div#manga-chapters-holder").attr("data-id") + val url = "https://$domain/wp-admin/admin-ajax.php" + val postData = postDataReq + mangaId + webClient.httpPost(url, postData).parseHtml() + } else { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + webClient.httpPost(url, emptyMap()).parseHtml() + } + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = i + 1f, + volume = 0, + branch = null, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + scanlator = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow(".related-manga") + return root.select("div.related-reading-wrap").mapNotNull { div -> + val a = div.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = emptySet(), + title = div.selectFirstOrThrow(".widget-title").text(), + authors = emptySet(), + coverUrl = div.selectFirst("img")?.src(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + protected open val selectBodyPage = "div.main-col-inner div.reading-content" + protected open val selectPage = "div.page-break, div.page-box" + protected open val selectRequiredLogin = ".content-blocked, .login-required" + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chapterProtector = doc.getElementById("chapter-protector-data") + if (chapterProtector == null) { + throw if (doc.selectFirst(selectRequiredLogin) != null) { + AuthRequiredException(source) + } else { + val root = doc.body().selectFirst(selectBodyPage) ?: throw ParseException( + "No image found, try to log in", + fullUrl, + ) + return root.select(selectPage).flatMap { div -> + div.selectOrThrow("img").map { img -> + val url = img.requireSrc().toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + } + } else { + + val chapterProtectorHtml = chapterProtector.attr("src") + .takeIf { it.startsWith("data:text/javascript;base64,") } + ?.substringAfter("data:text/javascript;base64,") + ?.let { + Base64.getDecoder().decode(it).decodeToString() + } + ?: chapterProtector.html() + + val password = chapterProtectorHtml.substringAfter("wpmangaprotectornonce='").substringBefore("';") + val chapterData = JSONObject( + chapterProtectorHtml.substringAfter("chapter_data='").substringBefore("';").replace("\\/", "/"), + ) + val unsaltedCiphertext = context.decodeBase64(chapterData.getString("ct")) + val salt = chapterData.getString("s").decodeHex() + val ciphertext = "Salted__".toByteArray(Charsets.UTF_8) + salt + unsaltedCiphertext + + val rawImgArray = CryptoAES(context).decrypt(context.encodeBase64(ciphertext), password) + val imgArrayString = rawImgArray.filterNot { c -> c == '[' || c == ']' || c == '\\' || c == '"' } + + return imgArrayString.split(",").map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + + WordSet( + " ago", "atrás", " hace", " publicado", " назад", " önce", " trước", "مضت", "قبل", + " h", " d", " días", " jour", " horas", " heure", " mins", " minutos", " minute", " mois", + ).endsWith(d) -> { + parseRelativeDate(d) + } + + WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút", "giây").startsWith(d) -> { + parseRelativeDate(d) + } + + WordSet("yesterday", "يوم واحد").startsWith(d) -> { + Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) // yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + WordSet("today").startsWith(d) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + WordSet("يومين").startsWith(d) -> { + Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -2) // day before yesterday + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + d.contains(Regex("""\b\d+ jour""")) -> { + parseRelativeDate(d) + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.parseSafe(it.joinToString(" ")) } + + else -> dateFormat.parseSafe(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان") + .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + + WordSet("menit", "dakika", "min", "minute", "minutes", "minuto", "mins", "นาที", "دقائق", "phút", "минут", "دقيقة") + .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + + WordSet("jam", "saat", "heure", "hora", "horas", "hour", "hours", "h", "ชั่วโมง", "giờ", "ore", "ساعة", "小时", "ساعات") + .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + + WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "d", "วัน", "ngày", "giorni", "أيام", "天", "день") + .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(date) -> + cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis + + WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois") + .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + + WordSet("year", "año", "năm") + .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + + else -> 0 + } + } + + private companion object { + + private fun createRequestTemplate() = + ("action=madara_load_more&page=0&template=madara-core%2Fcontent%2Fcontent-search&vars%5Bs%5D=&vars%5Bpaged%5D=1&vars%5Btemplate%5D=search&vars%5Bmeta_query%5D%5B0%5D%5Brelation%5D=AND&vars%5Bmeta_query%5D%5Brelation%5D=AND&vars%5Bpost_type%5D=wp-manga&vars%5Bpost_status%5D=publish&vars%5Bmanga_archives_item_layout%5D=default").split( + '&', + ).map { + val pos = it.indexOf('=') + it.substring(0, pos) to it.substring(pos + 1) + }.toMutableMap() + + fun String.decodeHex(): ByteArray { + check(length % 2 == 0) { "Must have an even length" } + + return chunked(2).map { it.toInt(16).toByte() }.toByteArray() + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/MadthemeParser.kt b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt new file mode 100644 index 00000000..3cbd1f01 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt @@ -0,0 +1,449 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapChapters +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.Interceptor +import okhttp3.Response +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet + +internal abstract class MadthemeParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 48, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + SortOrder.RATING, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + + @JvmField + protected val ongoing: Set = setOf( + "on going", + "ongoing", + ) + + @JvmField + protected val finished: Set = setOf( + "completed", + ) + + protected open val listUrl = "search/" + protected open val datePattern = "MMM dd, yyyy" + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + val fragment = url.fragment + + if (fragment != null && (fragment.startsWith("https://") || fragment.startsWith("http://"))) { + val cleanUrl = url.newBuilder().fragment(null).build() + val response = chain.proceed(request.newBuilder().url(cleanUrl).build()) + if (response.isSuccessful) { + return response + } + response.close() + return chain.proceed(request.newBuilder().url(fragment).build()) + } + + val response = chain.proceed(request) + if (!response.isSuccessful && fragment == IMAGE_REQUEST_FRAGMENT) { + response.close() + val fallbackUrl = url.newBuilder() + .host(IMAGE_FALLBACK_HOST) + .encodedPath(url.encodedPath.replaceFirst("/res/", "/")) + .fragment(null) + .build() + return chain.proceed(request.newBuilder().url(fallbackUrl).build()) + } + return response + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + append('/') + append(listUrl) + + append("?page=") + append(page.toString()) + + query?.let { + append("&q=") + append(query.urlEncoded()) + } + + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("updated_at") + SortOrder.ALPHABETICAL -> append("name") // On some sites without tags or searches, the alphabetical option is empty. + SortOrder.NEWEST -> append("created_at") + SortOrder.RATING -> append("rating") + else -> append("updated_at") + } + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { + append("&") + append("genre[]".urlEncoded()) + append("=") + append(it.key) + } + } + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> "all" + }, + ) + } + } + + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.book-item").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src(), + title = div.selectFirst("div.meta")?.selectFirst("div.title")?.text().orEmpty(), + altTitles = emptySet(), + rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = doc.body().select("div.meta div.genres span").mapToSet { span -> + MangaTag( + key = span.attr("class"), + title = span.text().toTitleCase(), + source = source, + ) + }, + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + return doc.select("div.genres .checkbox").mapNotNullToSet { checkbox -> + val key = checkbox.selectFirstOrThrow("input").attr("value") + val name = checkbox.selectFirst("span.radio__label")?.text() ?: key + MangaTag( + key = key, + title = name, + source = source, + ) + } + } + + protected open val selectDesc = "div.section-body.summary p.content" + protected open val selectState = "div.detail p:contains(Status) span" + protected open val selectAlt = "div.detail div.name h2" + protected open val selectTag = "div.detail p:contains(Genres) a" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(doc) } + + val desc = doc.selectFirst(selectDesc)?.html() + + val stateDiv = doc.selectFirst(selectState) + + val state = stateDiv?.let { + when (it.text().lowercase()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + + val alt = doc.body().select(selectAlt).text() + + val nsfw = doc.getElementById("adt-warning") != null + + manga.copy( + tags = doc.body().select(selectTag).mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().toTitleCase().replace(",", ""), + source = source, + ) + }, + description = desc?.nullIfEmpty(), + altTitles = setOfNotNull(alt.nullIfEmpty()), + state = state, + chapters = chaptersDeferred.await(), + contentRating = if (nsfw || manga.contentRating == ContentRating.ADULT) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + ) + } + + + protected open val selectDate = "div .chapter-update" + protected open val selectChapter = "ul#chapter-list li" + + protected open suspend fun getChapters(doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val slug = doc.selectFirstOrThrow("script:containsData(bookSlug)").data().substringAfter("bookSlug = \"") + .substringBefore("\";") + val docChapter = webClient.httpGet("https://$domain/api/manga/$slug/chapters?source=detail").parseHtml() + return docChapter.select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val dateText = li.selectFirst(selectDate)?.text() + MangaChapter( + id = generateUid(href), + title = li.selectFirst(".chapter-title")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + protected open val selectPage = "div#chapter-images img" + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.normalizedChapterUrl() + val doc = webClient.httpGet(fullUrl).parseHtml() + val known = HashSet() + val result = ArrayList() + // html parisng + doc.select(selectPage).forEach { img -> + result.addPage(known, img.resolveImageUrl()) + } + // js parsing + val regexPages = Regex("chapImages\\s*=\\s*['\"](.*?)['\"]") + val mainServer = doc.select("script").firstNotNullOfOrNull { script -> + Regex("mainServer\\s*=\\s*\"(.*?)\"").find(script.html())?.groupValues?.getOrNull(1) + } + val schemePrefix = if (mainServer?.startsWith("//") == true) "https:" else "" + val pages = doc.select("script").firstNotNullOfOrNull { script -> + regexPages.find(script.html())?.groupValues?.getOrNull(1) + }?.split(',') + pages?.forEach { url -> + val pageUrl = if (mainServer.isNullOrEmpty()) { + url.resolveChapterImageUrl() + } else { + "$schemePrefix$mainServer$url" + } + result.addPage(known, pageUrl) + } + return result + } + + override suspend fun getPageUrl(page: MangaPage): String { + val url = page.url.toAbsoluteUrl(domain) + return if ('#' in url) url else "$url#$IMAGE_REQUEST_FRAGMENT" + } + + private fun MutableList.addPage(known: MutableSet, url: String) { + if (known.add(url)) { + add( + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ), + ) + } + } + + private fun Element.resolveImageUrl(): String { + val primary = requireSrc() + val rawFallback = attr("onerror") + .substringAfter("this.src='", "") + .substringBefore("'") + .takeIf { it.isNotBlank() } + ?: return primary + val fallback = rawFallback.resolveChapterImageUrl() + .takeIf { it.startsWith("https://") || it.startsWith("http://") } + ?: return primary + + return if ("://s20." in primary) fallback else "$primary#$fallback" + } + + private fun String.resolveChapterImageUrl(): String { + val value = trim() + return when { + value.startsWith("https://") || value.startsWith("http://") -> value + value.startsWith("//") -> "https:$value" + value.contains("/manga/") && !value.contains("/wp-content/") -> + "https://$IMAGE_FALLBACK_HOST/manga${value.substringAfter("/manga")}" + value.startsWith("/") -> "https://$domain$value" + else -> value.toAbsoluteUrl(domain) + } + } + + private fun String.normalizedChapterUrl(): String { + val value = trim() + if (value.startsWith("https://") || value.startsWith("http://")) { + val schemeEnd = value.indexOf("://") + val pathStart = value.indexOf('/', startIndex = schemeEnd + 3) + if (pathStart == -1) return value + val prefix = value.substring(0, pathStart) + val path = value.substring(pathStart).replace(Regex("/{2,}"), "/") + return prefix + path + } + return value.toAbsoluteUrl(domain) + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + + WordSet(" ago", " h", " d").endsWith(d) -> { + parseRelativeDate(d) + } + + WordSet("today").startsWith(d) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.parseSafe(it.joinToString(" ")) } + + else -> dateFormat.parseSafe(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("second") + .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + + WordSet("min", "minute", "minutes") + .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + + WordSet("hour", "hours", "h") + .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + + WordSet("day", "days") + .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("month", "months") + .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + + WordSet("year") + .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + + else -> 0 + } + } + + private companion object { + private const val IMAGE_REQUEST_FRAGMENT = "image-request" + private const val IMAGE_FALLBACK_HOST = "sb.mbcdn.xyz" + } + +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt new file mode 100644 index 00000000..ca55945e --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -0,0 +1,276 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseSafe +import tsuki.util.removeSuffix + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +internal abstract class Manga18Parser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + ) + + @Volatile + private var genresCache: Set? = null + private val genresMutex = Mutex() + + private suspend fun getOrFetchGenres(): Set { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + genresCache = fetchAvailableTags() + genresCache!! + } + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl/").parseHtml() + return doc.select("div.grid_cate li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix('/').substringAfterLast('/') + MangaTag( + key = href, + title = a.text(), + source = source, + ) + } + } + + override fun getRequestHeaders() = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + protected open val ongoing: Set = setOf("On Going") + protected open val finished: Set = setOf("Completed") + + protected open val listUrl = "list-manga/" + protected open val tagUrl = "manga-list/" + protected open val datePattern = "dd-MM-yyyy" + + protected open val selectDesc = "div.detail_reviewContent" + protected open val selectState = "div.item:contains(Status) div.info_value" + protected open val selectAlt = "div.item:contains(Other name) div.info_value" + protected open val selectTag = "div.info_value > a[href*=/manga-list/]" + protected open val selectAuthor = "div.info_label:contains(author) + div.info_value, div.info_label:contains(autor) + div.info_value" + protected open val selectArtist = "div.info_label:contains(artist) + div.info_value" + protected open val selectDate = "div.item p" + protected open val selectChapter = "div.chapter_box li" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim().orEmpty() + val tag = filter.tags.firstOrNull() + + val url = buildString(120) { + append("https://") + append(domain) + append('/') + + when { + tag != null && query.isNotEmpty() -> { + throw IllegalArgumentException("Search is not supported with tags") + } + tag != null -> { + append(tagUrl) + append(tag.key) + append('/') + append(page) + } + query.isNotEmpty() -> { + append(listUrl) + append(page) + append("?search=") + append(query.urlEncoded()) + append("&order_by=latest") + } + else -> { + append(listUrl) + append(page) + } + } + if (query.isEmpty()) { + append("?order_by=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + SortOrder.ALPHABETICAL -> append("name") + else -> append("latest") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + protected open fun parseMangaList(doc: Document): List { + return doc.select("div.story_item").map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirst("div.mg_info") + ?.selectFirst("div.mg_name a") + ?.text() + ?: a.attr("title") + val cover = div.selectFirst("img")?.src() + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = cover, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val info = doc.body().selectFirstOrThrow("div.detail_listInfo") + val chaptersDeferred = async { getChapters(doc) } + val desc = doc.selectFirst(selectDesc)?.html()?.nullIfEmpty() + val state = info.selectFirst(selectState)?.text()?.let { + when (it) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + val alt = info.selectFirst(selectAlt)?.textOrNull()?.takeUnless { it == "Updating" } + val author = info.selectFirst(selectAuthor)?.textOrNull()?.takeUnless { it == "Updating" } + val artist = info.selectFirst(selectArtist)?.textOrNull()?.takeUnless { it == "Updating" } + val tags = doc.body().select(selectTag).mapToSet { a -> + val key = a.attr("href").removeSuffix('/').substringAfterLast('/') + MangaTag( + key = key, + title = a.text().toTitleCase(), + source = source, + ) + } + manga.copy( + tags = tags, + description = desc.nullIfEmpty(), + altTitles = setOfNotNull(alt), + authors = setOfNotNull(author, artist), + state = state, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun getChapters(doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val dateText = li.selectFirst(selectDate)?.text() + MangaChapter( + id = generateUid(href), + title = a.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = dateFormat.parseSafe(dateText), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val script = doc.selectFirstOrThrow("script:containsData(slides_p_path)") + val encoded = script.data() + .substringAfter('[') + .substringBefore(",]") + .replace("\"", "") + .split(",") + + return encoded.map { encodedUrl -> + val decoded = context.decodeBase64(encodedUrl).toString(Charsets.UTF_8) + val absoluteUrl = decoded.toAbsoluteUrl(domain) + MangaPage( + id = generateUid(absoluteUrl), + url = absoluteUrl, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt new file mode 100644 index 00000000..1cb7c034 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt @@ -0,0 +1,461 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.LinkResolver +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.attrOrNull +import tsuki.util.copy +import tsuki.util.getCookies +import tsuki.util.insertCookie +import tsuki.util.mimeType +import tsuki.util.parseRaw +import tsuki.util.runCatchingCancellable +import tsuki.util.toRelativeUrl + +import androidx.collection.ArrayMap +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Cookie +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.json.JSONObject +import org.jsoup.nodes.Document + +import java.text.SimpleDateFormat +import java.util.Base64 +import java.util.EnumSet + +internal abstract class MangaReaderParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int, + searchPageSize: Int, +) : PagedMangaParser(context, source, pageSize, searchPageSize), Interceptor { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set + get() = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrCreateTagMap().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + + protected open val listUrl = "/manga" + protected open val datePattern = "MMMM d, yyyy" + protected open val isNetShieldProtected = false + + protected var tagCache: ArrayMap? = null + protected val mutex = Mutex() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + + when { + + !query.isNullOrEmpty() -> { + append("/page/") + append(page.toString()) + append("/?s=") + append(query.urlEncoded()) + } + + else -> { + append(listUrl) + + append("/?order=") + append( + when (order) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + }, + ) + + filter.tags.forEach { + append("&") + append("genre[]".urlEncoded()) + append("=") + append(it.key) + } + + filter.tagsExclude.forEach { + append("&") + append("genre[]".urlEncoded()) + append("=-") + append(it.key) + } + + if (filter.states.isNotEmpty()) { + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + when (it) { + MangaState.ONGOING -> append("ongoing") + MangaState.FINISHED -> append("completed") + MangaState.PAUSED -> append("hiatus") + else -> append("") + } + } + } + + filter.types.oneOrThrowIfMany()?.let { + append("&type=") + append( + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + }, + ) + } + + append("&page=") + append(page.toString()) + } + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + protected open val selectMangaList = ".postbody .listupd .bs .bsx" + protected open val selectMangaListImg = "img.ts-post-image" + protected open val selectMangaListTitle = "div.tt" + + protected open fun parseMangaList(docs: Document): List { + return docs.select(selectMangaList).mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = it.selectFirst(selectMangaListTitle)?.text() ?: a.attr("title"), + altTitles = emptySet(), + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = it.selectFirst(selectMangaListImg)?.src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + protected open val selectChapter = "#chapterlist > ul > li" + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> + val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(url), + title = element.selectFirst(".chapternum")?.textOrNull(), + url = url, + number = index + 1f, + volume = 0, + scanlator = null, + uploadDate = dateFormat.parseSafe(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source, + ) + } + return parseInfo(docs, manga, chapters) + } + + protected open val detailsDescriptionSelector = "div.entry-content" + + open suspend fun parseInfo(docs: Document, manga: Manga, chapters: List): Manga { + /// set if is table + val tableMode = + docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr") + ?: docs.selectFirst("div.seriestucon") + + val tagMap = getOrCreateTagMap() + + val selectTag = if (tableMode != null) { + docs.select(".seriestugenre > a") + } else { + docs.select(".wd-full .mgen > a") + } + + val tags = selectTag.mapNotNullToSet { tagMap[it.text()] } + + val stateSelect = if (tableMode != null) { + tableMode.selectFirst(".infotable td:contains(Status)") + ?: tableMode.selectFirst(".infotable td:contains(Statut)") + ?: tableMode.selectFirst(".infotable td:contains(حالة العمل)") + ?: tableMode.selectFirst(".infotable td:contains(الحالة)") + ?: tableMode.selectFirst(".infotable td:contains(Estado)") + ?: tableMode.selectFirst(".infotable td:contains(สถานะ)") + ?: tableMode.selectFirst(".infotable td:contains(Stato )") + ?: tableMode.selectFirst(".infotable td:contains(Durum)") + ?: tableMode.selectFirst(".infotable td:contains(Statüsü)") + + } else { + docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)") + ?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)") + ?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )") + ?: docs.selectFirst(".tsinfo div:contains(Durum)") ?: docs.selectFirst(".tsinfo div:contains(Statüsü)") + ?: docs.selectFirst(".tsinfo div:contains(Statü)") ?: docs.selectFirst(".tsinfo div:contains(الحالة)") + } + + val state = if (tableMode != null) { + stateSelect?.lastElementSibling() + + } else { + stateSelect?.lastElementChild() + } + + val mangaState = state?.let { + when (it.text()) { + + "مستمرة", "En curso", "En Curso", "Ongoing", "OnGoing", "On going", "Ativo", "En Cours", "En cours", "En cours \uD83D\uDFE2", + "En cours de publication", "Đang tiến hành", "Em lançamento", "em lançamento", "Em Lançamento", "Онгоінг", "Publishing", + "Devam Ediyor", "Em Andamento", "In Corso", "Güncel", "Berjalan", "Продолжается", "Updating", "Lançando", "In Arrivo", "Emision", + "En emision", "مستمر", "Curso", "En marcha", "Publicandose", "Publicando", "连载中", "Devam ediyor", "Devam Etmekte", + -> MangaState.ONGOING + + "Completed", "Completo", "Complété", "Fini", "Achevé", "Terminé", "Terminé ⚫", "Tamamlandı", "Đã hoàn thành", "Hoàn Thành", + "مكتملة", "Завершено", "Finished", "Finalizado", "Completata", "One-Shot", "Bitti", "Tamat", "Completado", "Concluído", + "Concluido", "已完结", "Bitmiş", + -> MangaState.FINISHED + + "Canceled", "Cancelled", "Cancelado", "cancellato", "Cancelados", "Dropped", "Discontinued", "abandonné", "Abandonné", + -> MangaState.ABANDONED + + "Hiatus", "On Hold", "Pausado", "En espera", "En pause", "En Pause", "En attente", + -> MangaState.PAUSED + + else -> null + } + } + + val author = tableMode?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.textOrNull() + + val nsfw = docs.selectFirst(".restrictcontainer") != null + || docs.selectFirst(".info-right .alr") != null + || docs.selectFirst(".postbody .alr") != null + + val title = docs.selectFirst("h1.entry-title")?.text() + ?: docs.selectFirst("h1")?.text() + ?: docs.selectFirst(".entry-title")?.text() + ?: docs.selectFirst(".seriestucontent h1")?.text() + ?: docs.selectFirst(".postbody h1")?.text() + ?: docs.selectFirst("title")?.text()?.substringBefore(" - ")?.trim() + ?: manga.title + + return manga.copy( + title = title, + description = docs.selectFirst(detailsDescriptionSelector)?.text(), + state = mangaState, + authors = setOfNotNull(author), + contentRating = if (manga.contentRating == ContentRating.ADULT || nsfw) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List { + return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) + } + + protected open val encodedSrc = false + protected open val selectScript = "div.wrapper script" + protected open val selectPage = "div#readerarea img" + + protected open val selectTestScript = "script:containsData(ts_reader)" + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + + val test = docs.select(selectTestScript) + if (test.isEmpty() and !encodedSrc) { + return docs.select(selectPage).map { img -> + val url = img.requireSrc().toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } else { + val images = if (encodedSrc) { + val script = docs.select(selectScript) + var decode = "" + for (i in script) { + if (i.attr("src").startsWith("data:text/javascript;base64,")) { + decode = Base64.getDecoder().decode(i.attr("src").replace("data:text/javascript;base64,", "")) + .decodeToString() + if (decode.startsWith("ts_reader.run")) { + break + } + } + } + JSONObject(decode.substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + } else { + val script = docs.selectFirstOrThrow(selectTestScript) + JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + } + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + preview = null, + source = source, + ), + ) + } + return pages + } + } + + protected open suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + val tagMap = ArrayMap(tagElements.size) + for (el in tagElements) { + tagMap[el.text()] = MangaTag( + title = el.textOrNull()?.toTitleCase(sourceLocale) ?: continue, + key = el.selectFirst("input")?.attrOrNull("value") ?: continue, + source = source, + ) + } + tagCache = tagMap + return@withLock tagMap + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (!isNetShieldProtected) { + return response + } + val contentType = response.mimeType + if ( + contentType?.endsWith("/html") != false && + context.cookieJar.getCookies(domain).none { it.name.contains("NetShield") } + ) { + val cookie = runBlocking { response.copy().parseHtml().getNetShieldCookie() } ?: return response + context.cookieJar.insertCookie(domain, cookie) + return chain.proceed(response.request.newBuilder().build()).also { + response.closeQuietly() + } + } + return response + } + + @Suppress("DEPRECATION") + private suspend fun Document.getNetShieldCookie(): Cookie? = runCatchingCancellable { + val script = select("script").firstNotNullOfOrNull { s -> + s.html().takeIf { x -> x.contains("slowAES.decrypt") } + } ?: return@runCatchingCancellable null + val min = webClient.httpGet("https://$domain/min.js").parseRaw() + val res = context.evaluateJs(min + "\n\n" + script.replace(Regex("document.cookie\\s*=\\s*"), "return ")) + res?.let { + Cookie.parse(baseUri().toHttpUrl(), it) + } + }.getOrNull() + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val mangaSlug = link.pathSegments[1].nullIfEmpty() ?: return null + val url = "/manga/$mangaSlug/" + return resolver.resolveManga( + this, url, generateUid(url), + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt new file mode 100644 index 00000000..014d5b46 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -0,0 +1,414 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.ContentType +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded +import tsuki.util.mapChapters + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale + +abstract class MangaThemesia( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + + protected open val mangaDirectory = "manga" + protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US) + protected open val withoutAjax = false + protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" + protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" + protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num), div.bxcl li:not(a[data-bs-target='#lockedChapterModal'])" + protected open val pageSelector = "div#readerarea img" + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + @Volatile + private var genreCache: Set? = null + private val genreMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + + private suspend fun getOrFetchGenres(): Set { + genreCache?.let { return it } + return genreMutex.withLock { + genreCache ?: fetchGenres().also { genreCache = it } + } + } + + protected open suspend fun fetchGenres(): Set { + val doc = webClient.httpGet("https://$domain/$mangaDirectory/").parseHtml() + val genrez = doc.selectFirst("ul.genrez") ?: return emptySet() + return genrez.select("li").mapNotNull { li -> + val label = li.selectFirst("label")?.text()?.trim() ?: return@mapNotNull null + val input = li.selectFirst("input[type=checkbox]") ?: return@mapNotNull null + val value = input.attr("value").trim() + if (label.isBlank() || value.isBlank()) null + else MangaTag(key = value, title = label, source = source) + }.toSet() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim().orEmpty() + val url = buildString { + append("https://$domain/$mangaDirectory/") + if (query.isNotEmpty()) { + append("?s=${query.urlEncoded()}") + } else { + append("?page=$page") + } + filter.states.firstOrNull()?.let { + append("&status=") + append(when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "dropped" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + append("&type=") + append(when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + }) + } + filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } + filter.tagsExclude.forEach { append("&genre[]=-${it.key.urlEncoded()}") } + append("&order=") + append(when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ADDED -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "update" + }) + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + protected open fun parseMangaList(doc: Document): List { + return doc.select(searchSelector).mapNotNull { element -> + parseMangaElement(element) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1") + ?.text() ?: manga.title + + val description = doc.selectFirst( + ".desc, .entry-content[itemprop=description], .summary__content, .sinopsis, .entry-content p" + )?.text()?.trim().orEmpty() + + val coverUrl = doc.selectFirst( + ".thumb img, .infomanga img, .summary_image img, .cover img, img.wp-post-image" + )?.imgAttr() ?: manga.coverUrl + + val table = doc.selectFirst("table.infotable") + val hasTable = table != null + + val authors = if (hasTable) { + val authorCell = table.selectFirst("tr:has(td:contains(Author)) td:last-child") + val artistCell = table.selectFirst("tr:has(td:contains(Artist)) td:last-child") + listOfNotNull( + authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, + artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } + ).toSet() + } else { + doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Author) i, .spe span:contains(Author) a") + .map { it.text().trim() } + .filter { it.isNotBlank() } + .toSet() + } + + val tags = doc.select(".mgen a, .gnr a, .seriestugenre a, .genres-content a") + .mapNotNull { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + }.toSet() + + val statusText = if (hasTable) { + table.selectFirst("tr:has(td:contains(Status)) td:last-child")?.text() + } else { + doc.select(".imptdt:contains(Status) i, .tsinfo .imptdt:contains(Status) a, .fmed b:contains(Status)+span span") + .text() + .ifEmpty { + doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() + } + } + val state = parseStatus(statusText) + + val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() + ?: doc.selectFirst(".num")?.text()?.toFloatOrNull() + val normalizedRating = if (rating != null && rating > 0) rating / 10f else RATING_UNKNOWN + + val chapters = loadChapters(doc, manga.url) + + return manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + tags = tags, + state = state, + rating = normalizedRating, + chapters = chapters, + ) + } + + protected open fun parseStatus(text: String?): MangaState? { + val status = text?.lowercase() ?: return null + return when (status) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in paused -> MangaState.PAUSED + in abandoned -> MangaState.ABANDONED + else -> null + } + } + + protected open suspend fun loadChapters(doc: Document, mangaUrl: String): List { + val chapterElements = doc.select(chapterListSelector) + if (chapterElements.isEmpty() && !withoutAjax) { + return loadChaptersAjax(mangaUrl) + } + return parseChapters(chapterElements) + } + + protected open suspend fun loadChaptersAjax(mangaUrl: String): List { + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/" + val doc = webClient.httpPost(ajaxUrl, emptyMap()).parseHtml() + return parseChapters(doc.select(chapterListSelector)) + } + + private fun parseChapters(elements: Elements): List { + return elements.mapChapters(reversed = true) { i, element -> + val a = element.selectFirst("a") ?: return@mapChapters null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() + val dateStr = element.selectFirst(".chapterdate")?.text() + val number = extractChapterNumber(name) ?: (i + 1).toFloat() + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = number, + volume = 0, + uploadDate = parseChapterDate(dateStr), + scanlator = null, + branch = null, + source = source, + ) + } + } + + private fun extractChapterNumber(title: String): Float? { + return Regex("""(?i)(?:chapter|ch\.?)\s*([0-9]+(?:\.[0-9]+)?)""") + .find(title) + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + } + + protected open fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val clean = date.trim() + return when { + WordSet(" ago", "atrás", " hace", " назад", " önce", " trước", "مضت", "قبل").endsWith(clean) -> parseRelativeDate(clean) + WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút").startsWith(clean) -> parseRelativeDate(clean) + WordSet("yesterday", "يوم واحد").startsWith(clean) -> yesterday() + WordSet("today").startsWith(clean) -> today() + WordSet("يومين").startsWith(clean) -> dayBeforeYesterday() + else -> dateFormat.parseSafe(clean) + } + } + + private fun parseRelativeDate(text: String): Long { + val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0L + val cal = Calendar.getInstance() + return when { + WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان").anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут").anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giờ", "ore", "ساعة", "小时").anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis + WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois").anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year", "año", "năm").anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0L + } + } + + private fun yesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + private fun today() = Calendar.getInstance().apply { set(Calendar.HOUR_OF_DAY, 0); set(Calendar.MINUTE, 0) }.timeInMillis + private fun dayBeforeYesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -2) }.timeInMillis + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + // json image list + val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + val jsonMatch = jsonRegex.find(doc.html()) + if (jsonMatch != null) { + val jsonArray = JSONArray(jsonMatch.groupValues[1]) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + // direct + return doc.select(pageSelector).mapNotNull { img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(relatedSelector).mapNotNull { element -> + parseMangaElement(element) + } + } + + protected open fun parseMangaElement(element: Element): Manga? { + val a = element.selectFirst("a") ?: return null + val href = a.attrAsRelativeUrl("href") + val title = a.attr("title").ifBlank { a.text() } + val coverUrl = element.selectFirst("img")?.imgAttr() + return makeManga(href, title, coverUrl) + } + + private fun makeManga(href: String, title: String, coverUrl: String?): Manga { + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = coverUrl, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + + protected fun Element.imgAttr(): String { + for (attr in listOf("data-src", "data-lazy-src", "data-original", "data-cfsrc", "data-image", "src")) { + val value = attr(attr).trim().takeIf { it.isNotEmpty() } ?: continue + return value.toAbsoluteUrl(domain).substringBefore("?") // remove WP resize params + } + val srcset = attr("srcset") + if (srcset.isNotBlank()) { + return srcset.split(",").last().trim().split(" ").first().toAbsoluteUrl(domain) + } + return "" + } + + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) + + companion object { + val ongoing = setOf( + "ongoing", "on going", "publishing", "updating", "en curso", "ativo", "en cours", + "đang tiến hành", "em lançamento", "devam ediyor", "in corso", "güncel", "berjalan", + "продолжается", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة", + ) + val finished = setOf( + "completed", "complete", "finished", "finalizado", "terminé", "tamamlandı", + "đã hoàn thành", "hoàn thành", "مكتملة", "завершено", "completato", "one-shot", + "bitti", "tamat", "concluído", "已完结", "bitmiş", "achevé", + ) + val paused = setOf( + "hiatus", "on hold", "pausado", "en espera", "en pause", "en attente", + "durduruldu", "beklemede", "đang chờ", "متوقف", "заморожено", + ) + val abandoned = setOf( + "canceled", "cancelled", "cancelado", "cancellato", "dropped", "discontinued", + "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено", + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/MangaboxParser.kt b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt new file mode 100644 index 00000000..b14786c6 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt @@ -0,0 +1,459 @@ +@file:Suppress("DEPRECATION") + +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.FlexiblePagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toTitleCase +import tsuki.util.mapChapters +import tsuki.util.nullIfEmpty +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.toRelativeUrl + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.jsoup.nodes.Document +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet + +internal abstract class MangaboxParser( + context: MangaLoaderContext, + source: MangaParserSource, + pageSize: Int = 24, +) : FlexiblePagedMangaParser(context, source, pageSize) { + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) + override val searchQueryCapabilities: MangaSearchQueryCapabilities + get() = MangaSearchQueryCapabilities( + SearchCapability( + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class, QueryCriteria.Exclude::class), + isMultiple = true, + ), + SearchCapability( + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), + isMultiple = false, + ), + SearchCapability( + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = true, + ), + SearchCapability( + field = SearchableField.AUTHOR, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + isExclusive = true, + ), + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + @JvmField + protected val ongoing: Set = setOf( + "ongoing", + ) + + @JvmField + protected val finished: Set = setOf( + "completed", + ) + + protected open val listUrl = "/manga-list/latest-manga" + protected open val searchUrl = "/search/story/" + protected open val datePattern = "MMM dd,yy" + + override suspend fun getListPage(query: MangaSearchQuery, page: Int): List { + val titleQuery = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } + ?.value + ?.toString() + ?.trim() + + if (!titleQuery.isNullOrBlank()) { + val url = "https://$domain${searchUrl.trimEnd('/')}/${normalizeSearchQuery(titleQuery)}?page=$page" + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + val tagCriterion = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TAG } + + val tagKey = tagCriterion?.values?.firstOrNull()?.let { value -> + when (value) { + is MangaTag -> value.key + else -> value.toString().replace(" ", "-").lowercase(sourceLocale) + } + } + + if (!tagKey.isNullOrBlank()) { + val url = "https://$domain/genre/$tagKey?page=$page" + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + val url = "https://$domain$listUrl?page=$page" + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + protected open fun parseSearchResults(doc: Document): List { + val elements = doc.select(".panel_story_list .story_item") + .ifEmpty { doc.select(".story_item") } + .ifEmpty { doc.select("div.list-truyen-item-wrap") } + .ifEmpty { doc.select("div.list-comic-item-wrap") } + .ifEmpty { doc.select("div.content-genres-item") } + .ifEmpty { doc.select("div.itemupdate") } + .ifEmpty { doc.select("div.manga-item") } + .ifEmpty { doc.select(".item") } + .ifEmpty { doc.select("a[href*='/manga/']").mapNotNull { it.parent() ?: it } } + + return elements.mapNotNull { div -> + val linkElement = div.selectFirst(".story_name a") + ?: div.selectFirst("h3 a") + ?: div.selectFirst("h2 a") + ?: div.selectFirst(".slide-caption h3 a") + ?: div.selectFirst("a.cover") + ?: div.selectFirst("a[href*='/manga/']") + ?: if (div.tagName() == "a") div else null + + linkElement ?: return@mapNotNull null + + val href = linkElement.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + if (!href.contains("/manga/")) return@mapNotNull null + + val title = linkElement.text().trim() + .takeIf { it.isNotEmpty() } + ?: linkElement.attr("title").trim().takeIf { it.isNotEmpty() } + ?: div.selectFirst("h3, h2, h1")?.text()?.trim()?.takeIf { it.isNotEmpty() } + ?: return@mapNotNull null + + val img = linkElement.selectFirst("img") ?: div.selectFirst("img") + val coverUrl = img?.attr("data-src")?.nullIfEmpty() + ?: img?.attr("data-lazy-src")?.nullIfEmpty() + ?: img?.src() + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = sourceContentRating, + ) + } + } + + protected open fun normalizeSearchQuery(query: String): String { + return query.lowercase(sourceLocale) + .replace("[àáạảãâầấậẩẫăằắặẳẵ]".toRegex(), "a") + .replace("[èéẹẻẽêềếệểễ]".toRegex(), "e") + .replace("[ìíịỉĩ]".toRegex(), "i") + .replace("[òóọỏõôồốộổỗơờớợởỡ]".toRegex(), "o") + .replace("[ùúụủũưừứựửữ]".toRegex(), "u") + .replace("[ỳýỵỷỹ]".toRegex(), "y") + .replace("đ".toRegex(), "d") + .replace("""[^\p{L}\p{N}]+""".toRegex(), "_") + .replace("_+".toRegex(), "_") + .trim('_') + } + + protected open val selectTagMap = "div.panel-genres-list a:not(.genres-select)" + + protected open suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain$listUrl").parseHtml() + val tags = doc.select(selectTagMap).drop(1) + + return tags.mapToSet { a -> + val key = a.attr("href").removeSuffix('/').substringAfterLast('/') + val name = a.attr("title").replace(" Manga", "") + MangaTag( + key = key, + title = name, + source = source, + ) + } + } + + protected open val selectDesc = "div#noidungm, div#panel-story-info-description" + protected open val selectState = "li:contains(status), td:containsOwn(status) + td" + protected open val selectAlt = ".story-alternative, tr:has(.info-alternative) h2" + protected open val selectAut = "li:contains(author) a, td:contains(author) + td a" + protected open val selectTag = "div.manga-info-top li:contains(genres) a , td:containsOwn(genres) + td a" + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { getChapters(doc) } + val desc = doc.selectFirst(selectDesc)?.html() + val stateDiv = doc.select(selectState).text() + val state = stateDiv.let { + when (it.lowercase()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + val alt = doc.body().select(selectAlt).text().replace("Alternative : ", "").nullIfEmpty() + val authors = doc.body().select(selectAut).mapToSet { it.text() } + + manga.copy( + tags = doc.body().select(selectTag).mapToSet { a -> + val href = a.attr("href") + MangaTag( + key = href.substringAfterLast("category=").substringBefore("&") + .takeIf { it != href } + ?: href.removeSuffix("/").substringAfterLast("/"), + title = a.text().toTitleCase(), + source = source, + ) + }, + description = desc, + altTitles = setOfNotNull(alt), + authors = authors, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + protected open val selectDate = "span" + protected open val selectChapter = "div.chapter-list div.row, ul.row-content-chapter li" + + protected open suspend fun getChapters(doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val dateText = li.select(selectDate).last()?.text() + + MangaChapter( + id = generateUid(href), + title = a.text(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } + } + + protected open val selectPage = "div#vungdoc img, div.container-chapter-reader img" + protected open val otherDomain = "" + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val scriptPages = parseScriptPages(doc) + if (scriptPages.isNotEmpty()) { + return scriptPages + } + + val pages = doc.select(selectPage).mapNotNull { img -> + val url = img.requireSrc().toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + if (pages.isNotEmpty() || otherDomain.isBlank()) { + return pages + } + + val fallbackUrl = fullUrl.replace(domain, otherDomain) + val fallbackDoc = webClient.httpGet(fallbackUrl).parseHtml() + + return fallbackDoc.select(selectPage).map { img -> + val url = img.requireSrc().toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseScriptPages(doc: Document): List { + val content = doc.select("script:containsData(cdns =), script:containsData(chapterImages =)") + .joinToString("\n") { it.data() } + + if (content.isBlank()) { + return emptyList() + } + + val cdns = extractArray(content, cdnsRegex) + extractArray(content, backupImageRegex) + val chapterImages = extractArray(content, chapterImagesRegex) + + if (cdns.isEmpty() || chapterImages.isEmpty()) { + return emptyList() + } + + val baseUrl = cdns.first().toHttpUrlOrNull() ?: return emptyList() + + return chapterImages.map { imagePath -> + val url = baseUrl.newBuilder() + .encodedPath("/$imagePath".replace("//", "/")) + .build() + .toString() + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun extractArray(scriptContent: String, regex: Regex): List { + val match = regex.find(scriptContent) ?: return emptyList() + return match.groupValues[1] + .split(",") + .mapNotNull { raw -> + raw.trim() + .removeSurrounding("\"") + .removeSurrounding("'") + .replace("\\/", "/") + .removeSuffix("/") + .nullIfEmpty() + } + } + + protected fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + WordSet(" ago", " h", " d").endsWith(d) -> { + parseRelativeDate(d) + } + + WordSet("today").startsWith(d) -> { + Calendar.getInstance().apply { + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } + + date.contains(Regex("""\d(st|nd|rd|th)""")) -> date.split(" ").map { + if (it.contains(Regex("""\d\D\D"""))) { + it.replace(Regex("""\D"""), "") + } else { + it + } + }.let { dateFormat.parseSafe(it.joinToString(" ")) } + + else -> dateFormat.parseSafe(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("second") + .anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + + WordSet("min", "minute", "minutes") + .anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + + WordSet("hour", "hours", "h") + .anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + + WordSet("day", "days") + .anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + + WordSet("month", "months") + .anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + + WordSet("year") + .anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + + else -> 0 + } + } + + @Deprecated("Use getRequestHeaders() instead", ReplaceWith("getRequestHeaders()")) + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Accept", "image/webp,image/apng,image/*,*/*;q=0.8") + .add("Accept-Encoding", "gzip, deflate, br") + .add("Accept-Language", "en-US,en;q=0.9") + .build() + + companion object { + + private val cdnsRegex = Regex("""cdns\s*=\s*\[([^]]+)]""") + private val backupImageRegex = Regex("""backupImage\s*=\s*\[([^]]+)]""") + private val chapterImagesRegex = Regex("""chapterImages\s*=\s*\[([^]]+)]""") + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt new file mode 100644 index 00000000..d3b2084f --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt @@ -0,0 +1,384 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.ErrorMessages + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlDecode + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +internal abstract class ZeistMangaParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 12, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED), + ) + + @JvmField + protected val ongoing: Set = hashSetOf( + "ongoing", + "en curso", + "ativo", + "lançando", + "مستمر", + "devam ediyor", + "güncel", + "en emisión", + ) + + @JvmField + protected val finished: Set = hashSetOf( + "completed", + "completo", + "tamamlandı", + "finalizado", + ) + + @JvmField + protected val abandoned: Set = hashSetOf( + "cancelled", + "dropped", + "dropado", + "abandonado", + "cancelado", + "suspendido", + ) + + @JvmField + protected val paused: Set = hashSetOf( + "hiatus", + ) + + protected open val sateOngoing: String = "Ongoing" + protected open val sateFinished: String = "Completed" + protected open val sateAbandoned: String = "Cancelled" + + protected open val maxMangaResults: Int = 20 + + protected open val mangaCategory: String = "Series" + + protected open val datePattern = "yyyy-MM-dd" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val startIndex = maxMangaResults * (page - 1) + 1 + val query = filter.query + val url = buildString { + append("https://") + append(domain) + append("/feeds/posts/default/-/") + when { + + !query.isNullOrEmpty() -> { + append(mangaCategory) + append("?alt=json&orderby=published&max-results=") + append((maxMangaResults + 1).toString()) + append("&start-index=") + append(startIndex.toString()) + append("&q=label:") + append(mangaCategory) + append("+") + append(query.urlEncoded()) + } + + else -> { + + + if (filter.tags.isNotEmpty() && filter.states.isNotEmpty()) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_STATES_GENRES_NOT_SUPPORTED) + } + + if (filter.tags.isNotEmpty()) { + append(filter.tags.oneOrThrowIfMany()?.key.orEmpty()) + } else if (filter.states.isNotEmpty()) { + append( + filter.states.oneOrThrowIfMany().let { + when (it) { + MangaState.ONGOING -> sateOngoing + MangaState.FINISHED -> sateFinished + MangaState.ABANDONED -> sateAbandoned + else -> mangaCategory + } + }, + ) + } else { + append(mangaCategory) + } + + append("?alt=json&orderby=published&max-results=") + append((maxMangaResults + 1).toString()) + append("&start-index=") + append(startIndex.toString()) + } + } + } + + val json = webClient.httpGet(url).parseJson().getJSONObject("feed") + + return if (json.toString().contains("\"entry\":")) { + parseMangaList(json.getJSONArray("entry")) + } else { + emptyList() + } + } + + protected open fun parseMangaList(json: JSONArray): List { + return json.mapJSON { j -> + val name = j.getJSONObject("title").getString("\$t") + val href = + j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" } + .getString("href") + val urlImg = if (j.toString().contains("media\$thumbnail")) { + j.getJSONObject("media\$thumbnail").getStringOrNull("url") + ?.replace("""/s.+?-c/""".toRegex(), "/w600/") + ?.replace("""=s(?!.*=s).+?-c$""".toRegex(), "=w600") + ?.replace("""/s.+?-c-rw/""".toRegex(), "/w600/") + ?.replace("""=s(?!.*=s).+?-c-rw$""".toRegex(), "=w600") + } else { + Jsoup.parse(j.getJSONObject("content").getString("\$t")).selectFirst("img")?.attr("src") + } + Manga( + id = generateUid(href), + url = href.toRelativeUrl(domain), + publicUrl = href, + coverUrl = urlImg.orEmpty(), + title = name, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + protected open suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + return doc.selectFirstOrThrow("div.filter").select("ul li").mapToSet { + MangaTag( + key = it.selectFirstOrThrow("input").attr("value"), + title = it.selectFirstOrThrow("label").text().toTitleCase(sourceLocale), + source = source, + ) + } + } + + protected open val selectTags = "article div.mt-15 a, .info-genre a, dl:contains(Genre) dd a" + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val state = + doc.selectFirst("div.y6x11p:contains(Status) .dt") + ?: doc.selectFirst("div.y6x11p:contains(Estado) .dt") + ?: doc.selectFirst("ul.infonime li:contains(Status) span") + ?: doc.selectFirst("ul.infonime li:contains(Estado) span") + ?: doc.selectFirst("span.status-novel") + ?: doc.selectFirst("span[data-status]") + val mangaState = state?.text()?.lowercase().let { + when (it) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in abandoned -> MangaState.ABANDONED + in paused -> MangaState.PAUSED + else -> null + } + } + val author = doc.selectFirst("div.y6x11p:contains(الكاتب) .dt") + ?: doc.selectFirst("div.y6x11p:contains(Author) .dt") + ?: doc.selectFirst("dl:contains(Author) dd") + ?: doc.selectFirst("div.y6x11p:contains(Autor) .dt") + ?: doc.selectFirst("div.y6x11p:contains(Yazar) .dt") + ?: doc.selectFirst("ul.infonime li:contains(Author) span") + + + val desc = doc.getElementById("synopsis") ?: doc.getElementById("Sinopse") ?: doc.getElementById("sinopas") + ?: doc.selectFirst(".sinopsis") ?: doc.selectFirst(".sinopas") + val chaptersDeferred = async { loadChapters(manga.url, doc) } + manga.copy( + authors = author?.text()?.let { setOf(it) } ?: emptySet(), + tags = doc.select(selectTags).mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast("label/").substringBefore("?"), + title = a.text().toTitleCase(), + source = source, + ) + }, + description = desc?.text().orEmpty(), + state = mangaState, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun loadChapters(mangaUrl: String, doc: Document): List { + + val feed = if (doc.getElementById("myUL") != null) { + doc.requireElementById("myUL").selectFirstOrThrow("script").attr("src") + .substringAfterLast("/-/").substringBefore("?").urlDecode() + + } else if (doc.selectFirst("#latest > script") != null) { + val chapterRegex = """label\s*=\s*'([^']+)'""".toRegex() + val scriptSelector = "#latest > script" + val script = doc.selectFirstOrThrow(scriptSelector) + chapterRegex + .find(script.html()) + ?.groupValues?.get(1) + ?: doc.parseFailed("Failed to find chapter feed") + + } else if (doc.selectFirst("#clwd > script") != null) { + val chapterRegex = """clwd\.run\('([^']+)'""".toRegex() + val scriptSelector = "#clwd > script" + val script = doc.selectFirstOrThrow(scriptSelector) + chapterRegex + .find(script.html()) + ?.groupValues?.get(1) + ?: doc.parseFailed("Failed to find chapter feed") + + } else if (doc.selectFirst("#chapterlist") != null) { + doc.selectFirstOrThrow("#chapterlist").attr("data-post-title") + } else { + doc.selectFirstOrThrow("script:containsData(var label_chapter)").data() + .substringAfter("label_chapter = \"").substringBefore("\"") + } + + val url = buildString { + append("https://") + append(domain) + append("/feeds/posts/default/-/") + append(feed) + append("?alt=json&orderby=published&max-results=9999") + } + val json = + webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList() + .reversed() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return json.mapIndexedNotNull { i, j -> + val name = j.getJSONObject("title").getString("\$t") + val href = + j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" } + .getString("href") + val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") + val slug = mangaUrl.substringAfterLast('/') + val slugChapter = href.substringAfterLast('/') + if (slug == slugChapter) { + return@mapIndexedNotNull null + } + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = i + 1f, + volume = 0, + branch = null, + uploadDate = dateFormat.parseSafe(dateText), + scanlator = null, + source = source, + ) + } + } + + protected open val selectPage = + "div.check-box img, article#reader .separator img, article.container .separator img, #readarea img, #reader img, #readerarea img" + + override suspend fun getPages(chapter: MangaChapter): List { + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + return if (doc.selectFirst("script:containsData(chapterImage =)") != null) { + doc.selectFirstOrThrow("script:containsData(chapterImage =)").data() + .substringAfter("[").substringBefore("]") + .replace(" ", "").replace("\"", "") + .split(",").map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + } else if (doc.selectFirst("script:containsData(const content = )") != null) { + doc.selectFirstOrThrow("script:containsData(const content = )").data() + .substringAfter("`").substringBefore("`;").split("src=\"").drop(1) + + .map { img -> + val url = img.substringBefore("\"") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + + } else { + doc.select(selectPage).map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/MangaDexParser.kt b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt new file mode 100644 index 00000000..e49e3983 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt @@ -0,0 +1,540 @@ +@file:Suppress("DEPRECATION") + +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.FlexibleMangaParser +import tsuki.exception.ParseException +import tsuki.model.* +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria.* +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.model.search.SearchableField.* +import tsuki.network.CommonHeaders +import tsuki.util.* +import tsuki.util.json.* + +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +private const val PAGE_SIZE = 20 +private const val CHAPTERS_FIRST_PAGE_SIZE = 120 +private const val CHAPTERS_MAX_PAGE_SIZE = 500 +private const val CHAPTERS_PARALLELISM = 3 +private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks like a bug +private const val LOCALE_FALLBACK = "en" +private const val SERVER_DATA = "data" +private const val SERVER_DATA_SAVER = "data-saver" +private const val USER_AGENT = "Usagi/1 (Android)" + +@MangaSourceParser("MANGADEXORG", "MangaDex") +internal class MangaDexParser(context: MangaLoaderContext) : FlexibleMangaParser(context, MangaParserSource.MANGADEXORG) { + + override val configKeyDomain = ConfigKey.Domain("mangadex.org") + override val userAgentKey = ConfigKey.UserAgent(USER_AGENT) + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = request.newBuilder() + .header(CommonHeaders.USER_AGENT, USER_AGENT) + .build() + return chain.proceed(newRequest) + } + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + SERVER_DATA to "Original quality", + SERVER_DATA_SAVER to "Compressed quality", + ), + defaultValue = SERVER_DATA, + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.RELEVANCE, + ) + + @Deprecated("Too complex. Use filterCapabilities instead") + override val searchQueryCapabilities: MangaSearchQueryCapabilities + get() = MangaSearchQueryCapabilities( + SearchCapability( + field = TAG, + criteriaTypes = setOf(Include::class, Exclude::class), + isMultiple = true, + ), + SearchCapability( + field = TITLE_NAME, + criteriaTypes = setOf(Match::class), + isMultiple = false, + ), + SearchCapability( + field = STATE, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = AUTHOR, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = CONTENT_TYPE, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = CONTENT_RATING, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = DEMOGRAPHIC, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = ORIGINAL_LANGUAGE, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = LANGUAGE, + criteriaTypes = setOf(Include::class), + isMultiple = true, + ), + SearchCapability( + field = PUBLICATION_YEAR, + criteriaTypes = setOf(Match::class), + isMultiple = false, + ), + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = coroutineScope { + val localesDeferred = async { fetchAvailableLocales() } + val tagsDeferred = async { fetchAvailableTags() } + MangaListFilterOptions( + availableTags = tagsDeferred.await(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentRating = EnumSet.allOf(ContentRating::class.java), + availableDemographics = EnumSet.of( + Demographic.SHOUNEN, + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.JOSEI, + Demographic.NONE, + ), + availableLocales = localesDeferred.await(), + ) + } + + private fun SearchableField.toParamName(): String = when (this) { + TITLE_NAME -> "title" + TAG -> "includedTags[]" + AUTHOR -> "authors[]" + STATE -> "status[]" + CONTENT_TYPE -> "contentType[]" + CONTENT_RATING -> "contentRating[]" + DEMOGRAPHIC -> "publicationDemographic[]" + ORIGINAL_LANGUAGE -> "originalLanguage[]" + LANGUAGE -> "availableTranslatedLanguage[]" + PUBLICATION_YEAR -> "year" + } + + private fun Any?.toQueryParam(): String = when (this) { + is String -> urlEncoded() + is Locale -> if (language == "in") "id" else language + is MangaTag -> key + is MangaState -> when (this) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.ABANDONED -> "cancelled" + MangaState.PAUSED -> "hiatus" + else -> "" + } + + is ContentRating -> when (this) { + ContentRating.SAFE -> "safe" + // quick fix for double value + ContentRating.SUGGESTIVE -> "suggestive&contentRating[]=erotica" + ContentRating.ADULT -> "pornographic" + } + + is Demographic -> when (this) { + Demographic.SHOUNEN -> "shounen" + Demographic.SHOUJO -> "shoujo" + Demographic.SEINEN -> "seinen" + Demographic.JOSEI -> "josei" + Demographic.NONE -> "none" + else -> "" + } + + is SortOrder -> when (this) { + SortOrder.UPDATED -> "[latestUploadedChapter]=desc" + SortOrder.UPDATED_ASC -> "[latestUploadedChapter]=asc" + SortOrder.RATING -> "[rating]=desc" + SortOrder.RATING_ASC -> "[rating]=asc" + SortOrder.ALPHABETICAL -> "[title]=asc" + SortOrder.ALPHABETICAL_DESC -> "[title]=desc" + SortOrder.NEWEST -> "[year]=desc" + SortOrder.NEWEST_ASC -> "[year]=asc" + SortOrder.POPULARITY -> "[followedCount]=desc" + SortOrder.POPULARITY_ASC -> "[followedCount]=asc" + SortOrder.ADDED -> "[createdAt]=desc" + SortOrder.ADDED_ASC -> "[createdAt]=asc" + SortOrder.RELEVANCE -> "&order[relevance]=desc" + else -> "[latestUploadedChapter]=desc" + } + + else -> this.toString().urlEncoded() + } + + private fun StringBuilder.appendCriterion(field: SearchableField, value: Any?, paramName: String? = null) { + val param = paramName ?: field.toParamName() + if (param.isNotBlank()) { + append("&$param=") + append(value.toQueryParam()) + } + } + + @Deprecated("Too complex. Use getList with filter instead") + override suspend fun getList(query: MangaSearchQuery): List { + val url = buildString { + append("https://api.$domain/manga?limit=$PAGE_SIZE&offset=${query.offset}") + .append("&includes[]=cover_art&includes[]=author&includes[]=artist&includedTagsMode=AND&excludedTagsMode=OR") + + var hasContentRating = false + + query.criteria.forEach { criterion -> + when (criterion) { + is Include<*> -> { + if (criterion.field == CONTENT_RATING) { + hasContentRating = true + } + criterion.values.forEach { appendCriterion(criterion.field, it) } + } + + is Exclude<*> -> { + criterion.values.forEach { appendCriterion(criterion.field, it, "excludedTags[]") } + } + + is Match<*> -> { + appendCriterion(criterion.field, criterion.value) + } + + else -> { + // Not supported + } + } + } + + // If contentRating is not provided, add default values + if (!hasContentRating) { + append("&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic") + } + + append("&order") + append((query.order ?: defaultSortOrder).toQueryParam()) + } + + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + return json.mapJSON { jo -> jo.fetchManga(null) } + } + + override suspend fun getDetails(manga: Manga): Manga { + val mangaId = manga.url.removePrefix("/") + return getDetails(mangaId) + } + + override suspend fun resolveLink(link: HttpUrl): Manga? { + val regex = Regex("[0-9a-f\\-]{10,}", RegexOption.IGNORE_CASE) + val mangaId = link.pathSegments.find { regex.matches(it) } ?: return null + return getDetails(mangaId) + } + + private suspend fun getDetails(mangaId: String): Manga = coroutineScope { + val jsonDeferred = async { + webClient.httpGet( + "https://api.$domain/manga/${mangaId}?includes[]=artist&includes[]=author&includes[]=cover_art", + ).parseJson().getJSONObject("data") + } + val feedDeferred = async { loadChapters(mangaId) } + jsonDeferred.await().fetchManga(mapChapters(feedDeferred.await())) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val json = webClient.httpGet( + "https://api.$domain/at-home/server/${chapter.url}?forcePort443=false", + ).parseJson() + val chapterJson = json.getJSONObject("chapter") + val server = config[preferredServerKey] ?: SERVER_DATA + val pages = chapterJson.getJSONArray( + if (server == SERVER_DATA_SAVER) "dataSaver" else "data", + ) + val prefix = "${json.getString("baseUrl")}/$server/${chapterJson.getString("hash")}/" + return List(pages.length()) { i -> + val url = prefix + pages.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, // TODO prefix + dataSaver.getString(i), + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val tags = webClient.httpGet("https://api.${domain}/manga/tag").parseJson() + .getJSONArray("data") + return tags.mapJSONToSet { jo -> + MangaTag( + title = jo.getJSONObject("attributes").getJSONObject("name") + .firstStringValue() + .toTitleCase(Locale.ENGLISH), + key = jo.getString("id"), + source = source, + ) + } + } + + private suspend fun fetchAvailableLocales(): Set { + val head = webClient.httpGet("https://$domain/").parseHtml().head() + return head.getElementsByAttributeValue("property", "og:locale:alternate") + .mapNotNullToSet { meta -> + val raw = meta.attrOrNull("content") ?: return@mapNotNullToSet null + Locale(raw.substringBefore('_'), raw.substringAfter('_', "")) + } + } + + private fun JSONObject.fetchManga(chapters: List?): Manga { + val id = getString("id") + val attrs = getJSONObject("attributes") + val relations = getJSONArray("relationships").associateByKey("type") + val cover = relations["cover_art"] + ?.firstOrNull() + ?.getJSONObject("attributes") + ?.getString("fileName") + ?.let { + "https://uploads.$domain/covers/$id/$it" + } + val authors: Set = (relations["author"] ?: relations["artist"]) + ?.mapNotNullToSet { + it.getJSONObject("attributes")?.getStringOrNull("name") + }.orEmpty() + + return Manga( + id = generateUid(id), + title = requireNotNull(attrs.getJSONObject("title").selectByLocale()) { + "Title should not be null" + }, + altTitles = setOfNotNull(attrs.optJSONArray("altTitles")?.flatten()?.selectByLocale()), // TODO + url = id, + publicUrl = "https://$domain/title/$id", + rating = RATING_UNKNOWN, + contentRating = when (attrs.getStringOrNull("contentRating")) { + "pornographic" -> ContentRating.ADULT + "erotica", "suggestive" -> ContentRating.SUGGESTIVE + "safe" -> ContentRating.SAFE + else -> null + }, + coverUrl = cover?.plus(".256.jpg"), + largeCoverUrl = cover, + description = attrs.optJSONObject("description")?.selectByLocale(), + tags = attrs.getJSONArray("tags").mapJSONToSet { tag -> + MangaTag( + title = tag.getJSONObject("attributes") + .getJSONObject("name") + .firstStringValue() + .toTitleCase(), + key = tag.getString("id"), + source = source, + ) + }, + state = when (attrs.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + }, + authors = authors, + chapters = chapters, + source = source, + ) + } + + private fun JSONObject.firstStringValue() = entries().first().value + + private fun JSONObject.selectByLocale(): String? { + val preferredLocales = context.getPreferredLocales() + for (locale in preferredLocales) { + getStringOrNull(locale.language)?.let { return it } + getStringOrNull(locale.toLanguageTag())?.let { return it } + } + return getStringOrNull(LOCALE_FALLBACK) ?: entries().firstOrNull()?.value?.nullIfEmpty() + } + + private fun JSONArray.flatten(): JSONObject { + val result = JSONObject() + repeat(length()) { i -> + val jo = optJSONObject(i) + if (jo != null) { + for (key in jo.keys()) { + result.put(key, jo.get(key)) + } + } + } + return result + } + + private suspend fun loadChapters(mangaId: String): List { + val firstPage = loadChapters(mangaId, offset = 0, limit = CHAPTERS_FIRST_PAGE_SIZE) + if (firstPage.size >= firstPage.total) { + return firstPage.data + } + val tail = coroutineScope { + val leftCount = firstPage.total.coerceAtMost(CHAPTERS_MAX_COUNT) - firstPage.size + val pages = (leftCount / CHAPTERS_MAX_PAGE_SIZE.toFloat()).toIntUp() + val dispatcher = Dispatchers.Default.limitedParallelism(CHAPTERS_PARALLELISM) + List(pages) { page -> + val offset = page * CHAPTERS_MAX_PAGE_SIZE + firstPage.size + async(dispatcher) { + loadChapters(mangaId, offset, CHAPTERS_MAX_PAGE_SIZE) + } + }.awaitAll() + } + val result = ArrayList(firstPage.total) + result += firstPage.data + tail.flatMapTo(result) { it.data } + return result + } + + private suspend fun loadChapters(mangaId: String, offset: Int, limit: Int): Chapters { + val limitedLimit = when { + offset >= CHAPTERS_MAX_COUNT -> return Chapters(emptyList(), CHAPTERS_MAX_COUNT) + offset + limit > CHAPTERS_MAX_COUNT -> CHAPTERS_MAX_COUNT - offset + else -> limit + } + val url = buildString { + append("https://api.") + append(domain) + append("/manga/") + append(mangaId) + append("/feed") + append("?limit=") + append(limitedLimit) + append("&includes[]=scanlation_group&order[volume]=asc&order[chapter]=asc&offset=") + append(offset) + append("&contentRating[]=safe&contentRating[]=suggestive&contentRating[]=erotica&contentRating[]=pornographic") + } + val json = webClient.httpGet(url).parseJson() + if (json.getString("result") == "ok") { + return Chapters( + data = json.optJSONArray("data")?.asTypedList().orEmpty(), + total = json.getInt("total"), + ) + } else { + val error = json.optJSONArray("errors").mapJSON { jo -> + jo.getString("detail") + }.joinToString("\n") + throw ParseException(error, url) + } + } + + private fun mapChapters(list: List): List { + // 2022-01-02T00:27:11+00:00 + val dateFormat = SimpleDateFormat( + "yyyy-MM-dd'T'HH:mm:ss'+00:00'", + Locale.ROOT, + ) + val chaptersBuilder = ChaptersListBuilder(list.size) + val branchedChapters = HashMap, MangaChapter>>() + for (jo in list) { + val id = jo.getString("id") + val attrs = jo.getJSONObject("attributes") + if (!attrs.isNull("externalUrl")) { + continue + } + val number = attrs.getFloatOrDefault("chapter", 0f) + val volume = attrs.getIntOrDefault("volume", 0) + val locale = attrs.getStringOrNull("translatedLanguage")?.let { Locale.forLanguageTag(it) } + val lc = locale?.getDisplayName(locale)?.toTitleCase(locale) + val relations = jo.getJSONArray("relationships").associateByKey("type") + val team = + relations["scanlation_group"]?.firstOrNull()?.optJSONObject("attributes")?.getStringOrNull("name") + val branch = (list.indices).firstNotNullOf { i -> + val b = if (i == 0) lc else "$lc ($i)" + if (branchedChapters[b]?.get(volume to number) == null) b else null + } + val chapter = MangaChapter( + id = generateUid(id), + title = attrs.getStringOrNull("title"), + number = number, + volume = volume, + url = id, + scanlator = team, + uploadDate = dateFormat.parseSafe(attrs.getString("publishAt")), + branch = branch, + source = source, + ) + if (chaptersBuilder.add(chapter)) { + branchedChapters.getOrPut(branch, ::HashMap)[volume to number] = chapter + } + } + return chaptersBuilder.toList() + } + + private fun JSONArray.associateByKey(key: String): Map> { + val destination = LinkedHashMap>(length()) + repeat(length()) { i -> + val item = getJSONObject(i) + val keyValue = item.getString(key) + destination.getOrPut(keyValue) { mutableListOf() }.add(item) + } + return destination + } + + private class Chapters( + val data: List, + val total: Int, + ) { + + val size: Int + get() = data.size + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt new file mode 100644 index 00000000..9c169c50 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -0,0 +1,494 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.MangaParserAuthProvider + +import tsuki.util.* +import tsuki.model.* + +import okhttp3.Interceptor +import org.jsoup.Jsoup +import java.util.EnumSet +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + + +internal abstract class MangaFireParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val siteLang: String, +) : PagedMangaParser(context, source, 30), Interceptor, MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain("mangafire.to") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, // chapter update + SortOrder.POPULARITY, // most views + SortOrder.RATING, // rating score + SortOrder.NEWEST, // created manga + SortOrder.ALPHABETICAL, // title asc + SortOrder.RELEVANCE, // relevance sc + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + private val apiClient by lazy { + val newHttpClient = context.httpClient.newBuilder() + .addInterceptor { chain -> + val request = chain.request().newBuilder() + .header("Referer", "https://$domain/") + .header("Accept", "application/json") + .build() + chain.proceed(request) + } + .build() + OkHttpWebClient(newHttpClient, source) + } + + override fun intercept(chain: Interceptor.Chain) = chain.proceed( + chain.request().newBuilder() + .header("Referer", "https://$domain/") + .build() + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearRangeSupported = true, + ) + + companion object { + val GENRE_MAP = mapOf( + "Action" to "1", + "Adult" to "268929", + "Adventure" to "78", + "Avant Garde" to "3", + "Boys Love" to "4", + "Comedy" to "5", + "Crime" to "268921", + "Demons" to "77", + "Drama" to "6", + "Ecchi" to "7", + "Fantasy" to "79", + "Girls Love" to "9", + "Gourmet" to "10", + "Harem" to "11", + "Hentai" to "268930", + "Historical" to "268922", + "Horror" to "530", + "Isekai" to "13", + "Iyashikei" to "531", + "Josei" to "15", + "Kids" to "532", + "Magic" to "539", + "Magical Girls" to "268923", + "Mahou Shoujo" to "533", + "Martial Arts" to "534", + "Mature" to "268931", + "Mecha" to "19", + "Medical" to "268924", + "Military" to "535", + "Music" to "21", + "Mystery" to "22", + "Parody" to "23", + "Philosophical" to "268925", + "Psychological" to "536", + "Reverse Harem" to "25", + "Romance" to "26", + "School" to "73", + "Sci-Fi" to "28", + "Seinen" to "537", + "Shoujo" to "30", + "Shounen" to "31", + "Slice of Life" to "538", + "Smut" to "268932", + "Space" to "33", + "Sports" to "34", + "Super Power" to "75", + "Superhero" to "268926", + "Supernatural" to "76", + "Suspense" to "37", + "Thriller" to "38", + "Tragedy" to "268927", + "Vampire" to "39", + "Wuxia" to "268928" + ) + } + + private val tags by lazy { + GENRE_MAP.entries.map { (title, id) -> + MangaTag(title, id, source) + }.toSet() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, + MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER, + ), + availableDemographics = EnumSet.of(Demographic.SHOUNEN, Demographic.SHOUJO, Demographic.SEINEN, Demographic.JOSEI), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val urlBuilder = okhttp3.HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegments("api/titles") + .addQueryParameter("page", page.toString()) + .addQueryParameter("limit", "50") + + if (!filter.query.isNullOrBlank()) { + urlBuilder.addQueryParameter("keyword", filter.query) + } + + if (filter.yearFrom > 0) { + urlBuilder.addQueryParameter("year_from", filter.yearFrom.toString()) + } + if (filter.yearTo > 0) { + urlBuilder.addQueryParameter("year_to", filter.yearTo.toString()) + } + + filter.types.forEach { type -> + val value = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.OTHER -> "other" + else -> null + } + value?.let { urlBuilder.addQueryParameter("types[]", it) } + } + + filter.demographics.forEach { demo -> + val id = when (demo) { + Demographic.JOSEI -> "268919" + Demographic.SEINEN -> "268920" + Demographic.SHOUJO -> "268917" + Demographic.SHOUNEN -> "268918" + else -> null + } + id?.let { urlBuilder.addQueryParameter("demographics[]", it) } + } + + filter.tags.forEach { urlBuilder.addQueryParameter("genres_in[]", it.key) } + filter.tagsExclude.forEach { urlBuilder.addQueryParameter("genres_ex[]", it.key) } + + filter.states.forEach { state -> + val apiState = when (state) { + MangaState.ONGOING -> "releasing" + MangaState.FINISHED -> "finished" + MangaState.ABANDONED -> "discontinued" + MangaState.PAUSED -> "on_hiatus" + MangaState.UPCOMING -> "not_yet_released" + else -> null + } + apiState?.let { urlBuilder.addQueryParameter("statuses[]", it) } + } + + val sortParam = when (order) { + SortOrder.UPDATED -> "chapter_updated_at" to "desc" + SortOrder.POPULARITY -> "views_total" to "desc" + SortOrder.RATING -> "score" to "desc" + SortOrder.NEWEST -> "created_at" to "desc" + SortOrder.ALPHABETICAL -> "title" to "asc" + SortOrder.RELEVANCE -> "relevance" to "desc" + SortOrder.POPULARITY_WEEK -> "views_7d" to "desc" + SortOrder.POPULARITY_MONTH -> "views_30d" to "desc" + else -> null + } + + sortParam?.let { (field, dir) -> + urlBuilder.addQueryParameter("order[$field]", dir) + } + + val url = urlBuilder.build().toString() + + val response = apiClient.httpGet(url).parseJson() + val items = response.getJSONArray("items") + val mangas = mutableListOf() + for (i in 0 until items.length()) { + val obj = items.getJSONObject(i) + val hid = obj.getString("hid") + val slug = obj.optString("slug", null) + val title = obj.getString("title") + val poster = obj.optJSONObject("poster") + val cover = poster?.optString("large") + ?: poster?.optString("medium") + ?: poster?.optString("small") ?: "" + val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" + mangas.add( + Manga( + id = generateUid(urlPath), + url = urlPath, + publicUrl = "https://$domain$urlPath", + title = title, + coverUrl = cover, + source = source, + altTitles = emptySet(), + largeCoverUrl = null, + authors = emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + tags = emptySet(), + ) + ) + } + return mangas + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val result = coroutineScope { + val hid = extractHid(manga.url) + + val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() + val data = detailsJson.getJSONObject("data") + + val hasVolumes = data.optBoolean("hasVolumes", false) + + val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } + + val title = data.getString("title") + val poster = data.optJSONObject("poster") + val cover = poster?.optString("large") + ?: poster?.optString("medium") + ?: poster?.optString("small") + val synopsisHtml = data.optString("synopsisHtml", null) + val status = data.optString("status", null) + val type = data.optString("type", null) + val authorsList = data.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val artistsList = data.optJSONArray("artists")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val genres = data.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val themes = data.optJSONArray("themes")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + + val rawRating = data.optDouble("rating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" + + val genreList = buildList { + type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } + genres?.let { addAll(it) } + themes?.let { addAll(it) } + } + val genreTags = genreList.mapNotNull { name -> + tags.find { it.title == name } + }.toSet() + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + authors = (authorsList + artistsList).toSet(), + description = synopsisText.trim(), + rating = rating, + state = when (status?.lowercase()) { + "releasing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + "discontinued" -> MangaState.ABANDONED + "on_hiatus" -> MangaState.PAUSED + "not_yet_released" -> MangaState.UPCOMING + else -> null + }, + tags = genreTags, + altTitles = altTitlesArray.toSet(), + chapters = chapters, + ) + } + + detailsCache.put(manga.url, result) + return result + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + return size > 10 + } + } + + private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List { + val chapters = mutableListOf() + val base = "https://$domain/api/titles/$hid" + + var page = 1 + var lastPage: Int + do { + val json = apiClient.httpGet( + "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" + ).parseJson() + val items = json.getJSONArray("items") + val meta = json.optJSONObject("meta") + lastPage = meta?.optInt("lastPage", 1) ?: 1 + + for (i in 0 until items.length()) { + val ch = items.getJSONObject(i) + if (ch.getString("language") != siteLang) continue + + val id = ch.getInt("id") + val number = ch.getDouble("number").toFloat() + val name = ch.optString("name", null) + val createdAt = ch.optLong("createdAt", 0L) * 1000L + val type = ch.getString("type") + val chapterUrl = "/title/$hid/$id" + val displayName = buildString { + append("Ch. ") + append(number.toString().removeSuffix(".0")) + if (!name.isNullOrBlank()) append(" - $name") + } + chapters.add( + MangaChapter( + id = generateUid(chapterUrl), + title = displayName, + number = number, + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = createdAt, + branch = type, + source = source, + ) + ) + } + page++ + } while (page <= lastPage) + + if (hasVolumes) { + try { + val volJson = apiClient.httpGet("$base/volumes?language=$siteLang").parseJson() + val volItems = volJson.getJSONArray("items") + for (i in 0 until volItems.length()) { + val vol = volItems.getJSONObject(i) + if (vol.getString("language") != siteLang) continue + + val volId = vol.getInt("id") + val volNumber = vol.getDouble("number").toFloat() + val volName = vol.optString("name", "").takeIf { it.isNotBlank() } + val chapterCount = vol.optInt("chapterCount", 0) + + val title = buildString { + append("Vol. ") + append(volNumber.toString().removeSuffix(".0")) + if (volName != null) append(" - $volName") + } + val name = if (chapterCount > 0) "$chapterCount chapters" else "" + + chapters.add( + MangaChapter( + id = generateUid("/title/$hid/vol/$volId"), + title = title, + number = volNumber, + volume = 0, + url = "/title/$hid/vol/$volId", + scanlator = name, + uploadDate = 0L, + branch = "Volume", + source = source, + ) + ) + } + } catch (_: Exception) { + } + } + + val distinctBranches = chapters.map { it.branch }.distinct() + val useGroups = distinctBranches.size > 1 + + return chapters + .map { chapter -> + chapter.copy( + branch = if (useGroups) (chapter.branch ?: "").replaceFirstChar { it.uppercase() } else null + ) + } + .sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterId = chapter.url.substringAfterLast("/") // numeric ID + val isVolume = chapter.url.contains("/vol/") + val endpoint = if (isVolume) "volumes" else "chapters" + + val response = apiClient.httpGet("https://$domain/api/$endpoint/$chapterId").parseJson() + val pagesArray = response.getJSONObject("data").getJSONArray("pages") + val pages = ArrayList(pagesArray.length()) + for (i in 0 until pagesArray.length()) { + val pageObj = pagesArray.getJSONObject(i) + val url = pageObj.getString("url") + pages.add( + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + ) + } + return pages + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + override val authUrl: String get() = "https://$domain" + override suspend fun isAuthorized(): Boolean = true + override suspend fun getUsername(): String = "" + + + private fun extractHid(url: String): String { + val lastPart = url.removeSuffix("/").substringAfterLast("/") + return when { + lastPart.contains(".") -> lastPart.substringAfterLast(".") + lastPart.contains("-") -> lastPart.substringBefore("-") + else -> lastPart + } + } + + @MangaSourceParser("MANGAFIRE_EN", "MangaFire (English)", "en") + class English(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_EN, "en") + + @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish)", "es") + class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") + + @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire Spanish (Latin)", "es") + class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") + + @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") + class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") + + @MangaSourceParser("MANGAFIRE_JA", "MangaFire (Japanese)", "ja") + class Japanese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_JA, "ja") + + @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese)", "pt") + class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") + + @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire Portuguese (Brazil)", "pt") + class PortugueseBR(context: MangaLoaderContext) : + MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/MangaPlus.kt b/src/main/kotlin/tsuki/site/all/MangaPlus.kt new file mode 100644 index 00000000..0d150371 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/MangaPlus.kt @@ -0,0 +1,297 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.asTypedList +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNull +import tsuki.util.suspendlazy.suspendLazy + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONArray +import org.json.JSONObject + +import java.util.EnumSet +import java.util.UUID + +internal abstract class MangaPlusParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val sourceLang: String +): + SinglePageMangaParser(context, source), Interceptor { + + private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" + override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + private val extraHeaders = Headers.headersOf("Session-Token", UUID.randomUUID().toString()) + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + return when { + filter.query.isNullOrEmpty() -> { + when (order) { + SortOrder.POPULARITY -> getPopularList() + SortOrder.UPDATED -> getLatestList() + else -> getAllTitleList() + } + } + else -> getAllTitleList(filter.query) + } + } + + private suspend fun getPopularList(): List { + val json = apiCall("/title_list/ranking") + + return json.getJSONObject("titleRankingView") + .getJSONArray("titles") + .asTypedList() + .toMangaList() + } + + private suspend fun getLatestList(): List { + val json = apiCall("/title_list/updated") + + return json.getJSONObject("titleUpdatedView") + .getJSONArray("latestTitle") + .mapJSON { it.getJSONObject("title") } + .toMangaList() + } + + private val allTitleCache = suspendLazy { + apiCall("/title_list/allV2") + .getJSONObject("allTitlesViewV2") + .getJSONArray("AllTitlesGroup") + .mapJSON { it.getJSONArray("titles").asTypedList() } + .flatten() + } + + private suspend fun getAllTitleList(query: String? = null): List { + return allTitleCache.get().toMangaList(query) + } + + private fun List.toMangaList(query: String? = null): List { + return mapNotNull { + val language = it.getStringOrNull("language") ?: "ENGLISH" + + if (language != sourceLang) { + return@mapNotNull null + } + + val name = it.getString("name") + val author = it.getString("author") + .split('/') + .joinToString(transform = String::trim) + + if (query != null && !(name.contains(query, true) || author.contains(query, true))) { + return@mapNotNull null + } + + val titleId = it.getInt("titleId").toString() + + Manga( + id = generateUid(titleId), + url = titleId, + publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), + title = name, + coverUrl = it.getString("portraitImageUrl"), + altTitles = emptySet(), + authors = setOf(author), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + source = source, + tags = emptySet(), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = apiCall("/title_detailV3?title_id=${manga.url}") + .getJSONObject("titleDetailView") + val title = json.getJSONObject("title") + + val completed = json.getJSONObject("titleLabels") + .getString("releaseSchedule").let { + it == "DISABLED" || it == "COMPLETED" + } + + val hiatus = json.getStringOrNull("nonAppearanceInfo")?.contains("on a hiatus") == true + val author = title.getString("author") + .split("/").joinToString(transform = String::trim) + + return manga.copy( + title = title.getString("name"), + publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain), + coverUrl = title.getString("portraitImageUrl"), + authors = setOf(author), + description = buildString { + json.getString("overview").let(::append) + json.getStringOrNull("viewingPeriodDescription") + ?.takeIf { !completed } + ?.let { append("

", it) } + }, + chapters = parseChapters( + json.getJSONArray("chapterListGroup"), + title.getStringOrNull("language") ?: "ENGLISH", + ), + state = when { + completed -> MangaState.FINISHED + hiatus -> MangaState.PAUSED + else -> MangaState.ONGOING + }, + ) + } + + private fun parseChapters(chapterListGroup: JSONArray, language: String): List { + val chapterList = chapterListGroup + .asTypedList() + .flatMap { + it.optJSONArray("firstChapterList")?.asTypedList().orEmpty() + + it.optJSONArray("lastChapterList")?.asTypedList().orEmpty() + } + + return chapterList.mapChapters { _, chapter -> + val chapterId = chapter.getInt("chapterId").toString() + val subtitle = chapter.getStringOrNull("subTitle") ?: return@mapChapters null + + MangaChapter( + id = generateUid(chapterId), + url = chapterId, + title = subtitle, + number = chapter.getString("name") + .substringAfter("#") + .toFloatOrNull() ?: -1f, + volume = 0, + uploadDate = chapter.getInt("startTimeStamp") * 1000L, + branch = when (language) { + "PORTUGUESE_BR" -> "Portuguese (Brazil)" + else -> language.lowercase().toTitleCase() + }, + scanlator = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high") + .getJSONObject("mangaViewer") + .getJSONArray("pages") + + return pages.mapJSONNotNull { + val mangaPage = it.optJSONObject("mangaPage") + ?: return@mapJSONNotNull null + val url = mangaPage.getString("imageUrl") + val encryptionKey = mangaPage.getStringOrNull("encryptionKey") + MangaPage( + id = generateUid(url), + url = url + if (encryptionKey == null) "" else "#$encryptionKey", + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val encryptionKey = request.url.fragment + + if (encryptionKey.isNullOrEmpty()) { + return response + } + + return response.map { responseBody -> + val contentType = response.headers["Content-Type"] ?: "image/jpeg" + val image = responseBody.bytes().decodeXorCipher(encryptionKey) + image.toResponseBody(contentType.toMediaTypeOrNull()) + } + } + + private fun ByteArray.decodeXorCipher(key: String): ByteArray { + val keyStream = key.chunked(2) + .map { it.toInt(16) } + + return mapIndexed { i, byte -> byte.toInt() xor keyStream[i % keyStream.size] } + .map(Int::toByte) + .toByteArray() + } + + private suspend fun apiCall(url: String): JSONObject { + val newUrl = "$apiUrl$url".toHttpUrl().newBuilder() + .addQueryParameter("format", "json") + .build() + val response = webClient.httpGet(newUrl, extraHeaders).parseJson() + + val success = response.optJSONObject("success") + + return checkNotNull(success) { + val error = response.getJSONObject("error") + val reason = error.getJSONArray("popups") + .asTypedList() + .firstOrNull { it.getStringOrNull("language") == null } + + if (reason?.getStringOrNull("subject") == "Not Found" && url.contains("manga_viewer")) { + "This chapter has expired" + } else { + reason?.getStringOrNull("body") ?: "Unknown Error" + } + } + } + + @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") + class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH") + + @MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus (Spanish)", "es") + class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH") + + @MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus (French)", "fr") + class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH") + + @MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus (Indonesian)", "id") + class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN") + + @MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt") + class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR") + + @MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus (Russian)", "ru") + class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN") + + @MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus (Thai)", "th") + class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI") + + @MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus (Vietnamese)", "vi") + class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE") + + @MangaSourceParser("MANGAPLUSPARSER_DE", "MANGA Plus (German)", "de") + class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN") +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt new file mode 100644 index 00000000..8986773d --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -0,0 +1,891 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.HttpStatusException +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +private const val DOMAIN = "mangaball.net" + +internal abstract class MangaBallParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val siteLanguages: Set, +) : PagedMangaParser(context, source, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain(_root_ide_package_.tsuki.site.all.DOMAIN) + + private val showSuspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) + private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } + private val client: WebClient by lazy { + OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(this) + .build(), + source, + ) + } + + @Volatile + private var csrfToken: String? = null + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.RELEVANCE, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showSuspiciousContentKey) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = TAG_DEFINITIONS.mapTo(LinkedHashSet(TAG_DEFINITIONS.size)) { + MangaTag(key = it.id, title = it.title, source = source) + }, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ), + availableDemographics = EnumSet.of( + Demographic.SHOUNEN, + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.JOSEI, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + ensureAdultCookie() + val query = filter.query?.trim().orEmpty() + if (page == 1 && query.startsWith("http")) { + return resolveSearchUrl(query) + } + val isQueryOnly = query.isNotEmpty() && !filter.hasNonSearchOptions() + return when { + isQueryOnly && page == 1 -> smartSearch(query) + isQueryOnly -> advancedSearch(page - 1, order, filter) + else -> advancedSearch(page, order, filter) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + ensureAdultCookie() + val doc = client.httpGet(getMangaUrl(manga.url)).parseHtml() + updateCsrf(doc) + + val tags = doc.select("#comicDetail span[data-tag-id]").mapTo(LinkedHashSet()) { tag -> + MangaTag( + key = tag.attr("data-tag-id"), + title = tag.ownTextOrNull().orEmpty().trim(), + source = source, + ) + } + val altTitles = doc.select("div.alternate-name-container").text() + .split('/') + .mapNotNullTo(LinkedHashSet()) { it.trim().nullIfEmpty() } + val authors = doc.select("#comicDetail span[data-person-id]") + .mapNotNullTo(LinkedHashSet()) { it.text().trim().nullIfEmpty() } + val status = when (doc.selectFirst("span.badge-status")?.text()?.trim()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + "Cancelled" -> MangaState.ABANDONED + else -> null + } + val title = doc.selectFirst("#comicDetail h6")?.ownTextOrNull()?.trim()?.nullIfEmpty() ?: manga.title + val cover = doc.selectFirst("img.featured-cover")?.absUrl("src")?.nullIfEmpty() ?: manga.coverUrl + val description = doc.selectFirst("#descriptionContent p")?.wholeText()?.trim()?.nullIfEmpty() + val contentRating = when { + tags.any { it.title in ADULT_TAG_TITLES } -> ContentRating.ADULT + else -> manga.contentRating + } + + val result = manga.copy( + title = title, + altTitles = if (altTitles.isEmpty()) manga.altTitles else altTitles, + publicUrl = getMangaUrl(manga.url), + coverUrl = cover, + largeCoverUrl = cover, + tags = tags, + state = status, + authors = authors, + description = description, + chapters = getChapterList(manga.url), + contentRating = contentRating, + ) + detailsCache[manga.url] = result + return result + } + + override suspend fun getPages(chapter: MangaChapter): List { + ensureAdultCookie() + val doc = client.httpGet(getChapterUrl(chapter)).parseHtml() + updateCsrf(doc) + val script = doc.select("script") + .firstOrNull { it.data().contains("chapterImages") } + ?.data() + .orEmpty() + val rawImages = CHAPTER_IMAGES_REGEX.find(script)?.groupValues?.getOrNull(1).orEmpty() + if (rawImages.isEmpty()) { + return emptyList() + } + val images = JSONArray(rawImages) + return List(images.length()) { index -> + val imageUrl = images.getString(index) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + if (link.host != domain) { + return null + } + val pathType = link.pathSegments.firstOrNull() ?: return null + val slug = when (pathType) { + "title-detail" -> link.pathSegments.getOrNull(1) + "chapter-detail" -> resolveChapterSlug(link.toString()) + else -> null + } ?: return null + return getDetails(seedManga(slug)) + } + + override fun intercept(chain: Interceptor.Chain): Response { + ensureAdultCookie() + val request = chain.request() + var builder = request.newBuilder() + if ( + (request.url.host == domain || request.url.host.endsWith(".poke-black-and-white.net")) && + request.header("Referer").isNullOrEmpty() + ) { + builder = builder.header("Referer", "https://$domain/") + } + if (request.url.host == domain && request.url.pathSegments.firstOrNull() == "api") { + builder = builder + .header("X-Requested-With", "XMLHttpRequest") + .header("X-CSRF-TOKEN", getCsrf()) + } + var response = chain.proceed(builder.build()) + if (response.code == 403 && request.url.host == domain && request.url.pathSegments.firstOrNull() == "api") { + response.close() + refreshCsrf() + response = chain.proceed( + request.newBuilder() + .header("Referer", "https://$domain/") + .header("X-Requested-With", "XMLHttpRequest") + .header("X-CSRF-TOKEN", getCsrf()) + .build(), + ) + } + return response + } + + private suspend fun smartSearch(query: String): List { + val payload = mapOf("search_input" to query) + val json = postApi("$BASE_URL/api/v1/smart-search/search/".toHttpUrl(), payload).parseJson() + val mangaArray = json.getJSONObject("data").getJSONArray("manga") + return List(mangaArray.length()) { index -> + val item = mangaArray.getJSONObject(index) + val slug = extractSlug(item.getString("url")) + Manga( + id = generateUid(slug), + title = item.getString("title"), + altTitles = emptySet(), + url = slug, + publicUrl = getMangaUrl(slug), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = item.getString("img").nullIfEmpty(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun advancedSearch(page: Int, order: SortOrder, filter: MangaListFilter): List { + if (page < 1) { + return emptyList() + } + val payload = buildAdvancedPayload(page, order, filter) + val json = postApi("$BASE_URL/api/v1/title/search-advanced/".toHttpUrl(), payload).parseJson() + val data = json.optJSONArray("data") ?: return emptyList() + return List(data.length()) { index -> + val item = data.getJSONObject(index) + val slug = extractSlug(item.getString("url")) + Manga( + id = generateUid(slug), + title = item.getString("name"), + altTitles = emptySet(), + url = slug, + publicUrl = getMangaUrl(slug), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = item.optString("cover").nullIfEmpty(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private fun buildAdvancedPayload(page: Int, order: SortOrder, filter: MangaListFilter): String { + val includedTags = LinkedHashSet() + includedTags += filter.tags.map { it.key } + filter.types.oneOrThrowIfMany()?.let { type -> + includedTags += TYPE_TAG_IDS.getValue(type) + } + + return buildList { + addEncoded("search_input", filter.query?.trim().orEmpty()) + addEncoded("filters[sort]", sortValue(order)) + addEncoded("filters[page]", page.toString()) + includedTags.forEach { addEncoded("filters[tag_included_ids][]", it) } + addEncoded("filters[tag_included_mode]", "and") + filter.tagsExclude.forEach { addEncoded("filters[tag_excluded_ids][]", it.key) } + addEncoded("filters[tag_excluded_mode]", "or") + addEncoded("filters[contentRating]", "any") + addEncoded("filters[demographic]", demographicValue(filter.demographics.oneOrThrowIfMany())) + addEncoded("filters[person]", "any") + addEncoded("filters[publicationYear]", filter.year.takeIf { it > 0 }?.toString().orEmpty()) + addEncoded("filters[publicationStatus]", stateValue(filter.states.oneOrThrowIfMany())) + siteLanguages.forEach { addEncoded("filters[translatedLanguage][]", it) } + }.joinToString("&") + } + + private suspend fun getChapterList(mangaSlug: String): List { + val titleId = mangaSlug.substringAfterLast('-') + val json = postApi( + "$BASE_URL/api/v1/chapter/chapter-listing-by-title-id/".toHttpUrl(), + mapOf("title_id" to titleId), + ).parseJson() + val containers = json.getJSONArray("ALL_CHAPTERS") + val chapterCandidates = ArrayList(containers.length()) + for (i in 0 until containers.length()) { + val container = containers.getJSONObject(i) + val number = container.optDouble("number_float", 0.0).toFloat() + val translations = container.getJSONArray("translations") + for (j in 0 until translations.length()) { + val translation = translations.getJSONObject(j) + val language = translation.optString("language") + if (language !in siteLanguages) { + continue + } + val group = translation.getJSONObject("group") + val groupId = group.optString("_id") + val groupName = group.optString("name").trim() + val scanlator = buildString { + append(groupName) + if (!GROUP_ID_REGEX.matches(groupId)) { + append(" (") + append(groupId) + append(')') + } + }.nullIfEmpty() + val volume = translation.optInt("volume", 0) + val translationName = translation.optString("name").trim() + val numberText = number.toString().removeSuffix(".0") + val chapter = MangaChapter( + id = generateUid(translation.getString("id")), + title = buildChapterTitle(numberText, volume, translationName), + number = number, + volume = volume, + url = translation.getString("id"), + scanlator = scanlator, + uploadDate = chapterDateFormat.parseSafe(translation.optString("date")), + branch = language.takeIf { siteLanguages.size > 1 }, + source = source, + ) + chapterCandidates += ChapterCandidate(language = language, chapter = chapter) + } + } + val scanlatorCounts = chapterCandidates.groupingBy { + "${it.language}|${it.chapter.scanlator.orEmpty()}" + }.eachCount() + return chapterCandidates + .groupBy { "${it.language}|${it.chapter.number}" } + .values + .mapNotNull { candidates -> + candidates.maxWithOrNull { left, right -> + compareChapterCandidates( + left = left, + right = right, + scanlatorCounts = scanlatorCounts, + ) + }?.chapter + } + .sortedWith( + compareBy { it.number } + .thenBy { if (it.volume > 0) 1 else 0 } + .thenBy { it.uploadDate } + .thenBy { it.branch.orEmpty() } + .thenBy { it.title.orEmpty() }, + ) + } + + private fun buildChapterTitle(numberText: String, volume: Int, rawName: String): String { + val baseTitle = buildString { + if (volume > 0) { + append("Vol. ") + append(volume) + append(' ') + } + append("Chapter ") + append(numberText) + } + val name = rawName.trim() + if (name.isEmpty()) { + return baseTitle + } + val compactName = name.replace(Regex("\\s+"), " ") + val chapterPrefixRegex = Regex("^(Vol\\.?\\s*\\d+\\s+)?Ch(?:apter)?\\.?\\s*", RegexOption.IGNORE_CASE) + val normalized = chapterPrefixRegex.find(compactName)?.let { match -> + val volumePrefix = match.groups[1]?.value?.trim().orEmpty() + buildString { + if (volumePrefix.isNotEmpty()) { + append(volumePrefix.replaceFirst(Regex("(?i)^vol\\.?"), "Vol.")) + append(' ') + } + append("Chapter ") + append(compactName.removeRange(match.range)) + } + }?.trim() ?: compactName.trim() + return if (normalized.startsWith("Chapter $numberText", ignoreCase = true) || + normalized.startsWith("Vol. $volume Chapter $numberText", ignoreCase = true) + ) { + normalized + } else { + "$baseTitle: $normalized" + } + } + + private fun compareChapterCandidates( + left: ChapterCandidate, + right: ChapterCandidate, + scanlatorCounts: Map, + ): Int { + val leftChapter = left.chapter + val rightChapter = right.chapter + val leftScore = scanlatorCounts["${left.language}|${leftChapter.scanlator.orEmpty()}"] ?: 0 + val rightScore = scanlatorCounts["${right.language}|${rightChapter.scanlator.orEmpty()}"] ?: 0 + if (leftScore != rightScore) { + return leftScore.compareTo(rightScore) + } + if ((leftChapter.volume > 0) != (rightChapter.volume > 0)) { + return if (leftChapter.volume == 0) 1 else -1 + } + val leftTitle = leftChapter.title.orEmpty() + val rightTitle = rightChapter.title.orEmpty() + if (leftTitle.length != rightTitle.length) { + return rightTitle.length.compareTo(leftTitle.length) + } + if (leftChapter.uploadDate != rightChapter.uploadDate) { + return leftChapter.uploadDate.compareTo(rightChapter.uploadDate) + } + return rightChapter.id.compareTo(leftChapter.id) + } + + private suspend fun resolveSearchUrl(query: String): List { + val slug = when { + query.startsWith("https://") || query.startsWith("http://") -> { + val link = query.toHttpUrl() + when (link.pathSegments.firstOrNull()) { + "title-detail" -> link.pathSegments.getOrNull(1) + "chapter-detail" -> resolveChapterSlug(query) + else -> null + } + } + else -> null + } ?: return emptyList() + return listOf(getDetails(seedManga(slug))) + } + + private suspend fun resolveChapterSlug(chapterUrl: String): String? { + val doc = client.httpGet(chapterUrl).parseHtml() + updateCsrf(doc) + val yoastRaw = doc.selectFirst("script.yoast-schema-graph")?.data()?.nullIfEmpty() ?: return null + val graph = JSONObject(yoastRaw).optJSONArray("@graph") ?: return null + for (i in 0 until graph.length()) { + val item = graph.getJSONObject(i) + if (item.optString("@type") == "WebPage") { + val url = item.optString("url").nullIfEmpty() ?: continue + return url.toHttpUrl().pathSegments.getOrNull(1) + } + } + return null + } + + private fun seedManga(slug: String): Manga = Manga( + id = generateUid(slug), + title = slug.substringBeforeLast('-').replace('-', ' ').trim(), + altTitles = emptySet(), + url = slug, + publicUrl = getMangaUrl(slug), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + + private fun extractSlug(url: String): String = url.toAbsoluteUrl(domain).toHttpUrl().pathSegments[1] + + private fun getMangaUrl(slug: String): String = "$BASE_URL/title-detail/$slug/" + + private fun getChapterUrl(chapter: MangaChapter): String = "$BASE_URL/chapter-detail/${chapter.url}/" + + private fun getHeadersWithReferer(): Headers = getRequestHeaders().newBuilder() + .add("Referer", "$BASE_URL/") + .build() + + private fun getApiHeaders(): Headers = getHeadersWithReferer().newBuilder() + .add("X-Requested-With", "XMLHttpRequest") + .add("X-CSRF-TOKEN", getCsrf()) + .build() + + private suspend fun postApi(url: HttpUrl, form: Map): Response { + return try { + client.httpPost(url, form, getApiHeaders()) + } catch (e: HttpStatusException) { + if (e.statusCode != 403) throw e + refreshCsrf() + client.httpPost(url, form, getApiHeaders()) + } + } + + private suspend fun postApi(url: HttpUrl, payload: String): Response { + return try { + client.httpPost(url, payload, getApiHeaders()) + } catch (e: HttpStatusException) { + if (e.statusCode != 403) throw e + refreshCsrf() + client.httpPost(url, payload, getApiHeaders()) + } + } + + @Synchronized + private fun ensureAdultCookie() { + context.cookieJar.insertCookies(domain, "show18PlusContent=${config[showSuspiciousContentKey]}") + } + + @Synchronized + private fun getCsrf(): String { + if (csrfToken.isNullOrEmpty()) { + refreshCsrf() + if (csrfToken.isNullOrEmpty()) { + refreshCsrf() + } + if (csrfToken.isNullOrEmpty()) { + error("CSRF token not found after two attempts") + } + } + return csrfToken!! + } + + @Synchronized + private fun refreshCsrf(document: Document? = null) { + ensureAdultCookie() + if (document != null) { + updateCsrf(document) + if (!csrfToken.isNullOrEmpty()) return + } + + val request = Request.Builder() + .url(BASE_URL) + .headers(getHeadersWithReferer()) + .header("Cache-Control", "no-cache") + .build() + val response = rawHttpClient.newCall(request).execute() + updateCsrf(response.parseHtml()) + response.close() + } + + @Synchronized + private fun updateCsrf(document: Document) { + document.selectFirst("meta[name=csrf-token]")?.attr("content")?.trim()?.nullIfEmpty()?.let { + csrfToken = it + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + private fun MutableList.addEncoded(key: String, value: String) { + add("${key.urlEncoded()}=${value.urlEncoded()}") + } + + private fun sortValue(order: SortOrder): String = when (order) { + SortOrder.UPDATED -> "updated_chapters_desc" + SortOrder.POPULARITY -> "views_desc" + SortOrder.NEWEST -> "created_at_desc" + SortOrder.ALPHABETICAL -> "name_asc" + SortOrder.ALPHABETICAL_DESC -> "name_desc" + SortOrder.RELEVANCE -> "updated_chapters_desc" + else -> "updated_chapters_desc" + } + + private fun stateValue(state: MangaState?): String = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> "any" + } + + private fun demographicValue(demographic: Demographic?): String = when (demographic) { + Demographic.SHOUNEN -> "shounen" + Demographic.SHOUJO -> "shoujo" + Demographic.SEINEN -> "seinen" + Demographic.JOSEI -> "josei" + else -> "any" + } + + private data class ChapterCandidate( + val language: String, + val chapter: MangaChapter, + ) + + companion object { + + private const val BASE_URL = "https://${_root_ide_package_.tsuki.site.all.DOMAIN}" + private val GROUP_ID_REGEX = Regex("[a-z0-9]{24}") + private val CHAPTER_IMAGES_REGEX = Regex("""const\s+chapterImages\s*=\s*JSON\.parse\(`([^`]+)`\)""") + private val ADULT_TAG_TITLES = setOf( + "Adult", + "Ecchi", + "Mature", + "Smut", + "Hentai", + "Manhwa 18+", + "Sexual Violence", + ) + + private val TYPE_TAG_IDS = mapOf( + ContentType.COMICS to "68ecab8507ec62d87e62780f", + ContentType.MANGA to "68ecab1e07ec62d87e627806", + ContentType.MANHUA to "68ecab4807ec62d87e62780b", + ContentType.MANHWA to "68ecab3b07ec62d87e627809", + ) + + private data class TagDef( + val id: String, + val title: String, + ) + + private val TAG_DEFINITIONS = listOf( + TagDef("685148d115e8b86aae68e4f3", "Gore"), + TagDef("685146c5f3ed681c80f257e7", "Sexual Violence"), + TagDef("685148d115e8b86aae68e4ec", "4-Koma"), + TagDef("685148cf15e8b86aae68e4de", "Adaptation"), + TagDef("685148e915e8b86aae68e558", "Anthology"), + TagDef("685148fe15e8b86aae68e5a7", "Award Winning"), + TagDef("6851490e15e8b86aae68e5da", "Doujinshi"), + TagDef("6851498215e8b86aae68e704", "Fan Colored"), + TagDef("685148d615e8b86aae68e502", "Full Color"), + TagDef("685148d915e8b86aae68e517", "Long Strip"), + TagDef("6851493515e8b86aae68e64a", "Official Colored"), + TagDef("685148eb15e8b86aae68e56c", "Oneshot"), + TagDef("6851492e15e8b86aae68e633", "Self-Published"), + TagDef("685148d715e8b86aae68e50d", "Web Comic"), + TagDef("685146c5f3ed681c80f257e3", "Action"), + TagDef("689371f0a943baf927094f03", "Adult"), + TagDef("685146c5f3ed681c80f257e6", "Adventure"), + TagDef("685148ef15e8b86aae68e573", "Boys' Love"), + TagDef("685146c5f3ed681c80f257e5", "Comedy"), + TagDef("685148da15e8b86aae68e51f", "Crime"), + TagDef("685148cf15e8b86aae68e4dd", "Drama"), + TagDef("6892a73ba943baf927094e37", "Ecchi"), + TagDef("685146c5f3ed681c80f257ea", "Fantasy"), + TagDef("685148da15e8b86aae68e524", "Girls' Love"), + TagDef("685148db15e8b86aae68e527", "Historical"), + TagDef("685148da15e8b86aae68e520", "Horror"), + TagDef("685146c5f3ed681c80f257e9", "Isekai"), + TagDef("6851490d15e8b86aae68e5d4", "Magical Girls"), + TagDef("68932d11a943baf927094e7b", "Mature"), + TagDef("6851490c15e8b86aae68e5d2", "Mecha"), + TagDef("6851494e15e8b86aae68e66e", "Medical"), + TagDef("685148d215e8b86aae68e4f4", "Mystery"), + TagDef("685148e215e8b86aae68e544", "Philosophical"), + TagDef("685148d715e8b86aae68e507", "Psychological"), + TagDef("685148cf15e8b86aae68e4db", "Romance"), + TagDef("685148cf15e8b86aae68e4da", "Sci-Fi"), + TagDef("689f0ab1f2e66744c6091524", "Shounen Ai"), + TagDef("685148d015e8b86aae68e4e3", "Slice of Life"), + TagDef("689371f2a943baf927094f04", "Smut"), + TagDef("685148f515e8b86aae68e588", "Sports"), + TagDef("6851492915e8b86aae68e61c", "Superhero"), + TagDef("685148d915e8b86aae68e51e", "Thriller"), + TagDef("685148db15e8b86aae68e529", "Tragedy"), + TagDef("68932c3ea943baf927094e77", "User Created"), + TagDef("6851490715e8b86aae68e5c3", "Wuxia"), + TagDef("68932f68a943baf927094eaa", "Yaoi"), + TagDef("6896a885a943baf927094f66", "Yuri"), + TagDef("6851490d15e8b86aae68e5d5", "Aliens"), + TagDef("685148e715e8b86aae68e54b", "Animals"), + TagDef("68bf09ff8fdeab0b6a9bc2b7", "Comics"), + TagDef("685148d215e8b86aae68e4f8", "Cooking"), + TagDef("685148df15e8b86aae68e534", "Crossdressing"), + TagDef("685148d915e8b86aae68e519", "Delinquents"), + TagDef("685146c5f3ed681c80f257e4", "Demons"), + TagDef("685148d715e8b86aae68e505", "Genderswap"), + TagDef("685148d615e8b86aae68e501", "Ghosts"), + TagDef("685148d015e8b86aae68e4e8", "Gyaru"), + TagDef("685146c5f3ed681c80f257e8", "Harem"), + TagDef("68bfceaf4dbc442a26519889", "Hentai"), + TagDef("685148f215e8b86aae68e584", "Incest"), + TagDef("685148d715e8b86aae68e506", "Loli"), + TagDef("685148d915e8b86aae68e518", "Mafia"), + TagDef("685148d715e8b86aae68e509", "Magic"), + TagDef("68f5f5ce5f29d3c1863dec3a", "Manhwa 18+"), + TagDef("6851490615e8b86aae68e5c2", "Martial Arts"), + TagDef("685148e215e8b86aae68e541", "Military"), + TagDef("685148db15e8b86aae68e52c", "Monster Girls"), + TagDef("685146c5f3ed681c80f257e2", "Monsters"), + TagDef("685148d015e8b86aae68e4e4", "Music"), + TagDef("685148d715e8b86aae68e508", "Ninja"), + TagDef("685148d315e8b86aae68e4fd", "Office Workers"), + TagDef("6851498815e8b86aae68e714", "Police"), + TagDef("685148e215e8b86aae68e540", "Post-Apocalyptic"), + TagDef("685146c5f3ed681c80f257e1", "Reincarnation"), + TagDef("685148df15e8b86aae68e533", "Reverse Harem"), + TagDef("6851490415e8b86aae68e5b9", "Samurai"), + TagDef("685148d015e8b86aae68e4e7", "School Life"), + TagDef("685148d115e8b86aae68e4ed", "Shota"), + TagDef("685148db15e8b86aae68e528", "Supernatural"), + TagDef("685148cf15e8b86aae68e4dc", "Survival"), + TagDef("6851490c15e8b86aae68e5d1", "Time Travel"), + TagDef("6851493515e8b86aae68e645", "Traditional Games"), + TagDef("685148f915e8b86aae68e597", "Vampires"), + TagDef("685148e115e8b86aae68e53c", "Video Games"), + TagDef("6851492115e8b86aae68e602", "Villainess"), + TagDef("68514a1115e8b86aae68e83e", "Virtual Reality"), + TagDef("6851490c15e8b86aae68e5d3", "Zombies"), + ) + } + + @MangaSourceParser("MANGABALL_AR", "Manga Ball (Arabic)", "ar") + class Arabic(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_AR, setOf("ar")) + + @MangaSourceParser("MANGABALL_BG", "Manga Ball (Bulgarian)", "bg") + class Bulgarian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_BG, setOf("bg")) + + @MangaSourceParser("MANGABALL_BN", "Manga Ball (Bengali)", "bn") + class Bengali(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_BN, setOf("bn")) + + @MangaSourceParser("MANGABALL_CA", "Manga Ball (Catalan)", "ca") + class Catalan(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_CA, + setOf("ca", "ca-ad", "ca-es", "ca-fr", "ca-it", "ca-pt"), + ) + + @MangaSourceParser("MANGABALL_CS", "Manga Ball (Czech)", "cs") + class Czech(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_CS, setOf("cs")) + + @MangaSourceParser("MANGABALL_DA", "Manga Ball (Danish)", "da") + class Danish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_DA, setOf("da")) + + @MangaSourceParser("MANGABALL_DE", "Manga Ball (German)", "de") + class German(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_DE, setOf("de")) + + @MangaSourceParser("MANGABALL_EL", "Manga Ball (Greek)", "el") + class Greek(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_EL, setOf("el")) + + @MangaSourceParser("MANGABALL_EN", "Manga Ball (English)", "en") + class English(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_EN, setOf("en")) + + @MangaSourceParser("MANGABALL_ES", "Manga Ball (Spanish)", "es") + class Spanish(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_ES, + setOf("es", "es-ar", "es-mx", "es-es", "es-la", "es-419"), + ) + + @MangaSourceParser("MANGABALL_FA", "Manga Ball (Persian)", "fa") + class Persian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FA, setOf("fa")) + + @MangaSourceParser("MANGABALL_FI", "Manga Ball (Finnish)", "fi") + class Finnish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FI, setOf("fi")) + + @MangaSourceParser("MANGABALL_FR", "Manga Ball (French)", "fr") + class French(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_FR, setOf("fr")) + + @MangaSourceParser("MANGABALL_HE", "Manga Ball (Hebrew)", "he") + class Hebrew(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HE, setOf("he")) + + @MangaSourceParser("MANGABALL_HI", "Manga Ball (Hindi)", "hi") + class Hindi(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HI, setOf("hi")) + + @MangaSourceParser("MANGABALL_HU", "Manga Ball (Hungarian)", "hu") + class Hungarian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_HU, setOf("hu")) + + @MangaSourceParser("MANGABALL_ID", "Manga Ball (Indonesian)", "id") + class Indonesian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_ID, setOf("id")) + + @MangaSourceParser("MANGABALL_IT", "Manga Ball (Italian)", "it") + class Italian(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_IT, + setOf("it", "it-it"), + ) + + @MangaSourceParser("MANGABALL_IS", "Manga Ball (Icelandic)", "is") + class Icelandic(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_IS, + setOf("ib", "ib-is", "is"), + ) + + @MangaSourceParser("MANGABALL_JA", "Manga Ball (Japanese)", "ja") + class Japanese(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_JA, setOf("jp")) + + @MangaSourceParser("MANGABALL_KO", "Manga Ball (Korean)", "ko") + class Korean(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_KO, setOf("kr")) + + @MangaSourceParser("MANGABALL_KN", "Manga Ball (Kannada)", "kn") + class Kannada(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_KN, + setOf("kn", "kn-in", "kn-my", "kn-sg", "kn-tw"), + ) + + @MangaSourceParser("MANGABALL_ML", "Manga Ball (Malayalam)", "ml") + class Malayalam(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_ML, + setOf("ml", "ml-in", "ml-my", "ml-sg", "ml-tw"), + ) + + @MangaSourceParser("MANGABALL_MS", "Manga Ball (Malay)", "ms") + class Malay(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_MS, setOf("ms")) + + @MangaSourceParser("MANGABALL_NE", "Manga Ball (Nepali)", "ne") + class Nepali(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_NE, setOf("ne")) + + @MangaSourceParser("MANGABALL_NL", "Manga Ball (Dutch)", "nl") + class Dutch(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_NL, + setOf("nl", "nl-be"), + ) + + @MangaSourceParser("MANGABALL_NO", "Manga Ball (Norwegian)", "no") + class Norwegian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_NO, setOf("no")) + + @MangaSourceParser("MANGABALL_PL", "Manga Ball (Polish)", "pl") + class Polish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_PL, setOf("pl")) + + @MangaSourceParser("MANGABALL_PTBR", "Manga Ball (Portuguese)", "pt") + class PortugueseBrazil(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_PTBR, + setOf("pt-br", "pt-pt"), + ) + + @MangaSourceParser("MANGABALL_RO", "Manga Ball (Romanian)", "ro") + class Romanian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_RO, setOf("ro")) + + @MangaSourceParser("MANGABALL_RU", "Manga Ball (Russian)", "ru") + class Russian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_RU, setOf("ru")) + + @MangaSourceParser("MANGABALL_SK", "Manga Ball (Slovak)", "sk") + class Slovak(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SK, setOf("sk")) + + @MangaSourceParser("MANGABALL_SL", "Manga Ball (Slovenian)", "sl") + class Slovenian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SL, setOf("sl")) + + @MangaSourceParser("MANGABALL_SQ", "Manga Ball (Albanian)", "sq") + class Albanian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SQ, setOf("sq")) + + @MangaSourceParser("MANGABALL_SR", "Manga Ball (Serbian)", "sr") + class Serbian(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_SR, + setOf("sr", "sr-cyrl"), + ) + + @MangaSourceParser("MANGABALL_SV", "Manga Ball (Swedish)", "sv") + class Swedish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_SV, setOf("sv")) + + @MangaSourceParser("MANGABALL_TA", "Manga Ball (Tamil)", "ta") + class Tamil(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_TA, setOf("ta")) + + @MangaSourceParser("MANGABALL_TH", "Manga Ball (Thai)", "th") + class Thai(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_TH, + setOf("th", "th-hk", "th-kh", "th-la", "th-my", "th-sg"), + ) + + @MangaSourceParser("MANGABALL_TR", "Manga Ball (Turkish)", "tr") + class Turkish(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_TR, setOf("tr")) + + @MangaSourceParser("MANGABALL_UK", "Manga Ball (Ukrainian)", "uk") + class Ukrainian(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_UK, setOf("uk")) + + @MangaSourceParser("MANGABALL_VI", "Manga Ball (Vietnamese)", "vi") + class Vietnamese(context: MangaLoaderContext) : MangaBallParser(context, MangaParserSource.MANGABALL_VI, setOf("vi")) + + @MangaSourceParser("MANGABALL_ZH", "Manga Ball (Chinese)", "zh") + class Chinese(context: MangaLoaderContext) : MangaBallParser( + context, + MangaParserSource.MANGABALL_ZH, + setOf("zh", "zh-cn", "zh-hk", "zh-mo", "zh-sg", "zh-tw"), + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/ProjectSuki.kt b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt new file mode 100644 index 00000000..b3ac9881 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt @@ -0,0 +1,490 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + + +import tsuki.util.* +import tsuki.model.* + +import androidx.collection.LruCache +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.LinkedHashMap +import java.util.LinkedHashSet +import java.util.Locale +import java.text.SimpleDateFormat +import java.util.Calendar + +@MangaSourceParser("PROJECTSUKI", "ProjectSuki") +internal class ProjectSuki(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.PROJECTSUKI, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("projectsuki.com") + private val pagesCache = LruCache>(50) + override val defaultSortOrder: SortOrder = SortOrder.UPDATED + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + ) + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + return when { + !filter.query.isNullOrBlank() || !filter.isEmpty() -> search(page, filter) + order == SortOrder.UPDATED -> parseBookList( + webClient.httpGet("https://$domain/", getRequestHeaders()).parseHtml() + ) + else -> parseBookList( + webClient.httpGet( + "https://$domain/browse/${(page - 1).coerceAtLeast(0)}", + getRequestHeaders() + ).parseHtml() + ) + } + } + + private suspend fun search(page: Int, filter: MangaListFilter): List { + val url = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("page", (page - 1).coerceAtLeast(0).toString()) + .addQueryParameter("q", filter.query.orEmpty()) + + filter.states.firstOrNull()?.let { state -> + url.addQueryParameter("adv", "1") + url.addQueryParameter( + "status", + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> return@let + } + ) + } + + return parseBookList( + webClient.httpGet(url.build(), getRequestHeaders()).parseHtml() + ) + } + + private fun parseBookList(document: Document): List { + val result = LinkedHashMap() + + document.select("div.browse, div.item, div.book-item, .row .col-md-3, .row .col-sm-4, .row .col-xs-6") + .forEach { container -> + val bookAnchor = container.select("a[href]").firstOrNull { anchor -> + anchor.absUrl("href").toBookId() != null + } ?: return@forEach + + val bookId = bookAnchor.absUrl("href").toBookId() ?: return@forEach + if (bookId in result) return@forEach + + val title = extractBookTitle(container, bookAnchor, bookId) + val cover = extractBookCover(container, bookId) + val url = "/book/$bookId" + + val manga = Manga( + id = generateUid(url), + title = title, + altTitles = emptySet(), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + result[bookId] = manga + } + + return result.values.toList() + } + + private fun extractBookTitle(container: Element, anchor: Element, bookId: String): String { + return sequenceOf( + container.select(".details h4 a[href], h4 a[href], .details a[itemprop=title], a[itemprop=title]") + .firstOrNull { it.absUrl("href").toBookId() == bookId } + ?.text(), + container.select("h1, h2, h3, h4, .title, [itemprop=name]").firstOrNull()?.text(), + anchor.ownText(), + anchor.text(), + container.selectFirst("img[title]")?.attr("title"), + container.selectFirst("img[alt]")?.attr("alt"), + ).firstOrNull { it != null && it.isValidBookTitle(bookId) } + ?.trim() + ?: bookId + } + + private fun extractBookCover(container: Element, bookId: String): String { + return container.select("img").firstNotNullOfOrNull { it.imageSrc() } + ?: bookThumbnailUrl(bookId) + } + + private fun String.isValidBookTitle(bookId: String): Boolean { + val value = trim() + return value.isNotEmpty() && + !value.equals(bookId, ignoreCase = true) && + !value.equals("show more", ignoreCase = true) && + !value.all(Char::isDigit) + } + + override suspend fun getDetails(manga: Manga): Manga { + val document = webClient.httpGet( + manga.url.toAbsoluteUrl(domain), + getRequestHeaders() + ).parseHtml() + + val bookId = manga.url.toBookId() ?: document.location().toBookId() ?: manga.url.substringAfterLast('/') + + val fallbackTimestamp = document.selectFirst("meta[property='og:updated_time']") + ?.attr("content") + ?.toLongOrNull() + ?.times(1000L) + + val details = parseDetailsTable(document) + val title = document.selectFirst("h2[itemprop=title]")?.text()?.nullIfEmpty() + ?: document.selectFirst("h2")?.text()?.nullIfEmpty() + ?: manga.title + val cover = document.select("img").firstNotNullOfOrNull { it.imageSrc() }?.takeIf { bookId in it } + ?: manga.coverUrl + ?: bookThumbnailUrl(bookId) + + val description = buildDescription(document, details) + val authors = parseAuthors(document) + val state = parseState(details["status"]) + val genres = parseGenres(document) + val altTitles = setOfNotNull(details["alt titles"] ?: details["alternative titles"]) + val chapters = parseChapters(document, fallbackTimestamp) + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + altTitles = altTitles, + authors = authors, + state = state, + tags = genres, + contentRating = ContentRating.SAFE, + chapters = chapters, + ) + } + + private fun parseDetailsTable(document: Document): Map { + val map = LinkedHashMap() + document.select("div, li, tr, .row .col-md-6, .col-sm-6").forEach { row -> + val children = row.children() + if (children.size < 2) return@forEach + val key = children[0].text().trim().trim(':').lowercase(Locale.ROOT) + if (key !in DETAIL_KEYS) return@forEach + val value = children[1].text().trim().nullIfEmpty() ?: return@forEach + map[key] = value + } + return map + } + + private fun buildDescription(document: Document, details: Map): String? { + val body = document.selectFirst("#descriptionCollapse")?.wholeText()?.trim() + ?: document.select(".description").joinToString("\n\n") { it.wholeText().trim() } + .nullIfEmpty() + + val detailText = details.entries.joinToString("\n") { (key, value) -> + "${key.replaceFirstChar { it.titlecase(Locale.ROOT) }}: $value" + }.nullIfEmpty() + + return listOfNotNull(body, detailText).joinToString("\n\n").nullIfEmpty() + } + + private fun parseAuthors(document: Document): Set { + return document.select("a[href]").mapNotNullTo(LinkedHashSet()) { anchor -> + val href = anchor.absUrl("href") + if ("author=" in href || "artist=" in href) { + anchor.text().nullIfEmpty() + } else null + } + } + + private fun parseGenres(document: Document): Set { + return document.select("a[href]").mapNotNullTo(LinkedHashSet()) { anchor -> + val url = anchor.absUrl("href").toHttpUrlOrNullSafe() ?: return@mapNotNullTo null + val segments = url.pathSegments.filter { it.isNotBlank() } + if (segments.firstOrNull() != "genre") return@mapNotNullTo null + val key = segments.getOrNull(1)?.nullIfEmpty() ?: return@mapNotNullTo null + MangaTag( + key = key, + title = anchor.text().ifBlank { key }.replace('-', ' ').replaceFirstChar { it.titlecase(Locale.ROOT) }, + source = source, + ) + } + } + + private fun parseState(value: String?): MangaState? { + return when (value?.trim()?.lowercase(Locale.ROOT)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled", "canceled" -> MangaState.ABANDONED + else -> null + } + } + + private fun parseChapters(document: Document, fallbackTimestamp: Long? = null): List { + val chaptersMap = LinkedHashMap() + + val table = document.select("table").firstOrNull { table -> + table.select("tr").firstOrNull()?.let { row -> + row.select("td, th").any { cell -> + cell.text().trim().matches(Regex("added|date", RegexOption.IGNORE_CASE)) + } + } ?: false + } + + if (table != null) { + val headerRow = table.select("tr").first() + val headers = headerRow!!.select("td, th").map { it.text().trim().lowercase() } + + val dateCol = headers.indexOfFirst { it == "added" || it == "date" } + val chapterCol = headers.indexOfFirst { it.matches(Regex("chapter|ch\\.|name|#")) } + .takeIf { it >= 0 } ?: 0 + + val dataRows = table.select("tbody tr").ifEmpty { table.select("tr").drop(1) } + for (row in dataRows) { + val cells = row.select("td, th") + if (cells.size <= maxOf(chapterCol, dateCol)) continue + + val chapterCell = cells[chapterCol] + val link = chapterCell.selectFirst("a[href]") ?: continue + val parts = link.absUrl("href").toChapterParts() ?: continue + val key = "${parts.bookId}/${parts.chapterId}" + if (key in chaptersMap) continue + + val title = link.text().trim().nullIfEmpty() + ?: chapterCell.text().trim().nullIfEmpty() + val number = parseChapterNumber(title ?: "") + + val dateText = if (dateCol >= 0) cells[dateCol].text().trim() else "" + val uploadDate = parseChapterDate(dateText) ?: fallbackTimestamp ?: 0L + + chaptersMap[key] = MangaChapter( + id = generateUid(key), + title = title, + number = number, + volume = 0, + url = "/read/${parts.bookId}/${parts.chapterId}/1", + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } else { + document.select("a[href*=/read/]").forEach { anchor -> + val parts = anchor.absUrl("href").toChapterParts() ?: return@forEach + val key = "${parts.bookId}/${parts.chapterId}" + if (key in chaptersMap) return@forEach + + val title = anchor.text().trim().nullIfEmpty() + val number = parseChapterNumber(title ?: "") + chaptersMap[key] = MangaChapter( + id = generateUid(key), + title = title, + number = number, + volume = 0, + url = "/read/${parts.bookId}/${parts.chapterId}/1", + scanlator = null, + uploadDate = fallbackTimestamp ?: 0L, + branch = null, + source = source, + ) + } + } + + return chaptersMap.values.sortedBy { it.number } + } + + private fun parseChapterDate(text: String): Long? { + if (text.isBlank()) return null + + val relativeRegex = Regex("""(\d+)\s+(years?|months?|weeks?|days?|hours?|mins?|minutes?|seconds?|sec)\s+ago""", RegexOption.IGNORE_CASE) + relativeRegex.matchEntire(text.trim())?.let { match -> + val number = match.groupValues[1].toInt() + val unit = match.groupValues[2].lowercase() + val cal = Calendar.getInstance() + when { + unit.startsWith("year") -> cal.add(Calendar.YEAR, -number) + unit.startsWith("month") -> cal.add(Calendar.MONTH, -number) + unit.startsWith("week") -> cal.add(Calendar.DAY_OF_MONTH, -number * 7) + unit.startsWith("day") -> cal.add(Calendar.DAY_OF_MONTH, -number) + unit.startsWith("hour") -> cal.add(Calendar.HOUR, -number) + unit.startsWith("min") -> cal.add(Calendar.MINUTE, -number) + unit.startsWith("sec") -> cal.add(Calendar.SECOND, -number) + } + return cal.timeInMillis + } + + val formats = listOf( + SimpleDateFormat("MMMM dd, yyyy", Locale.US), + SimpleDateFormat("MMM dd, yyyy", Locale.US), + SimpleDateFormat("MMM d, yyyy", Locale.US), + SimpleDateFormat("yyyy-MM-dd", Locale.US), + ) + for (fmt in formats) { + try { + return fmt.parse(text)?.time + } catch (_: Exception) {} + } + + return null + } + + private fun parseChapterNumber(text: String): Float { + val match = CHAPTER_NUMBER_REGEX.find(text) ?: return 0f + val main = match.groupValues.getOrNull(1)?.toFloatOrNull() ?: return 0f + val sub = match.groupValues.getOrNull(2)?.toFloatOrNull() ?: 0f + return main + if (sub > 0f) sub / 10f.powDigits() else 0f + } + + private fun Float.powDigits(): Float { + var value = this + var divisor = 1f + while (value >= 1f) { + divisor *= 10f + value /= 10f + } + return divisor + } + + override suspend fun getPages(chapter: MangaChapter): List { + val cached = pagesCache.get(chapter.url) + if (cached != null) return cached + + val parts = chapter.url.toChapterParts() ?: return emptyList() + + val payload = JSONObject() + .put("bookid", parts.bookId) + .put("chapterid", parts.chapterId) + .put("first", true) + + val headers = getRequestHeaders().newBuilder() + .set("X-Requested-With", "XMLHttpRequest") + .set("Content-Type", "application/json;charset=UTF-8") + .build() + + val response = webClient.httpPost( + "https://$domain/callpage".toHttpUrl(), + payload, + headers + ).parseJson() + + val src = response.optString("src") + if (src.isEmpty()) return emptyList() + + val base = "https://$domain/images/gallery/" + val regex = Regex("""$base[^"'\s]+""") + + val seen = HashSet(16) + val pages = ArrayList(16) + + regex.findAll(src).forEach { match -> + val url = match.value + if (!seen.add(url)) return@forEach + + pages.add( + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + ) + } + + val result = pages.toList() + pagesCache.put(chapter.url, result) + + return result + } + + + private fun Element.imageSrc(): String? { + for (attr in IMAGE_ATTRS) { + val value = attr("abs:$attr").nullIfEmpty() ?: continue + return value.substringBefore(' ') + } + return attributes().firstOrNull { attr -> + "src" in attr.key && IMAGE_EXTENSIONS.any { ext -> ext in attr.value.lowercase(Locale.ROOT) } + }?.value?.substringBefore(' ')?.toAbsoluteUrl(domain) + } + + private fun bookThumbnailUrl(bookId: String): String = + "https://$domain/images/gallery/$bookId/thumb" + + private fun String.toBookId(): String? { + val url = toHttpUrlOrNullSafe() ?: return null + val segments = url.pathSegments.filter { it.isNotBlank() } + return if (segments.size >= 2 && segments[0].equals("book", ignoreCase = true)) { + segments[1] + } else null + } + + private fun String.toChapterParts(): ChapterParts? { + val url = runCatching { + if (startsWith("http")) this else "https://$domain/${trimStart('/')}" + }.getOrNull()?.toHttpUrlOrNullSafe() ?: return null + val segments = url.pathSegments.filter { it.isNotBlank() } + if (segments.size < 3 || !segments[0].equals("read", ignoreCase = true)) return null + return ChapterParts(segments[1], segments[2]) + } + + private fun String.toHttpUrlOrNullSafe(): HttpUrl? = + runCatching { toHttpUrl() }.getOrNull() + + private data class ChapterParts( + val bookId: String, + val chapterId: String, + ) + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + + private companion object { + private val DETAIL_KEYS = setOf( + "alt titles", "alternative titles", "author", "authors", + "artist", "artists", "status", "origin", "release year", + "views", "official", "purchase", "genre", "genres" + ) + private val IMAGE_ATTRS = arrayOf("src", "data-src", "data-lazy-src", "srcset") + private val IMAGE_EXTENSIONS = setOf(".jpg", ".png", ".jpeg", ".webp", ".avif", ".tiff") + private val CHAPTER_NUMBER_REGEX = Regex("""(?:chapter|ch\.?)\s*(\d+)(?:\s*[.,-]\s*(\d+))?""", RegexOption.IGNORE_CASE) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt new file mode 100644 index 00000000..00526da8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -0,0 +1,459 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.getStringOrNull + +import androidx.collection.arraySetOf +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Element +import java.util.EnumSet + +private const val ID_SEARCH_PREFIX = "id:" + +private const val webtoons = "webtoons" +private const val canvas = "canvas" + +internal abstract class WebtoonsParser( + context: MangaLoaderContext, + source: MangaParserSource, +) : PagedMangaParser(context, source, 20) { + + override val configKeyDomain = ConfigKey.Domain("webtoons.com") + + private val mobileApiDomain = "m.webtoons.com" + private val staticDomain = "webtoon-phinf.pstatic.net" + + private val contentKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + _root_ide_package_.tsuki.site.all.webtoons to "Webtoons", + _root_ide_package_.tsuki.site.all.canvas to "Canvas", + ), + defaultValue = _root_ide_package_.tsuki.site.all.webtoons, + ) + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36") + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(contentKey) + } + + override suspend fun getPageUrl(page: MangaPage): String { + return page.url.toAbsoluteUrl(staticDomain) + } + + private val languageCode: String + get() = when (val tag = sourceLocale.toLanguageTag()) { + "in" -> "id" + "zh" -> "zh-hant" + else -> tag + } + + private suspend fun fetchEpisodes(titleNo: Long, type: String): List { + val baseUrl = "https://$mobileApiDomain/api/v1/$type/$titleNo/episodes" + val url = if (type == "canvas") { + "$baseUrl?pageSize=99999&readingLanguageCode=$languageCode" + } else { + "$baseUrl?pageSize=99999" + } + val json = webClient.httpGet(url).parseJson() + + val episodeList = json.optJSONObject("result")?.optJSONArray("episodeList") + ?: throw ParseException("No episodes found for title $titleNo", url) + + return episodeList.mapChapters { _, jo -> + val episodeTitle = jo.getStringOrNull("episodeTitle") ?: "" + val episodeNo = jo.getInt("episodeNo") + val viewerLink = jo.getString("viewerLink") + val hasBgm = jo.optBoolean("hasBgm", false) + + MangaChapter( + id = generateUid("$titleNo-$episodeNo"), + title = buildString { + append(org.jsoup.parser.Parser.unescapeEntities(episodeTitle, false)) + append(" (ch. $episodeNo)") + if (hasBgm) append(" ♫") + }, + number = episodeNo.toFloat(), + volume = 0, + url = viewerLink, + uploadDate = jo.getLong("exposureDateMillis"), + branch = null, + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val titleNo = manga.url.toLong() + val detailsUrl = manga.publicUrl.ifBlank { + "https://$domain/$languageCode/drama/placeholder/list?title_no=$titleNo" + } + + val doc = webClient.httpGet(detailsUrl).parseHtml() + + val title = doc.select("meta[property='og:title']").attr("content") + .ifEmpty { doc.select("h1.subj, h3.subj").text().ifEmpty { manga.title } } + + val description = listOf( + doc.select("meta[property='og:description']").attr("content"), + doc.select("#_asideDetail p.summary").text(), + doc.select(".detail_header .summary").text(), + ).firstOrNull { it.isNotBlank() }.orEmpty() + + val coverUrl = doc.select("meta[property=\"og:image\"]").attr("content").let { url -> + if (url.isNotBlank()) url.toAbsoluteUrl(staticDomain) else manga.coverUrl + } + + val author = listOf( + doc.select("meta[property='com-linewebtoon:webtoon:author']").attr("content"), + doc.select(".detail_header .info .author:nth-of-type(1)").firstOrNull()?.ownText(), + doc.select(".author_area").text(), + ).firstOrNull { !it.isNullOrBlank() && it != "null" } + + val artist = listOf( + doc.select(".detail_header .info .author:nth-of-type(2)").firstOrNull()?.ownText(), + doc.select(".author_area").text().takeIf { it != author }, + ).firstOrNull { !it.isNullOrBlank() } ?: author + + val genreElements = doc.select(".detail_header .info .genre").ifEmpty { + doc.select("h2.genre") + } + val genres = genreElements.map { it.text() }.toSet() + + val dayInfo = doc.select("#_asideDetail p.day_info").text().ifEmpty { + doc.select(".day_info").text() + } + val state = when { + dayInfo.contains("UP") || dayInfo.contains("EVERY") || dayInfo.contains("NOUVEAU") -> MangaState.ONGOING + dayInfo.contains("END") || dayInfo.contains("COMPLETED") || dayInfo.contains("TERMINÉ") -> MangaState.FINISHED + else -> null + } + + val type = if ("/canvas/" in detailsUrl) "canvas" else "webtoon" + val chapters = async { fetchEpisodes(titleNo, type) }.await() + + Manga( + id = generateUid(titleNo), + title = title, + altTitles = emptySet(), + url = "$titleNo", + publicUrl = detailsUrl, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + largeCoverUrl = null, + tags = genres.map { genre -> MangaTag(title = genre, key = genre.lowercase(), source = source) }.toSet(), + authors = setOfNotNull(author, artist).filter { it != "null" }.toSet(), + description = description, + state = state, + chapters = chapters, + source = source, + ) + } + + private fun getSortOrderParam(order: SortOrder): String { + return when (order) { + SortOrder.POPULARITY -> "MANA" + SortOrder.RATING -> "LIKEIT" + SortOrder.UPDATED -> "UPDATE" + else -> "MANA" + } + } + + private fun availableTags() = arraySetOf( + MangaTag("Action", "action", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Drama", "drama", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Horror", "horror", source), + MangaTag("Romance", "romance", source), + MangaTag("Sci-Fi", "sf", source), + MangaTag("Slice of Life", "slice_of_life", source), + MangaTag("Sports", "sports", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Historical", "historical", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Superhero", "super_hero", source), + MangaTag("Heartwarming", "heartwarming", source), + MangaTag("Graphic Novel", "graphic_novel", source), + MangaTag("Informative", "tiptoon", source), + ) + + private val genreUrlMap: Map = availableTags().associate { + it.title.lowercase() to it.key + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query.orEmpty() + val currentContent = config[contentKey] ?: _root_ide_package_.tsuki.site.all.webtoons + + var directTitleNo: Long? = null + var directType: String? = null + + when { + query.startsWith("https://") -> { + val url = query.toHttpUrl() + if (!url.host.endsWith("webtoons.com")) { + throw Exception("Unsupported domain: ${url.host}") + } + directTitleNo = url.queryParameter("title_no")?.toLongOrNull() + ?: throw Exception("Missing 'title_no' parameter in URL") + val path = url.pathSegments.filter(String::isNotEmpty) + if (path.size < 2) throw Exception("Invalid URL path") + if (path[0] != languageCode) { + throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") + } + directType = if (path.getOrNull(1) == "canvas") _root_ide_package_.tsuki.site.all.canvas else _root_ide_package_.tsuki.site.all.webtoons + } + + query.startsWith(_root_ide_package_.tsuki.site.all.ID_SEARCH_PREFIX) -> { + val idPart = query.removePrefix(_root_ide_package_.tsuki.site.all.ID_SEARCH_PREFIX) + val parts = idPart.split(":") + val (type, lang, titleNo) = when (parts.size) { + 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[0]}")) + 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[1]}")) + 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[2]}")) + else -> throw Exception("Invalid id search format") + } + if (lang != languageCode) return emptyList() + if (type !in listOf( + _root_ide_package_.tsuki.site.all.webtoons, + _root_ide_package_.tsuki.site.all.canvas + )) { + throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") + } + directTitleNo = titleNo + directType = type + } + + query.toLongOrNull() != null -> { + directTitleNo = query.toLong() + directType = currentContent + } + } + + if (directTitleNo != null && directType != null) { + val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" + val manga = getDetails(Manga( + id = generateUid(directTitleNo), + title = "", + altTitles = emptySet(), + url = directTitleNo.toString(), + publicUrl = detailsUrl, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + )) + return listOf(manga) + } + + val selectedGenre = filter.tags.firstOrNull() + + val url = buildString { + append("https://$domain/$languageCode") + when { + query.isNotBlank() -> { + append("/search") + var cleanQuery = query + var searchType: String? = null + when { + cleanQuery.startsWith("type:originals ") -> { + searchType = "originals" + cleanQuery = cleanQuery.removePrefix("type:originals ").trim() + } + cleanQuery.startsWith("type:canvas ") -> { + searchType = "canvas" + cleanQuery = cleanQuery.removePrefix("type:canvas ").trim() + } + } + if (searchType == null) { + searchType = when (currentContent) { + _root_ide_package_.tsuki.site.all.webtoons -> "originals" + _root_ide_package_.tsuki.site.all.canvas -> "canvas" + else -> null + } + } + if (searchType != null) append("/$searchType") + append("?keyword=${cleanQuery.urlEncoded()}") + if (page > 1) append("&page=$page") + append("&mode=$currentContent") + } + selectedGenre != null -> { + val genreUrlPath = genreUrlMap[selectedGenre.key] ?: selectedGenre.key + append("/genres/$genreUrlPath") + append("?sortOrder=${getSortOrderParam(order)}") + if (page > 1) append("&page=$page") + append("&mode=$currentContent") + } + else -> { + val rankingType = when { + currentContent == _root_ide_package_.tsuki.site.all.canvas -> "canvas" + else -> when (order) { + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "trending" + SortOrder.UPDATED -> "originals" + else -> "popular" + } + } + append("/ranking/$rankingType") + if (page > 1) append("?page=$page") + append("&mode=$currentContent") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".webtoon_list li a, .card_wrap .card_item a") + .map { element -> createMangaFromElement(element, source as MangaParserSource, selectedGenre) } + } + + private fun createMangaFromElement( + element: Element, + source: MangaParserSource, + selectedGenre: MangaTag? = null, + ): Manga { + val href = element.absUrl("href") + val titleNo = extractTitleNoFromUrl(href) + val title = element.select(".title, .card_title").text() + val thumbnailUrl = element.select("img").attr("src") + + return Manga( + id = generateUid(titleNo), + title = title, + altTitles = emptySet(), + url = titleNo.toString(), + publicUrl = href, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = thumbnailUrl.toAbsoluteUrl(staticDomain), + tags = selectedGenre?.let { setOf(it) } ?: emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = null, + source = source, + ) + } + + private fun extractTitleNoFromUrl(url: String): Long { + return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() + ?: throw ParseException("Could not extract title_no from URL: $url", url) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = try { + val absUrl = chapter.url.toAbsoluteUrl(domain) + webClient.httpGet(absUrl).parseHtml() + } catch (e: Exception) { + throw ParseException("Failed to get pages for chapter: ${chapter.title}", chapter.url, e) + } + + val images = doc.select("div#_imageList > img").mapIndexedNotNull { i, element -> + val rawUrl = element.attr("data-url").takeIf { it.isNotBlank() } + ?: element.attr("src").takeIf { it.contains(staticDomain) } + ?: return@mapIndexedNotNull null + + MangaPage( + id = generateUid("${chapter.id}-$i"), + url = rawUrl, + preview = null, + source = source, + ) + }.toMutableList() + + if (images.isEmpty()) { + val motionPages = fetchMotionToonPages(doc) + if (motionPages.isNotEmpty()) { + images.addAll(motionPages) + } + } + + if (images.isEmpty()) { + throw ParseException("No images found in chapter.", chapter.url) + } + return images + } + + private suspend fun fetchMotionToonPages(doc: org.jsoup.nodes.Document): List { + val docString = doc.toString() + val docUrlRegex = Regex("documentURL:.*?'(.*?)'") + val motionToonPathRegex = Regex("jpg:.*?'(.*?)\\{") + + val docUrl = docUrlRegex.find(docString)?.groupValues?.get(1) ?: return emptyList() + val motionToonPath = motionToonPathRegex.find(docString)?.groupValues?.get(1) ?: return emptyList() + + return try { + val motionToonResponse = webClient.httpGet(docUrl).parseJson() + val imagesMap = motionToonResponse.optJSONObject("assets")?.optJSONObject("images") + imagesMap?.let { map -> + map.keySet().filter { it.contains("layer") }.mapIndexedNotNull { i, key -> + val relPath = map.optString(key, null) ?: return@mapIndexedNotNull null + MangaPage( + id = generateUid("motion-$i"), + url = motionToonPath + relPath, + preview = null, + source = source, + ) + } + } ?: emptyList() + } catch (_: Exception) { + emptyList() + } + } + + @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", type = ContentType.MANGA) + class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_EN) + + @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", type = ContentType.MANGA) + class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ID) + + @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es", type = ContentType.MANGA) + class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ES) + + @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", type = ContentType.MANGA) + class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_FR) + + @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", type = ContentType.MANGA) + class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_TH) + + @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", type = ContentType.MANGA) + class Chinese(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ZH) + + @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", type = ContentType.MANGA) + class German(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_DE) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt b/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt new file mode 100644 index 00000000..77be444c --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt @@ -0,0 +1,46 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix + +import org.jsoup.nodes.Element +import java.util.Locale + +@MangaSourceParser("ASMHENTAI", "AsmHentai", type = ContentType.HENTAI) +internal class AsmHentai(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.ASMHENTAI, "asmhentai.com") { + + override val selectGallery = ".preview_item" + override val selectGalleryLink = ".image a" + override val selectGalleryImg = ".image img" + override val pathTagUrl = "/tags/?page=" + override val selectTags = ".tags_page ul.tags" + override val selectAuthor = "div.tags:contains(Artists:) .tag_list a span.tag" + override val idImg = "fimg" + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale.CHINESE, + Locale("tr"), + ), + ) + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".tag")?.html()?.substringBefore("<") ?: it.html().substringBefore("<") + MangaTag( + key = key, + title = name, + source = source, + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt b/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt new file mode 100644 index 00000000..68619e29 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt @@ -0,0 +1,19 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.GalleryParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) +internal class BuonDua(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( + "buondua.com", + "buondua.us", + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt b/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt new file mode 100644 index 00000000..dbc978ff --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt @@ -0,0 +1,90 @@ +package tsuki.site.all.hentais + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.SortOrder + +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import java.util.EnumSet +import java.util.Locale + + +@MangaSourceParser("HENTAIENVY", "HentaiEnvy", type = ContentType.HENTAI) +internal class HentaiEnvy(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIENVY, "hentaienvy.com", pageSize = 24) { + override val selectGalleryLink = "a" + override val selectGalleryTitle = "div.title" + override val selectTags = ".tags_items" + override val selectTag = ".gt_right_tags ul:contains(Tags:)" + override val selectAuthor = ".gt_right_tags ul:contains(Artists:) a" + override val selectLanguageChapter = ".gt_right_tags ul:contains(Languages:) a" + override val idImg = "fimg" + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.JAPANESE, + Locale.CHINESE, + Locale("es"), + Locale("ru"), + Locale("ko"), + Locale.GERMAN, + Locale("pt"), + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("/search/?s_key=") + append(query?.urlEncoded()) + append("&") + } + + else -> { + if (filter.tags.isNotEmpty()) { + if (filter.locale != null) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) + } + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + if (order == SortOrder.POPULARITY) { + append("/popular") + } + append("/?") + } + } else if (filter.locale != null) { + val locale = filter.locale + append("/language/") + append(locale?.toLanguagePath()) + append("/?") + } else { + append("/?") + } + } + } + append("page=") + append(page) + + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt new file mode 100644 index 00000000..0d25ec78 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt @@ -0,0 +1,136 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.SortOrder + +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("HENTAIFOX", "HentaiFox", type = ContentType.HENTAI) +internal class HentaiFox(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIFOX, "hentaifox.com") { + override val selectGallery = ".lc_galleries .thumb, .related_galleries .thumb" + override val pathTagUrl = "/tags/popular/pag/" + override val selectTags = ".list_tags" + override val selectTag = "ul.tags" + override val selectLanguageChapter = "ul.languages a.tag_btn" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + ) + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("/search/?q=") + append(query?.urlEncoded()) + if (page > 1) { + append("&page=") + append(page.toString()) + } + } + + else -> { + if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { + append("/search/?q=") + append(buildQuery(filter.tags, filter.locale)) + if (page > 1) { + append("&page=") + append(page.toString()) + } + + if (order == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else if (filter.tags.isNotEmpty()) { + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + } + append("/") + if (order == SortOrder.POPULARITY) { + append("popular/") + } + + if (page > 1) { + append("/pag/") + append(page.toString()) + append("/") + } + } else if (filter.locale != null) { + val locale = filter.locale + append("/language/") + append(locale?.toLanguagePath()) + append("/") + if (order == SortOrder.POPULARITY) { + append("popular/") + } + + if (page > 1) { + append("/pag/") + append(page.toString()) + append("/") + } + } else { + if (page > 2) { + append("/pag/") + append(page.toString()) + append("/") + } else if (page > 1) { + append("/page/") + append(page.toString()) + append("/") + } + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".list_tag")?.text() ?: it.html().substringBefore("<") + MangaTag( + key = key, + title = name, + source = source, + ) + } + + private fun buildQuery(tags: Collection, language: Locale?): String { + val joiner = StringUtil.StringJoiner(" ") + tags.forEach { tag -> + joiner.add(tag.key) + } + language?.let { lc -> + joiner.add(lc.toLanguagePath()) + } + return joiner.complete() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt new file mode 100644 index 00000000..fadc5a10 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt @@ -0,0 +1,798 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseBytes +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.runCatchingCancellable +import tsuki.util.selectFirstOrThrow +import tsuki.util.splitByWhitespace +import tsuki.util.toAbsoluteUrl +import tsuki.util.toCamelCase +import tsuki.util.toTitleCase +import tsuki.util.urlDecode + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedList +import java.util.Locale +import kotlin.math.min + +@OptIn(ExperimentalUnsignedTypes::class) +@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) +internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) { + override val configKeyDomain = ConfigKey.Domain("hitomi.la") + + private val cdnDomain = "gold-usergeneratedcontent.net" + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private val ltnBaseUrl get() = "https://ltn.$cdnDomain" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.POPULARITY_YEAR, + ) + + private val localeMap: Map = mapOf( + Locale.forLanguageTag("id") to "indonesian", + Locale.forLanguageTag("jv") to "javanese", + Locale.forLanguageTag("ca") to "catalan", + Locale.forLanguageTag("ceb") to "cebuano", + Locale.forLanguageTag("cs") to "czech", + Locale.forLanguageTag("da") to "danish", + Locale.forLanguageTag("de") to "german", + Locale.forLanguageTag("et") to "estonian", + Locale.ENGLISH to "english", + Locale.forLanguageTag("es") to "spanish", + Locale.forLanguageTag("eo") to "esperanto", + Locale.forLanguageTag("fr") to "french", + Locale.forLanguageTag("it") to "italian", + Locale.forLanguageTag("hi") to "hindi", + Locale.forLanguageTag("hu") to "hungarian", + Locale.forLanguageTag("pl") to "polish", + Locale.forLanguageTag("pt") to "portuguese", + Locale.forLanguageTag("vi") to "vietnamese", + Locale.forLanguageTag("tr") to "turkish", + Locale.forLanguageTag("ru") to "russian", + Locale.forLanguageTag("uk") to "ukrainian", + Locale.forLanguageTag("ar") to "arabic", + Locale.KOREAN to "korean", + Locale.CHINESE to "chinese", + Locale.JAPANESE to "japanese", + ) + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableLocales = localeMap.keys, + ) + + private fun Locale?.getSiteLang(): String = when (this) { + null -> "all" + else -> localeMap[this] ?: "all" + } + + private suspend fun fetchAvailableTags(): Set = coroutineScope { + ('a'..'z').map { alphabet -> + async { + val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() + + doc.select(".posts > li").mapNotNull { element -> + val num = + element.ownText().let { + Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 + } + + if (num > 100) { + val url = element.selectFirst("a") + val href = + url?.attrAsRelativeUrl("href") + ?: return@mapNotNull null + + MangaTag( + title = url.ownText().toTagTitle(), + key = href.tagUrlToTag(), + source = source, + ) + } else { + null + } + } + } + }.awaitAll().flatten().toSet() + } + + private var cachedSearchIds: List = emptyList() + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List = when { + filter.query.isNullOrEmpty() -> { + if (filter.tags.isEmpty()) { + when (order) { + SortOrder.POPULARITY_TODAY -> { + getGalleryIDsFromNozomi( + "popular", + "today", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_WEEK -> { + getGalleryIDsFromNozomi( + "popular", + "week", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_MONTH -> { + getGalleryIDsFromNozomi( + "popular", + "month", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_YEAR -> { + getGalleryIDsFromNozomi( + "popular", + "year", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + else -> { + getGalleryIDsFromNozomi(null, "index", filter.locale.getSiteLang(), offset.nextOffsetRange()) + } + } + } else { + if (offset == 0) { + cachedSearchIds = + hitomiSearch( + filter.tags.joinToString(" ") { it.key }, + order, + filter.locale.getSiteLang(), + ).toList() + } + cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) + } + } + + else -> { + if (offset == 0) { + cachedSearchIds = hitomiSearch(filter.query ?: "", order).toList() + } + cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) + } + }.toMangaList() + + private fun Int.nextOffsetRange(): LongRange { + val bytes = this * 4L + return bytes.until(bytes + 100L) + } + + private suspend fun hitomiSearch( + query: String, + sortByPopularity: SortOrder = SortOrder.UPDATED, + language: String = "all", + ): Set = + coroutineScope { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .splitByWhitespace() + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList() + val negativeTerms = LinkedList() + + for (term in terms) { + if (term.startsWith("-")) { + negativeTerms.push(term.removePrefix("-")) + } else if (term.isNotBlank()) { + positiveTerms.push(term) + } + } + + val positiveResults = positiveTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val negativeResults = negativeTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val results = when { + sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) + sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) + sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) + sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) + sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) + else -> emptySet() + }.toMutableSet() + + fun filterPositive(newResults: Set) { + when { + results.isEmpty() -> results.addAll(newResults) + else -> results.retainAll(newResults) + } + } + + fun filterNegative(newResults: Set) { + results.removeAll(newResults) + } + + // positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + // negative results + negativeResults.forEach { + filterNegative(it.await()) + } + + results + } + + // search.js + private suspend fun getGalleryIDsForQuery( + query: String, + language: String = "all", + ): Set { + query.replace("_", " ").let { + if (it.indexOf(':') > -1) { + val sides = it.split(":") + val ns = sides[0] + var tag = sides[1] + + var area: String? = ns + var lang = language + when (ns) { + "female", "male" -> { + area = "tag" + tag = it + } + + "language" -> { + area = null + lang = tag + tag = "index" + } + } + + return getGalleryIDsFromNozomi(area, tag, lang) + } + + val key = hashTerm(it) + val node = getGalleryNodeAtAddress(0) + val data = bSearch(key, node) ?: return emptySet() + + return getGalleryIDsFromData(data) + } + } + + private suspend fun getGalleryIDsFromData(data: Pair): Set { + val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" + val (offset, length) = data + require(length in 1..100000000) { + "Length $length is too long" + } + + val inbuf = getRangedResponse(url, offset.until(offset + length)) + + val galleryIDs = mutableSetOf() + + val buffer = + ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + + val numberOfGalleryIDs = buffer.int + + val expectedLength = numberOfGalleryIDs * 4 + 4 + + require(numberOfGalleryIDs in 1..10000000) { + "number_of_galleryids $numberOfGalleryIDs is too long" + } + require(inbuf.size == expectedLength) { + "inbuf.byteLength ${inbuf.size} != expected_length $expectedLength" + } + + for (i in 0.until(numberOfGalleryIDs)) + galleryIDs.add(buffer.int) + + return galleryIDs + } + + private suspend fun bSearch( + key: UByteArray, + node: Node, + ): Pair? { + fun compareArrayBuffers( + dv1: UByteArray, + dv2: UByteArray, + ): Int { + val top = min(dv1.size, dv2.size) + + for (i in 0.until(top)) { + if (dv1[i] < dv2[i]) { + return -1 + } else if (dv1[i] > dv2[i]) { + return 1 + } + } + + return 0 + } + + fun locateKey( + key: UByteArray, + node: Node, + ): Pair { + for (i in node.keys.indices) { + val cmpResult = compareArrayBuffers(key, node.keys[i]) + + if (cmpResult <= 0) { + return Pair(cmpResult == 0, i) + } + } + + return Pair(false, node.keys.size) + } + + fun isLeaf(node: Node): Boolean { + for (subnode in node.subNodeAddresses) + if (subnode != 0L) { + return false + } + + return true + } + + if (node.keys.isEmpty()) { + return null + } + + val (there, where) = locateKey(key, node) + if (there) { + return node.datas[where] + } else if (isLeaf(node)) { + return null + } + + val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) + return bSearch(key, nextNode) + } + + private suspend fun getGalleryIDsFromNozomi( + area: String?, + tag: String, + language: String, + range: LongRange? = null, + ): Set { + val nozomiAddress = when (area) { + null -> "$ltnBaseUrl/$tag-$language.nozomi" + else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" + } + + val bytes = getRangedResponse(nozomiAddress, range) + val nozomi = mutableSetOf() + + val arrayBuffer = ByteBuffer + .wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + return nozomi + } + + private val galleriesIndexVersion = suspendLazy { + webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() + } + + private data class Node( + val keys: List, + val datas: List>, + val subNodeAddresses: List, + ) + + private fun decodeNode(data: ByteArray): Node { + val buffer = ByteBuffer + .wrap(data) + .order(ByteOrder.BIG_ENDIAN) + + val uData = data.toUByteArray() + + val numberOfKeys = buffer.int + val keys = ArrayList() + + for (i in 0.until(numberOfKeys)) { + val keySize = buffer.int + + check(keySize in 1..32) { "Invalid key size $keySize" } + + keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) + buffer.position(buffer.position() + keySize) + } + + val numberOfDatas = buffer.int + val datas = ArrayList>() + + for (i in 0.until(numberOfDatas)) { + val offset = buffer.long + val length = buffer.int + + datas.add(Pair(offset, length)) + } + + val numberOfSubNodeAddresses = 16 + 1 + val subNodeAddresses = ArrayList() + + for (i in 0.until(numberOfSubNodeAddresses)) { + val subNodeAddress = buffer.long + subNodeAddresses.add(subNodeAddress) + } + + return Node(keys, datas, subNodeAddresses) + } + + private suspend fun getGalleryNodeAtAddress(address: Long): Node { + val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index" + + val nodedata = getRangedResponse(url, address.until(address + 464)) + + return decodeNode(nodedata) + } + + private suspend fun getRangedResponse( + url: String, + range: LongRange? = null, + ): ByteArray { + val rangeHeaders = when (range) { + null -> Headers.headersOf() + else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") + } + + return webClient.httpGet(url, rangeHeaders).parseBytes() + } + + private fun hashTerm(term: String): UByteArray { + return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray() + } + + private fun sha256(data: ByteArray): ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) + } + + private suspend fun Collection.toMangaList(): List = coroutineScope { + map { id -> + async { + runCatchingCancellable { + val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { + val baseUri = it.request.url.toString() + val html = it.parseRaw() + Jsoup.parse(rewriteTnPaths(html), baseUri) + } + + Manga( + id = generateUid(id.toString()), + title = doc.selectFirstOrThrow("h1").text(), + url = id.toString(), + coverUrl = + "https:" + + doc.selectFirstOrThrow("picture > img") + .attr("data-src"), + publicUrl = + doc.selectFirstOrThrow("h1 > a") + .attrAsRelativeUrl("href") + .toAbsoluteUrl(domain), + authors = emptySet(), + tags = emptySet(), + contentRating = ContentRating.ADULT, + rating = RATING_UNKNOWN, + altTitles = emptySet(), + state = null, + source = source, + ) + }.getOrNull() + } + }.awaitAll().filterNotNull() + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + val author = + json.optJSONArray("artists") + ?.mapJSON { it.getString("artist").toCamelCase() } + ?.joinToString() + + return manga.copy( + title = json.getString("title"), + largeCoverUrl = + json.getJSONArray("files").getJSONObject(0).let { + val hash = it.getString("hash") + val imageId = imageIdFromHash(hash) + val subDomain = 'a' + subdomainOffset(imageId) + + "https://${subDomain}tn.$cdnDomain/webpbigtn/${thumbPathFromHash(hash)}/$hash.webp" + }, + authors = setOfNotNull(author), + publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), + tags = + buildSet + { + json.optJSONArray("characters") + ?.mapToTags("character") + ?.let(::addAll) + json.optJSONArray("tags") + ?.mapToTags("tag") + ?.let(::addAll) + json.optJSONArray("artists") + ?.mapToTags("artist") + ?.let(::addAll) + json.optJSONArray("parodys") + ?.mapToTags("parody") + ?.let(::addAll) + json.optJSONArray("groups") + ?.mapToTags("group") + ?.let(::addAll) + }, + chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + url = manga.url, + title = json.getStringOrNull("title"), + scanlator = json.getString("type").toTitleCase(), + number = 1f, + volume = 0, + branch = json.getString("language_localname"), + source = source, + uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), + ), + ), + ) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) + + private fun JSONArray.mapToTags(key: String): Set { + val tags = ArraySet(length()) + mapJSON { + MangaTag( + title = + it.getString(key).toCamelCase().let { title -> + if (it.getStringOrNull("female")?.toIntOrNull() == 1) { + "$title ♀" + } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { + "$title ♂" + } else { + title + } + }, + key = it.getString("url").tagUrlToTag(), + source = source, + ).let(tags::add) + } + return tags + } + + private fun String.tagUrlToTag(): String { + val urlContent = this.split("/") + val ns = urlContent[1] + val tag = + urlContent[2] + .substringBeforeLast("-") + .urlDecode() + .replace(" ", "_") + + return if (tag.split(":")[0] in listOf("female", "male")) { + tag + } else { + "$ns:$tag" + } + } + + override suspend fun getRelatedManga(seed: Manga): List { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + + // any better way to get List from this json? + return json.getJSONArray("related").let { + 0.until(it.length()).map { i -> it.getInt(i) } + }.toMangaList() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + + return json.getJSONArray("files").mapJSON { image -> + val hash = image.getString("hash") + val commonId = commonImageId() + val imageId = imageIdFromHash(hash) + val subDomain = subdomainOffset(imageId) + 1 + val thumbSubdomain = 'a' + subdomainOffset(imageId) + MangaPage( + id = generateUid(hash), + url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", + preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", + source = source, + ) + } + } + +// / ---> + + private var scriptLastRetrieval: Long = -1L + private val mutex = Mutex() + private var subdomainOffsetDefault = 0 + private val subdomainOffsetMap = mutableMapOf() + private var commonImageId = "" + + private suspend fun refreshScript() = mutex.withLock { + if (scriptLastRetrieval == -1L || (scriptLastRetrieval + 60000) < System.currentTimeMillis()) { + val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() + + subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt() + val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt() + + subdomainOffsetMap.clear() + Regex("case (\\d+):").findAll(ggScript).forEach { + val case = it.groupValues[1].toInt() + subdomainOffsetMap[case] = o + } + + commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1] + + scriptLastRetrieval = System.currentTimeMillis() + } + } + + // m <-- gg.js + private suspend fun subdomainOffset(imageId: Int): Int { + refreshScript() + return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault + } + + // b <-- gg.js + private suspend fun commonImageId(): String { + refreshScript() + return commonImageId + } + + // s <-- gg.js + private fun imageIdFromHash(hash: String): Int { + val match = Regex("(..)(.)$").find(hash) + return match!!.groupValues.let { it[2] + it[1] }.toInt(16) + } + + // real_full_path_from_hash <-- common.js + private fun thumbPathFromHash(hash: String): String { + return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") + } + + // rewrite_tn_paths <-- common.js + private suspend fun rewriteTnPaths(html: String): String { + val thumbUrlRegex = Regex( + """(?//)(?[a-z0-9.-]+\.(?:hitomi\.la|${Regex.escape(cdnDomain)}))/(?(?:avif|webp)?(?:small)?(?:big|small|medium)tn/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}\.(?:webp|avif|gif|png|jpe?g))""", + ) + + var resultHtml = html + thumbUrlRegex.findAll(html).forEach { matchResult -> + val originalUrl = matchResult.value + val groups = matchResult.groups + + val pathAfterHost = groups["pathAfterHost"]?.value ?: return@forEach + val newTnSubdomain = subdomainFromURL(originalUrl, "tn") + val correctedUrl = "${groups["protocol"]!!.value}$newTnSubdomain.$cdnDomain/$pathAfterHost" + + if (originalUrl != correctedUrl) { + resultHtml = resultHtml.replace(originalUrl, correctedUrl) + } + } + return resultHtml + } + + private suspend fun subdomainFromURL(url: String, base: String?): String { + val resultSubdomain = base ?: "b" + + // This regex extracts the last 3 hex characters from the hash in the URL + // The hash is 64 characters, so we look for the 61st character onward + val hashRegex = Regex("""/([0-9a-f]{61}[0-9a-f]{3})[./]""") + val fullHashMatch = hashRegex.find(url) + ?: // If no hash is found, default to "a" + base (typically "atn") + return "a$resultSubdomain" + + val fullHash = fullHashMatch.groupValues[1] + + val lastThreeChars = fullHash.takeLast(3) + val lastDigit = lastThreeChars.last() + val lastTwoDigits = lastThreeChars.take(2) + + val imageId = "$lastDigit$lastTwoDigits".toIntOrNull(16) + + return if (imageId != null) { + ('a' + subdomainOffset(imageId)).toString() + resultSubdomain + } else { + "a$resultSubdomain" + } + } + + private fun String.toTagTitle(): String { + return toCamelCase() + .replace("♂", "(male)") + .replace("♀", "(female)") + } +} diff --git a/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt b/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt new file mode 100644 index 00000000..1f41ff03 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt @@ -0,0 +1,248 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Element +import java.util.ArrayList +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI) +internal class ImHentai(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.IMHENTAI, 20) { + + override val availableSortOrders: Set = + EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING) + + override val configKeyDomain = ConfigKey.Domain("imhentai.xxx") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale("es"), + Locale.FRENCH, + Locale("kr"), + Locale.GERMAN, + Locale("ru"), + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.COMICS, + ContentType.IMAGE_SET, + ContentType.ARTIST_CG, + ContentType.GAME_CG, + ), + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val url = buildString { + append("https://") + append(domain) + append("/search/?page=") + append(page.toString()) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("&key=") + append(query?.urlEncoded()) + } + + else -> { + + if (filter.tags.isNotEmpty()) { + append("&key=") + filter.tags.joinTo(this, separator = ",") { it.key } + } + + var types = "&m=1&d=1&w=1&i=1&a=1&g=1" + if (filter.types.isNotEmpty()) { + types = "&m=0&d=0&w=0&i=0&a=0&g=0" + filter.types.forEach { + when (it) { + ContentType.MANGA -> types = types.replace("&m=0", "&m=1") + ContentType.DOUJINSHI -> types = types.replace("&d=0", "&d=1") + ContentType.COMICS -> types = types.replace("&w=0", "&w=1") + ContentType.IMAGE_SET -> types = types.replace("&i=0", "&i=1") + ContentType.ARTIST_CG -> types = types.replace("&a=0", "&a=1") + ContentType.GAME_CG -> types = types.replace("&g=0", "&g=1") + else -> {} + } + } + } + append(types) + + + var lang = "&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1" + filter.locale?.let { + lang = "&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0" + lang = lang.replace("${it.language}=0", "${it.language}=1") + } + append(lang) + + when (order) { + SortOrder.UPDATED -> append("<=1&pp=0") + SortOrder.POPULARITY -> append("<=0&pp=1") + SortOrder.RATING -> append("<=0&pp=0") + else -> append("<=1&pp=0") + } + } + } + + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.galleries div.thumb").map { div -> + val a = div.selectFirstOrThrow(".inner_thumb a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = a.selectFirst("img")?.src(), + title = div.selectFirst(".caption")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + //Tags are deliberately reduced because there are too many and this slows down the application. + //only the most popular ones are taken. + private suspend fun fetchAvailableTags(): Set { + return coroutineScope { + (1..3).map { page -> + async { fetchTagsPage(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } + + private suspend fun fetchTagsPage(page: Int): Set { + val url = "https://$domain/tags/popular/?page=$page" + val root = webClient.httpGet(url).parseHtml() + return root.parseTags() + } + + private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet { + val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') + MangaTag( + key = href, + title = it.selectFirstOrThrow("h3.list_tag").text().toTitleCase(sourceLocale), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val author = doc.selectFirst("li:contains(Artists) a.tag")?.ownTextOrNull() + manga.copy( + tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet { + val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') + MangaTag( + key = href, + title = it.ownText().toTitleCase(sourceLocale), + source = source, + ) + }, + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = null, + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0, + branch = doc.selectFirst("li:contains(Language) a.tag")?.ownTextOrNull()?.toTitleCase(sourceLocale), + source = source, + ), + ), + ) + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.related") + return root.select("div.thumb").mapNotNull { div -> + val a = div.selectFirstOrThrow(".inner_thumb a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = a.selectFirst("img")?.src(), + title = div.selectFirst(".caption")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() + val baseImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img") + val baseUrl = baseImg.selectFirstParentOrThrow("a").attrAsRelativeUrl("href").replace("/1/", + "/$/" + ) + val baseThumbUrl = baseImg.src()?.replace("/1t.", $$"/$t.") + val pages = ArrayList(totalPages) + repeat(totalPages) { i -> + val url = baseUrl.replace("$", (i + 1).toString()) + pages.add( + MangaPage( + id = generateUid(url), + url = url, + preview = baseThumbUrl?.replace("$", (i + 1).toString()), + source = source, + ), + ) + } + return pages + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val img = doc.body().requireElementById("gimg") + return img.requireSrc() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt b/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt new file mode 100644 index 00000000..01df3f33 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt @@ -0,0 +1,35 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.network.CommonHeaders +import tsuki.parsers.GalleryParser + +import okhttp3.Interceptor +import okhttp3.Response + +@MangaSourceParser("KIUTAKU", "Kiutaku", type = ContentType.OTHER) +internal class Kiutaku(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.KIUTAKU, "kiutaku.com") { + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url.toString() + + val headers = if (url.contains("wp-content")) { + request.headers.newBuilder() + .removeAll(CommonHeaders.REFERER) + .build() + } else { + request.headers + } + + val newRequest = request.newBuilder() + .headers(headers) + .build() + + return chain.proceed(newRequest) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt b/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt new file mode 100644 index 00000000..da7863a5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt @@ -0,0 +1,15 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.parsers.MadaraParser + +import java.util.Locale + +@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) +internal class MangaCrazy(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { + override val sourceLocale: Locale = Locale.ENGLISH +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt b/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt new file mode 100644 index 00000000..a0b4eb28 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt @@ -0,0 +1,152 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import java.util.EnumSet + +@MangaSourceParser("MISSKON", "MissKon", type = ContentType.HENTAI) +internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MISSKON, 24) { + + override val configKeyDomain = ConfigKey.Domain("misskon.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities(isSearchSupported = true) + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/page/$page/") + append("?s=") + append((filter.query?.urlEncoded() ?: "")) + } + order == SortOrder.POPULARITY -> { + append("/top3/") + } + else -> { + append("/page/$page") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("article.item-list").map { article -> + val titleEl = article.selectFirst(".post-box-title")!! + val href = titleEl.selectFirst("a")?.attrAsRelativeUrl("href") + ?: article.parseFailed("Cannot find manga link") + + Manga( + id = generateUid(href), + title = titleEl.text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = article.selectFirst(".post-thumbnail img")?.absUrl("data-src").orEmpty(), + tags = setOf(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val postInnerEl = doc.selectFirst("article > .post-inner")!! + + return manga.copy( + tags = postInnerEl.select(".post-tag > a").mapToSet { a -> + MangaTag( + key = a.text().lowercase(), + title = a.text(), + source = source + ) + }, + chapters = listOf( + MangaChapter( + id = manga.id, + title = "Oneshot", // 1 album, idk + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0L, + branch = null, + source = source + ) + ) + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val basePageUrl = doc.selectFirst("link[rel=canonical]")?.absUrl("href") + ?: chapter.url.toAbsoluteUrl(domain) + + val pages = mutableListOf() + val pageLinks = doc.select("div.post-inner div.page-link:nth-child(1) .post-page-numbers") + + if (pageLinks.isEmpty()) { + // Single page gallery + return doc.select("div.post-inner > div.entry > p > img") + .mapNotNull { img -> img.absUrl("data-src") } + .mapIndexed { i, url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + // Multi-page gallery + pageLinks.forEachIndexed { index, pageEl -> + val pageDoc = when (index) { + 0 -> doc + else -> { + val url = "$basePageUrl${pageEl.text()}/" + webClient.httpGet(url).parseHtml() + } + } + + pages.addAll( + pageDoc.select("div.post-inner > div.entry > p > img") + .mapNotNull { img -> img.absUrl("data-src") } + .map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + ) + } + return pages + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt b/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt new file mode 100644 index 00000000..2f4ccaf8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt @@ -0,0 +1,235 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MULTPORN", "Multporn", type = ContentType.HENTAI) +internal class Multporn(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MULTPORN, 42) { + + override val configKeyDomain = ConfigKey.Domain("multporn.net") + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add(CommonHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36") + .add(CommonHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=UTF-8") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + init { + setFirstPage(0) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableLocales = setOf( + Locale("en"), + Locale("de"), + Locale("ru"), + Locale("zh"), + Locale("es"), + ), + availableContentTypes = EnumSet.of( + ContentType.COMICS, + ContentType.HENTAI, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search?search_api_views_fulltext=") + val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> + part.urlEncoded() + } + append(encodedQuery) + append("&undefined=Search") + append("&page=$page") + } + + filter.tags.isNotEmpty() -> { + val tag = filter.tags.first() + append("/category/") + append(tag.key) + + append("?sort_by=") + append( + when (order) { + SortOrder.NEWEST -> "created" + else -> "title" // default + } + ) + + append("&page=0,") + append(page) + } + + else -> { + append("/new") + append("?type=") + if (filter.types.isNotEmpty()) { + filter.types.oneOrThrowIfMany()?.let { + append( + when (it) { + ContentType.COMICS -> "1" + ContentType.HENTAI -> "2" + else -> "All" // all + }, + ) + } + } else append("All") + + + filter.locale?.let { + append("&language=") + append( + when (it) { + Locale("en") -> "1" + Locale("de") -> "2" + Locale("ru") -> "3" + Locale("zh") -> "4" + Locale("es") -> "5" + else -> "All" + } + ) + } + + append("&field_user_discription_value=All") + + append("&sort_by=") + append( + when (order) { + SortOrder.NEWEST -> "created&sort_order=DESC" + SortOrder.NEWEST_ASC -> "created&sort_order=ASC" + SortOrder.UPDATED -> "changed&sort_order=DESC" + SortOrder.UPDATED_ASC -> "changed&sort_order=ASC" + else -> "created&sort_order=DESC" // default + } + ) + + append("&undefined=Apply") + append("&page=$page") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".masonry-item").map { div -> + val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") + val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() + Manga( + id = generateUid(href), + title = div.select(".views-field-title").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = coverUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val authors = (doc.select(".field:has(.field-label:contains(Author:)) .links a").map { it.text() } + + parseUnlabelledAuthorNames(doc)).distinct() + + val tags = listOf("Tags", "Section", "Characters") + .flatMap { type -> + doc.select(".field:has(.field-label:contains($type:)) .links a").map { it.text() } + } + .distinct() + .map { tag -> + MangaTag( + title = tag, + key = tag.lowercase().replace(" ", "_"), + source = source, + ) + }.toSet() + + val isOngoing = doc.select(".field .links a").any { it.text() == "Ongoings" } + + return manga.copy( + authors = authors.toSet(), + tags = tags, + description = buildString { + append("Pages: ") + append(doc.select(".jb-image img").size) + append("\n\n") + doc.select(".field:has(.field-label:contains(Section:)) .links a").joinTo(this, prefix = "Section: ") { it.text() } + doc.select(".field:has(.field-label:contains(Characters:)) .links a").joinTo(this, prefix = "\n\nCharacters: ") { it.text() } + }, + state = if (isOngoing) MangaState.ONGOING else MangaState.FINISHED, + chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + title = "Oneshot", + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ) + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".jb-image img").map { img -> + val url = img.attrAsAbsoluteUrl("src") + .replace("/styles/juicebox_2k/public", "") + .substringBefore("?") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseUnlabelledAuthorNames(document: Document): List { + val authorClasses = listOf( + "field-name-field-author", + "field-name-field-authors-gr", + "field-name-field-img-group", + "field-name-field-hentai-img-group", + "field-name-field-rule-63-section" + ) + return authorClasses.flatMap { className -> + document.select(".$className a").map { it.text().trim() } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt new file mode 100644 index 00000000..fdc94f7f --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt @@ -0,0 +1,1079 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.MangaParserAuthProvider + +import tsuki.model.* +import tsuki.util.* + +import org.json.JSONObject +import java.util.EnumSet +import java.util.Locale + + +// removed first server, it has always problems +private const val TWO_IMG_SERVER = "2" +private const val THREE_IMG_SERVER = "3" +private const val FOUR_IMG_SERVER = "4" + +@MangaSourceParser("NHENTAI", "NHentai.net", type = ContentType.HENTAI) +internal class NHentaiParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.NHENTAI, 25), + MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain("nhentai.net") + + private var imageServer: String = "" + private var thumbServer: String = "" + + private var displayFullTitle: Boolean = true + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, // latest (default) + SortOrder.POPULARITY, // popular (all time) + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + _root_ide_package_.tsuki.site.all.hentais.TWO_IMG_SERVER to "First server", + _root_ide_package_.tsuki.site.all.hentais.THREE_IMG_SERVER to "Second server", + _root_ide_package_.tsuki.site.all.hentais.FOUR_IMG_SERVER to "Third server", + ), + defaultValue = _root_ide_package_.tsuki.site.all.hentais.TWO_IMG_SERVER, + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + private val selectedServer: String + get() = config[preferredServerKey].toString() + + companion object { + val LANG_MAP = mapOf( + "en" to "english", + "ja" to "japanese", + "zh" to "chinese", + ) + } + + private val nhTagMap: Map by lazy { + val raw = """ +2937=big breasts +35762=sole female +35763=sole male +8010=group +14283=anal +19440=lolicon +24201=stockings +10314=schoolgirl uniform +13720=nakadashi +29859=blowjob +8378=glasses +20905=full color +32341=shotacon +27553=rape +15658=bondage +23895=yaoi +27473=mosaic censorship +13989=ahegao +22942=incest +21712=males only +1207=milf +19018=dark skin +22945=double penetration +25614=paizuri +20035=x-ray +779=futanari +23237=tankoubon +21572=multi-work series +20525=defloration +14971=sex toys +8653=netorare +3735=swimsuit +19954=yuri +15348=ffm threesome +8368=full censorship +15408=femdom +29224=impregnation +29013=dilf +85295=twintails +31044=collar +85288=ponytail +24380=pantyhose +9260=cheating +28031=sister +16828=hairy +31880=bbm +30555=big penis +15782=crossdressing +31775=tentacles +27384=mind break +19175=bikini +8739=story arc +30473=muscle +24102=lactation +7752=schoolboy uniform +20617=mind control +9083=big ass +29023=tomgirl +81774=kemonomimi +1590=sweating +9162=masturbation +7256=mmf threesome +28550=teacher +190=maid +8693=uncensored +19899=exhibitionism +6343=pregnant +8050=females only +6817=unusual pupils +25871=lingerie +10988=anthology +20282=footjob +15853=mother +15785=harem +14072=huge breasts +30035=gender bender +1643=kissing +130025=anal intercourse +1033=handjob +12824=condom +31386=catgirl +10476=urination +3666=garter belt +26130=fingering +81707=beauty mark +22079=drugs +105833=gloves +4435=gag +25601=small breasts +5820=piercing +12695=prostitution +16228=demon girl +7155=cunnilingus +22950=tanlines +832=elf +31012=blindfold +17773=kimono +2820=scat +29182=blackmail +23132=bunny girl +32484=stomach deformation +2515=virginity +27063=filming +7142=bbw +21989=inflation +88846=horns +104227=tail +26953=bukkake +28800=bloomers +25050=gyaru +24676=rimjob +23632=big areolae +16533=sleeping +73750=bald +18567=monster +35972=sole dickgirl +18328=thigh high boots +5810=strap-on +29565=school swimsuit +32996=deepthroat +370=business suit +7550=monster girl +1067=inseki +50585=webtoon +12523=bestiality +27697=leotard +30645=dick growth +29631=inverted nipples +29366=tomboy +24412=bodysuit +15492=scanmark +9406=enema +35970=dickgirl on dickgirl +29399=daughter +18613=military +11941=replaced +6525=nurse +9661=cervix penetration +33129=slave +4573=corruption +5529=urethra insertion +10542=snuff +683=squirting +51399=crotch tattoo +122908=very long hair +7838=magical girl +24726=apron +23183=breast expansion +20074=latex +28426=hairy armpits +27217=guro +31285=fox girl +106119=no penetration +24764=drunk +9990=prostate massage +35968=dickgirl on male +2956=old man +32752=shibari +6900=miko +2153=wings +706=birth +10794=breast feeding +14069=ryona +25822=smell +5357=humiliation +5962=spanking +2531=transformation +21538=bike shorts +31101=incomplete +32745=chikan +16236=shemale +36957=bisexual +26952=tall girl +25663=oppai loli +7995=big nipples +32602=fisting +106733=hair buns +1088=bdsm +21283=masked face +15225=blowjob face +2633=leg lock +27378=artbook +35971=male on dickgirl +27112=tiara +107705=facial hair +24933=eyepatch +4549=torture +30206=tribadism +1037=oni +89056=hidden sex +13136=facesitting +3391=nun +25766=gokkun +5200=pegging +17531=cosplaying +28521=voyeurism +19479=nipple fuck +17349=tracksuit +22221=blood +50505=oyakodon +50486=tail plug +560=twins +23965=chloroform +15425=vore +25457=possession +129668=eye-covering bang +24984=orgasm denial +144644=extraneous ads +28589=hotpants +17752=foot licking +32282=piss drinking +19390=cousin +32589=feminization +11376=body modification +20362=gyaru-oh +28778=large insertions +27720=smegma +10811=double vaginal +3614=triple penetration +3455=chastity belt +2452=scar +31319=yandere +7354=amputee +28335=giantess +26848=waitress +28349=cbt +24967=sumata +104893=vtuber +8516=emotionless sex +26380=demon +17591=robot +17801=solo action +13640=frottage +25996=gaping +23035=aunt +23967=huge penis +31846=body writing +25744=cheerleader +24708=cowgirl +25085=swinging +18322=brother +101724=leash +10354=milking +97795=pixie cut +11089=body swap +32224=eggs +10606=pasties +3947=onahole +14573=tall man +10604=dog +14362=low lolicon +15242=lab coat +4935=farting +13468=shimapan +5620=double anal +14138=freckles +50390=josou seme +15119=dog girl +93324=fishnets +22025=prolapse +15471=asphyxiation +21774=human pet +31337=kunoichi +15712=eyemask +30126=big clit +92409=thick eyebrows +109360=cumflation +7208=catboy +31687=randoseru +24529=bride +19561=big balls +24450=chinese dress +121738=focus anal +22967=diaper +29347=miniguy +29001=parasite +25296=armpit licking +6220=orc +7546=witch +30895=sunglasses +7372=corset +28119=nose hook +8429=machine +7684=armpit sex +14516=wolf girl +15045=niece +13882=tutor +8391=public use +30811=christmas +104245=small penis +266=sundress +17501=phimosis +17800=tickling +25794=widow +7288=vomit +1215=unusual teeth +72471=dickgirls only +107503=soushuuhen +138044=exposed clothing +1352=slime +31986=age regression +23917=long tongue +24115=angel +114993=shimaidon +13722=moral degeneration +26898=age progression +27120=selfcest +7577=vampire +17676=ghost +88103=clothed female nude male +13515=coach +141098=nipple stimulation +9116=unbirth +5936=time stop +18420=all the way through +72139=clothed paizuri +27530=ball sucking +16518=coprophagia +28869=stuck in wall +2527=bandages +24621=insect +11399=metal armor +106006=large tattoo +3843=fundoshi +20120=multiple paizuri +8400=goblin +129321=mesuiki +124610=mouth mask +10693=dougi +31371=mecha girl +21450=minigirl +10685=double blowjob +118056=petplay +20789=policewoman +3031=underwater +31173=first person perspective +78262=shaved head +19064=pubic stubble +14280=bunny boy +25949=gothic lolita +23463=wrestling +16947=horse +11247=skinsuit +11073=living clothes +30786=watermarked +23073=assjob +52826=dark sclera +107478=drill hair +23225=non-h +109930=domination loss +20170=poor grammar +138200=gender change +16759=artistcg +80978=nudity only +15749=oil +30176=petrification +25848=human cattle +559=ttf threesome +14010=snake girl +11276=multiple penises +90671=original +18024=touhou project +1841=kantai collection +35605=fate grand order +20925=the idolmaster +972=granblue fantasy +78245=azur lane +17137=neon genesis evangelion +3185=love live +391=girls und panzer +11219=pokemon +15021=sailor moon +4505=mahou shoujo lyrical nanoha +128408=blue archive +10222=fate stay night +27431=to love-ru +13159=naruto +123503=genshin impact +3984=sword art online +3603=street fighter +22174=one piece +16285=puella magi madoka magica +91195=princess connect +12232=my hero academia +3163=king of fighters +26172=k-on +7259=touken ranbu +19080=code geass +37544=love live sunshine +17077=cardcaptor sakura +27547=the melancholy of haruhi suzumiya +13508=final fantasy vii +10954=shingeki no kyojin +25430=vocaloid +32687=free +4577=toheart2 +22146=dead or alive +20025=gochuumon wa usagi desu ka +8485=dragon ball z +5037=bleach +3218=bakemonogatari +12624=ore no imouto ga konna ni kawaii wake ga nai +37109=kono subarashii sekai ni syukufuku o +4369=monster hunter +127065=hololive +74788=girls frontline +24886=fate kaleid liner prisma illya +6999=toaru kagaku no railgun +22032=boku wa tomodachi ga sukunai +18350=ragnarok online +21674=dragon quest iii +14345=ojamajo doremi +7832=darkstalkers +24135=ah my goddess +32394=samurai spirits +1283=queens blade +16639=haikyuu +13924=yu-gi-oh +79467=kimetsu no yaiba +18238=danganronpa +26336=yu-gi-oh zexal +16984=persona 4 +18569=kuroko no basuke +1910=smile precure +30587=sakura taisen +16166=mahou sensei negima +12285=ranma 12 +8470=infinite stratos +32363=toaru majutsu no index +22708=saki +8708=to heart +108082=arknights +16707=detective conan +22210=guilty gear +947=gundam seed destiny +22677=tenchi muyo +23429=pretty cure +18512=strike witches +31027=lucky star +7408=league of legends +394=love hina +23201=kanon +27704=amagami +127052=nijisanji +70802=kemono friends +52098=persona 5 +22215=super robot wars +27567=hayate no gotoku +35251=osomatsu-san +7633=pripara +34823=ensemble stars +37914=re zero kara hajimeru isekai seikatsu +74918=bang dream +15041=martian successor nadesico +24783=dragon ball +120519=love live nijigasaki high school idol club +2803=love plus +5085=senki zesshou symphogear +28474=zero no tsukaima +15197=gundam build fighters +15427=dragon quest iv +1163=rozen maiden +23859=yu-gi-oh arc-v +75023=dragon quest xi +2112=dungeon ni deai o motomeru no wa machigatteiru darou ka +36418=voiceroid +28281=mitsudomoe +11624=the legend of zelda +14694=fullmetal alchemist +16847=dragon quest v +2497=urusei yatsura +5671=tengen toppa gurren lagann +22754=amagi brilliant park +20606=tsukihime +5165=gundam build fighters try +4114=macross frontier +20763=inazuma eleven +14550=sister princess +19083=jojos bizarre adventure +21052=fate hollow ataraxia +29922=teitoku +51810=gudao +16643=producer +13848=reimu hakurei +25125=asuka langley soryu +17279=sakuya izayoi +10496=patchouli knowledge +37739=shielder +3206=shinji ikari +38068=gran +4675=sanae kochiya +21779=rei ayanami +14040=fate testarossa +3870=flandre scarlet +23902=remilia scarlet +21688=atago +11373=marisa kirisame +35128=kashima +17154=sakura kinomoto +31462=satori komeiji +30080=kaga +10802=alice margatroid +17017=aya shameimaru +17862=yukari yakumo +5340=shimakaze +18935=nanoha takamachi +18896=shirou emiya +16555=rin tosaka +16130=rito yuuki +15890=reisen udongein inaba +7724=takao +27060=jeanne darc +78989=jeanne alter +7718=naruto uzumaki +5337=nami +22975=chun-li +17502=illyasviel von einzbern +20111=tifa lockhart +21131=youmu konpaku +18026=kazuto kirigaya +92923=shikikan +29856=saber +71442=minamoto no raikou +1843=asuna yuuki +51419=gudako +7488=mai shiranui +9835=koishi komeiji +16916=kasumi +30026=maki nishikino +143975=sensei +26906=izuku midoriya +37275=scathach +7696=momiji inubashiri +38039=astolfo +27794=mikoto misaka +20062=hamakaze +78285=artoria pendragon +34860=katsuki bakugou +3328=homura akemi +37687=djeeta +32200=suzuya +21108=rin shibuya +35964=nico yazawa +27494=levi ackerman +609=eren jaeger +11920=sakura haruno +20427=sailor mercury +24714=chino kafuu +31456=mikan yuuki +866=koyomi araragi +12149=kyousuke kousaka +277=haruka nanase +19926=haruna +3763=haruhi suzumiya +26427=mio akiyama +25439=hinata hyuga +17811=ran yakumo +14857=kongou +18548=kotori minami +32364=rider +15641=madoka kaname +2613=hong meiling +491=makoto tachibana +20702=koakuma +15315=tomoyo daidouji +10730=shigure +14265=touma kamijou +80311=bb +4203=mami tomoe +37706=kazuma satou +33070=umi sonoda +27172=yuyuko saigyouji +3353=yuuka kazami +2078=nagato +6311=arisu tachibana +647=belldandy +9274=maya +24889=sena kashiwazaki +15125=golden darkness +6555=sailor jupiter +25695=mika jougasaki +50929=shuten douji +33077=sailor mars +8293=minami nitta +7451=lelouch vi britannia +389=rika jougasaki +22469=prinz eugen +16108=azusa nakano +12812=tenryuu +7311=ami mizuno +6642=byakuren hijiri +7097=suwako moriya +19172=miki hoshii +9657=ayane +29433=c.c. +25220=sakura matou +14499=tsunade +10665=tenshi hinanai +16564=miku hatsune +29190=kallen stadtfeld +3312=kirino kousaka +1234=yuki nagato +26261=ranma saotome +19002=rin kaenbyou +12748=nico robin +32765=rin matsuoka +4241=fumika sagisawa +1729=tamaki kousaka +23997=ruri gokou +29684=sailor venus +19160=nitori kawashiro +27302=uzuki shimamura +23216=android 18 +8489=hibiki +7333=suguha kirigaya +1267=kodaka hasegawa +2345=morrigan aensland +9371=yamato +26087=inazuma +27532=archer +26587=miho nishizumi +12346=utsuho reiuji +37108=megumin +22407=takane shijou +15914=sasuke uchiha +2774=kyouko sakura +80930=abigail williams +81288=gudao | ritsuka fujimaru +73756=nightingale +6109=eri ayase +27492=akagi +17899=sakura kasugano +32137=cirno +11760=yui kotegawa +75029=eli ayase +11740=sailor moon +49158=narmaya +29693=ikazuchi +126586=aether +20918=iori minase +24832=misato katsuragi +2883=kasen ibara +6932=souji okita +28555=tamamo-no-mae +14428=kokoa hoto +26783=taihou +12763=rumia +401=nakoruru +72475=musashi miyamoto +23122=maho nishizumi +29188=eirin yagokoro +466=usagi tsukino +29638=kyon +15995=makoto kino +11744=amatsukaze +6175=cammy white +30331=ichika orimura +23473=mikuru asahina +28807=ruri hoshino +2572=hatate himekaidou +15291=chen +23386=fujiwara no mokou +9237=shoukaku +28763=tewi inaba +23851=gilgamesh +10672=aqua +9702=ro-500 +31074=keine kamishirasawa +32443=charlotte dunois +2196=sayaka miki +1645=zuikaku +5925=akatsuki +4196=hestia +33171=shiho nishizumi +19534=hayate yagami +79507=belfast +12433=kaede takagaki +12872=warrior +8170=len kagamine +50415=rem +14409=momoka sakurai +2211=mari illustrious makinami +99075=kokkoro +1907=rei hino +15651=miyu edelfelt +26169=musashi +8053=lum +50596=you watanabe +9883=kagami hiiragi +24509=darjeeling +11992=lala satalin deviluke +32683=hachiman hikigaya +31076=kuroko shirai +20836=red saber +12902=isuzu sento +10379=bianca whitaker +16181=nozomi toujou +27774=bismarck +28219=yui hirasawa +1271=momo velia deviluke +49852=subaru natsuki +5918=shinobu oshino +28056=link +25605=rangiku matsumoto +35313=cagliostro +18453=hero +75102=nozomi tojo +20722=mutsu +29170=yuma tsukumo +9486=nue houjuu +33049=ritsuko akizuki +23626=murakumo +20323=tsumugi kotobuki +16566=ritsu tainaka +14016=yuu narukami +11609=yoko ritona +107011=chloe von einzbern +52132=riko sakurauchi +32114=onpu segawa +11924=kagerou imaizumi + """.trimIndent() + raw.lineSequence() + .mapNotNull { line -> + val parts = line.split('=') + if (parts.size == 2 && parts[0].isNotBlank() && parts[1].isNotBlank()) { + parts[0] to parts[1] + } else null + }.toMap() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = nhTagMap.map { (id, name) -> + MangaTag( + title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, + key = id, // numeric tag ID + source = source, + ) + }.toSet(), + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale.CHINESE, + ), + ) + + private suspend fun ensureNhConfig() { + if (imageServer.isBlank() || thumbServer.isBlank()) { + try { + val json = webClient.httpGet("https://$domain/api/v2/config").parseJson() + val imgServers = json.getJSONArray("image_servers") + .let { (0 until it.length()).map { i -> it.getString(i) } } + val thbServers = json.getJSONArray("thumb_servers") + .let { (0 until it.length()).map { i -> it.getString(i) } } + imageServer = imgServers[selectedServer.toInt() - 1] + thumbServer = thbServers[selectedServer.toInt() - 1] + } catch (_: Exception) { + imageServer = "https://i$selectedServer.nhentai.net" + thumbServer = "https://t$selectedServer.nhentai.net" + } + } + } + + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + ensureNhConfig() + + // Direct ID search + val directId = extractGalleryId(filter.query) + if (directId != null) { + return try { + listOf(fetchMangaById(directId)) + } catch (_: Exception) { + emptyList() + } + } + + val selectedTag = filter.tags.firstOrNull() + if (selectedTag != null) { + val url = "https://$domain/api/v2/galleries/tagged?tag_id=${selectedTag.key}&page=$page" + val json = webClient.httpGet(url).parseJson() + val items = json.getJSONArray("result") + return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } + } + + val lang = filter.locale?.let { loc -> + LANG_MAP[loc.language] + } + + val url = when { + // Search with language + lang != null -> { + val sort = when (order) { + SortOrder.POPULARITY -> "popular" + else -> "date" + } + "https://$domain/api/v2/search?query=language:$lang&sort=$sort&page=$page" + } + // Latest + order == SortOrder.UPDATED -> "https://$domain/api/v2/galleries?page=$page" + // Popular (all / today / week / month) + else -> { + val sort = when (order) { + SortOrder.POPULARITY_TODAY -> "popular-today" + SortOrder.POPULARITY_WEEK -> "popular-week" + SortOrder.POPULARITY_MONTH -> "popular-month" + SortOrder.POPULARITY -> "popular" + else -> "popular" + } + "https://$domain/api/v2/search?query=%22%22&sort=$sort&page=$page" + } + } + + val json = webClient.httpGet(url).parseJson() + val items = json.getJSONArray("result") + return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } + } + + private fun extractGalleryId(query: String?): Int? { + if (query.isNullOrBlank()) return null + val trimmed = query.trim() + return when { + trimmed.startsWith("id:") -> trimmed.removePrefix("id:").toIntOrNull() + trimmed.all(Char::isDigit) -> trimmed.toIntOrNull() + else -> null + } + } + + private suspend fun fetchMangaById(id: Int): Manga { + ensureNhConfig() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + return json.toMangaDetails() + } + + override suspend fun getDetails(manga: Manga): Manga { + ensureNhConfig() + val id = manga.url.removeSurrounding("/g/", "/").toInt() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + return json.toMangaDetails().copy( + coverUrl = manga.coverUrl?.ifBlank { + "${thumbServer}/${json.getJSONObject("cover").optString("path")}" + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + ensureNhConfig() + val id = chapter.url.removeSurrounding("/g/", "/").toInt() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + val pagesArray = json.getJSONArray("pages") + return (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + val path = pageObj.getString("path") + MangaPage( + id = generateUid(path), + url = "$imageServer/$path", + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val titleObj = optJSONObject("title") + val rawTitle = titleObj?.optString("english") + ?: titleObj?.optString("pretty") + ?: optString("english_title") + ?: optString("japanese_title") + ?: "Gallery $id" + + val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() + val thumbPath = optJSONObject("thumbnail")?.optString("path") + ?: optString("thumbnail").takeIf { !it.isNullOrBlank() } + ?: "galleries/$id/thumb.webp" + + return Manga( + id = generateUid("/g/$id/"), + title = title, + altTitles = emptySet(), + url = "/g/$id/", + publicUrl = "https://$domain/g/$id/", + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = "$thumbServer/$thumbPath", + tags = extractTags(), + state = null, + authors = emptySet(), + source = source, + ) + } + + private fun JSONObject.toMangaDetails(): Manga { + val id = getInt("id") + val titleObj = optJSONObject("title") + val rawTitle = titleObj?.optString("english") + ?: titleObj?.optString("japanese") + ?: titleObj?.optString("pretty") + ?: "Gallery $id" + val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() + + val thumbPath = getJSONObject("thumbnail").getString("path") + val coverPath = getJSONObject("cover").optString("path", thumbPath) + val numPages = optInt("num_pages") + val uploadDate = optLong("upload_date") * 1000 + + val tags = extractTags() + val artists = getTagsOfType("artist") + val groups = getTagsOfType("group") + val author = artists.joinToString(", ").ifBlank { groups.joinToString(", ") } + + var langName: String? + var translated = false + val languageCandidates = mutableListOf() + + for (tag in tags) { + val parts = tag.key.split(":", limit = 2) + if (parts.size == 2 && parts[0] == "language") { + val name = parts[1] + when { + name.equals("translated", ignoreCase = true) -> translated = true + else -> languageCandidates.add(name) + } + } + } + + langName = languageCandidates.firstOrNull { it.equals("chinese", ignoreCase = true) } + ?: languageCandidates.firstOrNull { it.equals("japanese", ignoreCase = true) } + ?: languageCandidates.firstOrNull { it.equals("english", ignoreCase = true) } + ?: languageCandidates.firstOrNull() + + val description = buildString { + langName?.let { + val displayLang = it.replaceFirstChar { c -> c.uppercase() } + append(if (translated) "$displayLang (translated)" else displayLang) + append("\n") + } + append("Pages: $numPages") + } + + return Manga( + id = generateUid("/g/$id/"), + title = title, + altTitles = listOfNotNull(titleObj?.optString("japanese")).toSet(), + url = "/g/$id/", + publicUrl = "https://$domain/g/$id/", + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = "$thumbServer/$thumbPath", + largeCoverUrl = "$imageServer/$coverPath", + tags = tags, + state = null, + authors = setOfNotNull(author.ifBlank { null }), + description = description, + chapters = listOf( + MangaChapter( + id = generateUid("/g/$id/"), + title = title, + number = 1f, + volume = 0, + url = "/g/$id/", + scanlator = groups.joinToString(", ").ifBlank { null }, + uploadDate = uploadDate, + branch = null, + source = source, + ) + ), + source = source, + ) + } + + private fun JSONObject.extractTags(): Set { + val tagsArray = optJSONArray("tags") ?: return emptySet() + return (0 until tagsArray.length()).map { i -> + val tag = tagsArray.getJSONObject(i) + val name = tag.getString("name") + MangaTag( + title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, + key = "${tag.getString("type")}:$name", + source = source, + ) + }.toSet() + } + + private fun JSONObject.getTagsOfType(type: String): List { + val tagsArray = optJSONArray("tags") ?: return emptyList() + return (0 until tagsArray.length()).mapNotNull { i -> + val tag = tagsArray.getJSONObject(i) + if (tag.getString("type") == type) tag.getString("name") else null + } + } + + private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") + private fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() + + override val authUrl: String get() = "https://$domain" + override suspend fun isAuthorized(): Boolean = true + override suspend fun getUsername(): String = "" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt new file mode 100644 index 00000000..7c1d4a8b --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt @@ -0,0 +1,187 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.ErrorMessages +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("NHENTAIXXX", "NHentai.xxx", type = ContentType.HENTAI) +internal class NHentaiXXXParser(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.NHENTAIXXX, "nhentai.xxx", 25) { + + val supportedImageFormats = setOf("jpg", "webp", "jpeg", "png", "bmp") + + override val selectGallery = "div.galleries_box .gallery_item, #related-container .gallery_item" + override val selectGalleryLink = "a" + override val selectGalleryTitle = ".caption" + override val pathTagUrl = "/tags/popular?page=" + override val selectTitle = "h1" + override val selectTags = "div.tags_items" + override val selectTag = ".tags:contains(Tags)" + override val selectAuthor = ".tags:contains(Artists) span.tag_name" + override val selectLanguageChapter = + ".tags:contains(Languages) a:not([href=\"/language/translated/\"]) span.tag_name" + override val idImg = "fimg" + + override val availableSortOrders: Set = + EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf(Locale.ENGLISH, Locale.JAPANESE, Locale.CHINESE), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + + val tags = filter.tags + val lang = filter.locale + if (tags.isNotEmpty() && lang != null) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) + } + + when { + lang != null -> { + append("/language/") + append(lang.toLanguagePath()) + append("/?") + } + + else -> { + append("/search/?key=") + + val joiner = StringUtil.StringJoiner("+") + tags.forEach { tag -> + joiner.add(tag.title) + } + + if (!query.isNullOrEmpty()) { + joiner.add(query.urlEncoded()) + } + append(joiner.complete()) + } + } + + when (order) { + SortOrder.POPULARITY, SortOrder.RELEVANCE -> append("&sort=popular") + SortOrder.UPDATED -> {} + else -> {} + } + + if (page > 1) { + append("&page=") + append(page.toString()) + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(doc: Document): List { + return doc.select(selectGallery).map { div -> + val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(selectGalleryTitle).text().cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = div.selectFirstOrThrow(selectGalleryImg).src(), + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = null, + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() + val firstPageUrl = doc.requireElementById(idImg).requireSrc() + return (1..totalPages).map { + val url = replacePageNumber(firstPageUrl, it) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + return if (hasError(page.url)) { + findAlternativeUrl(page.url) ?: page.url + } else { + page.url + } + } + + private suspend fun hasError(url: String): Boolean { + return try { + webClient.httpHead(url) + false + } catch (e: Exception) { + true + } + } + + private suspend fun findAlternativeUrl(originalUrl: String): String? { + val lastSegment = originalUrl.substringAfterLast("/") + val oldFormat = lastSegment.substringAfterLast(".", "") + + return supportedImageFormats.firstNotNullOfOrNull { format -> + val newUrl = originalUrl.replace(".$oldFormat", ".$format") + try { + webClient.httpHead(newUrl) + newUrl + } catch (e: Exception) { + null // Skip errors and continue checking other formats + } + } + } + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".tag_name")?.text() ?: it.text() + MangaTag( + key = key, + title = name.toTitleCase(sourceLocale), + source = source, + ) + } + + private fun replacePageNumber(url: String, newPageNumber: Int): String { + val lastSegment = url.substringAfterLast("/") + val extension = lastSegment.substringAfterLast(".", "") + + return if (extension.isNotEmpty()) { + url.substringBeforeLast("/") + "/$newPageNumber.$extension" + } else { + url.substringBeforeLast("/") + "/$newPageNumber" + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt b/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt new file mode 100644 index 00000000..10cfbc5b --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt @@ -0,0 +1,14 @@ +package tsuki.site.all.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) +internal class Novelcrow(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { + override val tagPrefix = "comic-genre/" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt new file mode 100644 index 00000000..7e126923 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt @@ -0,0 +1,275 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("TEAMXNOVEL", "TeamXNovel", "ar") +internal class TeamXNovel(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.TEAMXNOVEL, 10) { + + override val availableSortOrders: Set = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override val configKeyDomain = ConfigKey.Domain("olympustaff.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/?search=") + append(query.urlEncoded()) + append("&page=") + append(page) + } + + else -> { + + if (order == SortOrder.UPDATED) { + if (filter.tags.isNotEmpty() || filter.demographics.isNotEmpty()) { + throw IllegalArgumentException("Updated sorting does not support other sorting filters") + } + append("/?page=") + append(page.toString()) + } else { + append("/series?page=") + append(page.toString()) + + filter.tags.oneOrThrowIfMany()?.let { + append("&genre=") + append(it.key) + } + + filter.types.forEach { + append("&type=") + append( + when (it) { + ContentType.MANGA -> "مانجا ياباني" + ContentType.MANHWA -> "مانهوا كورية" + ContentType.MANHUA -> "مانها صيني" + else -> "" + }, + ) + } + + filter.states.oneOrThrowIfMany()?.let { + append("status=") + append( + when (it) { + MangaState.ONGOING -> "مستمرة" + MangaState.FINISHED -> "مكتمل" + MangaState.ABANDONED -> "متوقف" + else -> "مستمرة" + }, + ) + } + } + } + } + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.listupd .bs .bsx").ifEmpty { + doc.select("div.post-body .box") + }.map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(".tt, h3").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = div.selectFirstOrThrow("img").src()?.replace("thumbnail_", ""), + tags = emptySet(), + state = when (div.selectFirst(".status")?.text()) { + "مستمرة" -> MangaState.ONGOING + "مكتمل" -> MangaState.FINISHED + "متوقف" -> MangaState.ABANDONED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/series").parseHtml() + return doc.requireElementById("select_genre").select("option").mapToSet { + MangaTag( + key = it.attr("value"), + title = it.text().toTitleCase(sourceLocale), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val mangaUrl = manga.url.toAbsoluteUrl(domain) + val maxPageChapterSelect = doc.select(".pagination .page-item a") + var maxPageChapter = 1 + if (!maxPageChapterSelect.isNullOrEmpty()) { + maxPageChapterSelect.map { + val i = it.attr("href").substringAfterLast("=").toInt() + if (i > maxPageChapter) { + maxPageChapter = i + } + } + } + return manga.copy( + state = when (doc.selectFirstOrThrow(".full-list-info:contains(الحالة:) a").text()) { + "مستمرة" -> MangaState.ONGOING + "مكتمل" -> MangaState.FINISHED + "متوقف" -> MangaState.ABANDONED + else -> null + }, + tags = doc.select(".review-author-info a").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast("="), + title = a.text(), + source = source, + ) + }, + description = doc.selectFirstOrThrow(".review-content").html(), + chapters = run { + if (maxPageChapter == 1) { + parseChapters(doc) + } else { + coroutineScope { + val result = ArrayList(parseChapters(doc)) + result.ensureCapacity(result.size * maxPageChapter) + (2..maxPageChapter).map { i -> + async { + loadChapters(mangaUrl, i) + } + }.awaitAll() + .flattenTo(result) + result + } + } + }.reversed(), + ) + } + + private suspend fun loadChapters(baseUrl: String, page: Int): List { + return parseChapters(webClient.httpGet("$baseUrl?page=$page").parseHtml().body()) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd hh:mm:ss", Locale.getDefault()) + + private fun parseChapters(root: Element): List { + // Try current website structure first (enhanced-chapters-grid) + val chapters = root.select("div.enhanced-chapters-grid a.chapter-link").ifEmpty { + // Fallback to potential alternative structures + root.select("div.chapter-card a").ifEmpty { + root.select("div.eplister ul a").ifEmpty { + root.select("#chapter-contact .eplister ul li a") + } + } + } + + return chapters.map { element -> + val url = element.attrAsRelativeUrl("href") + + // Extract chapter number and title from current structure + val chpNum = element.select(".chapter-number").text() + val chpTitle = element.select(".chapter-title").text() + + // Fallback to old structure if current selectors are empty + val finalChpNum = chpNum.ifEmpty { + element.select("div.chapter-info div.chapter-number").text().ifEmpty { + element.select("div.epl-num").text() + } + } + val finalChpTitle = chpTitle.ifEmpty { + element.select("div.chapter-info div.chapter-title").text().ifEmpty { + element.select("div.epl-title").text() + } + } + + val title = when { + finalChpNum.isNotBlank() -> "$finalChpNum - $finalChpTitle" + else -> finalChpTitle + } + + // Extract date from current structure, fallback to old structures + val dateText = element.select(".chapter-date").text().ifEmpty { + element.select("div.chapter-info div.chapter-date").text().ifEmpty { + element.select("div.epl-date").text() + } + }.replace(Regex("^\\s*\\S+\\s+"), "") // Remove icon text if present + + // Try to extract chapter number from data attribute or URL + val chapterNumber = element.select("div.chapter-card").attr("data-number").toFloatOrNull() + ?: finalChpNum.filter { it.isDigit() || it == '.' }.toFloatOrNull() + ?: url.substringAfterLast('/').toFloatOrNull() + ?: 0f + + MangaChapter( + id = generateUid(url), + title = title, + number = chapterNumber, + volume = 0, + url = url, + scanlator = null, + uploadDate = dateFormat.parseSafe(dateText), + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select(".image_list img, .image_list canvas").map { img -> + val url = when { + img.hasAttr("src") -> img.requireSrc().toRelativeUrl(domain) + else -> img.attrAsRelativeUrl("data-src") + } + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/AquaManga.kt b/src/main/kotlin/tsuki/site/en/AquaManga.kt new file mode 100644 index 00000000..307f99a7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/AquaManga.kt @@ -0,0 +1,200 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("AQUAMANGA", "AquaManga", "en") +internal class AquaManga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.AQUAMANGA, "aquareader.org", 20) { + + override val withoutAjax = true + override val stylePage = "" + + override val selectChapter = ".aqua-ch-item" + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = setOf( + MangaTag("Academy", "academy", source), + MangaTag("Action", "action", source), + MangaTag("Adaptation", "adaptation", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Crime", "crime", source), + MangaTag("Cultivation", "cultivation", source), + MangaTag("Delinquents", "delinquents", source), + MangaTag("Demons", "demons", source), + MangaTag("Drama", "drama", source), + MangaTag("Dungeons", "dungeons", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Game", "game", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Magic", "magic", source), + MangaTag("Manga", "manga", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Military", "military", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Murim", "murim", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Necromancer", "necromancer", source), + MangaTag("Ninja", "ninja", source), + MangaTag("Office Workers", "office-workers", source), + MangaTag("OP-MC", "op-mc", source), + MangaTag("Overpowered", "overpowered", source), + MangaTag("Philosophical", "philosophical", source), + MangaTag("Post-Apocalyptic", "post-apocalyptic", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Rebirth", "rebirth", source), + MangaTag("Regression", "regression", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Returner", "returner", source), + MangaTag("Revenge", "revenge", source), + MangaTag("Romance", "romance", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Slice-of-Life", "slice-of-life", source), + MangaTag("Sports", "sports", source), + MangaTag("Super Power", "super-power", source), + MangaTag("Superhero", "superhero", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Survival", "survival", source), + MangaTag("System", "system", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tower", "tower", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Vampire", "vampire", source), + MangaTag("Video Games", "video-games", source), + MangaTag("Villainess", "villainess", source), + MangaTag("Virtual Reality", "virtual-reality", source), + MangaTag("Voilence", "voilence", source), + MangaTag("Webcomic", "webcomic", source), + MangaTag("Wuxia", "wuxia", source), + MangaTag("Zombies", "zombies", source), + ), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirstOrThrow(".aqua-series-info__title").text() + val thumbnail = doc.selectFirstOrThrow(".aqua-series-cover__img").requireSrc() + val description = doc.selectFirst(".aqua-series-synopsis")?.html().orEmpty() + val status = doc.selectFirst(".aqua-series-meta__status")?.text() + val genres = doc.select(".aqua-series-genre-pill").map { it.text() }.toSet() + val author = doc.selectFirst(".aqua-series-info__creator-value a")?.ownText() + val artist = doc.selectFirst(".aqua-series-info__creator-value a")?.ownText() + + val tags = genres.mapTo(mutableSetOf()) { MangaTag(title = it, key = it, source = source) } + val statusText = status?.lowercase().orEmpty() + val state = when (statusText) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + in abandoned -> MangaState.ABANDONED + in paused -> MangaState.PAUSED + else -> null + } + + val chapters = getChapters(manga, doc) + return manga.copy( + title = title, + coverUrl = thumbnail, + description = description, + tags = tags, + state = state, + authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet(), + chapters = chapters, + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List { + return doc.select(selectChapter).mapChapters(reversed = true) { i, el -> + val a = el.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val name = el.selectFirstOrThrow(".aqua-ch-item__name").text() + val dateText = el.selectFirst(".aqua-ch-item__time")?.text()?.trim() + MangaChapter( + id = generateUid(href), + url = href + stylePage, + title = name, + number = i + 1f, + volume = 0, + uploadDate = parseChapterDate(dateText), + source = source, + scanlator = null, + branch = null, + ) + } + } + + private fun parseChapterDate(text: String?): Long { + if (text.isNullOrBlank()) return 0L + + val relativeRegex = Regex( + """(\d+)\s+(years?|months?|weeks?|days?|hours?|mins?|minutes?|sec(?:onds?)?)\s+ago""", + RegexOption.IGNORE_CASE + ) + val match = relativeRegex.matchEntire(text) + if (match != null) { + val number = match.groupValues[1].toInt() + val unit = match.groupValues[2].lowercase() + val cal = Calendar.getInstance() + when { + unit.startsWith("year") -> cal.add(Calendar.YEAR, -number) + unit.startsWith("month") -> cal.add(Calendar.MONTH, -number) + unit.startsWith("week") -> cal.add(Calendar.DAY_OF_MONTH, -number * 7) + unit.startsWith("day") -> cal.add(Calendar.DAY_OF_MONTH, -number) + unit.startsWith("hour") -> cal.add(Calendar.HOUR, -number) + unit.startsWith("min") -> cal.add(Calendar.MINUTE, -number) + unit.startsWith("sec") -> cal.add(Calendar.SECOND, -number) + } + return cal.timeInMillis + } + + val formats = listOf( + SimpleDateFormat("MMM d, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, + SimpleDateFormat("MMM dd, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, + SimpleDateFormat("MMMM d, yyyy", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") }, + ) + for (fmt in formats) { + try { + return fmt.parse(text)?.time ?: 0L + } catch (_: Exception) {} + } + + return 0L + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/AsuraScan.kt b/src/main/kotlin/tsuki/site/en/AsuraScan.kt new file mode 100644 index 00000000..d9ecc879 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/AsuraScan.kt @@ -0,0 +1,354 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.toJSONArrayOrNull +import tsuki.util.json.toJSONObjectOrNull + +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import java.util.TreeMap + +@MangaSourceParser("ASURASCANS", "AsuraComic", "en") +internal class AsuraScansParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ASURASCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("asurascans.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL_DESC, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + MangaState.UPCOMING, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = "https://$domain/browse".toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + + if (!filter.query.isNullOrBlank()) { + addQueryParameter("search", filter.query) + } + + if (filter.tags.isNotEmpty()) { + addQueryParameter("genres", filter.tags.joinToString(",") { it.key }) + } + + filter.states.oneOrThrowIfMany()?.let { + addQueryParameter( + "status", + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.ABANDONED -> "dropped" + MangaState.PAUSED -> "hiatus" + else -> throw IllegalArgumentException("$it not supported") + }, + ) + } + + filter.types.oneOrThrowIfMany()?.let { + addQueryParameter( + "types", + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> throw IllegalArgumentException("$it not supported") + }, + ) + } + + addQueryParameter( + "sort", + when (order) { + SortOrder.RATING -> "rating" + SortOrder.UPDATED -> "" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL_DESC -> "desc" + SortOrder.ALPHABETICAL -> "asc" + else -> "update" + }, + ) + }.build() + val doc = webClient.httpGet(url).parseHtml() + return doc.select("#series-grid .series-card").mapNotNull { card -> + val link = card.selectFirst("a[href*=/comics/]") ?: return@mapNotNull null + val href = link.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = card.selectFirst("img")?.src(), + title = card.selectFirst("h3")?.text()?.trim().orEmpty(), + altTitles = emptySet(), + rating = card.selectFirst("div.absolute.top-2.right-2 span")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = when (card.select("div.p-3 span").lastOrNull()?.text()?.trim()?.lowercase(Locale.ENGLISH)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "dropped" -> MangaState.ABANDONED + "coming soon" -> MangaState.UPCOMING + else -> null + }, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + private val availableTags by lazy(LazyThreadSafetyMode.NONE) { + ASURA_GENRES.mapTo(LinkedHashSet(ASURA_GENRES.size)) { title -> + MangaTag( + key = title.toAsuraGenreKey(), + title = title, + source = source, + ) + } + } + + private val tagMap by lazy(LazyThreadSafetyMode.NONE) { + availableTags.associateByTo(LinkedHashMap(availableTags.size)) { + it.title.lowercase(Locale.ENGLISH) + } + } + + private val regexDate = """(\d+)(st|nd|rd|th)""".toRegex() + private val chapterDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.US) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val selectTag = doc.select("div[class^=space] > div.flex > button.text-white") + val tags = selectTag.mapNotNullToSet { element -> + val title = element.text().trim().nullIfEmpty() ?: return@mapNotNullToSet null + tagMap[title.lowercase(Locale.ENGLISH)] ?: MangaTag( + key = title.toAsuraGenreKey(), + title = title, + source = source, + ) + } + val author = doc.selectFirst("div.grid > div:has(h3:eq(0):containsOwn(Author)) > h3:eq(1)")?.text().orEmpty() + val cutoffTime = System.currentTimeMillis() - CHAPTER_HIDE_WINDOW_MS + return manga.copy( + title = doc.selectFirst("article h1") + ?.text() + ?.trim() + ?.takeIf { it.isNotEmpty() } + ?: manga.title, + altTitles = doc.selectFirst("#alt-titles") + ?.text() + .orEmpty() + .split('•', '\n') + .mapNotNullToSet { it.trim().nullIfEmpty() }, + description = doc.selectFirst("#description-text") + ?.html() + ?.trim() + ?.takeIf { it.isNotEmpty() } + ?: doc.selectFirst("span.font-medium.text-sm")?.text().orEmpty(), + tags = tags, + authors = setOf(author), + chapters = doc.select("a.group[href*=/chapter/]").mapChapters(reversed = true) { i, a -> + val urlRelative = a.attrAsRelativeUrl("href") + val titleElement = a.selectFirst("span.font-medium") ?: a.selectFirst("span") + val chapterLabel = titleElement?.text()?.trim()?.takeIf { it.isNotEmpty() } + val chapterTitle = a.selectFirst("span.text-sm.text-white\\/50") + ?.text() + ?.trim() + ?.takeIf { it.isNotEmpty() } + val fullTitle = when { + chapterLabel != null && chapterTitle != null -> "$chapterLabel - $chapterTitle" + chapterLabel != null -> chapterLabel + else -> chapterTitle + } + val chapterNumber = chapterNumberRegex.find(chapterLabel.orEmpty()) + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + ?: i + 1f + val dateText = a.selectFirst("span.text-sm.text-white\\/40") + ?.text() + ?.trim() + MangaChapter( + id = generateUid(urlRelative), + title = fullTitle, + number = chapterNumber, + volume = 0, + url = urlRelative, + scanlator = null, + uploadDate = parseChapterDate(chapterDateFormat, dateText), + branch = null, + source = source, + ) + }.filter { chapter -> + chapter.uploadDate == 0L || chapter.uploadDate <= cutoffTime + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + doc.selectFirst("astro-island[component-url*='ChapterReader']")?.attr("props")?.let { props -> + pageUrlRegex.findAll(props.replace(""", "\"")) + .map { it.groupValues[1] } + .distinct() + .toList() + .takeIf { it.isNotEmpty() } + ?.let { urls -> + return urls.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + } + + val scripts = doc.selectOrThrow("script") + val sb = StringBuilder() + for (script in scripts) { + val raw = script.data().substringBetween("self.__next_f.push(", ")", "").trim() + if (raw.isEmpty()) continue + val ja = raw.toJSONArrayOrNull() ?: continue + for (i in 0 until ja.length()) { + (ja.opt(i) as? String)?.let { sb.append(it) } + } + } + val lines = sb.toString().split('\n') + val pages = TreeMap() + for (line in lines) { + val obj = line.substringAfter(':').toJSONObjectOrNull() ?: continue + if (obj.has("order") && obj.has("url")) { + pages[obj.getInt("order")] = obj.getString("url") + } + } + return pages.values.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun String.toAsuraGenreKey(): String { + return trim() + .lowercase(Locale.ENGLISH) + .replace(asuraGenreKeyRegex, "-") + .trim('-') + } + + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val value = date?.trim().orEmpty() + if (value.isEmpty()) return 0L + val lower = value.lowercase(Locale.ENGLISH) + return when { + lower == "last week" -> Calendar.getInstance().apply { + add(Calendar.WEEK_OF_YEAR, -1) + }.timeInMillis + + lower == "yesterday" -> Calendar.getInstance().apply { + add(Calendar.DAY_OF_MONTH, -1) + }.timeInMillis + + lower.endsWith(" ago") -> parseRelativeDate(lower) + else -> synchronized(dateFormat) { + dateFormat.parseSafe(value.replace(regexDate, "$1")) + } + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0L + val cal = Calendar.getInstance() + return when { + WordSet("second", "seconds").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + WordSet("minute", "minutes", "min", "mins").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis + WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year", "years").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0L + } + } + + private companion object { + private const val CHAPTER_HIDE_WINDOW_MS = 6L * 60L * 60L * 1000L + private val chapterNumberRegex = Regex("""Chapter\s+(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) + private val pageUrlRegex = Regex(""""url":\s*\[0,\s*"([^"]+)"""") + private val ASURA_GENRES = listOf( + "Action", + "Adventure", + "Comedy", + "Crazy MC", + "Demon", + "Dungeons", + "Fantasy", + "Game", + "Genius MC", + "Isekai", + "Magic", + "Murim", + "Mystery", + "Necromancer", + "Overpowered", + "Regression", + "Reincarnation", + "Revenge", + "Romance", + "School Life", + "Sci-fi", + "Shoujo", + "Shounen", + "System", + "Tower", + "Tragedy", + "Villain", + "Violence", + ) + private val asuraGenreKeyRegex = Regex("[^a-z0-9]+") + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt new file mode 100644 index 00000000..52817815 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -0,0 +1,459 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.util.EnumSet +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + +@MangaSourceParser("ATSUMARU", "Atsumaru", "en") +internal class Atsumaru(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ATSUMARU, pageSize = 40) { + + override val configKeyDomain = ConfigKey.Domain("atsu.moe") + private val baseUrl = "https://atsu.moe" + + private val apiHeaders: Headers by lazy { + Headers.Builder() + .add("Accept", "*/*") + .add("Referer", baseUrl) + .add("Content-Type", "application/json") + .build() + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, // trending (views) + SortOrder.UPDATED, // recently added (dateAdded) + SortOrder.ADDED, // recently new added mangas + SortOrder.RELEVANCE // chapters count + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + private val allGenres = listOf( + Genre("Action", "39"), Genre("Adult", "46"), Genre("Adventure", "37"), + Genre("Boys Love", "180"), Genre("Comedy", "6"), Genre("Drama", "31"), + Genre("Fantasy", "36"), Genre("Girls Love", "4"), Genre("Hentai", "10"), + Genre("Historical", "45"), Genre("Horror", "44"), Genre("Martial Arts", "29"), + Genre("Mystery", "32"), Genre("Psychological", "18"), Genre("Romance", "9"), + Genre("Sci-Fi", "1"), Genre("Slice of Life", "7"), Genre("Smut", "41"), + Genre("Supernatural", "22"), Genre("Thriller", "19"), Genre("Tragedy", "5") + ) + + private val allTags = listOf( + Tag("Blackmail", "285"), Tag("Cooking", "669"), Tag("Crimes", "288"), + Tag("Crossdressing", "167"), Tag("Murder", "250"), Tag("Prostitution", "366"), + Tag("Swordplay", "337"), Tag("Working", "248"), + Tag("Josei", "43"), Tag("Seinen", "8"), Tag("Shoujo", "40"), Tag("Shounen", "38"), + Tag("Otaku", "264"), Tag("Tsundere", "313"), Tag("Yandere", "315"), + Tag("Animal Characteristics", "274"), Tag("Beautiful Female Lead", "72"), + Tag("Big Breasts", "123"), Tag("Flat Chest", "320"), Tag("Glasses-Wearing Male Lead", "71"), + Tag("Handsome Male Lead", "68"), Tag("Kemonomimi", "279"), Tag("MILF", "339"), + Tag("Small Breasts", "124"), Tag("Young Male Lead", "787"), + Tag("Adult Cast", "159"), Tag("Bisexual", "382"), Tag("Ensemble Cast", "362"), + Tag("Female Lead", "59"), Tag("Male Lead", "58"), Tag("Non-Human Protagonist", "247"), + Tag("Primarily Adult Cast", "158"), Tag("Primarily Female Cast", "333"), + Tag("Primarily Male Cast", "335"), Tag("Primarily Teen Cast", "334"), + Tag("Strong Female Lead", "69"), Tag("Strong Male Lead", "67"), + Tag("Adapted to Anime", "166"), Tag("Based on a Light Novel", "76"), + Tag("Based on a Novel", "75"), Tag("Based on a Video Game", "77"), + Tag("Based on a Web Novel", "74"), Tag("College", "257"), Tag("Company", "1205"), + Tag("Countryside", "415"), Tag("Europe", "405"), Tag("Foreign", "336"), + Tag("High School", "162"), Tag("Hospital", "760"), Tag("Japan", "225"), + Tag("School", "107"), Tag("School Clubs", "356"), + Tag("Amnesia", "283"), Tag("Appearance Different from Personality", "651"), + Tag("Caught in the Act", "874"), Tag("Dead Family Member", "831"), + Tag("Family Drama", "848"), Tag("Flashbacks", "449"), Tag("Gender Bender", "12"), + Tag("Love Triangle", "125"), Tag("Male Lead Falls in Love First", "653"), + Tag("Misunderstandings", "647"), Tag("Past Plays a Big Role", "648"), + Tag("Reincarnation", "126"), Tag("Secret Identity", "260"), + Tag("Time Manipulation", "311"), Tag("Time Skip", "172"), Tag("Time Travel", "249"), + Tag("Tragic Past", "898"), Tag("Weak to Strong", "1064"), + Tag("Delinquents", "239"), Tag("Detectives", "240"), Tag("Idols", "281"), + Tag("Maids", "116"), Tag("Office Lady", "312"), Tag("Office Worker", "429"), + Tag("School Girl", "788"), Tag("Teachers", "175"), + Tag("Age Gap", "106"), Tag("Childhood Friends", "97"), Tag("Coworkers", "286"), + Tag("Female Harem", "163"), Tag("Friends to Lovers", "243"), Tag("Friendship", "242"), + Tag("Harem", "20"), Tag("Heterosexual", "108"), Tag("Incest", "174"), + Tag("Infidelity", "231"), Tag("Interspecies Relationship", "308"), + Tag("Love-Hate Relationship", "889"), Tag("Master-Servant Relationship", "406"), + Tag("Older Female Younger Male", "114"), Tag("Older Male Younger Female", "649"), + Tag("Older Uke Younger Seme", "880"), Tag("Siblings", "254"), + Tag("Student-Student Relationship", "573"), Tag("Student-Teacher Relationship", "177"), + Tag("Twins", "253"), + Tag("Chinese Ambience", "588"), Tag("European Ambience", "450"), + Tag("Fantasy World", "642"), Tag("Feudal Japan", "606"), Tag("Game Elements", "399"), + Tag("Game World", "641"), Tag("Isekai", "94"), Tag("Isekaied Into a Novel", "258"), + Tag("Mecha", "11"), Tag("Mythology", "259"), Tag("Urban", "338"), + Tag("Urban Fantasy", "261"), + Tag("Anal Intercourse", "100"), Tag("Bondage", "280"), Tag("Boobjob", "381"), + Tag("Borderline H", "448"), Tag("Cunnilingus", "171"), Tag("Defloration", "306"), + Tag("Dubious Consent", "985"), Tag("Ecchi", "21"), Tag("Erotica", "14"), + Tag("Exhibitionism", "287"), Tag("Group Intercourse", "373"), + Tag("Handjob", "303"), Tag("Lolicon", "28"), Tag("Masturbation", "161"), + Tag("Mature", "15"), Tag("Nakadashi", "169"), Tag("Netorare", "232"), + Tag("Nudity", "109"), Tag("Oral Intercourse", "99"), Tag("Outdoor Intercourse", "307"), + Tag("Public Intercourse", "103"), Tag("Rape", "95"), Tag("Sex Addict", "650"), + Tag("Sex Toys", "289"), Tag("Shotacon", "35"), Tag("Teens Love", "374"), + Tag("Threesome", "173"), Tag("Virginity", "369"), + Tag("Animals", "278"), Tag("Cats", "284"), Tag("Demons", "160"), + Tag("Ghosts", "229"), Tag("Gods", "176"), Tag("Monsters", "395"), + Tag("Non-human", "547"), Tag("Vampires", "252"), + Tag("21st century", "132"), Tag("Betrayal", "403"), Tag("Bullying", "235"), + Tag("Cohabitation", "228"), Tag("Coming of Age", "117"), Tag("Danmei", "305"), + Tag("Depression", "1090"), Tag("Family Life", "282"), Tag("Female Empowerment", "1816"), + Tag("Forbidden Love", "699"), Tag("Gore", "262"), Tag("Gourmet", "2"), + Tag("Harlequin", "304"), Tag("Jealousy", "881"), Tag("LGBTQ+", "326"), + Tag("Love Confession", "882"), Tag("Marriage", "360"), Tag("Mature Romance", "241"), + Tag("Medical", "350"), Tag("Military", "230"), Tag("Music", "27"), + Tag("Nobility", "127"), Tag("Obsessive Love", "893"), Tag("Orphans", "237"), + Tag("Religion", "498"), Tag("Reunion", "984"), Tag("Revenge", "227"), + Tag("Royalty", "128"), Tag("School Life", "42"), Tag("Shoujo Ai", "47"), + Tag("Shounen Ai", "23"), Tag("Special Ability", "883"), Tag("Sports", "30"), + Tag("Suicide", "309"), Tag("Super Powers", "236"), Tag("Unrequited Love", "226"), + Tag("Violence", "830"), Tag("War", "238"), Tag("Yaoi", "16"), Tag("Yuri", "33"), + Tag("4-Koma", "105"), Tag("Anthology", "113"), Tag("Chinese Novels", "1112"), + Tag("Collection of Stories", "111"), Tag("Doujinshi", "24"), Tag("Episodic", "115"), + Tag("Full color", "57"), Tag("Korean Novels", "1111"), Tag("Light Novel", "466"), + Tag("Longstrip", "93"), Tag("One Shot", "110"), Tag("Web Comic", "428"), + Tag("Web Novel", "427"), Tag("Magic", "121") + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = (allGenres.map { (name, id) -> MangaTag(name, "genre:$id", source) } + + allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List = getSearchPage(page, order, filter) + + private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query ?: "*" + val url = "$baseUrl/collections/manga/documents/search".toHttpUrl().newBuilder().apply { + addQueryParameter("q", query) + if (query != "*") { + addQueryParameter("query_by", "title,englishTitle,otherNames,authors") + addQueryParameter("query_by_weights", "4,3,2,1") + addQueryParameter("num_typos", "4,3,2,1") + } + addQueryParameter("page", page.toString()) + addQueryParameter("per_page", pageSize.toString()) + + val filterParts = mutableListOf("hidden:!=true", "views:>0") + + val ratings = filter.contentRating + when { + ratings.contains(ContentRating.ADULT) && !ratings.contains(ContentRating.SAFE) -> { + filterParts.add("isAdult:=true") + } + ratings.contains(ContentRating.SAFE) && !ratings.contains(ContentRating.ADULT) -> { + filterParts.add("isAdult:=false") + } + // both or none: no filter shows all + else -> { /* no isAdult filter */ } + } + filterParts.add("(mbContentRating:=[`Safe`,`Suggestive`,`Erotica`] || mbContentRating:!=*)") + + val types = filter.types.mapNotNull { ct -> + when (ct) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manwha" + ContentType.MANHUA -> "Manhua" + ContentType.OTHER -> "OEL" + else -> null + } + } + if (types.isNotEmpty()) { + filterParts.add("type:=[${types.joinToString(",") { "`$it`" }}]") + } + + val statusVals = filter.states.mapNotNull { st -> + when (st) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + MangaState.ABANDONED -> "Canceled" + else -> null + } + } + if (statusVals.isNotEmpty()) { + filterParts.add("status:=[${statusVals.joinToString(",") { "`$it`" }}]") + } + + val includedGenres = mutableListOf() + val excludedGenres = mutableListOf() + val includedTags = mutableListOf() + val excludedTags = mutableListOf() + + for (tag in filter.tags) { + val key = tag.key + when { + key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) + } + } + for (tag in filter.tagsExclude) { + val key = tag.key + when { + key.startsWith("genre:") -> excludedGenres.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + } + } + + if (includedGenres.isNotEmpty()) { + filterParts.add(includedGenres.joinToString(" && ") { "genreIds:=`$it`" }) + } + if (excludedGenres.isNotEmpty()) { + filterParts.add("genreIds:!=[${excludedGenres.joinToString(",") { "`$it`" }}]") + } + if (includedTags.isNotEmpty()) { + filterParts.add(includedTags.joinToString(" && ") { "tagIds:=`$it`" }) + } + if (excludedTags.isNotEmpty()) { + filterParts.add("tagIds:!=[${excludedTags.joinToString(",") { "`$it`" }}]") + } + + addQueryParameter("filter_by", filterParts.joinToString(" && ")) + + val sort = when (order) { + SortOrder.POPULARITY -> "trending:desc" + SortOrder.UPDATED -> "released:desc" + SortOrder.ADDED -> "dateAdded:desc" + SortOrder.RELEVANCE -> "chapterCount:desc" + else -> "released:desc" + } + addQueryParameter("sort_by", sort) + addQueryParameter("sort_by", sort) + }.build() + + val json = webClient.httpGet(url, apiHeaders).parseJson() + val hits = json.optJSONArray("hits") ?: return emptyList() + return (0 until hits.length()).map { i -> + hits.getJSONObject(i).getJSONObject("document").toManga() + } + } + + // limit tags to 20 + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val result = coroutineScope { + val mangaId = manga.url.substringAfterLast("/") + + val pageDeferred = async { + webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() + } + val chaptersDeferred = async { + webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() + } + + val pageJson = pageDeferred.await() + val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga + + val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } + val description = mangaPage.optString("synopsis") ?: manga.description + + val rawRating = mangaPage.optDouble("avgRating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val posterObj = mangaPage.optJSONObject("poster") + val posterImage = posterObj?.optString("mediumImage") + val coverUrl = if (!posterImage.isNullOrEmpty()) { + "https://$domain/static/$posterImage" + } else manga.coverUrl + + val authors = mangaPage.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() + } ?: emptySet() + + val statusText = mangaPage.optString("status").orEmpty() + val state = when (statusText.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> manga.state + } + + val tagSet = mutableSetOf() + mangaPage.optJSONArray("genres")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) + } + } + mangaPage.optJSONArray("tags")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) + } + } + + val chaptersJson = chaptersDeferred.await() + val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() + + val scanlators = mutableMapOf() + mangaPage.optJSONArray("scanlators")?.let { scanArr -> + for (i in 0 until scanArr.length()) { + val sc = scanArr.optJSONObject(i) ?: continue + val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue + val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue + scanlators[id] = name + } + } + + val chapters = (0 until chaptersArray.length()).map { i -> + val ch = chaptersArray.getJSONObject(i) + val chId = ch.getString("id") + val number = ch.optDouble("number", 0.0).toFloat() + val chTitle = ch.optString("title") + val scanId = ch.optString("scanlationMangaId").nullIfEmpty() + val scanName = scanId?.let { scanlators[it] } + val createdAt = ch.optLong("createdAt", 0L) + + MangaChapter( + id = generateUid("$mangaId/$chId"), + title = chTitle.ifBlank { "Chapter $number" }, + number = number, + volume = 0, + url = "$mangaId/$chId", + uploadDate = createdAt, + source = source, + scanlator = scanName, + branch = scanName, + ) + }.reversed() + + val groupedChapters = if (chapters.map { it.branch }.distinct().size > 1) { + chapters.map { it.copy(branch = it.branch ?: "Unknown") } + } else { + chapters.map { it.copy(branch = null, scanlator = null) } + } + + val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() + } ?: emptySet() + + manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + state = state, + tags = tagSet, + rating = rating, + chapters = groupedChapters, + altTitles = altTitles, + ) + } + + detailsCache.put(manga.url, result) + return result + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { + return size > 10 + } + } + + override suspend fun getRelatedManga(seed: Manga): List { + val mangaId = seed.url.substringAfterLast("/") + val pageJson = webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() + val mangaPage = pageJson.optJSONObject("mangaPage") ?: return emptyList() + val similar = mangaPage.optJSONArray("similarManga") ?: return emptyList() + return (0 until similar.length()).map { i -> + similar.getJSONObject(i).toManga() + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val (mangaId, chapterId) = chapter.url.split("/") + val url = "$baseUrl/api/read/chapter".toHttpUrl().newBuilder() + .addQueryParameter("mangaId", mangaId) + .addQueryParameter("chapterId", chapterId) + .build() + val json = webClient.httpGet(url, apiHeaders).parseJson() + val pagesArray = json.getJSONObject("readChapter").getJSONArray("pages") + return (0 until pagesArray.length()).map { i -> + val page = pagesArray.getJSONObject(i) + val imagePath = page.getString("image") + val fullUrl = when { + imagePath.startsWith("http") -> imagePath + imagePath.startsWith("//") -> "https:$imagePath" + else -> "https://$domain/static/${imagePath.removePrefix("/").removePrefix("static/")}" + } + MangaPage(id = generateUid(fullUrl), url = fullUrl, preview = null, source = source) + } + } + + private fun JSONObject.toManga(): Manga { + val id = getString("id") + val title = optString("title").ifEmpty { optString("englishTitle", "Unknown") } + + val imageRaw: Any? = when { + has("poster") -> get("poster") + has("image") -> getString("image") + else -> null + } + + val imagePath: String? = when (imageRaw) { + is String -> imageRaw.takeIf { it.isNotBlank() } + is JSONObject -> { + imageRaw.optString("image").nullIfEmpty() + ?: imageRaw.optString("mediumImage").nullIfEmpty() + ?: imageRaw.optString("smallImage").nullIfEmpty() + } + else -> null + }?.removePrefix("/")?.removePrefix("static/") + + val coverUrl = if (!imagePath.isNullOrBlank()) { + when { + imagePath.startsWith("http") -> imagePath + imagePath.startsWith("//") -> "https:$imagePath" + else -> "https://$domain/static/$imagePath" + } + } else null + + return Manga( + id = generateUid(id), + title = title, + altTitles = emptySet(), + url = "/manga/$id", + publicUrl = "$baseUrl/manga/$id", + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + private data class Genre(val name: String, val id: String) + private data class Tag(val name: String, val id: String) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt new file mode 100644 index 00000000..3ab015a0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -0,0 +1,254 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.FormBody +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import org.json.JSONObject +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) +internal class Batcave(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { + + override val configKeyDomain = ConfigKey.Domain("batcave.biz") + + private var publishers: List> = emptyList() + private var genres: List> = emptyList() + private var filterParseFailed = false + + private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.ENGLISH) + + private val apiClient: OkHttpWebClient by lazy { + val httpClient = context.httpClient.newBuilder() + .addInterceptor(::refererInterceptor) + .addInterceptor(::dleGuardInterceptor) + .build() + OkHttpWebClient(httpClient, source) + } + + private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } + + private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { + val request = chain.request() + val url = request.url.toString() + val referer = when { + url.contains("readcomicsonline.ru") -> "https://readcomicsonline.ru/" + else -> "https://$domain/" + } + return chain.proceed(request.newBuilder().header("Referer", referer).build()) + } + + private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { + val originalRequest = chain.request() + val response = chain.proceed(originalRequest) + if (response.request.url.pathSegments.firstOrNull() != "_c") { + return response + } + response.close() + val url = if (originalRequest.method == "GET") { + originalRequest.url.toString() + } else { + "https://$domain/" + } + context.requestBrowserAction(this, url) + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = emptySet(), + ) + } + + private fun parseFilterValues(filterItems: JSONObject, key: String): List> { + val obj = filterItems.optJSONObject(key) ?: return emptyList() + val values = obj.optJSONArray("values") ?: return emptyList() + val result = mutableListOf>() + for (i in 0 until values.length()) { + val item = values.getJSONObject(i) + result.add(item.getString("value") to item.getInt("id")) + } + return result + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + if (query != null) return searchManga(query, page) + + val urlBuilder = StringBuilder().apply { + append("https://$domain/ComicList/") + if (filter.yearFrom != YEAR_UNKNOWN) append("y[from]=${filter.yearFrom}/") + if (filter.yearTo != YEAR_UNKNOWN) append("y[to]=${filter.yearTo}/") + val pIds = filter.tags.filter { it.key.startsWith("p_") }.map { it.key.removePrefix("p_") } + val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } + if (pIds.isNotEmpty()) append("p=${pIds.joinToString(",")}/") + if (gIds.isNotEmpty()) append("g=${gIds.joinToString(",")}/") + append("sort") + if (page > 1) append("/page/$page/") + } + val url = urlBuilder.toString() + + val sortPair = when (order) { + SortOrder.POPULARITY -> "rating" to "desc" + SortOrder.UPDATED -> "date" to "desc" + else -> "" to "" + } + + return if (sortPair.first.isEmpty()) { + parseMangaList(apiClient.httpGet(url).parseHtml()) + } else { + val formBody = FormBody.Builder() + .add("dlenewssortby", sortPair.first) + .add("dledirection", sortPair.second) + .add("set_new_sort", "dle_sort_xfilter") + .add("set_direction_sort", "dle_direction_xfilter") + .build() + val request = Request.Builder().url(url).post(formBody).build() + val response = rawHttpClient.newCall(request).execute() + parseMangaList(response.parseHtml()) + } + } + + private suspend fun searchManga(query: String, page: Int): List { + val encoded = query.urlEncoded() + val url = if (page == 1) "https://$domain/search/$encoded/" + else "https://$domain/search/$encoded/page/$page/" + return parseMangaList(apiClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(doc: org.jsoup.nodes.Document): List { + return doc.select("#dle-content > .readed").map { element -> + val a = element.selectFirst(".readed__title > a") ?: return@map null + val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.absUrl("href"), + title = a.ownText(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title + val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") + val description = doc.selectFirst("div.page__text")?.text() + val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() + val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { + "Ongoing" -> MangaState.ONGOING + else -> MangaState.FINISHED + } + + val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() + ?: throw ParseException("Chapter data script not found", manga.url) + val json = JSONObject( + script.substringAfter("window.__DATA__ = ").substringBeforeLast(";").trim() + ) + val newsId = json.getLong("news_id") + val chaptersArray = json.getJSONArray("chapters") + val chapters = (0 until chaptersArray.length()).map { i -> + val ch = chaptersArray.getJSONObject(i) + MangaChapter( + id = generateUid("/reader/$newsId/${ch.getInt("id")}"), + url = "/reader/$newsId/${ch.getInt("id")}", + title = ch.optString("title"), + number = ch.optDouble("posi", 0.0).toFloat(), + volume = 0, + scanlator = null, + uploadDate = dateFormat.parseSafe(ch.optString("date")), + branch = null, + source = source, + ) + }.reversed() + + val result = manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + description = description, + authors = setOfNotNull(author), + state = state, + tags = emptySet(), + chapters = chapters, + ) + detailsCache[manga.url] = result + return result + } + + override suspend fun getPages(chapter: MangaChapter): List { + val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) + val id = Regex("^\\d+").find(rawId)?.value ?: rawId + val jsonBody = JSONObject().apply { + put("news_id", newsId) + put("chapter_id", id) + }.toString() + + val request = Request.Builder() + .url("https://$domain/engine/ajax/controller.php?mod=api&action=reader/getChapterData") + .header("Referer", "https://$domain/") + .header("Content-Type", "application/json") + .post( + jsonBody + .toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) + ) + .build() + + val response = rawHttpClient.newCall(request).execute() + val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) + val data = json.getJSONObject("data") + val images = data.getJSONArray("images") + + return (0 until images.length()).map { i -> + var img = images.getString(i).trim() + if (!img.startsWith("http")) img = "https://$domain$img" + MangaPage(id = generateUid(img), url = img, preview = null, source = source) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private fun SimpleDateFormat.parseSafe(date: String?): Long = + if (date.isNullOrBlank()) 0L else runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/CoffeeManga.kt b/src/main/kotlin/tsuki/site/en/CoffeeManga.kt new file mode 100644 index 00000000..d3dbb49e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/CoffeeManga.kt @@ -0,0 +1,11 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("COFFEEMANGA", "CoffeeManga", "en") +internal class CoffeeManga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.COFFEEMANGA, "coffeemanga.ink") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/DemonicScan.kt b/src/main/kotlin/tsuki/site/en/DemonicScan.kt new file mode 100644 index 00000000..06c59f4b --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/DemonicScan.kt @@ -0,0 +1,247 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import androidx.collection.arraySetOf +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("DEMONICSCANS", "DemonicScans", "en") +internal class DemonicScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DEMONICSCANS, 25) { + + override val configKeyDomain = ConfigKey.Domain("demonicscans.org") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val hasQuery = !filter.query.isNullOrEmpty() + val isNewest = order == SortOrder.NEWEST && !hasQuery && filter.tags.isEmpty() + val isAlpha = order == SortOrder.ALPHABETICAL && !hasQuery && filter.tags.isEmpty() + val isAlphaDesc = order == SortOrder.ALPHABETICAL_DESC && !hasQuery && filter.tags.isEmpty() + val url = when { + hasQuery -> "https://$domain/search.php?manga=" + (filter.query ?: "") + isNewest -> "https://$domain/lastupdates.php?list=$page" + isAlpha -> buildString { + append("https://$domain/advanced.php") + append("?list=$page") + append("&status=all") + append("&orderby=NAME ASC") + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&genre[]=") + append(tag.key) + } + } + } + isAlphaDesc -> buildString { + append("https://$domain/advanced.php") + append("?list=$page") + append("&status=all") + append("&orderby=NAME DESC") + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&genre[]=") + append(tag.key) + } + } + } + else -> buildString { + append("https://$domain/advanced.php") + append("?list=$page") + append("&status=all") + append("&orderby=VIEWS DESC") + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&genre[]=") + append(tag.key) + } + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + val selector = when { + hasQuery -> "body > a[href]" + isNewest -> "div#updates-container > div.updates-element" + else -> "div#advanced-content > div.advanced-element" + } + + return doc.select(selector).map { element -> + when { + isNewest -> parseNewestManga(element) + else -> parseNormalManga(element, hasQuery) + } + } + } + + private fun parseNewestManga(element: Element): Manga { + val info = element.selectFirst("div.updates-element-info")!! + val anchor = info.selectFirst("a")!! + val href = anchor.attr("href").let { + if (it.startsWith("http", ignoreCase = true)) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" + } + return Manga( + id = generateUid(href), + title = anchor.ownText(), + altTitles = emptySet(), + url = href, + publicUrl = href, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = element.selectFirst("div.thumb img")?.attr("src")?.let { url -> + if (url.startsWith("http", ignoreCase = true)) url else "https://$domain$url" + }, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source + ) + } + + private fun parseNormalManga(element: Element, hasQuery: Boolean): Manga { + val anchor = if (hasQuery) element else element.selectFirst("a")!! + val href = anchor.attr("href").let { + if (it.startsWith("http", ignoreCase = true)) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" + } + return Manga( + id = generateUid(href), + title = if (hasQuery) + element.selectFirst("div.seach-right > div")?.text().orEmpty() + else + element.selectFirst("h1")?.ownText().orEmpty(), + altTitles = emptySet(), + url = href, + publicUrl = href, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = anchor.selectFirst("img")?.attr("src")?.let { url -> + if (url.startsWith("http", ignoreCase = true)) url else "https://$domain$url" + }, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url).parseHtml() + val info = doc.selectFirst("div#manga-info-container") + val title = info?.selectFirst("h1.big-fat-titles")?.ownText().orEmpty() + val thumbnail = info?.selectFirst("div#manga-page img")?.attrAsAbsoluteUrlOrNull("src") + val genre = info?.select("div.genres-list > li")?.joinToString { it.text() } ?: "" + val description = info?.selectFirst("div#manga-info-rightColumn > div > div.white-font")?.textOrNull() + val author = info?.select("div#manga-info-stats > div:has(> li:eq(0):contains(Author)) > li:eq(1)")?.textOrNull() + val statusText = info?.select("div#manga-info-stats > div:has(> li:eq(0):contains(Status)) > li:eq(1)")?.text() + val state = when (statusText) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + } + + val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { i, el -> + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) + val href = el.attr("href").let { + if (it.startsWith("http")) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" + } + val date = el.selectFirst("span")?.text() + MangaChapter( + id = generateUid(href), + title = el.ownText(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + uploadDate = dateFormat.parseSafe(date), + branch = null, + source = source + ) + } + + return manga.copy( + title = title, + coverUrl = thumbnail, + tags = genre.split(", ").filter { it.isNotBlank() }.mapToSet { + MangaTag(title = it.lowercase().replace(" ", "-").toTitleCase(sourceLocale), key = it, source) + }, + description = description, + state = state, + authors = if (author.isNullOrBlank()) emptySet() else setOf(author), + chapters = chapters + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url).parseHtml() + return doc.select("div > img.imgholder").map { + val url = it.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + private fun fetchTags() = arraySetOf( + MangaTag("Action", "1", source), + MangaTag("Adventure", "2", source), + MangaTag("Comedy", "3", source), + MangaTag("Cooking", "34", source), + MangaTag("Doujinshi", "25", source), + MangaTag("Drama", "4", source), + MangaTag("Ecchi", "19", source), + MangaTag("Fantasy", "5", source), + MangaTag("Gender Bender", "30", source), + MangaTag("Harem", "10", source), + MangaTag("Historical", "28", source), + MangaTag("Horror", "8", source), + MangaTag("Isekai", "33", source), + MangaTag("Josei", "31", source), + MangaTag("Martial Arts", "6", source), + MangaTag("Mature", "22", source), + MangaTag("Mecha", "32", source), + MangaTag("Mystery", "15", source), + MangaTag("One Shot", "26", source), + MangaTag("Psychological", "11", source), + MangaTag("Romance", "12", source), + MangaTag("School Life", "13", source), + MangaTag("Sci-fi", "16", source), + MangaTag("Seinen", "17", source), + MangaTag("Shoujo", "14", source), + MangaTag("Shoujo Ai", "23", source), + MangaTag("Shounen", "7", source), + MangaTag("Shounen Ai", "29", source), + MangaTag("Slice of Life", "21", source), + MangaTag("Smut", "27", source), + MangaTag("Sports", "20", source), + MangaTag("Supernatural", "9", source), + MangaTag("Tragedy", "18", source), + MangaTag("Webtoons", "24", source), + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt new file mode 100644 index 00000000..105bd162 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -0,0 +1,25 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaReaderParser + +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource + +@MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") +internal class EnThunderScans(context: MangaLoaderContext) : + MangaReaderParser( + context, + MangaParserSource.ENTHUNDERSCANS, + "en-thunderscans.com", + pageSize = 30, + searchPageSize = 10, + ) { + override val listUrl = "/comics" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isTagsExclusionSupported = false, + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/FlameComics.kt b/src/main/kotlin/tsuki/site/en/FlameComics.kt new file mode 100644 index 00000000..651fa5bc --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/FlameComics.kt @@ -0,0 +1,272 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.asTypedList +import tsuki.util.json.entries +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNull +import tsuki.util.json.toStringSet +import tsuki.util.suspendlazy.suspendLazy + +import androidx.collection.ArraySet +import okhttp3.HttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.util.EnumSet +import java.util.concurrent.TimeUnit + +@MangaSourceParser("FLAMECOMICS", "FlameComics", "en") +internal class FlameComics(context: MangaLoaderContext) : + SinglePageMangaParser(context, MangaParserSource.FLAMECOMICS) { + + private val commonPrefix = suspendLazy(initializer = ::fetchCommonPrefix) + private val removeSpecialCharsRegex = Regex("[^A-Za-z0-9 ]") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override val configKeyDomain = ConfigKey.Domain("flamecomics.xyz") + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + .build() + + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") + + val allManga = json.mapJSONNotNull { jo -> + parseManga(jo).takeIf { it.tags.matches(filter) } + } + + // Filter by search if provided + val query = filter.query + val filteredManga = if (!query.isNullOrEmpty()) { + val normalizedQuery = removeSpecialCharsRegex.replace(query.lowercase(), "") + allManga.filter { manga -> + val titles = mutableListOf(manga.title) + titles.addAll(manga.altTitles) + + titles.any { title -> + normalizedQuery in removeSpecialCharsRegex.replace(title.lowercase(), "") + } + } + } else { + allManga + } + + return filteredManga + .let { list -> + when (order) { + SortOrder.ALPHABETICAL -> list.sortedBy { it.title } + else -> list + } + } + } + + override suspend fun getDetails(manga: Manga): Manga = getDetailsImpl(manga.url.toLong()) + + override suspend fun getPages(chapter: MangaChapter): List { + val (seriesId, token) = chapter.url.split('?') + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("series") + .addPathSegment(seriesId) + .addPathSegment("$token.json") + .addQueryParameter("id", seriesId) + .addQueryParameter("token", token) + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") + .getJSONObject("chapter") + .getJSONObject("images") + .entries() + return json.map { (i, jo) -> + MangaPage( + id = generateUid("$i|$token"), + url = imageUrl(seriesId, token + "/" + jo.getString("name"), 1920), + preview = imageUrl(seriesId, token + "/" + jo.getString("name"), 128), + source = source, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val seriesId = link.pathSegments.lastOrNull()?.toLongOrNull() ?: return null + return getDetailsImpl(seriesId) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + private suspend fun fetchCommonPrefix(): String { + val raw = webClient.httpGet(urlBuilder().build()).parseRaw() + val regex = Regex("/_next/static/([^/]+)/_buildManifest\\.js") + val prefix = raw.findGroupValue(regex) + return checkNotNull(prefix) { + "Unable to find common prefix in page. Looking for pattern: /_next/static/([^/]+)/_buildManifest\\.js\n" + + "Page content sample: ${raw.take(1000)}" + } + } + + private fun imageUrl(seriesId: Any, url: String, width: Int) = urlBuilder() + .addPathSegment("_next") + .addPathSegment("image") + .addQueryParameter( + "url", + urlBuilder("cdn") + .addPathSegment("uploads") + .addPathSegment("images") + .addPathSegment("series") + .addPathSegment(seriesId.toString()) + .addPathSegments(url) + .build().toString(), + ) + .addQueryParameter("w", width.toString()) + .addQueryParameter("q", "100") + .build() + .toString() + + private suspend fun fetchAvailableTags(): Set { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + .build() + return webClient.httpGet(url).parseJson() + .getJSONObject("pageProps") + .getJSONArray("series") + .mapJSONNotNull { it.getStringOrNull("categories") } + .flatMapTo(ArraySet()) { + JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } + } + } + + private fun parseManga(jo: JSONObject): Manga { + // Try different possible field names for series ID, including string conversion + val seriesId = jo.getLongOrDefault("series_id", -1L).takeIf { it != -1L } + ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } // Handle novels + ?: jo.getStringOrNull("series_id")?.toLongOrNull() + ?: jo.getStringOrNull("novel_id")?.toLongOrNull() // Handle novels as string + ?: jo.getLongOrDefault("id", -1L).takeIf { it != -1L } + ?: jo.getStringOrNull("id")?.toLongOrNull() + ?: jo.getLongOrDefault("seriesId", -1L).takeIf { it != -1L } + ?: jo.getStringOrNull("seriesId")?.toLongOrNull() + ?: jo.getLongOrDefault("series", -1L).takeIf { it != -1L } + ?: jo.getStringOrNull("series")?.toLongOrNull() + ?: throw IllegalArgumentException( + "No valid series ID found. Available fields: ${jo.keys().asSequence().joinToString()}\n" + + "series_id value: ${jo.optString("series_id", "MISSING")}\n" + + "novel_id value: ${jo.optString("novel_id", "MISSING")}\n" + + "JSON sample: ${jo.toString().take(500)}" + ) + val cover = jo.getStringOrNull("cover") + val author = jo.getStringOrNull("author") + return Manga( + id = generateUid(seriesId), + title = jo.getString("title"), + altTitles = jo.getStringOrNull("altTitles")?.let { + JSONArray(it).toStringSet() + }.orEmpty(), + url = seriesId.toString(), + publicUrl = "https://${domain}/series/$seriesId", + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = if (cover != null) { + imageUrl(seriesId, cover, 384) + } else { + null + }, + tags = jo.getStringOrNull("categories")?.let { + JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } + }.orEmpty(), + state = when (jo.getStringOrNull("status")) { + "Dropped" -> MangaState.ABANDONED + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + "Ongoing" -> MangaState.ONGOING + else -> null + }, + authors = setOfNotNull(author), + largeCoverUrl = if (cover != null) { + imageUrl(seriesId, cover, 720) + } else { + null + }, + description = jo.getStringOrNull("description"), + source = source, + ) + } + + private suspend fun getDetailsImpl(seriesId: Long): Manga { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("series") + .addPathSegment("$seriesId.json") + .addQueryParameter("id", seriesId.toString()) + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps") + val series = json.getJSONObject("series") + val chapters = json.getJSONArray("chapters") + return parseManga(series).copy( + chapters = chapters.mapJSON { jo -> + val chapterId = jo.getLong("chapter_id") + val number = jo.getFloatOrDefault("chapter", 0f) + MangaChapter( + id = generateUid(longOf(seriesId.toInt(), chapterId.toInt())), + title = jo.getStringOrNull("name"), + number = number, + volume = 0, + url = seriesId.toString() + "?" + jo.getStringOrNull("token").orEmpty(), + scanlator = null, + uploadDate = TimeUnit.SECONDS.toMillis(jo.getLongOrDefault("release_date", 0L)), + branch = jo.getStringOrNull("language"), + source = source, + ) + }.reversed(), + ) + } + + private fun Set.matches(filter: MangaListFilter): Boolean { + if (filter.tags.isNotEmpty() && !containsAll(filter.tags)) { + return false + } + for (tag in filter.tagsExclude) { + if (contains(tag)) { + return false + } + } + return true + } + + private fun String.toMangaTag() = MangaTag( + title = this.toTitleCase(sourceLocale), + key = this.lowercase().trim(), + source = source, + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/KaliScan.kt b/src/main/kotlin/tsuki/site/en/KaliScan.kt new file mode 100644 index 00000000..4efd9148 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/KaliScan.kt @@ -0,0 +1,211 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadthemeParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +@MangaSourceParser("KALISCAN", "kaliscan.io", "en") +internal class KaliScan(context: MangaLoaderContext) : + MadthemeParser(context, MangaParserSource.KALISCAN, "kaliscan.io", pageSize=24) { + + override val selectDesc = ".summary .content, .summary .content ~ p" + override val selectState = ".detail .meta > p > strong:contains(Status) ~ a" + override val selectAlt = ".detail h2" + override val selectTag = ".detail .meta > p > strong:contains(Genres) ~ a" + override val selectDate = ".chapter-update" + override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isAuthorSearchSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + append("/search") + + append("?page=") + append(page.toString()) + + filter.query?.let { + append("&q=") + append(it.urlEncoded()) + } + + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("updated_at") + SortOrder.ALPHABETICAL -> append("name") + SortOrder.NEWEST -> append("created_at") + SortOrder.RATING -> append("rating") + else -> append("updated_at") + } + + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&include[]=") + append(tag.key) + } + } + + if (filter.tagsExclude.isNotEmpty()) { + filter.tagsExclude.forEach { tag -> + append("&exclude[]=") + append(tag.key) + } + } + + append("&include_mode=and") + append("&bookmark=off") + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> "all" + }, + ) + } ?: append("&status=all") + + filter.author?.takeIf { it.isNotBlank() }?.let { author -> + append("&author=") + append(author.urlEncoded()) + } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List { + return doc.select(".book-detailed-item").map { div -> + val link = div.selectFirstOrThrow("a") + val href = link.attrAsRelativeUrl("href") + val title = link.attr("title").ifEmpty { + div.selectFirst(".title")?.text() ?: "" + } + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = div.selectFirst("img")?.attr("data-src")?.ifEmpty { + div.selectFirst("img")?.attr("src") + }, + title = title, + altTitles = emptySet(), + rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst(".detail h1")?.text() ?: manga.title + + val authors = doc.select(".detail .meta > p > strong:contains(Authors) ~ a") + .map { it.text().trim(',', ' ') } + .toSet() + + val tags = doc.select(selectTag).mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().trim(',', ' ').toTitleCase(), + source = source, + ) + } + + val altNames = doc.selectFirst(selectAlt)?.text() + ?.split(',', ';') + ?.mapNotNull { it -> it.trim().takeIf { it != title } } + ?.toSet() ?: emptySet() + + val description = doc.select(selectDesc).text() + val statusText = doc.selectFirst(selectState)?.text()?.lowercase() ?: "" + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + val nsfw = doc.selectFirst("#adt-warning") != null + + val coverUrl = doc.selectFirst("#cover img")?.attr("data-src") + + val chapters = doc.select(selectChapter).mapChapters(reversed = true) + { i, element -> + val link = element.selectFirst("a") ?: return@mapChapters null + val href = link.attrAsRelativeUrl("href") + val chapterTitle = element.selectFirst(".chapter-title")?.text()?.trim() ?: return@mapChapters null + val dateText = element.selectFirst(selectDate)?.text()?.trim() + + MangaChapter( + id = generateUid(href), + url = href, + title = chapterTitle, + uploadDate = parseChapterDate( + SimpleDateFormat(datePattern, sourceLocale), + dateText, + ), + source = source, + number = i + 1f, + volume = 0, + scanlator = null, + branch = null, + ) + } + + return manga.copy( + title = title, + altTitles = altNames, + authors = authors, + tags = tags, + description = description, + state = state, + largeCoverUrl = coverUrl, + chapters = chapters, + contentRating = if (nsfw || isNsfwSource) ContentRating.ADULT else ContentRating.SAFE, + ) + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + + return doc.selectFirst(".checkbox-group.genres")?.select(".checkbox-wrapper")?.mapNotNullToSet { element -> + val input = element.selectFirst("input") ?: return@mapNotNullToSet null + val key = input.attr("value").takeIf { it.isNotEmpty() } ?: return@mapNotNullToSet null + val label = element.selectFirst(".radio__label")?.text() ?: key + + MangaTag( + key = key, + title = label, + source = source, + ) + } ?: emptySet() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/KingofShojo.kt b/src/main/kotlin/tsuki/site/en/KingofShojo.kt new file mode 100644 index 00000000..63cc6a90 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/KingofShojo.kt @@ -0,0 +1,11 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("KINGOFSHOJO", "King of Shojo", "en") +internal class KingofShojo(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KINGOFSHOJO, "kingofshojo.com") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/LikeManga.kt b/src/main/kotlin/tsuki/site/en/LikeManga.kt new file mode 100644 index 00000000..8169e037 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/LikeManga.kt @@ -0,0 +1,29 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.CommonHeaders +import tsuki.parsers.LikeMangaParser + +import tsuki.model.MangaParserSource + +import okhttp3.Interceptor +import okhttp3.Response + +private const val USER_AGENT = "Usagi/1 (Android)" + +@MangaSourceParser("LIKEMANGA", "LikeManga", "en") +internal class LikeManga(context: MangaLoaderContext) : + LikeMangaParser(context, MangaParserSource.LIKEMANGA, "likemanga.ink") { + override val userAgentKey = ConfigKey.UserAgent(USER_AGENT) + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = request.newBuilder() + .header(CommonHeaders.USER_AGENT, USER_AGENT) + .build() + + return chain.proceed(newRequest) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaFox.kt b/src/main/kotlin/tsuki/site/en/MangaFox.kt new file mode 100644 index 00000000..36dafa32 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaFox.kt @@ -0,0 +1,240 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAFOX", "MangaFox", "en") +internal class MangaFox(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAFOX, pageSize = 24) { + + private val baseUrl = "fanfox.net" + private val mobileUrl = "https://m.fanfox.net" + private val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH) + + override val configKeyDomain = ConfigKey.Domain(baseUrl) + + private val apiClient: OkHttpWebClient by lazy { + val httpClient = context.httpClient.newBuilder() + .addInterceptor { chain -> + val request = chain.request().newBuilder() + .header("Referer", "https://$baseUrl/") + .build() + chain.proceed(request) + } + .build() + OkHttpWebClient(httpClient, source) + } + + override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true + ) + + @Volatile + private var tagsCache: Set? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = getOrFetchTags() + ) + } + + private suspend fun getOrFetchTags(): Set { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTags().also { tagsCache = it } + } + } + + private fun fetchTags(): Set { + return listOf( + "action", "adventure", "comedy", "drama", "fantasy", + "harem", "romance", "school-life", "shounen", "shoujo", + "seinen", "sci-fi", "slice-of-life", "supernatural", + "ecchi", "mystery", "psychological", "historical", + "martial-arts", "mecha", "sports" + ).map { + MangaTag(key = it, title = it.replace("-", " "), source = source) + }.toSet() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildUrl(page, filter, order) + val doc = apiClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun buildUrl(page: Int, filter: MangaListFilter, order: SortOrder): String { + val query = filter.query?.trim().orEmpty() + val tag = filter.tags.firstOrNull()?.key + return when { + !tag.isNullOrBlank() && query.isEmpty() -> + "https://m.fanfox.net/search/cate/$tag" + query.isNotEmpty() -> + "https://m.fanfox.net/search?k=${query.urlEncoded()}" + else -> + buildString { + append("https://$baseUrl/directory/") + if (page > 1) append("$page.html") + if (order == SortOrder.UPDATED) append("?latest") + } + } + } + + private fun parseMangaList(doc: Document): List { + val searchItems = doc.select(".post-list li") + if (searchItems.isNotEmpty()) { + return searchItems.mapNotNull { el -> + val a = el.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = a.attr("title").ifBlank { a.selectFirst(".cover-info .title")?.text() ?: a.text() }, + url = href, + publicUrl = a.absUrl("href"), + coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + source = source + ) + } + } + + val dirSelectors = listOf( + "ul.manga-list-1-list li", + "ul.manga-list-4-list li", + "ul.manga-list-2-list li" + ) + for (sel in dirSelectors) { + val items = doc.select(sel) + if (items.isNotEmpty()) { + return items.mapNotNull { el -> + val a = el.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = a.attr("title").ifBlank { a.text() }, + url = href, + publicUrl = a.absUrl("href"), + coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + source = source + ) + } + } + } + return emptyList() + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() + val info = doc.selectFirst(".detail-info-right")!! + + val chapters = doc.select("ul.detail-main-list li").mapIndexedNotNull { i, el -> + val a = el.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("p")?.text() ?: a.ownText() + val rawDate = el.text() + .let { text -> + Regex("""(Today|Yesterday|\w{3}\s\d{1,2},\s\d{4}|\d{4}-\d{2}-\d{2})""") + .find(text)?.value + } + val parsedDate = parseChapterDate(rawDate) + MangaChapter( + id = generateUid(href), + title = name, + number = extractChapterNumber(name), + volume = 0, + url = href, + uploadDate = parsedDate.takeIf { it > 0L } + ?: (System.currentTimeMillis() - (i * 86_400_000L)), + scanlator = null, + branch = null, + source = source + ) + }.sortedBy { it.number } + + return manga.copy( + title = doc.selectFirst("h1")?.text() ?: manga.title, + authors = info.select(".detail-info-right-say a").mapToSet { it.text() }, + tags = info.select(".detail-info-right-tag-list a").mapToSet { + MangaTag(key = it.text().lowercase(), title = it.text(), source = source) + }, + description = info.selectFirst("p.fullcontent")?.text(), + state = parseStatus(info.selectFirst(".detail-info-right-title-tip")?.text().orEmpty()), + coverUrl = doc.selectFirst(".detail-info-cover-img")?.attr("abs:src").orEmpty(), + chapters = chapters + ) + } + + private fun parseStatus(status: String): MangaState? = when { + status.contains("Ongoing", true) -> MangaState.ONGOING + status.contains("Completed", true) -> MangaState.FINISHED + else -> null + } + + private fun extractChapterNumber(name: String): Float { + return Regex("""(?i)(chapter|ch\.)\s*(\d+(\.\d+)?)""") + .find(name) + ?.groupValues + ?.getOrNull(2) + ?.toFloatOrNull() + ?: 0f + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val clean = date.trim() + return when { + "Today" in clean -> Calendar.getInstance().timeInMillis + "Yesterday" in clean -> Calendar.getInstance().apply { add(Calendar.DATE, -1) }.timeInMillis + clean.matches(Regex("""\d{4}-\d{2}-\d{2}""")) -> runCatching { + val parts = clean.split("-") + Calendar.getInstance().apply { set(parts[0].toInt(), parts[1].toInt() - 1, parts[2].toInt()) }.timeInMillis + }.getOrDefault(0L) + else -> runCatching { dateFormat.parse(clean)?.time ?: 0L }.getOrDefault(0L) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val mobilePath = chapter.url.replace("/manga/", "/roll_manga/") + val url = "$mobileUrl$mobilePath" + val doc = apiClient.httpGet(url).parseHtml() + return doc.select("#viewer img").map { + val imgUrl = it.attr("abs:data-original") + MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + override suspend fun getRelatedManga(seed: Manga): List = emptyList() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaGG.kt b/src/main/kotlin/tsuki/site/en/MangaGG.kt new file mode 100644 index 00000000..58d27c86 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaGG.kt @@ -0,0 +1,153 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import kotlinx.coroutines.awaitAll + + +@MangaSourceParser("MANGAGG", "MangaGG", "en") +internal class MangaGG(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAGG, "mangagg.com") { + + override val tagPrefix = "genre/" + override val datePattern = "MM/dd/yyyy" + + override suspend fun loadChapters( + mangaUrl: String, + document: Document, + ): List = coroutineScope { + + val baseUrl = mangaUrl + .toAbsoluteUrl(domain) + .removeSuffix('/') + + val chapters = mutableListOf() + + var page = 1 + var finished = false + + while (!finished) { + + val jobs = (page until page + 5).map { currentPage -> + async { + fetchChapterPage( + baseUrl, + currentPage, + ) + } + } + + val results = jobs.awaitAll() + + finished = results.all { + it.isEmpty() + } + + results.forEach { + chapters.addAll(it) + } + + page += 5 + } + + chapters + .distinctBy { + it.id + } + .reversed() + .mapIndexed { index, chapter -> + chapter.copy( + number = index + 1f + ) + } + } + + + private suspend fun fetchChapterPage( + mangaUrl: String, + page: Int, + ): List { + + val url = + "$mangaUrl/ajax/chapters/?t=$page" + + return runCatching { + + webClient + .httpPost( + url, + emptyMap(), + ) + .parseHtml() + .let(::parseChapterPage) + + }.getOrDefault(emptyList()) + } + + + private fun parseChapterPage( + doc: Document, + ): List { + + val dateFormat = + SimpleDateFormat( + datePattern, + sourceLocale, + ) + + return doc + .select(selectChapter) + .map { li -> + parseChapterItem( + li, + dateFormat, + ) + } + } + + + private fun parseChapterItem( + li: org.jsoup.nodes.Element, + dateFormat: SimpleDateFormat, + ): MangaChapter { + + val a = li.selectFirstOrThrow("a") + + val href = a.attrAsRelativeUrl("href") + + val dateText = + li.selectFirst("a.c-new-tag") + ?.attr("title") + ?: li.selectFirst(selectDate) + ?.text() + + val name = + a.selectFirst("p") + ?.text() + ?: a.ownText() + + return MangaChapter( + id = generateUid(href), + title = name, + number = 0f, + volume = 0, + url = href + stylePage, + uploadDate = parseChapterDate( + dateFormat, + dateText, + ), + source = source, + scanlator = null, + branch = null, + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaGeko.kt b/src/main/kotlin/tsuki/site/en/MangaGeko.kt new file mode 100644 index 00000000..92a25d2f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaGeko.kt @@ -0,0 +1,245 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAKEKO", "MangaGeko", "en") +internal class MangaGeko(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAKEKO, 30) { + + override val configKeyDomain = ConfigKey.Domain("www.mgeko.cc") + private val baseUrl = "https://$domain" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, // recently_added + SortOrder.UPDATED, // latest + SortOrder.POPULARITY, // popular_all_time + SortOrder.RATING, // rating + SortOrder.ALPHABETICAL, // az + SortOrder.ALPHABETICAL_DESC, // za + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + private val dateFormats = listOf( + SimpleDateFormat("MMMM dd, yyyy, h:mm a", Locale.ENGLISH), + SimpleDateFormat("MMMM dd, yyyy, h a", Locale.ENGLISH), + ) + + private val allGenres = setOf( + "Action", "Adventure", "Comedy", "Cooking", "Manga", "Drama", "Fantasy", + "Gender bender", "Harem", "Historical", "Horror", "Isekai", "Josei", + "Manhua", "Manhwa", "Martial arts", "Mature", "Mecha", "Medical", + "Mystery", "One shot", "Psychological", "Romance", "School life", + "Sci fi", "Seinen", "Shoujo", "Shounen", "Slice of life", "Sports", + "Supernatural", "Tragedy", "Webtoons", "Ladies", + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val tags = allGenres.map { MangaTag(it, it.lowercase().replace(" ", "_"), source) }.toSet() + return MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + if (!filter.query.isNullOrEmpty()) { + return searchManga(page, filter.query!!) + } + return browseManga(page, order, filter) + } + + private suspend fun searchManga(page: Int, query: String): List { + val url = "$baseUrl/search/".toHttpUrl().newBuilder() + .addQueryParameter("search", query) + .addQueryParameter("results", page.toString()) + .build() + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".novel-item").mapNotNull { it.toManga() } + } + + private suspend fun browseManga(page: Int, order: SortOrder, filter: MangaListFilter): List { + val sort = when (order) { + SortOrder.NEWEST -> "recently_added" + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "popular_all_time" + SortOrder.RATING -> "rating" + SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" + else -> "latest" + } + + val url = "$baseUrl/browse-comics/data/".toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("sort", sort) + + val safeMode = if (ContentRating.ADULT !in filter.contentRating) "1" else "0" + addQueryParameter("safe_mode", safeMode) + + if (filter.tags.isNotEmpty()) { + addQueryParameter("include_genres", filter.tags.joinToString(",") { it.key.replace("_", " ") }) + } + if (filter.tagsExclude.isNotEmpty()) { + addQueryParameter("exclude_genres", filter.tagsExclude.joinToString(",") { it.key.replace("_", " ") }) + } + + if (filter.states.isNotEmpty()) { + val status = when (filter.states.first()) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + else -> null + } + status?.let { addQueryParameter("status", it) } + } + + if (filter.types.isNotEmpty()) { + val type = when (filter.types.first()) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Webtoon" + else -> null + } + type?.let { addQueryParameter("type", it) } + } + }.build() + + val response = webClient.httpGet(url) + val json = JSONObject(response.body.string()) + val htmlFragment = json.getString("results_html") + val doc = Jsoup.parseBodyFragment(htmlFragment, baseUrl) + return doc.select(".comic-card").map { it.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val header = doc.selectFirstOrThrow(".novel-header") + + val title = header.selectFirst(".novel-title")?.text()?.trim() ?: manga.title + val cover = header.selectFirst(".cover img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + } ?: manga.coverUrl + + val author = header.selectFirst(".author a")?.attr("title")?.trim()?.takeIf { + it.lowercase() != "updating" + } + + val state = when { + header.selectFirst("div.header-stats strong.completed") != null -> MangaState.FINISHED + header.selectFirst("div.header-stats strong.ongoing") != null -> MangaState.ONGOING + else -> null + } + + val tags = doc.select(".categories a[href*=genre]").mapToSet { a -> + val name = a.ownText().trim().split(" ").joinToString(" ") { word -> + word.lowercase().replaceFirstChar { it.uppercase() } + } + MangaTag(name, name.lowercase().replace(" ", "_"), source) + } + + val chapters = fetchChapters(manga.url) + + return manga.copy( + title = title, + altTitles = emptySet(), + coverUrl = cover, + largeCoverUrl = cover, + state = state, + authors = setOfNotNull(author), + description = null, + tags = tags, + chapters = chapters, + ) + } + + private suspend fun fetchChapters(mangaUrl: String): List { + val doc = webClient.httpGet(mangaUrl.toAbsoluteUrl(domain) + "all-chapters/").parseHtml() + return doc.select("ul.chapter-list > li").mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val url = a.attrAsRelativeUrl("href") + val titleEl = li.selectFirst(".chapter-title, .chapter-number") ?: return@mapNotNull null + val chapterName = titleEl.ownText() + val dateStr = li.selectFirst(".chapter-update")?.attr("datetime") + ?.replace(".", "")?.replace("Sept", "Sep") ?: "" + val date = dateFormats.firstNotNullOfOrNull { fmt -> fmt.parseSafe(dateStr) } ?: 0L + MangaChapter( + id = generateUid(url), + title = "Chapter $chapterName", + number = 0f, + volume = 0, + url = url, + uploadDate = date, + source = source, + scanlator = null, + branch = null, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#chapter-reader img").mapNotNull { img -> + val src = img.absUrl("src") + if (src.isEmpty()) return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun Element.toManga(): Manga { + val titleEl = selectFirst(".comic-card__title a") ?: selectFirst(".novel-title") + val title = titleEl?.text()?.trim() ?: "" + val href = selectFirst("a")?.attrAsRelativeUrl("href") ?: "" + val cover = selectFirst(".comic-card__cover img, .novel-cover img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + } ?: "" + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaJinx.kt b/src/main/kotlin/tsuki/site/en/MangaJinx.kt new file mode 100644 index 00000000..c64acdfc --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaJinx.kt @@ -0,0 +1,211 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadthemeParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +@MangaSourceParser("MANGAJINX", "MangaJinx", "en") +internal class MangaJinx(context: MangaLoaderContext) : + MadthemeParser(context, MangaParserSource.MANGAJINX, "mgjinx.com", pageSize=24) { + + override val selectDesc = ".summary .content, .summary .content ~ p" + override val selectState = ".detail .meta > p > strong:contains(Status) ~ a" + override val selectAlt = ".detail h2" + override val selectTag = ".detail .meta > p > strong:contains(Genres) ~ a" + override val selectDate = ".chapter-update" + override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isAuthorSearchSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + append("/search") + + append("?page=") + append(page.toString()) + + filter.query?.let { + append("&q=") + append(it.urlEncoded()) + } + + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("updated_at") + SortOrder.ALPHABETICAL -> append("name") + SortOrder.NEWEST -> append("created_at") + SortOrder.RATING -> append("rating") + else -> append("updated_at") + } + + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&include[]=") + append(tag.key) + } + } + + if (filter.tagsExclude.isNotEmpty()) { + filter.tagsExclude.forEach { tag -> + append("&exclude[]=") + append(tag.key) + } + } + + append("&include_mode=and") + append("&bookmark=off") + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> "all" + }, + ) + } ?: append("&status=all") + + filter.author?.takeIf { it.isNotBlank() }?.let { author -> + append("&author=") + append(author.urlEncoded()) + } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List { + return doc.select(".book-detailed-item").map { div -> + val link = div.selectFirstOrThrow("a") + val href = link.attrAsRelativeUrl("href") + val title = link.attr("title").ifEmpty { + div.selectFirst(".title")?.text() ?: "" + } + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = div.selectFirst("img")?.attr("data-src")?.ifEmpty { + div.selectFirst("img")?.attr("src") + }, + title = title, + altTitles = emptySet(), + rating = div.selectFirst("div.meta span.score")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst(".detail h1")?.text() ?: manga.title + + val authors = doc.select(".detail .meta > p > strong:contains(Authors) ~ a") + .map { it.text().trim(',', ' ') } + .toSet() + + val tags = doc.select(selectTag).mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().trim(',', ' ').toTitleCase(), + source = source, + ) + } + + val altNames = doc.selectFirst(selectAlt)?.text() + ?.split(',', ';') + ?.mapNotNull { it -> it.trim().takeIf { it != title } } + ?.toSet() ?: emptySet() + + val description = doc.select(selectDesc).text() + val statusText = doc.selectFirst(selectState)?.text()?.lowercase() ?: "" + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + val nsfw = doc.selectFirst("#adt-warning") != null + + val coverUrl = doc.selectFirst("#cover img")?.attr("data-src") + + val chapters = doc.select(selectChapter).mapChapters(reversed = true) + { i, element -> + val link = element.selectFirst("a") ?: return@mapChapters null + val href = link.attrAsRelativeUrl("href") + val chapterTitle = element.selectFirst(".chapter-title")?.text()?.trim() ?: return@mapChapters null + val dateText = element.selectFirst(selectDate)?.text()?.trim() + + MangaChapter( + id = generateUid(href), + url = href, + title = chapterTitle, + uploadDate = parseChapterDate( + SimpleDateFormat(datePattern, sourceLocale), + dateText, + ), + source = source, + number = i + 1f, + volume = 0, + scanlator = null, + branch = null, + ) + } + + return manga.copy( + title = title, + altTitles = altNames, + authors = authors, + tags = tags, + description = description, + state = state, + largeCoverUrl = coverUrl, + chapters = chapters, + contentRating = if (nsfw || isNsfwSource) ContentRating.ADULT else ContentRating.SAFE, + ) + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + + return doc.selectFirst(".checkbox-group.genres")?.select(".checkbox-wrapper")?.mapNotNullToSet { element -> + val input = element.selectFirst("input") ?: return@mapNotNullToSet null + val key = input.attr("value").takeIf { it.isNotEmpty() } ?: return@mapNotNullToSet null + val label = element.selectFirst(".radio__label")?.text() ?: key + + MangaTag( + key = key, + title = label, + source = source, + ) + } ?: emptySet() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt new file mode 100644 index 00000000..caa9c109 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -0,0 +1,322 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNullToSet +import tsuki.util.json.mapJSONToSet + +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import okhttp3.Interceptor +import okhttp3.Response + +@MangaSourceParser("MANGAK", "MangaK", "en", ContentType.MANGA) +internal class MangaK(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAK, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("mangak.io") + + private val apiUrl = "https://api.mangak.io" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (!response.isSuccessful && request.url.host.matches(IMAGE_FALLBACK_REGEX)) { + response.close() + val newUrl = request.url.newBuilder().host("rx.rzyn.net").build() + return chain.proceed(request.newBuilder().url(newUrl).build()) + } + return response + } + + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, +// MangaState.PAUSED, +// MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + availableDemographics = EnumSet.of( + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.SHOUNEN, + Demographic.JOSEI, + ), + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), + ) + + private suspend fun fetchTags(): Set { + val json = webClient.httpGet("$apiUrl/genres").parseJson() + return json.getJSONObject("data").getJSONArray("items").mapJSONToSet { item -> + MangaTag( + title = item.getString("name").toTitleCase(sourceLocale), + key = item.getString("slug"), + source = source, + ) + } + } + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val allExcludedTags = filter.tagsExclude.map { it.key }.toMutableSet() + val query = filter.query + val url = buildString { + append(apiUrl) + append("/titles/search?page=") + append(page) + append("&limit=") + append(pageSize) + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "newest" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "rating" + else -> "latest" + }, + ) + + if (!query.isNullOrEmpty()) { + append("&q=") + // api does not like commans and character limits to 60-70 + append(query.replace(",", "").urlEncoded()) + } + + if (filter.tags.isNotEmpty()) { + append("&genres=") + append(filter.tags.joinToString(",") { it.key }) + } + + if (allExcludedTags.isNotEmpty()) { + append("&exclude=") + append(allExcludedTags.joinToString(",")) + } + + filter.types.oneOrThrowIfMany()?.let { + append("&type=") + append( + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> null + } ?: return@let + ) + } + + filter.contentRating.oneOrThrowIfMany()?.let { + append("&content_rating=") + append( + when (it) { + ContentRating.SAFE -> "safe" + ContentRating.SUGGESTIVE -> "suggestive" + ContentRating.ADULT -> "pornographic" + } + ) + } + + filter.demographics.oneOrThrowIfMany()?.let { + append("&demographic=") + append( + when (it) { + Demographic.SHOUJO -> "shoujo" + Demographic.SEINEN -> "seinen" + Demographic.SHOUNEN -> "shounen" + Demographic.JOSEI -> "josei" + else -> null + } ?: return@let + ) + } + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" +// MangaState.PAUSED -> "hiatus" +// MangaState.ABANDONED -> "cancelled" + else -> null + } ?: return@let + ) + } + } + + val json = webClient.httpGet(url).parseJson() + return json.getJSONObject("data").getJSONArray("items").mapJSON { item -> + item.toManga() + } + } + + private fun JSONObject.toManga(): Manga { + val relativeUrl = getString("url") + return Manga( + id = generateUid(getString("id")), + title = getString("name"), + altTitles = parseAltTitles(), + url = getString("id"), + publicUrl = "https://$domain$relativeUrl", + rating = optDouble("rating", 0.0).toRating(), + contentRating = optString("content_rating").nullIfEmpty().toContentRating(), + coverUrl = optString("cover").nullIfEmpty(), + tags = parseTags(), + state = optString("status").toMangaState(), + authors = emptySet(), + description = optString("summary").nullIfEmpty(), + source = source, + ) + } + + + private fun String?.toContentRating() = when (this) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + + + private fun JSONObject.parseAltTitles(): Set { + val result = mutableSetOf() + optJSONArray("alt_names")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.optString("name")?.nullIfEmpty()?.let { result.add(it) } + } + } + optString("alt_name").nullIfEmpty()?.let { result.add(it) } + return result + } + + + private fun JSONObject.parseTags(): Set = + optJSONArray("genres")?.mapJSONNotNullToSet { genre -> + val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null + MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) + }.orEmpty() + + + override suspend fun getDetails(manga: Manga): Manga { + val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() + .getJSONObject("data").getJSONObject("title") + val cv = json.optLong("cv") + val authors = json.optJSONArray("authors")?.mapJSONNotNullToSet { + it.optString("name").nullIfEmpty() + }.orEmpty() + val artists = json.optJSONArray("artists")?.mapJSONNotNullToSet { + it.optString("name").nullIfEmpty() + }.orEmpty() + return manga.copy( + title = json.optString("name").nullIfEmpty() ?: manga.title, + altTitles = json.parseAltTitles().ifEmpty { manga.altTitles }, + description = json.optString("summary").nullIfEmpty() ?: manga.description, + tags = json.parseTags().ifEmpty { manga.tags }, + authors = authors + artists, + state = json.optString("status").toMangaState() ?: manga.state, + rating = json.optDouble("rating", 0.0).toRating(), + chapters = fetchChapters(manga.url, cv), + ) + } + + + private suspend fun fetchChapters(id: String, cv: Long): List { + val json = webClient.httpGet("$apiUrl/titles/$id/chapters?cv=$cv").parseJson() + val chapters = json.getJSONObject("data").getJSONArray("chapters").mapJSON { it } + val total = chapters.size + return chapters.mapIndexed { index, chapter -> + MangaChapter( + id = generateUid(chapter.getString("id")), + title = chapter.optString("name").nullIfEmpty(), + number = chapter.optDouble("chapter_number", Double.NaN).takeUnless { it.isNaN() }?.toFloat() + ?: (total - index).toFloat(), + volume = 0, + url = chapter.getString("url"), + scanlator = null, + uploadDate = dateFormat.parseSafe(chapter.optString("updated_at").substringBefore('.')), + branch = null, + source = source, + ) + }.reversed() + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet("https://$domain${chapter.url}").parseHtml() + val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() + ?: throw ParseException("Cannot find chapter data", chapter.url) + val images = JSONObject(raw) + .getJSONObject("props") + .getJSONObject("pageProps") + .getJSONObject("initialChapter") + .getJSONArray("images") + return (0 until images.length()).map { i -> + val imageUrl = images.getString(i) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + + private fun Double.toRating() = if (this > 0.0) (this / 5.0).toFloat() else RATING_UNKNOWN + + private fun String?.toMangaState() = when (this?.lowercase(Locale.US)) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + } + + companion object { + private val IMAGE_FALLBACK_REGEX = Regex("rx\\.qvzr[a-z]\\.org") + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaKatana.kt b/src/main/kotlin/tsuki/site/en/MangaKatana.kt new file mode 100644 index 00000000..898d646e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaKatana.kt @@ -0,0 +1,398 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +private const val two = "?sv=mk" +private const val three = "?sv=3" + +@MangaSourceParser("MANGAKATANA", "MangaKatana", "en") +internal class MangaKatana(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.MANGAKATANA, pageSize = 24) { + + private val baseUrl = "https://mangakatana.com" + + override val configKeyDomain = ConfigKey.Domain("mangakatana.com") + + override val availableSortOrders = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + )!! + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + _root_ide_package_.tsuki.site.en.two to "First server", + _root_ide_package_.tsuki.site.en.three to "Second server", + ), + defaultValue = _root_ide_package_.tsuki.site.en.two, + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = availableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + ), + ) + } + + private fun MangaState.toMKStatus(): Int { + return when (this) { + MangaState.ONGOING -> 1 + MangaState.FINISHED -> 2 + MangaState.ABANDONED -> 0 + else -> 1 + } + } + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true + ) + + private val dateFormat = SimpleDateFormat("MMM-dd-yyyy", Locale.US) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + + val url = buildUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div#book_list > div.item").map { element -> + + val link = element.selectFirst("div.text > h3 > a")!! + val thumb = element.selectFirst("img")!! + + Manga( + id = generateUid(link.absUrl("href")), + title = link.ownText().ifEmpty { link.text() }, + url = link.absUrl("href"), + publicUrl = link.absUrl("href"), + + coverUrl = thumb.absUrl("src"), + + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + + source = source + ) + } + } + + private fun buildUrl( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): String { + val query = filter.query?.trim().orEmpty() + val tags = filter.tags.map { it.key }.filter { it.isNotBlank() } + val status = filter.states.firstOrNull() + + return when { + query.isNotEmpty() -> { + if (page == 1) { + "$baseUrl/?search=${query.urlEncoded()}&search_by=m_name" + } else { + "$baseUrl/page/$page?search=${query.urlEncoded()}&search_by=m_name" + } + } + + status != null -> { + "$baseUrl/genres/?" + + "filter=1" + + "&include_mode=and" + + "&bookmark_opts=off" + + "&chapters=1" + + "&order=latest" + + "&status=${status.toMKStatus()}" + + "&page=$page" + } + + tags.isNotEmpty() -> + "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" + + else -> when (order) { + SortOrder.NEWEST -> "$baseUrl/new-manga?page=$page" + SortOrder.UPDATED -> "$baseUrl/latest/page/$page" + else -> "$baseUrl/latest/page/$page" + } + } + } + + override suspend fun getDetails(manga: Manga): Manga { + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() + + val chapters = doc.select("tr:has(.chapter)") + .mapNotNull { tr -> + val a = tr.selectFirst("a") ?: return@mapNotNull null + val href = a.absUrl("href") + + if (href.isBlank() || href == "#" || !href.startsWith("http")) return@mapNotNull null + + MangaChapter( + id = generateUid(href), + title = a.text(), + number = extractChapterNumber(href), + volume = 0, + url = href, + uploadDate = parseDate(tr.selectFirst(".update_time")?.text()), + scanlator = null, + branch = null, + source = source + ) + } + .sortedBy { it.number } + + return manga.copy( + title = doc.selectFirst("h1")?.text() + ?: doc.selectFirst("h1.heading")?.text() + ?: manga.title, + + authors = doc.select(".author").eachText().toSet(), + + tags = doc.select(".genres a").map { + MangaTag( + key = it.text().lowercase(), + title = it.text(), + source = source + ) + }.toSet(), + + description = buildString { + append(doc.select(".summary > p").text()) + val alt = doc.select(".alt_name").text() + if (alt.isNotBlank()) { + append("\n\nAlt name(s): $alt") + } + }.trim(), + + state = parseStatus(doc.select(".value.status").text()), + + coverUrl = doc.select("div.media div.cover img").attr("abs:src"), + chapters = chapters + ) + } + + private fun parseStatus(status: String): MangaState? { + return when { + status.contains("Ongoing", true) -> MangaState.ONGOING + status.contains("Completed", true) -> MangaState.FINISHED + else -> null + } + } + + private fun extractChapterNumber(url: String): Float { + return Regex("""/c(\d+(\.\d+)?)""") + .find(url) + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + ?: 0f + } + + private fun parseDate(text: String?): Long { + if (text.isNullOrBlank()) return 0L + return runCatching { + dateFormat.parse(text)?.time ?: 0L + }.getOrDefault(0L) + } + + + private fun availableTags() = setOf( + MangaTag("4 koma", "4-koma", source), + MangaTag("Action", "action", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Artbook", "artbook", source), + MangaTag("Award winning", "award-winning", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Drama", "drama", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Erotica", "erotica", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Loli", "loli", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("One shot", "one-shot", source), + MangaTag("Overpowered MC", "overpowered-mc", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Romance", "romance", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Sexual violence", "sexual-violence", source), + MangaTag("Shota", "shota", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujo-ai", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Shounen Ai", "shounen-ai", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Sports", "sports", source), + MangaTag("Super power", "super-power", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Survival", "survival", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Yuri", "yuri", source), + ) + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + override suspend fun getPages(chapter: MangaChapter): List { + + val url = applyServer(chapter.url) + val doc = webClient.httpGet(url).parseHtml() + + parseThzq(doc)?.let { + if (it.isNotEmpty()) return it + } + parseHtmlImages(doc)?.let { + if (it.isNotEmpty()) return it + } + val fallbackDoc = webClient.httpGet(chapter.url).parseHtml() + parseThzq(fallbackDoc)?.let { + if (it.isNotEmpty()) return it + } + + return emptyList() + } + + private fun applyServer(url: String): String { + val server = config[preferredServerKey] + return url + server + } + + private fun parseThzq(doc: Document): List? { + + val script = doc.select("script") + .asSequence() + .map { it.data() } + .firstOrNull { it.contains("thzq") } + ?: return null + + val start = script.indexOf("var thzq") + if (start == -1) return null + + val arrStart = script.indexOf('[', start) + if (arrStart == -1) return null + + var depth = 0 + var end = -1 + + for (i in arrStart until script.length) { + when (script[i]) { + '[' -> depth++ + ']' -> { + if (--depth == 0) { + end = i + break + } + } + } + } + + if (end == -1) return null + + val raw = script.substring(arrStart, end) + + val urls = ArrayList(32) + + var i = 0 + while (i < raw.length) { + if (raw[i] == '\'' || raw[i] == '"') { + val quote = raw[i] + val startIdx = ++i + + while (i < raw.length && raw[i] != quote) { + i++ + } + + if (i > startIdx) { + val url = raw.substring(startIdx, i) + if (url.startsWith("http")) { + urls.add(url) + } + } + } + i++ + } + + if (urls.isEmpty()) return null + + return urls.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + private fun parseHtmlImages(doc: Document): List? { + + val pages = doc.select("div.wrap_img img") + .asSequence() + .mapNotNull { img -> + + val url = img.attr("data-src") + .ifEmpty { img.attr("src") } + + if (url.isEmpty() || !url.startsWith("http")) return@mapNotNull null + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + .toList() + + return pages.ifEmpty { null } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/MangaTown.kt b/src/main/kotlin/tsuki/site/en/MangaTown.kt new file mode 100644 index 00000000..4cc000b8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaTown.kt @@ -0,0 +1,377 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANGATOWN", "MangaTown", "en") +internal class MangaTown(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGATOWN, 30) { + + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isYearSupported = false, + ) + + + @Volatile + private var allTagsCache: List? = null + + private suspend fun getAllTags(): List { + if (allTagsCache == null) { + allTagsCache = fetchAvailableTags().toList() + } + return allTagsCache!! + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getAllTags().toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableDemographics = EnumSet.of( + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.SHOUNEN, + Demographic.JOSEI, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHUA, + ContentType.MANHWA, + ), + ) + + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim() + val hasFilters = filter.types.isNotEmpty() || filter.demographics.isNotEmpty() || + filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || + filter.states.isNotEmpty() + + val url = when { + hasFilters -> { + val builder = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("name_method", "cw") + .addQueryParameter("name", query ?: "") + .addQueryParameter("author_method", "cw") + .addQueryParameter("author", "") + .addQueryParameter("artist_method", "cw") + .addQueryParameter("artist", "") + + builder.addQueryParameter("type", + if (filter.types.size == 1) when (filter.types.first()) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } else "" + ) + + builder.addQueryParameter("demographic", + if (filter.demographics.size == 1) when (filter.demographics.first()) { + Demographic.SEINEN -> "SEINEN" + Demographic.SHOUJO -> "SHOUJO" + Demographic.SHOUNEN -> "SHOUNEN" + Demographic.JOSEI -> "JOSEI" + else -> "" + } else "" + ) + + if (filter.states.size == 1) { + when (filter.states.first()) { + MangaState.FINISHED -> builder.addQueryParameter("is_completed", "1") + MangaState.ONGOING -> builder.addQueryParameter("is_completed", "0") + else -> builder.addQueryParameter("is_completed", "") + } + } else { + builder.addQueryParameter("is_completed", "") + } + + val allTags = getAllTags() + val includedKeys = filter.tags.map { it.key }.toSet() + val excludedKeys = filter.tagsExclude.map { it.key }.toSet() + for (tag in allTags) { + val value = when (tag.key) { + in includedKeys -> "1" + in excludedKeys -> "2" + else -> "0" + } + builder.addQueryParameter("genres[${tag.title}]", value) + } + + builder.addQueryParameter("released_method", "eq") + builder.addQueryParameter("released", "") + builder.addQueryParameter("rating_method", "eq") + builder.addQueryParameter("rating", "") + builder.addQueryParameter("advopts", "1") + builder.addQueryParameter("page", page.toString()) + + builder.build().toString() + } + !query.isNullOrEmpty() -> { + // Simple search + "https://$domain/search?page=$page&name=${query.urlEncoded()}" + } + order == SortOrder.UPDATED -> { + "https://$domain/latest/$page.htm" + } + else -> { + // Directory browse + buildString { + append("https://$domain/directory/") + append("0-") + append( + if (filter.tags.isNotEmpty()) filter.tags.oneOrThrowIfMany()?.key ?: "0" + else "0" + ) + append("-0-") + append( + if (filter.states.isNotEmpty()) { + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> "0" + } + } ?: "0" + } else "0" + ) + append("-0-0/") + append(page).append(".htm") + append( + when (order) { + SortOrder.POPULARITY -> "" + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + else -> "?last_chapter_time.za" + } + ) + } + } + } + + val doc = webClient.httpGet(url, getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("ul.manga_pic_list") ?: return emptyList() + val manga = root.select("li") + if (manga.isEmpty()) return emptyList() + + return manga.mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + val author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim() + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + source = source, + altTitles = emptySet(), + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + authors = setOfNotNull(author), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), + source = source, + ) + }.orEmpty(), + url = href, + contentRating = null, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } + + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("section.main") + ?.selectFirst("div.article_content") ?: return manga + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + + if (root.select("div.chapter_content:contains(has been licensed)").isNotEmpty()) { + return manga.copy(state = MangaState.ABANDONED, chapters = bypassLicensedChapters(manga)) + } + + return manga.copy( + tags = manga.tags + (info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), + source = source, + ) + }.orEmpty()), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val linkEl = li.selectFirst("a") ?: return@mapChapters null + val href = linkEl.attrAsRelativeUrl("href") + val name = buildString { + append(linkEl.text()) + li.select("span").filter { span -> + !span.hasClass("time") && !span.hasClass("new") + }.forEach { span -> + if (isNotEmpty()) append(" ") + append(span.text()) + } + } + MangaChapter( + id = generateUid(href), + url = href, + source = source, + number = i + 1f, + volume = 0, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + title = name.nullIfEmpty(), + scanlator = null, + branch = null, + ) + } ?: emptyList(), + ) + } + + + private suspend fun bypassLicensedChapters(manga: Manga): List { + val subdomain = "m." + domain.removePrefix("www.") + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(subdomain), getRequestHeaders()).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { a.ownText() } + MangaChapter( + id = generateUid(href), + url = href, + source = source, + number = i + 1f, + volume = 0, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + title = name.nullIfEmpty(), + scanlator = null, + branch = null, + ) + } + } + + + private fun parseChapterDate(dateFormat: SimpleDateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.parseSafe(date) + } + } + + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-") + MangaTag( + source = source, + key = key, + title = a.text().toTitleCase(), + ) + } + } + + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl, getRequestHeaders()).parseHtml() + val pageSelect = doc.body().selectFirst("div.page_select select") ?: return emptyList() + val options = pageSelect.select("option").filterNot { it.attr("value").endsWith("featured.html") } + if (options.isNotEmpty()) { + return options.mapIndexed { _, option -> + val href = option.attrAsRelativeUrl("value") + MangaPage( + id = generateUid(href), + url = href, + preview = null, + source = source, + ) + } + } + val imgElements = doc.select("div#viewer img") + return imgElements.map { img -> + val src = img.attrAsAbsoluteUrl("src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + + override suspend fun getPageUrl(page: MangaPage): String { + if (page.url.startsWith("http")) return page.url + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + return doc.selectFirst("div#viewer img")?.attrAsAbsoluteUrl("src") + ?: throw Exception("Could not find image") + } + + + private fun String.nullIfEmpty(): String? = ifEmpty { null } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt new file mode 100644 index 00000000..393d01a4 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt @@ -0,0 +1,474 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("MANGADOTNET", "Mangadot.net", "en", ContentType.MANGA) +internal class Mangadotnet(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGADOTNET, 20) { + + override val configKeyDomain = ConfigKey.Domain("mangadot.net") + private val baseUrl = "https://mangadot.net" + private val apiBase = "$baseUrl/api" + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) + + private val genreNames = setOf( + "Josei", "Seinen", "Shoujo", "Shounen", + "Action", "Adventure", "Comedy", "Drama", "Fantasy", + "Historical", "Horror", "Mecha", "Mystery", "Psychological", + "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", + "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", + "Medical", "Military", "Music", "School Life", "Webtoon", + "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", + "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", + "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", + "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", + "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", + "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", + "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", + "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", + "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", + "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", + "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", + "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", + "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", + "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", + "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", + "Wuxia", "Yaoi", "Yuri", "Zombies" + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showAdultKey) + } + + private fun decodeRsc(flat: JSONArray): Any? { + val cache = arrayOfNulls(flat.length()) + val nil = Any() + fun resolve(i: Int): Any? { + if (i < 0 || i >= flat.length()) return null + val cached = cache[i] + if (cached != null) return if (cached === nil) null else cached + val result: Any? = when (val el = flat.opt(i)) { + JSONObject.NULL -> null + is String, is Number, is Boolean -> el + is JSONArray -> (0 until el.length()).mapTo(mutableListOf()) { j -> resolve(el.optInt(j, -1)) } + is JSONObject -> { + val map = mutableMapOf() + for (key in el.keys()) { + val actualKey = if (key.startsWith("_")) flat.optString(key.removePrefix("_").toInt(), key) else key + map[actualKey] = resolve(el.optInt(key, -1)) + } + map + } + else -> null + } + cache[i] = result ?: nil + return result + } + return resolve(0) + } + + @Suppress("UNCHECKED_CAST") + private suspend fun fetchRscData(url: String, route: String): Map? { + val flat = webClient.httpGet(url).parseJsonArray() + val decoded = decodeRsc(flat) ?: return null + val routeValue = (decoded as? Map)?.get(route) as? Map ?: return null + return routeValue.asMap("data") + } + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchWithFiltersSupported = true, + ) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, // latest + SortOrder.POPULARITY, // most viewed + SortOrder.RATING, // top rated + SortOrder.ALPHABETICAL, // a-z + SortOrder.RELEVANCE, // most tracked + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val tags = genreNames.map { name -> MangaTag(name, name, source) }.toSet() + return MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + availableDemographics = EnumSet.of(Demographic.JOSEI, Demographic.SEINEN, Demographic.SHOUJO, Demographic.SHOUNEN), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || + filter.states.isNotEmpty() || filter.types.isNotEmpty() || filter.demographics.isNotEmpty()) { + return getSearchPage(page, order, filter) + } + return getBrowsePage(page, order) + } + + private suspend fun getBrowsePage(page: Int, order: SortOrder): List { + val sortParam = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "rating" + SortOrder.ALPHABETICAL -> "alphabetical" + SortOrder.RELEVANCE -> "tracked" + else -> null + } + val adultParam = if (config[showAdultKey]) "both" else "0" + val url = "$baseUrl/view-all/latest-updates.data".toHttpUrl().newBuilder().apply { + addQueryParameter("adult", adultParam) + if (page > 1) addQueryParameter("page", page.toString()) + sortParam?.let { addQueryParameter("sortBy", it) } + addQueryParameter("_routes", "pages/ViewAllPage") + }.build().toString() + return parseViewAllPage(url) + } + + private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val adultParam = if (config[showAdultKey]) "both" else "0" + val url = "$baseUrl/search.data".toHttpUrl().newBuilder().apply { + addQueryParameter("adult", adultParam) + if (!filter.query.isNullOrBlank()) addQueryParameter("search", filter.query!!) + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", "56") + + val (sortBy, sortOrder) = when (order) { + SortOrder.UPDATED -> "latest" to "desc" + SortOrder.ALPHABETICAL -> "alphabetical" to "asc" + SortOrder.POPULARITY -> "views" to "desc" + SortOrder.RATING -> "rating" to "desc" + else -> "relevance" to "desc" + } + addQueryParameter("sortBy", sortBy) + addQueryParameter("sortOrder", sortOrder) + + if (filter.states.size == 1) { + addQueryParameter("status", filter.states.first().toApiStatus()) + } + + filter.types.forEach { ct -> + when (ct) { + ContentType.MANGA -> addQueryParameter("origin", "JP") + ContentType.MANHWA -> addQueryParameter("origin", "KR") + ContentType.MANHUA -> addQueryParameter("origin", "CN") + ContentType.OTHER -> addQueryParameter("origin", "ONESHOT") + else -> {} + } + } + + filter.demographics.forEach { demographic -> + addQueryParameter("genre", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) + } + + filter.tags.forEach { tag -> addQueryParameter("genre", tag.key) } + filter.tagsExclude.forEach { tag -> addQueryParameter("genre", "-${tag.key}") } + + addQueryParameter("_routes", "pages/SearchPage") + }.build().toString() + + val dataMap = fetchRscData(url, "pages/SearchPage") ?: return emptyList() + return parseMangaList(dataMap) + } + + private suspend fun parseViewAllPage(url: String): List { + val viewAllData = fetchRscData(url, "pages/ViewAllPage") ?: return emptyList() + val mangaListMap = viewAllData.asMap("data") ?: return emptyList() + return parseMangaList(mangaListMap) + } + + private fun parseMangaList(data: Map): List { + val list = data["manga_list"] as? List<*> ?: data["results"] as? List<*> ?: return emptyList() + return list.filterIsInstance>().map { parseMangaFromList(it) } + } + + private fun parseMangaFromList(data: Map): Manga { + val id = (data["id"] as? Number)?.toInt()?.toString() ?: "" + val title = data["title"] as? String ?: "" + val photo = data["photo"] as? String + val coverUrl = photo?.let { + when { + it.startsWith("/") -> "$baseUrl$it" + it.startsWith("http") -> it + else -> null + } + } + return Manga( + id = generateUid(id), + url = id, + publicUrl = "$baseUrl/manga/$id", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + @Suppress("UNCHECKED_CAST") + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" + val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return manga + + val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return manga + val title = mangaInfo["title"] as? String ?: manga.title + val description = mangaInfo["description"] as? String + val photo = mangaInfo["photo"] as? String + val coverUrl = photo?.let { + when { + it.startsWith("/") -> "$baseUrl$it" + it.startsWith("http") -> it + else -> null + } + } ?: manga.coverUrl + val bannerImage = mangaInfo["banner_image"] as? String + val largeCoverUrl = bannerImage?.let { + when { + it.startsWith("/") -> "$baseUrl$it" + it.startsWith("http") -> it + else -> null + } + } + + val genres = (mangaInfo["genres"] as? List<*>)?.filterIsInstance()?.mapNotNull { it.trim().ifBlank { null } }.orEmpty() + val altTitles = (mangaInfo["alt_titles"] as? List<*>)?.filterIsInstance()?.mapNotNull { it.trim().ifBlank { null } }?.toSet().orEmpty() + val origin = mangaInfo["country_of_origin"] as? String + val authorRaw = mangaInfo["authors"] as? String + val author = parseJsonArrayString(authorRaw)?.joinToString(", ") ?: authorRaw + + val statusText = mangaInfo["status_text"] as? String + val status = mangaInfo["status"] as? String + val state = when { + !statusText.isNullOrBlank() -> when (statusText.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + } + !status.isNullOrBlank() -> when (status.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + } + else -> null + } + val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { MangaTag(key = it, title = it, source = source) } + when (origin) { + "JP" -> tagSet.add(MangaTag("Manga", "Manga", source)) + "KR" -> tagSet.add(MangaTag("Manhwa", "Manhwa", source)) + "CN" -> tagSet.add(MangaTag("Manhua", "Manhua", source)) + } + + val tagsArray = mangaInfo["tags"] as? List> + tagsArray?.flatMap { category -> + (category["tags"] as? List> ?: emptyList()) + .mapNotNull { it["name"] as? String } + }?.distinct()?.take(20)?.forEach { tagSet.add(MangaTag(it, it, source)) } + + val rawRating = (mangaInfo["avg_rating"] as? Number)?.toDouble() ?: -1.0 + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val chapters = fetchChapters(manga.url) + + val result = manga.copy( + title = title, + coverUrl = coverUrl, + largeCoverUrl = largeCoverUrl, + rating = rating, + description = description, + altTitles = altTitles, + tags = tagSet, + state = state, + authors = setOfNotNull(author).filterTo(mutableSetOf()) { it.isNotBlank() }, + chapters = chapters, + ) + + detailsCache[manga.url] = result + return result + } + + private fun parseJsonArrayString(raw: String?): List? { + if (raw.isNullOrBlank()) return null + return try { + val arr = JSONArray(raw) + (0 until arr.length()).mapNotNull { arr.optString(it) } + } catch (_: Exception) { null } + } + + private suspend fun fetchChapters(mangaId: String): List { + val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=en").parseJsonArray() + val allChapters = (0 until response.length()).map { response.getJSONObject(it) } + + val chaptersByTeam = mutableMapOf>() + for (ch in allChapters) { + val team = listOf( + ch.optString("group_name", ""), + ch.optString("scanlator_name", "") + ).firstOrNull { it.isNotBlank() && !it.equals("null", ignoreCase = true) } + ?: "Unknown" + chaptersByTeam.getOrPut(team) { mutableListOf() }.add(ch) + } + + val allNumbers = allChapters.map { it.optDouble("chapter_number", 0.0).toFloat() }.distinct().sorted() + val builder = ChaptersListBuilder(allChapters.size * chaptersByTeam.size) + + for ((team, teamChapters) in chaptersByTeam) { + val mapByNum = teamChapters.associateBy { it.optDouble("chapter_number", 0.0).toFloat() } + for (num in allNumbers) { + val chapter = mapByNum[num] ?: continue + val chId = chapter.getString("id") + val src = chapter.optString("source", "user") + val name = chapter.optString("chapter_title", "").nullIfEmpty() + val date = chapter.optString("date_added", "").nullIfEmpty() + val title = buildString { + val numStr = num.toString().removeSuffix(".0") + append("Chapter $numStr") + val cleanName = name?.trim()?.takeIf { it.isNotEmpty() && !it.equals("null", ignoreCase = true) } + if (cleanName != null && !cleanName.contains(numStr)) { + append(": ").append(cleanName) + } + } + val chapterUrl = JSONObject().apply { + put("id", chId) + put("source", src) + }.toString() + builder.add( + MangaChapter( + id = generateUid("$team-$chId"), + title = title, + number = num, + volume = 0, + url = chapterUrl, + uploadDate = date?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = team, + branch = team, + ) + ) + } + } + + val volumesResponse = runCatching { + webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=en").parseJsonArray() + }.getOrNull() + + if (volumesResponse != null) { + val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } + .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } + + for (vol in volumes) { + if (vol.optString("language", "en") != "en") continue + val volId = vol.getInt("id") + val volNum = vol.optDouble("volume_number", 0.0).toFloat() + val volDate = vol.optString("date_added", "").nullIfEmpty() + val volSource = vol.optString("source", "user") + val volGroup = vol.optString("group_name", "").nullIfEmpty() + ?: vol.optString("scanlator_name", "").nullIfEmpty() + ?: "Unknown" + val volumeUrl = JSONObject().apply { + put("id", volId) + put("source", volSource) + }.toString() + builder.add( + MangaChapter( + id = generateUid("vol-$volId"), + title = "Volume ${volNum.toString().removeSuffix(".0")}", + number = volNum, + volume = 0, + url = volumeUrl, + uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = volGroup, + branch = "Volume", + ) + ) + } + } + + return builder.toList() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterData = JSONObject(chapter.url) + val chapterId = chapterData.getString("id") + val source = chapterData.optString("source", "user") + val segment = if (source == "user") "uploads" else "chapters" + val response = webClient.httpGet("$baseUrl/api/$segment/$chapterId/images").parseJson() + val data = response.optJSONObject("data") + val imagesArray = if (data != null && data.has("images")) data.getJSONArray("images") else response.getJSONArray("images") + return (0 until imagesArray.length()).mapNotNull { i -> + val img = imagesArray.getJSONObject(i) + val imgUrl = img.optString("url", "").nullIfEmpty() ?: return@mapNotNull null + val fullUrl = when { + imgUrl.startsWith("/") -> "$baseUrl$imgUrl" + imgUrl.startsWith("http") -> imgUrl + else -> return@mapNotNull null + } + MangaPage( + id = generateUid(fullUrl), + url = fullUrl, + preview = null, + source = this.source, + ) + } + } + + + @Suppress("UNCHECKED_CAST") + override suspend fun getRelatedManga(seed: Manga): List { + val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" + val data = fetchRscData(url, "pages/MangaDetailPage") ?: return emptyList() + + val related = mutableListOf() + (data["suggestions"] as? List<*>)?.filterIsInstance>()?.mapTo(related) { parseMangaFromList(it) } + val relationsData = data.asMap("relationsData") + val relations = relationsData?.asMap("relations") as? Map>> + relations?.values?.forEach { list -> list.mapTo(related) { parseMangaFromList(it) } } + + return related + } + + private fun MangaState.toApiStatus() = when (this) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + else -> "" + } + + @Suppress("UNCHECKED_CAST") + private fun Map.asMap(key: String): Map? = this[key] as? Map + private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Mangago.kt b/src/main/kotlin/tsuki/site/en/Mangago.kt new file mode 100644 index 00000000..f99b5778 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mangago.kt @@ -0,0 +1,688 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.bitmap.Bitmap +import tsuki.bitmap.Rect + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Interceptor +import okhttp3.Response +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import javax.crypto.Cipher +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import kotlin.math.min + +@MangaSourceParser("MANGAGO", "Mangago", "en") +internal class MangagoParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAGO, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("www.mangago.me") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + private fun getAvailableTags(): Set = setOf( + MangaTag("Yaoi", "Yaoi", source), + MangaTag("Doujinshi", "Doujinshi", source), + MangaTag("Shounen Ai", "Shounen Ai", source), + MangaTag("Shoujo", "Shoujo", source), + MangaTag("Yuri", "Yuri", source), + MangaTag("Romance", "Romance", source), + MangaTag("Fantasy", "Fantasy", source), + MangaTag("Comedy", "Comedy", source), + MangaTag("Smut", "Smut", source), + MangaTag("Adult", "Adult", source), + MangaTag("School Life", "School Life", source), + MangaTag("Mystery", "Mystery", source), + MangaTag("One Shot", "One Shot", source), + MangaTag("Ecchi", "Ecchi", source), + MangaTag("Shounen", "Shounen", source), + MangaTag("Martial Arts", "Martial Arts", source), + MangaTag("Shoujo Ai", "Shoujo Ai", source), + MangaTag("Supernatural", "Supernatural", source), + MangaTag("Drama", "Drama", source), + MangaTag("Action", "Action", source), + MangaTag("Adventure", "Adventure", source), + MangaTag("Harem", "Harem", source), + MangaTag("Historical", "Historical", source), + MangaTag("Horror", "Horror", source), + MangaTag("Josei", "Josei", source), + MangaTag("Mature", "Mature", source), + MangaTag("Mecha", "Mecha", source), + MangaTag("Psychological", "Psychological", source), + MangaTag("Sci-fi", "Sci-fi", source), + MangaTag("Seinen", "Seinen", source), + MangaTag("Slice Of Life", "Slice Of Life", source), + MangaTag("Sports", "Sports", source), + MangaTag("Gender Bender", "Gender Bender", source), + MangaTag("Tragedy", "Tragedy", source), + MangaTag("Bara", "Bara", source), + MangaTag("Shotacon", "Shotacon", source), + MangaTag("Webtoons", "Webtoons", source), + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query.orEmpty() + if (!query.isNullOrEmpty()) { + val url = buildString { + append("https://") + append(domain) + append("/r/l_search?name=") + append(query.urlEncoded()) + append("&page=") + append(page) + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + val url = when (order) { + SortOrder.NEWEST -> { + buildString { + append("https://") + append(domain) + append("/list/new/") + if (filter.tags.isNotEmpty()) { + filter.tags.joinTo(this, ",") { it.key } + } else { + append("all") + } + append("/") + append(page) + append("/") + } + } + else -> { + buildString { + append("https://") + append(domain) + append("/genre/") + if (filter.tags.isNotEmpty()) { + filter.tags.joinTo(this, ",") { it.key } + } else { + append("all") + } + append("/") + append(page) + append("/?") + val states = filter.states + val showFinished = states.isEmpty() || states.contains(MangaState.FINISHED) + val showOngoing = states.isEmpty() || states.contains(MangaState.ONGOING) + append("f=").append(if (showFinished) "1" else "0") + append("&o=").append(if (showOngoing) "1" else "0") + append("&sortby=") + when (order) { + SortOrder.POPULARITY -> append("view") + SortOrder.UPDATED -> append("update_date") + else -> append("update_date") + } + append("&e=") + if (filter.tagsExclude.isNotEmpty()) { + filter.tagsExclude.joinTo(this, ",") { it.key } + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(doc: Document): List { + return doc.select(".box, .updatesli, .pic_list > li").mapNotNull { element -> + val linkElement = element.selectFirst(".thm-effect") ?: return@mapNotNull null + val href = linkElement.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val title = linkElement.attr("title").ifEmpty { + linkElement.selectFirst("h2 a")?.text() ?: return@mapNotNull null + } + val thumbnailElem = linkElement.selectFirst("img") ?: return@mapNotNull null + val thumbnailUrl = thumbnailElem.attr("abs:data-src").ifEmpty { + thumbnailElem.attr("abs:src") + } + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = thumbnailUrl, + largeCoverUrl = null, + description = null, + tags = emptySet(), + authors = emptySet(), + state = null, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val infoBlock = doc.requireElementById("information") + + val title = doc.selectFirstOrThrow(".w-title h1").text() + val thumbnail = infoBlock.selectFirstOrThrow("img").absUrl("src") + + var description = infoBlock.selectFirst(".manga_summary")?.let { summary -> + summary.selectFirst("font")?.remove() + summary.text() + }.orEmpty() + + var author: String? = null + var genres = emptySet() + var state: MangaState? = null + + infoBlock.select(".manga_info li, .manga_right tr").forEach { el -> + when (el.selectFirst("b, label")?.text()?.lowercase()) { + "alternative:" -> description += "\n\n${el.text()}" + "status:" -> state = when (el.selectFirstOrThrow("span").text().lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + "author(s):", "author:" -> author = el.select("a").joinToString { it.text() } + "genre(s):" -> genres = el.select("a").mapToSet { + MangaTag( + key = it.text(), + title = it.text(), + source = source, + ) + } + } + } + + val chapters = parseChapterList(doc) + + return manga.copy( + title = title, + coverUrl = thumbnail, + description = description, + authors = author?.let { setOf(it) } ?: emptySet(), + tags = genres, + state = state, + chapters = chapters, + ) + } + + private data class ChapterParseData( + val name: String, + val url: String, + val dateUpload: Long, + val scanlator: String?, + ) + + private fun parseChapterList(doc: Document): List { + val rawChapters = doc.select("table#chapter_table > tbody > tr, table.uk-table > tbody > tr") + .mapNotNull { element -> + val link = element.selectFirst("a.chico") ?: return@mapNotNull null + val url = link.attrAsAbsoluteUrlOrNull("href") ?: return@mapNotNull null + val name = link.text().trim() + + val dateText = element.select("td:last-child").text().trim() + val dateUpload = runCatching { + dateFormat.parse(dateText)?.time + }.getOrNull() ?: 0L + + val scanlator = element.selectFirst("td.no a, td.uk-table-shrink a")?.text()?.trim() + ?.ifEmpty { null } + + ChapterParseData(name, url, dateUpload, scanlator) + } + .reversed() + + return buildChapterList(rawChapters) + } + + private fun buildChapterList(chapters: List): List { + val scanlatorCounts = mutableMapOf() + for (chapter in chapters) { + val scanlator = chapter.scanlator ?: extractTitleSuffix(chapter.name) ?: continue + scanlatorCounts[scanlator] = (scanlatorCounts[scanlator] ?: 0) + 1 + } + val preferredScanlator = scanlatorCounts.maxByOrNull { it.value }?.key + + val regularChapters = mutableMapOf() + val specialChapters = mutableListOf() + + for (chapter in chapters) { + val chapterNum = extractChapterNumber(chapter.name) + if (chapterNum != null) { + val existing = regularChapters[chapterNum] + if (existing == null) { + regularChapters[chapterNum] = chapter + } else { + val existingSource = existing.scanlator ?: extractTitleSuffix(existing.name) + val newSource = chapter.scanlator ?: extractTitleSuffix(chapter.name) + + val existingIsPreferred = existingSource == preferredScanlator + val newIsPreferred = newSource == preferredScanlator + + if (newIsPreferred && !existingIsPreferred) { + regularChapters[chapterNum] = chapter + } else if (!newIsPreferred && existingIsPreferred) { + } else { + if (chapter.name.length > existing.name.length) { + regularChapters[chapterNum] = chapter + } + } + } + } else { + specialChapters.add(chapter) + } + } + + val result = mutableListOf() + + val sortedRegular = regularChapters.entries.sortedBy { it.key } + for ((chapterNum, chapter) in sortedRegular) { + result.add( + MangaChapter( + id = generateUid(chapter.url), + url = chapter.url, + title = chapter.name, + number = chapterNum, + volume = 0, + uploadDate = chapter.dateUpload, + scanlator = chapter.scanlator, + branch = null, + source = source, + ), + ) + } + + val baseSpecialNumber = (regularChapters.keys.maxOrNull() ?: 0f) + 10000f + for ((index, chapter) in specialChapters.withIndex()) { + result.add( + MangaChapter( + id = generateUid(chapter.url), + url = chapter.url, + title = chapter.name, + number = baseSpecialNumber + index, + volume = 0, + uploadDate = chapter.dateUpload, + scanlator = chapter.scanlator, + branch = null, + source = source, + ), + ) + } + + return result + } + + private fun extractTitleSuffix(title: String): String? { + val regex = Regex("""(?:ch\.?|chapter)\s*\d+(?:\.\d+)?\s*[:\-]\s*(.+)""", RegexOption.IGNORE_CASE) + return regex.find(title)?.groupValues?.get(1)?.trim()?.takeIf { it.isNotEmpty() } + } + + private fun extractChapterNumber(title: String): Float? { + return try { + val regex = Regex("""(?:ch\.?|chapter|vol\.?\s*\d+\s+ch\.?)\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) + val match = regex.find(title) + match?.groupValues?.get(1)?.toFloat() + } catch (e: Exception) { + null + } + } + + private val jsCache = mutableMapOf>() + private val maxCacheTime = 1000 * 60 * 5 // 5 minutes + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val pageDropdown = doc.select("div.controls ul#dropdown-menu-page") + if (pageDropdown.isNotEmpty()) { + val pagesCount = pageDropdown.select("li").size + + val cleanUrl = fullUrl.removeSuffix("/") + val lastSegment = cleanUrl.substringAfterLast("/") + + val isPgFormat = lastSegment.startsWith("pg-") + val pageNumber = lastSegment.toIntOrNull() + val isPageNumber = pageNumber != null && pageNumber < 1000 + + val baseUrl: String + val buildBatchUrl: (Int) -> String + + if (isPgFormat) { + baseUrl = cleanUrl.substringBeforeLast("/") + buildBatchUrl = { batchNum -> "$baseUrl/pg-$batchNum/" } + } else if (isPageNumber) { + baseUrl = cleanUrl.substringBeforeLast("/") + buildBatchUrl = { batchNum -> "$baseUrl/$batchNum/" } + } else { + baseUrl = cleanUrl + buildBatchUrl = { batchNum -> "$baseUrl/$batchNum/" } + } + + val batchSize = 5 + val allImages = mutableListOf() + var batchStart = 1 + + val js = getDeobfuscatedJS(doc) + val cols = getColsFromDoc(doc) ?: "" + + while (allImages.size < pagesCount) { + val batchUrl = buildBatchUrl(batchStart) + val batchDoc = if (batchStart == 1) doc else webClient.httpGet(batchUrl).parseHtml() + val batchImages = decryptImageList(batchDoc, batchUrl) + + if (batchImages.isEmpty()) { + break + } + + allImages.addAll(batchImages) + batchStart += batchSize + + if (batchStart > pagesCount + batchSize) { + break + } + } + + return allImages.take(pagesCount).mapIndexed { index, imageUrl -> + val url = if (imageUrl.contains("cspiclink") && js != null) { + val descramblingKey = getDescramblingKey(js, imageUrl) + "$imageUrl#desckey=$descramblingKey&cols=$cols" + } else { + imageUrl + } + + MangaPage( + id = generateUid("$fullUrl-$index"), + url = url, + preview = null, + source = source, + ) + } + } + + val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() + + if (imgsrcsScript != null) { + val images = decryptImageList(doc, fullUrl) + val cols = getColsFromDoc(doc) ?: "" + val js = getDeobfuscatedJS(doc) ?: throw Exception("Could not get JS") + + return images.mapIndexed { index, imageUrl -> + val url = if (imageUrl.contains("cspiclink")) { + val descramblingKey = getDescramblingKey(js, imageUrl) + "$imageUrl#desckey=$descramblingKey&cols=$cols" + } else { + imageUrl + } + + if (url.isBlank()) { + throw Exception("Final image URL at index $index is blank after processing") + } + + MangaPage( + id = generateUid("$fullUrl-$index"), + url = url, + preview = null, + source = source, + ) + } + } + + throw Exception("Could not find pages") + } + + override suspend fun getPageUrl(page: MangaPage): String { + if (page.url.isBlank()) { + throw Exception("Page URL is blank in getPageUrl") + } + return page.url + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + private suspend fun decryptImageList(doc: Document, sourceUrl: String): List { + val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() + ?: throw Exception("Could not find imgsrcs") + + val imgsrcRaw = IMG_SRCS_REGEX.find(imgsrcsScript)?.groupValues?.get(1) + ?: throw Exception("Could not extract imgsrcs") + + val imgsrcs = context.decodeBase64(imgsrcRaw) + + val deobfChapterJs = getDeobfuscatedJS(doc) ?: throw Exception("Could not deobfuscate JS") + + val key = findHexEncodedVariable(deobfChapterJs, "key").decodeHex() + val iv = findHexEncodedVariable(deobfChapterJs, "iv").decodeHex() + + val cipher = Cipher.getInstance("AES/CBC/NoPadding") + val keySpec = SecretKeySpec(key, "AES") + cipher.init(Cipher.DECRYPT_MODE, keySpec, IvParameterSpec(iv)) + val decryptedBytes = cipher.doFinal(imgsrcs) + + var imageList = String(decryptedBytes, Charsets.UTF_8).trimEnd('\u0000') + imageList = unscrambleImageList(imageList, deobfChapterJs) + + return imageList.split(",") + .map { it.trim() } + .filter { it.isNotBlank() } + .map { url -> + if (url.startsWith("https://") && url.contains("/_") || url.contains("https://iweb_")) { + url.replaceFirst("https://", "http://") + } else { + url + } + } + } + + private suspend fun getDeobfuscatedJS(doc: Document): String? { + val chapterJsUrl = doc.select("script[src*=chapter.js]").firstOrNull()?.absUrl("src") ?: return null + + val now = System.currentTimeMillis() + val cached = jsCache[chapterJsUrl] + if (cached != null && now - cached.second < maxCacheTime) { + return cached.first + } + + val obfuscatedChapterJs = webClient.httpGet(chapterJsUrl).parseRaw() + val deobf = deobfuscateSoJsonV4(obfuscatedChapterJs) + + jsCache[chapterJsUrl] = Pair(deobf, now) + return deobf + } + + private fun getColsFromDoc(doc: Document): String? { + val chapterJsUrl = doc.select("script[src*=chapter.js]").firstOrNull()?.absUrl("src") + val cached = if (chapterJsUrl != null) jsCache[chapterJsUrl] else null + val js = cached?.first ?: return null + return COLS_REGEX.find(js)?.groupValues?.get(1) + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + val fragment = response.request.url.fragment + + if (fragment == null || !fragment.contains("desckey=")) { + return response + } + + return context.redrawImageResponse(response) { bitmap -> + val key = fragment.substringAfter("desckey=").substringBefore("&") + val cols = fragment.substringAfter("cols=").toIntOrNull() ?: return@redrawImageResponse bitmap + unscrambleImage(bitmap, key, cols) + } + } + + private fun unscrambleImage(bitmap: Bitmap, key: String, cols: Int): Bitmap { + val width = bitmap.width + val height = bitmap.height + + val result = context.createBitmap(width, height) + + val unitWidth = width / cols + val unitHeight = height / cols + + val keyArray = key.split("a") + + for (idx in 0 until cols * cols) { + val keyval = keyArray.getOrNull(idx)?.takeIf { it.isNotEmpty() }?.toIntOrNull() ?: 0 + + val heightY = keyval / cols + val dy = heightY * unitHeight + val dx = (keyval - heightY * cols) * unitWidth + + val widthY = idx / cols + val sy = widthY * unitHeight + val sx = (idx - widthY * cols) * unitWidth + + val w = min(unitWidth, width - dx) + val h = min(unitHeight, height - dy) + + val srcRect = Rect(sx, sy, sx + w, sy + h) + val dstRect = Rect(dx, dy, dx + w, dy + h) + + result.drawBitmap(bitmap, srcRect, dstRect) + } + + return result + } + + private fun deobfuscateSoJsonV4(jsf: String): String { + if (!jsf.startsWith("['sojson.v4']")) { + throw Exception("Obfuscated code header mismatch. Expected sojson.v4") + } + + val splitRegex = Regex("[a-zA-Z]+") + val args = jsf.substring(240, jsf.length - 59).split(splitRegex) + + return args.map { it.toInt().toChar() }.joinToString("") + } + + private fun findHexEncodedVariable(input: String, variable: String): String { + val regex = Regex("""var $variable\s*=\s*CryptoJS\.enc\.Hex\.parse\("([0-9a-zA-Z]+)"\)""") + return regex.find(input)?.groupValues?.get(1) + ?: throw Exception("Could not find variable: $variable") + } + + private fun String.decodeHex(): ByteArray { + check(length % 2 == 0) { "Must have an even length" } + return chunked(2).map { it.toInt(16).toByte() }.toByteArray() + } + + private fun unscrambleImageList(imageList: String, js: String): String { + var imgList = imageList + + val keyLocations = KEY_LOCATION_REGEX.findAll(js) + .map { it.groupValues[1].toInt() } + .distinct() + .sorted() + .toList() + + if (keyLocations.isEmpty()) { + return imgList + } + + val unscrambleKey = try { + keyLocations.map { loc -> + if (loc >= imgList.length) { + throw NumberFormatException("Position $loc is beyond string length") + } + imgList[loc].toString().toInt() + } + } catch (e: NumberFormatException) { + return imgList + } + + keyLocations.forEachIndexed { idx, loc -> + imgList = imgList.removeRange((loc - idx)..(loc - idx)) + } + + imgList = imgList.unscramble(unscrambleKey) + + return imgList + } + + private fun String.unscramble(keys: List): String { + var s = this + keys.reversed().forEach { key -> + for (i in s.length - 1 downTo key) { + if (i % 2 != 0) { + val sourceIdx = i - key + if (sourceIdx >= 0) { + val temp = s[sourceIdx] + s = s.substring(0, sourceIdx) + s[i] + s.substring(sourceIdx + 1) + s = s.substring(0, i) + temp + s.substring(i + 1) + } + } + } + } + return s + } + + private suspend fun getDescramblingKey(deobfChapterJs: String, imageUrl: String): String { + val imgkeys = deobfChapterJs + .substringAfter("var renImg = function(img,width,height,id){", "") + .substringBefore("key = key.split(", "") + .split("\n") + .filter { line -> JS_FILTERS.none { filter -> line.contains(filter) } } + .joinToString("\n") + .replace("img.src", "url") + + if (imgkeys.isEmpty()) { + throw Exception("Failed to extract image key extraction code from chapter.js") + } + + val js = """ + function replacePos(strObj, pos, replacetext) { + var str = strObj.substr(0, pos) + replacetext + strObj.substring(pos + 1, strObj.length); + return str; + } + function getDescramblingKey(url) { $imgkeys; return key; } + getDescramblingKey("$imageUrl"); + """.trimIndent() + + val result = context.evaluateJs("https://$domain/", js) + + if (result.isNullOrEmpty()) { + throw Exception("Failed to evaluate JavaScript to get descrambling key. The image format or site structure may have changed.") + } + + return result + } + + private companion object { + private val IMG_SRCS_REGEX = Regex("""var imgsrcs\s*=\s*['"]([a-zA-Z0-9+=/]+)['"]""") + private val COLS_REGEX = Regex("""var\s*widthnum\s*=\s*heightnum\s*=\s*(\d+);""") + private val KEY_LOCATION_REGEX = Regex("""str\.charAt\(\s*(\d+)\s*\)""") + private val JS_FILTERS = listOf("jQuery", "document", "getContext", "toDataURL", "getImageData", "width", "height") + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Mangaread.kt b/src/main/kotlin/tsuki/site/en/Mangaread.kt new file mode 100644 index 00000000..8bf4f0a9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mangaread.kt @@ -0,0 +1,14 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGAREAD", "MangaRead", "en") +internal class MangaRead(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAREAD, "www.mangaread.org") { + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Mangataro.kt b/src/main/kotlin/tsuki/site/en/Mangataro.kt new file mode 100644 index 00000000..96937325 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mangataro.kt @@ -0,0 +1,845 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import okhttp3.HttpUrl +import java.security.MessageDigest +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.ArrayList +import java.util.Calendar +import java.util.EnumSet +import java.util.HashSet +import java.util.LinkedHashSet +import java.util.Locale + +@MangaSourceParser("MANGATAROORG", "Mangataro", "en", ContentType.MANGA) +internal class Mangataro(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGATAROORG, 48) { + + override val configKeyDomain = ConfigKey.Domain("mangataro.org") + + private val chapterNumberRegex = Regex("""(\d+(?:\.\d+)?)""") + + private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isYearSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = genreTags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + ) + + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val payload = JSONObject().apply { + put("page", page) + put("search", filter.query.orEmpty()) + + // Year + val yearArray = JSONArray() + if (filter.year > 0) yearArray.put(filter.year.toString()) + put("years", yearArray.toString()) + + // Genres (tags) + put("genres", filter.tags.toGenrePayload()) + + // Types + val typeArray = JSONArray() + filter.types.forEach { ct -> + val apiType = when (ct) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.OTHER -> "Novel" + else -> null + } + if (apiType != null) typeArray.put(apiType) + } + put("types", typeArray.toString()) + + // Statuses + val statusArray = JSONArray() + filter.states.forEach { state -> + val apiStatus = when (state) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + else -> null + } + if (apiStatus != null) statusArray.put(apiStatus) + } + put("statuses", statusArray.toString()) + + put("sort", order.toApiSort()) + put("genreMatchMode", "any") + } + val json = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) + return List(json.length()) { index -> + val item = json.getJSONObject(index) + item.toManga() + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val detailUrl = manga.url.urlWithoutFragment().toAbsoluteUrl(domain) + val document = fetchDocument(detailUrl) + val jsonLd = document.findSeriesJsonLd() + val description = document.extractDescription(jsonLd) + val altTitles = document.extractAltTitles(jsonLd) + val tags = document.extractTags(jsonLd) + val authors = document.extractAuthors(jsonLd) + val state = document.extractStatus() + val rating = document.extractRating(jsonLd) + val mangaId = extractMangaId(manga.url) + ?: document.selectFirst("[data-manga-id]")?.attr("data-manga-id") + ?: jsonLd?.optString("identifier")?.nullIfEmpty() + ?: throw ParseException("Unable to determine manga id", detailUrl) + val chapters = loadChapters(mangaId, detailUrl) + return manga.copy( + description = description, + tags = tags.ifEmpty { manga.tags }, + authors = authors.ifEmpty { manga.authors }, + altTitles = altTitles.ifEmpty { manga.altTitles }, + state = state, + rating = rating, + chapters = chapters, + ) + } + + private suspend fun loadChapters(mangaId: String, referer: String): List { + val headers = Headers.headersOf("Referer", referer) + val result = ArrayList() + val seenIds = HashSet() + var offset = 0 + val limit = 500 + var expectedTotal = Int.MAX_VALUE + while (offset < expectedTotal) { + val (token, timestamp) = generateChapterToken() + val requestUrl = buildChapterUrl(mangaId, offset, limit, token, timestamp) + val raw = webClient.httpGet(requestUrl, headers).parseRaw().trim() + if (raw.isEmpty()) { + break + } + val json = runCatching { JSONObject(raw) }.getOrNull() ?: break + if (!json.optBoolean("success", false)) { + break + } + expectedTotal = json.optInt("total", expectedTotal) + val chaptersArray = json.optJSONArray("chapters") ?: break + if (chaptersArray.length() == 0) { + break + } + for (i in 0 until chaptersArray.length()) { + val item = chaptersArray.getJSONObject(i) + val key = item.optString("id").ifEmpty { item.optString("url") } + if (!seenIds.add(key)) { + continue + } + val fallbackNumber = (offset + i + 1).toFloat() + result.add(item.toMangaChapter(fallbackNumber)) + } + offset += chaptersArray.length() + if (!json.optBoolean("has_more", false)) { + break + } + } + return result + } + + private fun buildChapterUrl( + mangaId: String, + offset: Int, + limit: Int, + token: String, + timestamp: String, + ): String { + return HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegment("auth") + .addPathSegment("manga-chapters") + .addQueryParameter("manga_id", mangaId) + .addQueryParameter("offset", offset.toString()) + .addQueryParameter("limit", limit.toString()) + .addQueryParameter("order", "ASC") + .addQueryParameter("_t", token) + .addQueryParameter("_ts", timestamp) + .build() + .toString() + } + + private fun generateChapterToken(): Pair { + val timestamp = (System.currentTimeMillis() / 1000L).toString() + val hourKey = ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter) + val secret = "mng_ch_$hourKey" + val hashInput = timestamp + secret + val token = MessageDigest + .getInstance("MD5") + .digest(hashInput.toByteArray()) + .toHex() + .substring(0, 16) + return token to timestamp + } + + private fun ByteArray.toHex(): String { + if (isEmpty()) { + return "" + } + val chars = CharArray(size * 2) + var index = 0 + for (element in this) { + val unsigned = element.toInt() and 0xFF + chars[index++] = Character.forDigit((unsigned ushr 4) and 0xF, 16) + chars[index++] = Character.forDigit(unsigned and 0xF, 16) + } + return String(chars) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val chapterId = chapterUrl.extractChapterId() + ?: fetchDocument(chapterUrl).body().attr("data-chapter-id").nullIfEmpty() + if (chapterId != null) { + val pages = loadChapterPages(chapterId, chapterUrl) + if (pages.isNotEmpty()) { + return pages + } + } + val document = fetchDocument(chapterUrl) + val containers = document.select("div.comic-image-container img") + val seen = HashSet(containers.size) + return containers.mapNotNull { element -> + val imageUrl = element.resolveImageUrl() + ?.takeUnless { it.startsWith("data:") } + ?: return@mapNotNull null + if (!seen.add(imageUrl)) { + return@mapNotNull null + } + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun loadChapterPages(chapterId: String, referer: String): List { + val headers = Headers.headersOf( + "Accept", + "application/json", + "Referer", + referer, + ) + val raw = webClient.httpGet("https://$domain/auth/chapter-content?chapter_id=$chapterId", headers) + .parseRaw() + .trim() + if (raw.isEmpty()) { + return emptyList() + } + val json = raw.toJsonObjectOrNull() ?: return emptyList() + if (!json.optBoolean("success", false) || json.optString("chapter_type") == "text") { + return emptyList() + } + val images = json.optJSONArray("images") ?: return emptyList() + return List(images.length()) { index -> + val imageUrl = images.getString(index) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private fun JSONObject.toManga(): Manga { + val id = optString("id") + val rawUrl = optString("url") + val relativeUrl = if (rawUrl.isBlank()) "/" else rawUrl.toRelativeUrl(domain) + val internalUrl = relativeUrl.withMangaId(id) + val publicUrl = rawUrl.nullIfEmpty() ?: relativeUrl.toAbsoluteUrl(domain) + return Manga( + id = generateUid(internalUrl), + title = optString("title"), + altTitles = emptySet(), + url = internalUrl, + publicUrl = publicUrl, + rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover").nullIfEmpty(), + tags = emptySet(), + state = optString("status").toMangaState(), + authors = emptySet(), + description = optString("description").nullIfEmpty(), + source = source, + ) + } + + private fun JSONObject.toMangaChapter(fallbackNumber: Float): MangaChapter { + val chapterUrl = optString("url") + val rawChapter = optString("chapter") + val chapterLabel = rawChapter.nullIfEmpty() ?: fallbackNumber.toChapterLabel() + val rawTitle = optString("title").trim() + val title = when { + rawTitle.equals("N/A", ignoreCase = true) -> "Chapter $chapterLabel" + rawTitle.isEmpty() -> null + else -> rawTitle + } + val number = chapterNumberRegex.find(rawChapter)?.value?.toFloatOrNull() + return MangaChapter( + id = generateUid(chapterUrl), + title = title, + number = number ?: fallbackNumber, + volume = 0, + url = chapterUrl.toRelativeUrl(domain), + scanlator = null, + uploadDate = parseChapterDate(optString("date")), + branch = null, + source = source, + ) + } + + private fun Float.toChapterLabel(): String = if (this % 1f == 0f) { + toInt().toString() + } else { + replaceTrailingZeros() + } + + private fun Float.replaceTrailingZeros(): String { + val text = toString() + val trimmed = text.trimEnd('0') + return if (trimmed.endsWith('.')) { + trimmed.dropLast(1) + } else { + trimmed + } + } + + private fun String?.toMangaState(): MangaState? { + val normalized = this?.lowercase(Locale.US)?.trim() ?: return null + return when { + normalized.contains("ongoing") -> MangaState.ONGOING + normalized.contains("complete") || normalized.contains("completed") -> MangaState.FINISHED + normalized.contains("hiatus") -> MangaState.PAUSED + normalized.contains("canceled") || normalized.contains("cancelled") -> MangaState.ABANDONED + normalized.contains("upcoming") || normalized.contains("tba") -> MangaState.UPCOMING + else -> null + } + } + + private fun parseChapterDate(raw: String?): Long { + val value = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L + if (!value.endsWith("ago")) { + return 0L + } + val amount = chapterNumberRegex.find(value)?.value?.toFloatOrNull() ?: return 0L + val calendar = Calendar.getInstance() + when { + value.contains("second") -> calendar.add(Calendar.SECOND, -amount.toInt()) + value.contains("minute") -> calendar.add(Calendar.MINUTE, -amount.toInt()) + value.contains("hour") -> calendar.add(Calendar.HOUR_OF_DAY, -amount.toInt()) + value.contains("day") -> calendar.add(Calendar.DAY_OF_YEAR, -amount.toInt()) + value.contains("week") -> calendar.add(Calendar.WEEK_OF_YEAR, -amount.toInt()) + value.contains("month") -> calendar.add(Calendar.MONTH, -amount.toInt()) + value.contains("year") -> calendar.add(Calendar.YEAR, -amount.toInt()) + else -> return 0L + } + return calendar.timeInMillis + } + + private fun SortOrder.toApiSort(): String = when (this) { + SortOrder.POPULARITY -> "popular_desc" + SortOrder.UPDATED -> "post_desc" + SortOrder.NEWEST -> "release_desc" + SortOrder.ALPHABETICAL -> "title_desc" + else -> "post_desc" + } + + private fun Document.extractDescription(jsonLd: JSONObject?): String? { + val fromJson = jsonLd?.optString("description")?.nullIfEmpty() + if (fromJson != null) { + return fromJson + } + return selectFirst("div[data-description], div#description, div#synopsis, div:matchesOwn((?i)description)") + ?.nextElementSibling() + ?.text() + ?.nullIfEmpty() + ?: selectFirst("meta[name='description']")?.attr("content")?.nullIfEmpty() + ?: selectFirst("meta[property='og:description']")?.attr("content")?.nullIfEmpty() + } + + private fun Document.extractAltTitles(jsonLd: JSONObject?): Set { + val alt = mutableSetOf() + jsonLd?.opt("alternateName")?.let { value -> + when (value) { + is JSONArray -> { + repeat(value.length()) { index -> + value.optString(index).nullIfEmpty()?.let(alt::add) + } + } + is String -> value.nullIfEmpty()?.let(alt::add) + } + } + if (alt.isEmpty()) { + selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { alt.addAll(it) } + } + return alt + } + + private fun Document.extractTags(jsonLd: JSONObject?): Set { + val tags = LinkedHashSet() + jsonLd?.opt("genre")?.let { genre -> + when (genre) { + is JSONArray -> repeat(genre.length()) { index -> + genre.optString(index).nullIfEmpty()?.let { addTag(tags, it) } + } + is String -> genre.nullIfEmpty()?.let { addTag(tags, it) } + } + } + if (tags.isEmpty()) { + select("a[href*='/genre/']").mapNotNullToSet { anchor -> + anchor.text().nullIfEmpty()?.let { title -> + val href = anchor.attr("href").trimEnd('/') + val slug = href.substringAfterLast('/').substringBefore('?').nullIfEmpty() + ?: return@let null + MangaTag( + key = slug, + title = title, + source = source, + ) + } + } + .takeIf { it.isNotEmpty() } + ?.let(tags::addAll) + } + return tags + } + + private fun addTag(target: MutableSet, value: String) { + val normalized = value.trim().nullIfEmpty() ?: return + target.add( + MangaTag( + key = normalized.lowercase(Locale.US).replace(' ', '-'), + title = normalized, + source = source, + ), + ) + } + + private fun Document.extractAuthors(jsonLd: JSONObject?): Set { + val authors = LinkedHashSet() + fun addName(value: Any?) { + when (value) { + is JSONArray -> repeat(value.length()) { index -> addName(value.opt(index)) } + is JSONObject -> addName(value.opt("name")) + is String -> value.nullIfEmpty()?.let(authors::add) + } + } + addName(jsonLd?.opt("author")) + addName(jsonLd?.opt("creator")) + if (authors.isEmpty()) { + select("a[href*='/author/'], a[href*='/artist/']").forEach { anchor -> + anchor.text().nullIfEmpty()?.let(authors::add) + } + } + return authors + } + + private fun Document.extractStatus(): MangaState? { + val label = select("*").firstOrNull { element -> + element.ownText().trim().equals("Status", ignoreCase = true) + } + val statusText = label?.let { element -> + element.findSiblingText()?.nullIfEmpty() + } + ?: selectFirst("[data-status]")?.attr("data-status")?.nullIfEmpty() + ?: selectFirst("span.status, div.status")?.text()?.nullIfEmpty() + return statusText.toMangaState() + } + + private fun Document.extractRating(jsonLd: JSONObject?): Float { + jsonLd?.optJSONObject("aggregateRating")?.optDouble("ratingValue")?.takeIf { !it.isNaN() }?.let { + return it.toFloat() + } + return selectFirst("svg ~ span.font-semibold")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN + } + + private fun Document.findSeriesJsonLd(): JSONObject? { + for (script in select("script[type=application/ld+json]")) { + val raw = script.data().trim() + if (raw.isEmpty()) { + continue + } + val obj = raw.toJsonObjectOrNull() + if (obj != null) { + if (obj.matchesSeriesType()) { + return obj + } + continue + } + val array = raw.toJsonArrayOrNull() ?: continue + repeat(array.length()) { index -> + val candidate = array.optJSONObject(index) ?: return@repeat + if (candidate.matchesSeriesType()) { + return candidate + } + } + } + return null + } + + private fun String.toJsonObjectOrNull(): JSONObject? = runCatching { JSONObject(this) }.getOrNull() + private fun String.toJsonArrayOrNull(): JSONArray? = runCatching { JSONArray(this) }.getOrNull() + private fun JSONObject.matchesSeriesType(): Boolean { + val type = optString("@type") + if (type.isNullOrEmpty()) { + return false + } + return type.equals("ComicSeries", true) || + type.equals("ComicStory", true) || + type.equals("Book", true) || + type.equals("CreativeWorkSeries", true) + } + + private fun String?.withMangaId(id: String?): String { + val value = this?.nullIfEmpty() ?: return "/" + val cleanId = id?.nullIfEmpty() ?: return value + return if (value.contains('#')) { + if (value.contains("mid=")) value else "$value&mid=$cleanId" + } else { + "$value#mid=$cleanId" + } + } + + private fun extractMangaId(rawUrl: String): String? { + val fragment = rawUrl.substringAfter('#', "") + if (fragment.isNotEmpty()) { + fragment.split('&').firstOrNull { it.startsWith("mid=") }?.let { part -> + return part.substringAfter("mid=").nullIfEmpty() + } + } + val query = rawUrl.substringAfter('?', "") + if (query.isNotEmpty()) { + query.split('&').firstOrNull { it.startsWith("mid=") || it.startsWith("mangaId=") }?.let { part -> + return part.substringAfter('=').nullIfEmpty() + } + } + return null + } + + private fun String.urlWithoutFragment(): String = substringBefore('#') + + private fun String.extractChapterId(): String? = substringBefore('#') + .substringBefore('?') + .substringAfterLast('-') + .takeIf { it.all(Char::isDigit) } + ?.nullIfEmpty() + + private suspend fun postJsonArray(url: String, payload: JSONObject): JSONArray { + val raw = webClient.httpPost(url, payload).parseRaw().trim() + if (raw.isEmpty()) { + return JSONArray() + } + return raw.toJsonArrayOrNull() ?: JSONArray() + } + + private suspend fun fetchDocument(url: String): Document = webClient.httpGet(url).parseHtml() + + private fun Collection.toGenrePayload(): String { + if (isEmpty()) { + return "[]" + } + val ids = LinkedHashSet(size) + for (tag in this) { + val numericKey = tag.key.toIntOrNull()?.toString() + ?: genreTags.firstOrNull { ref -> + ref.title.equals(tag.title, ignoreCase = true) + }?.key?.toIntOrNull()?.toString() + if (numericKey != null) { + ids.add(numericKey) + } + } + if (ids.isEmpty()) { + return "[]" + } + return buildString(ids.size * 4) { + append('[') + ids.forEachIndexed { index, value -> + if (index > 0) { + append(',') + } + append(value) + } + append(']') + } + } + + private fun tag(id: String, title: String) = MangaTag(title = title, key = id, source = source) + + private fun Element.findSiblingText(): String? { + nextElementSibling()?.text()?.nullIfEmpty()?.let { return it } + val parent = parent() ?: return null + val children = parent.children() + val index = children.indexOf(this) + if (index == -1) { + return null + } + for (i in index + 1 until children.size) { + val text = children[i].text().nullIfEmpty() + if (text != null) { + return text + } + } + return null + } + + private fun Element.resolveImageUrl(): String? { + return attrAsAbsoluteUrlOrNull("data-src") + ?: attrAsAbsoluteUrlOrNull("data-lazy-src") + ?: attrAsAbsoluteUrlOrNull("src") + } + + private val genreTags by lazy { + setOf( + tag("7", "Action"), + tag("12", "Adventure"), + tag("17", "Award Winning"), + tag("80", "Boys Love"), + tag("20", "Comedy"), + tag("8", "Drama"), + tag("40", "Ecchi"), + tag("71", "Erotica"), + tag("2", "Fantasy"), + tag("34", "Full Color"), + tag("49", "Girls Love"), + tag("64", "Harem"), + tag("18", "Historical"), + tag("44", "Horror"), + tag("3", "Isekai"), + tag("43", "Josei"), + tag("1172", "Long Strip"), + tag("14", "Manga"), + tag("33", "Manhua"), + tag("6", "Manhwa"), + tag("4", "Martial Arts"), + tag("24", "Mystery"), + tag("48", "One-shot"), + tag("26", "Psychological"), + tag("5", "Reincarnation"), + tag("29", "Romance"), + tag("10", "School"), + tag("15", "Sci-Fi"), + tag("19", "Seinen"), + tag("42", "Shoujo"), + tag("13", "Shounen"), + tag("47", "Slice of Life"), + tag("25", "Sports"), + tag("9", "Supernatural"), + tag("32", "Time Travel"), + tag("39", "Webtoon"), + tag("2094", "4-Koma"), + tag("1050", "Abandoned Children"), + tag("2386", "Ability Steal"), + tag("2397", "Absent Parents"), + tag("4012", "Academy"), + tag("999", "Accelerated Growth"), + tag("1351", "Adaptation"), + tag("1000", "Adapted to Anime"), + tag("5072", "Adapted to Drama CD"), + tag("1001", "Adapted to Game"), + tag("5069", "Adapted to Manga"), + tag("963", "Adapted to Manhua"), + tag("1002", "Adapted to Manhwa"), + tag("5070", "Adapted to Visual Novel"), + tag("1058", "Adopted Children"), + tag("88", "Adult"), + tag("16", "Adult Cast"), + tag("2861", "Age Regression"), + tag("964", "Alchemy"), + tag("965", "Aliens"), + tag("966", "Alternate World"), + tag("2862", "Ancient Times"), + tag("1135", "Animals"), + tag("2096", "Anthology"), + tag("93", "Anthropomorphic"), + tag("5071", "Appearance Different from Actual Age"), + tag("1037", "Aristocracy"), + tag("2387", "Army"), + tag("1005", "Army Building"), + tag("1047", "Arranged Marriage"), + tag("2401", "Arrogant Characters"), + tag("4551", "Artbook"), + tag("4004", "Artifacts"), + tag("2863", "Assassins"), + tag("98", "Avant Garde"), + tag("5279", "Battle Academy"), + tag("2388", "Battle Competition"), + tag("1137", "BD"), + tag("968", "Beast Companions"), + tag("969", "Beautiful Female Lead"), + tag("2864", "Betrayal"), + tag("971", "Blacksmith"), + tag("4237", "Boys' Love"), + tag("1038", "Brotherhood"), + tag("2398", "Business Management"), + tag("1006", "Calm Protagonist"), + tag("2392", "Cautious Protagonist"), + tag("94", "CGDCT"), + tag("4929", "Character growth"), + tag("51", "Childcare"), + tag("4818", "Children's"), + tag("2393", "Cold Love Interests"), + tag("2394", "Cold Protagonist"), + tag("92", "Combat Sports"), + tag("2118", "Cooking"), + tag("1753", "Crime"), + tag("81", "Crossdressing"), + tag("2395", "Cruel Characters"), + tag("1039", "Cunning Protagonist"), + tag("4817", "Cute Stuffs"), + tag("4930", "Dark"), + tag("1044", "Death of Loved Ones"), + tag("52", "Delinquents"), + tag("1376", "Demons"), + tag("975", "Dense Protagonist"), + tag("58", "Detective"), + tag("4008", "Devoted love interests"), + tag("79", "Doujinshi"), + tag("1040", "Dragons"), + tag("1048", "Drugs"), + tag("4795", "Dungeon"), + tag("1011", "Dungeons"), + tag("66", "Educational"), + tag("976", "Elemental Magic"), + tag("4010", "Elves"), + tag("4005", "Family"), + tag("59", "Gag Humor"), + tag("36", "Game"), + tag("979", "Game Elements"), + tag("106", "Gender Bender"), + tag("2093", "Genderswap"), + tag("1134", "Ghosts"), + tag("4739", "Gyaru"), + tag("76", "High Stakes Game"), + tag("82", "Hentai"), + tag("4932", "Human nonhuman relationship"), + tag("95", "Idols (Female)"), + tag("101", "Idols (Male)"), + tag("109", "Indonesian"), + tag("91", "Iyashikei"), + tag("107", "Kids"), + tag("4006", "Kingdoms"), + tag("1020", "Level System"), + tag("55", "Light Novel"), + tag("4207", "Loli"), + tag("89", "Lolicon"), + tag("1173", "Web Comic"), + tag("41", "Love Polygon"), + tag("100", "Love Status Quo"), + tag("2061", "Mafia"), + tag("35", "Magic"), + tag("2095", "Magical Girls"), + tag("68", "Magical Sex Shift"), + tag("97", "Mahou Shoujo"), + tag("4933", "Male"), + tag("4792", "Mangataro Exclusive"), + tag("1414", "Monster Girls"), + tag("37", "Monsters"), + tag("1024", "Multiple POV"), + tag("38", "Murim"), + tag("83", "Music"), + tag("53", "Mythology"), + tag("4601", "Ninja"), + tag("103", "OEL"), + tag("2062", "Office Workers"), + tag("4834", "Official Colored"), + tag("4210", "Oneshot"), + tag("50", "Organized Crime"), + tag("70", "Otaku Culture"), + tag("21", "Parody"), + tag("72", "Performing Arts"), + tag("105", "Pets"), + tag("63", "Philosophical"), + tag("1136", "Police"), + tag("1557", "Post-Apocalyptic"), + tag("102", "Racing"), + tag("4014", "Reincarnated in another world"), + tag("61", "Reverse Harem"), + tag("28", "Romantic Subtext"), + tag("60", "Samurai"), + tag("73", "School Life"), + tag("1413", "Self-Published"), + tag("4116", "Sexual Violence"), + tag("90", "Shoujo Ai"), + tag("104", "Shounen Ai"), + tag("54", "Showbiz"), + tag("4009", "Skill books"), + tag("85", "Smut"), + tag("84", "Space"), + tag("77", "Strategy Game"), + tag("22", "Super Power"), + tag("4098", "Superhero"), + tag("56", "Survival"), + tag("75", "Suspense"), + tag("995", "Sword And Magic"), + tag("27", "Team Sports"), + tag("591", "Thai"), + tag("1341", "Thriller"), + tag("4742", "Traditional Games"), + tag("65", "Tragedy"), + tag("4011", "Transported to another world"), + tag("110", "Urban Fantasy"), + tag("45", "Vampire"), + tag("2102", "Vampires"), + tag("31", "Video Game"), + tag("1366", "Video Games"), + tag("62", "Villainess"), + tag("4485", "Virtual Reality"), + tag("57", "Visual Arts"), + tag("69", "Workplace"), + tag("1350", "Wuxia"), + tag("2391", "Xianxia"), + tag("962", "Xuanhuan"), + tag("86", "Yaoi"), + tag("99", "Yuri"), + tag("1342", "Zombies"), + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Manhuaus.kt b/src/main/kotlin/tsuki/site/en/Manhuaus.kt new file mode 100644 index 00000000..f169d4b1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Manhuaus.kt @@ -0,0 +1,80 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.MangaListFilterCapabilities +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import androidx.collection.arraySetOf + +@MangaSourceParser("MANHUAUS", "Manhuaus", "en") +internal class Manhuaus(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.MANHUAUS, "manhuaus.com") { + + override val postReq = false + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isAuthorSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + private fun availableTags() = arraySetOf( + MangaTag("Action", "action", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Anime", "anime", source), + MangaTag("Cartoon", "cartoon", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Comic", "comic", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Detective", "detective", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Drama", "drama", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Josei", "josei", source), + MangaTag("Live Action", "live-action", source), + MangaTag("Manga", "manga", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mature", "mature", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Mystery", "mystery", source), + MangaTag("One Shot", "one-shot", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Romance", "romance", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujo-ai", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Shounen Ai", "shounen-ai", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Smut", "smut", source), + MangaTag("Soft Yaoi", "soft-yaoi", source), + MangaTag("Soft Yuri", "soft-yuri", source), + MangaTag("Sports", "sports", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Yuri", "yuri", source), + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ManhwaNex.kt b/src/main/kotlin/tsuki/site/en/ManhwaNex.kt new file mode 100644 index 00000000..7e34539f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ManhwaNex.kt @@ -0,0 +1,15 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWANEX", "ManhwaNex", "en") +internal class ManhwaNex(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.MANHWANEX, "manhwanex.com") { + + override val postReq = false + +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ManhwaTop.kt b/src/main/kotlin/tsuki/site/en/ManhwaTop.kt new file mode 100644 index 00000000..9d9506d7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ManhwaTop.kt @@ -0,0 +1,11 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWATOP", "ManhwaTop", "en") +internal class ManhwaTop(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANHWATOP, "manhwatop.com") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ManhwaZ.kt b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt new file mode 100644 index 00000000..9d28b2cd --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt @@ -0,0 +1,123 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +@MangaSourceParser("MANHWAZ", "ManhwaZ", "en", ContentType.MANGA) +internal class Manhwaz(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANHWAZ, "manhwaz.com", 40) { + + override val listUrl = "genre/manhwa" + override val tagPrefix = "genre/" + override val withoutAjax = true + override val selectTestAsync = "div.list-chapter" + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableStates = emptySet(), + availableContentRating = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search?s=") + append(query.urlEncoded()) + append("&page=") + append(page.toString()) + } + + else -> { + + val tag = filter.tags.oneOrThrowIfMany() + if (filter.tags.isNotEmpty()) { + append("/$tagPrefix") + append(tag?.key.orEmpty()) + append("?page=") + append(page.toString()) + append("&") + } else { + append("/$listUrl") + append("?page=") + append(page.toString()) + append("&") + } + + append("m_orderby=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + SortOrder.NEWEST -> append("new") + SortOrder.RATING -> append("rating") + else -> append("latest") + } + } + } + } + val doc = webClient.httpGet(url).parseHtml() + + return doc.select("div.row.c-tabs-item__content").ifEmpty { + doc.select("div.page-item-detail") + }.map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") + val summary = div.selectFirst(".tab-summary") ?: div.selectFirst(".item-summary") + val author = summary?.selectFirst(".mg_author")?.selectFirst("a")?.ownText() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src(), + title = (summary?.selectFirst("h3") ?: summary?.selectFirst("h4"))?.text().orEmpty(), + altTitles = emptySet(), + rating = div.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, + tags = summary?.selectFirst(".mg_genres")?.select("a")?.mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text().ifEmpty { return@mapNotNullToSet null }.toTitleCase(), + source = source, + ) + }.orEmpty(), + authors = setOfNotNull(author), + state = when (summary?.selectFirst(".mg_status")?.selectFirst(".summary-content")?.ownText() + ?.lowercase().orEmpty()) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + }, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/en/Mgread.kt b/src/main/kotlin/tsuki/site/en/Mgread.kt new file mode 100644 index 00000000..31f21338 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mgread.kt @@ -0,0 +1,299 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.UserAgents + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import okhttp3.Interceptor +import okhttp3.Response +import java.util.Locale +import java.util.TimeZone + +// TODO +// add filter options + +@MangaSourceParser("MGREADIO", "Mgread.io", "en") +internal class MgreadIo(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MGREADIO, 24) { + + override val configKeyDomain = ConfigKey.Domain("mgread.io") + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request().newBuilder() + .header("Referer", "https://$domain/") + .header("Origin", "https://$domain") + .build() + return chain.proceed(request) + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US) + private val restChapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US).apply { + timeZone = TimeZone.getTimeZone("GMT+7") + } + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + + if (query != null) { + val url = "https://$domain/wp-json/initlise/v1/search".toHttpUrl().newBuilder() + .addQueryParameter("term", query) + .addQueryParameter("page", page.toString()) + .build() + val jsonArray = JSONArray(webClient.httpGet(url).parseRaw()) + return (0 until jsonArray.length()).mapNotNull { i -> + val obj = jsonArray.getJSONObject(i) + val title = obj.getString("title").let { org.jsoup.Jsoup.parse(it).text() } + val mangaUrl = obj.getString("url").trim().takeIf(String::isNotEmpty) ?: return@mapNotNull null + val thumb = obj.optString("thumb", null) + Manga( + id = generateUid(mangaUrl), + title = title, + url = mangaUrl, + publicUrl = mangaUrl.toAbsoluteUrl(domain), + coverUrl = thumb, + source = source, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + ) + }.filterNot { it.isAnime() } + } + + val slug = if (order == SortOrder.POPULARITY) "manga-ranking" else "recently-updated" + val url = if (page == 1) "https://$domain/$slug/" else "https://$domain/$slug/page/$page/" + val doc = webClient.httpGet(url).parseHtml() + return parseMangaGrid(doc).filterNot { it.isAnime() } + } + + private fun parseMangaGrid(doc: Document): List { + return doc.select(".manga-item-grid").map { element -> + val titleElement = element.selectFirst("h2 a[href*='/manga/']") + ?: element.selectFirst("a[href*='/manga/']:not([href*='/chapter-'])") + ?: return@map null + val href = titleElement.absUrl("href") + Manga( + id = generateUid(href), + title = titleElement.text(), + url = href.toHttpUrl().encodedPath, + publicUrl = href, + coverUrl = element.selectFirst("img")?.imageUrl(), + source = source, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + ) + }.filterNotNull() + } + + private fun Manga.isAnime(): Boolean { + val t = title.lowercase() + return t.startsWith("anime -") || t.startsWith("anime –") || + url.substringAfter("/manga/", "").startsWith("anime-") + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("#manga-title")?.ownText() + ?: doc.selectFirst("meta[property=og:title]")?.attr("content")?.substringBefore(" [Ch.")?.trim() + ?: manga.title + + val cover = doc.selectFirst(".story-cover img")?.imageUrl() + ?: doc.selectFirst("meta[property=og:image]")?.attr("content") + ?: manga.coverUrl + + val descElement = doc.selectFirst("#manga-description") + val descriptionText = descElement?.wholeText()?.trim() + ?: doc.selectFirst("meta[name=description]")?.attr("content")?.trim() + + val genre = doc.select("#genre-tags a[href*='/genre/']") + .joinToString { it.ownText().ifEmpty { it.text() } } + + val statusText = doc.selectFirst("#manga-status")?.text()?.lowercase(Locale.US)?.trim() + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "season end", "source hiatus", "caught up" -> MangaState.PAUSED + "dropped" -> MangaState.ABANDONED + else -> null + } + + val metaRow = doc.selectFirst("#manga-title + div") + val metadata = buildList { + metaRow?.ownText()?.substringBefore("Chapters")?.trim()?.takeIf(String::isNotEmpty)?.let { + add("Chapters: $it") + } + doc.selectFirst("#comic-othername")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Alternative title: $it") + } + doc.selectFirst(".init-review-info")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Rating: $it") + } + metaRow?.selectFirst(".init-plugin-suite-view-count-number")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Views: $it") + } + doc.selectFirst("#last-updated")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Last updated: $it") + } + } + + val description = buildString { + if (descriptionText != null) append(descriptionText) + if (metadata.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + metadata.joinTo(this, separator = "\n") + } + }.trim().takeIf(String::isNotEmpty) + + val tags = genre.split(",").mapNotNull { it.trim().takeIf(String::isNotEmpty) }.map { name -> + MangaTag(name, name.lowercase(), source) + }.toSet() + + val mangaId = doc.selectFirst("#manga-title[data-id], #chapter-search-input[data-manga-id]") + ?.attr("data-id")?.ifEmpty { doc.selectFirst("#chapter-search-input[data-manga-id]")?.attr("data-manga-id") } + ?.toIntOrNull() + + val chapters = if (mangaId != null) { + fetchChaptersApi(mangaId, manga.url) + } else { + doc.select(".chapter-list .chapter-item").mapNotNull { chapterFromElement(it) } + } + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + tags = tags, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChaptersApi(mangaId: Int, mangaPath: String): List { + val result = mutableListOf() + var page = 1 + do { + val url = "https://$domain/wp-json/initmanga/v1/chapters".toHttpUrl().newBuilder() + .addQueryParameter("manga_id", mangaId.toString()) + .addQueryParameter("paged", page.toString()) + .addQueryParameter("per_page", "50") + .build() + val json = webClient.httpGet(url).parseJson() + val items = json.optJSONArray("items") ?: break + val totalPages = json.optInt("total_pages", 1) + for (i in 0 until items.length()) { + val item = items.getJSONObject(i) + val number = item.optDouble("number", -1.0).toFloat() + val title = item.optString("title", "") + val slug = item.optString("slug", "") + val createdAt = item.optString("created_at", "") + val chapterName = if (title.isNotEmpty()) "Chapter ${number.toInt()} - $title" else "Chapter ${number.toInt()}" + val cleanPath = mangaPath.substringBefore("/chapter/").trimEnd('/') + result.add( + MangaChapter( + id = generateUid("$cleanPath/$slug/"), + title = chapterName, + number = number, + url = "$cleanPath/$slug/", + uploadDate = restChapterDateFormat.parseSafe(createdAt), + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + ) + } + page++ + } while (page <= totalPages) + return result.sortedBy { it.number } + } + + private fun chapterFromElement(element: Element): MangaChapter? { + val urlElement = element.selectFirst("a[href*='/chapter-']") ?: return null + val chapterUrl = urlElement.absUrl("href") + val name = element.selectFirst("h3")?.text() + ?: chapterUrl.substringBeforeLast('/').substringAfterLast('/').replace('-', ' ').replaceFirstChar { it.uppercase() } + val number = Regex("""/chapter-(\d+(?:\.\d+)?)/""").find(chapterUrl)?.groupValues?.get(1)?.toFloatOrNull() ?: -1f + val dateStr = element.selectFirst("time[datetime]")?.attr("datetime") + val date = dateStr?.parseChapterDate() ?: 0L + return MangaChapter( + id = generateUid(chapterUrl.toHttpUrl().encodedPath), + title = name, + number = number, + url = chapterUrl.toHttpUrl().encodedPath, + uploadDate = date, + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#chapter-content img[src]").mapIndexed { _, img -> + val src = img.absUrl("src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun String?.parseChapterDate(): Long { + if (isNullOrBlank()) return 0L + val normalized = this + .replace(Regex("""([+-]\d{2}):(\d{2})$"""), "$1$2") + .replace(Regex("""Z$"""), "+0000") + return chapterDateFormat.parseSafe(normalized) + } + + private fun Element.imageUrl(): String? { + return when (normalName()) { + "meta" -> attr("content") + else -> attr("abs:data-src").ifEmpty { + attr("abs:data-lazy-src").ifEmpty { attr("abs:src") } + } + }.takeIf(String::isNotEmpty) + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/Novel24h.kt b/src/main/kotlin/tsuki/site/en/Novel24h.kt new file mode 100644 index 00000000..479d4d3e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Novel24h.kt @@ -0,0 +1,17 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("TWENTYFOURHNOVEL", "24HNovel", "en") +internal class Novel24hParser(context: MangaLoaderContext) : + MadaraParser( + context = context, + source = MangaParserSource.TWENTYFOURHNOVEL, + domain = "24hnovel.com" + ) { + override val withoutAjax = true +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/RoliaScan.kt b/src/main/kotlin/tsuki/site/en/RoliaScan.kt new file mode 100644 index 00000000..8f50d8c0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/RoliaScan.kt @@ -0,0 +1,797 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.security.MessageDigest +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.ArrayList +import java.util.Calendar +import java.util.EnumSet +import java.util.HashSet +import java.util.LinkedHashSet +import java.util.Locale + +@MangaSourceParser("ROLIASCAN", "Rolia Scan", "en") +internal class RoliaScan(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ROLIASCAN, API_PAGES_PER_PAGE * API_PAGE_SIZE) { + + override val configKeyDomain = ConfigKey.Domain("roliascan.com") + + private val chapterNumberRegex = Regex("""(\d+(?:\.\d+)?)""") + + private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) + + private val genreTags by lazy { + ROLIA_TAGS.mapTo(LinkedHashSet(ROLIA_TAGS.size)) { (title, id) -> + tag(id.toString(), title) + } + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = genreTags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val startApiPage = (page - 1) * API_PAGES_PER_PAGE + 1 + val endApiPage = startApiPage + API_PAGES_PER_PAGE - 1 + val result = ArrayList(pageSize) + val seenIds = HashSet(pageSize * 2) + + for (apiPage in startApiPage..endApiPage) { + val payload = JSONObject().apply { + put("page", apiPage) + put("search", filter.query.orEmpty()) + put("years", "[]") + put("genres", filter.tags.toGenrePayload()) + put("types", filter.types.toTypePayload()) + put("statuses", filter.states.toStatusPayload()) + put("sort", order.toApiSort()) + put("genreMatchMode", "any") + } + val pageJson = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) + val rawSize = pageJson.length() + if (rawSize == 0) { + break + } + repeat(rawSize) { index -> + val item = pageJson.optJSONObject(index) ?: return@repeat + if (item.optString("type").equals("Novel", ignoreCase = true)) { + return@repeat + } + val rawUrl = item.optString("url").trim() + if (rawUrl.isEmpty()) { + return@repeat + } + val uniqueId = item.optString("id").nullIfEmpty() ?: rawUrl + if (!seenIds.add(uniqueId)) { + return@repeat + } + result.add(item.toManga()) + } + if (rawSize < API_PAGE_SIZE) { + break + } + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val detailUrl = manga.url.urlWithoutFragment().toAbsoluteUrl(domain) + val document = fetchDocument(detailUrl) + val jsonLd = document.findSeriesJsonLd() + val description = document.extractDescription(jsonLd) + val altTitles = document.extractAltTitles(jsonLd) + val tags = document.extractTags(jsonLd) + val authors = document.extractAuthors(jsonLd) + val state = document.extractStatus() + val rating = document.extractRating(jsonLd) + val mangaId = extractMangaId(manga.url) + ?: document.selectFirst("[data-manga-id]")?.attr("data-manga-id") + ?: jsonLd?.optString("identifier")?.nullIfEmpty() + ?: throw ParseException("Unable to determine manga id", detailUrl) + val chapters = loadChapters(mangaId, detailUrl) + return manga.copy( + description = description, + tags = if (tags.isEmpty()) manga.tags else tags, + authors = if (authors.isEmpty()) manga.authors else authors, + altTitles = if (altTitles.isEmpty()) manga.altTitles else altTitles, + state = state, + rating = rating, + chapters = chapters, + ) + } + + private suspend fun loadChapters(mangaId: String, referer: String): List { + val headers = Headers.headersOf("Referer", referer) + val result = ArrayList() + val seenIds = HashSet() + var offset = 0 + val limit = CHAPTER_LIMIT + var expectedTotal = Int.MAX_VALUE + while (offset < expectedTotal) { + val (token, timestamp) = generateChapterToken() + val requestUrl = buildChapterUrl(mangaId, offset, limit, token, timestamp) + val raw = webClient.httpGet(requestUrl, headers).parseRaw().trim() + if (raw.isEmpty()) { + break + } + val json = runCatching { JSONObject(raw) }.getOrNull() ?: break + if (json.has("success") && !json.optBoolean("success", false)) { + break + } + expectedTotal = json.optInt("total", expectedTotal) + val chaptersArray = json.optJSONArray("chapters") ?: break + if (chaptersArray.length() == 0) { + break + } + for (i in 0 until chaptersArray.length()) { + val item = chaptersArray.getJSONObject(i) + val chapterUrl = item.optString("url").nullIfEmpty() ?: continue + val chapterLanguage = item.optString("language").nullIfEmpty() + if (chapterLanguage != null && !chapterLanguage.equals("en", ignoreCase = true)) { + continue + } + val key = item.optString("id").ifEmpty { chapterUrl } + if (!seenIds.add(key)) { + continue + } + val fallbackNumber = (offset + i + 1).toFloat() + result.add(item.toMangaChapter(chapterUrl, fallbackNumber)) + } + offset += chaptersArray.length() + if (json.has("has_more") && !json.optBoolean("has_more", false)) { + break + } + if (!json.has("has_more") && chaptersArray.length() < limit) { + break + } + } + return result.reversed() + } + + private fun buildChapterUrl( + mangaId: String, + offset: Int, + limit: Int, + token: String, + timestamp: String, + ): String { + return HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegment("auth") + .addPathSegment("manga-chapters") + .addQueryParameter("manga_id", mangaId) + .addQueryParameter("offset", offset.toString()) + .addQueryParameter("limit", limit.toString()) + .addQueryParameter("order", "DESC") + .addQueryParameter("_t", token) + .addQueryParameter("_ts", timestamp) + .build() + .toString() + } + + private fun generateChapterToken(): Pair { + val timestamp = (System.currentTimeMillis() / 1000L).toString() + val hourKey = ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter) + val secret = "mng_ch_$hourKey" + val hashInput = timestamp + secret + val token = MessageDigest + .getInstance("MD5") + .digest(hashInput.toByteArray()) + .toHex() + .substring(0, 16) + return token to timestamp + } + + private fun ByteArray.toHex(): String { + if (isEmpty()) { + return "" + } + val chars = CharArray(size * 2) + var index = 0 + for (element in this) { + val unsigned = element.toInt() and 0xFF + chars[index++] = Character.forDigit((unsigned ushr 4) and 0xF, 16) + chars[index++] = Character.forDigit(unsigned and 0xF, 16) + } + return String(chars) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterId = chapter.url + .toAbsoluteUrl(domain) + .substringBefore('?') + .substringAfterLast('/') + .substringAfterLast('-') + .nullIfEmpty() + ?: return emptyList() + + val requestUrl = HttpUrl.Builder() + .scheme("https") + .host(domain) + .addPathSegment("auth") + .addPathSegment("chapter-content") + .addQueryParameter("chapter_id", chapterId) + .build() + .toString() + + val raw = webClient.httpGet(requestUrl).parseRaw().trim() + if (raw.isEmpty()) { + return emptyList() + } + + val images = runCatching { + JSONObject(raw).optJSONArray("images") + ?: JSONArray(raw) + }.getOrNull() ?: return emptyList() + + val seen = HashSet(images.length()) + val result = ArrayList(images.length()) + repeat(images.length()) { index -> + val imageUrl = images.optString(index).nullIfEmpty() ?: return@repeat + val absoluteUrl = if (imageUrl.startsWith("http", ignoreCase = true)) { + imageUrl + } else { + imageUrl.toAbsoluteUrl(domain) + } + if (!seen.add(absoluteUrl)) { + return@repeat + } + result.add( + MangaPage( + id = generateUid(absoluteUrl), + url = absoluteUrl, + preview = null, + source = source, + ), + ) + } + return result + } + + private fun JSONObject.toManga(): Manga { + val id = optString("id") + val rawUrl = optString("url") + val relativeUrl = if (rawUrl.isBlank()) "/" else rawUrl.toRelativeUrl(domain) + val internalUrl = relativeUrl.withMangaId(id) + val publicUrl = rawUrl.nullIfEmpty() ?: relativeUrl.toAbsoluteUrl(domain) + return Manga( + id = generateUid(internalUrl), + title = optString("title"), + altTitles = emptySet(), + url = internalUrl, + publicUrl = publicUrl, + rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover").nullIfEmpty(), + tags = emptySet(), + state = optString("status").toMangaState(), + authors = emptySet(), + description = optString("description").nullIfEmpty(), + source = source, + ) + } + + private fun JSONObject.toMangaChapter(chapterUrl: String, fallbackNumber: Float): MangaChapter { + val rawChapter = optString("chapter") + val chapterLabel = rawChapter.nullIfEmpty() ?: fallbackNumber.toChapterLabel() + val rawTitle = optString("title").trim() + val title = when { + rawTitle.equals("N/A", ignoreCase = true) -> "Chapter $chapterLabel" + rawTitle == "—" -> "Chapter $chapterLabel" + rawTitle.isEmpty() -> null + else -> rawTitle + } + val number = chapterNumberRegex.find(rawChapter)?.value?.toFloatOrNull() + val scanlator = optString("group_name") + .trim() + .nullIfEmpty() + ?.takeUnless { it.equals("N/A", ignoreCase = true) || it == "—" } + return MangaChapter( + id = generateUid(chapterUrl), + title = title, + number = number ?: fallbackNumber, + volume = 0, + url = chapterUrl.toRelativeUrl(domain), + scanlator = scanlator, + uploadDate = parseChapterDate(optString("date")), + branch = null, + source = source, + ) + } + + private fun Float.toChapterLabel(): String = if (this % 1f == 0f) { + toInt().toString() + } else { + replaceTrailingZeros() + } + + private fun Float.replaceTrailingZeros(): String { + val text = toString() + val trimmed = text.trimEnd('0') + return if (trimmed.endsWith('.')) { + trimmed.dropLast(1) + } else { + trimmed + } + } + + private fun String?.toMangaState(): MangaState? { + val normalized = this?.lowercase(Locale.US)?.trim() ?: return null + return when { + normalized.contains("ongoing") -> MangaState.ONGOING + normalized.contains("complete") || normalized.contains("completed") -> MangaState.FINISHED + normalized.contains("hiatus") -> MangaState.PAUSED + normalized.contains("canceled") || normalized.contains("cancelled") -> MangaState.ABANDONED + normalized.contains("upcoming") || normalized.contains("tba") -> MangaState.UPCOMING + else -> null + } + } + + private fun parseChapterDate(raw: String?): Long { + val value = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L + if (!value.endsWith("ago")) { + return 0L + } + val amount = chapterNumberRegex.find(value)?.value?.toFloatOrNull() ?: return 0L + val calendar = Calendar.getInstance() + when { + value.contains("second") -> calendar.add(Calendar.SECOND, -amount.toInt()) + value.contains("minute") -> calendar.add(Calendar.MINUTE, -amount.toInt()) + value.contains("hour") -> calendar.add(Calendar.HOUR_OF_DAY, -amount.toInt()) + value.contains("day") -> calendar.add(Calendar.DAY_OF_YEAR, -amount.toInt()) + value.contains("week") -> calendar.add(Calendar.WEEK_OF_YEAR, -amount.toInt()) + value.contains("month") -> calendar.add(Calendar.MONTH, -amount.toInt()) + value.contains("year") -> calendar.add(Calendar.YEAR, -amount.toInt()) + else -> return 0L + } + return calendar.timeInMillis + } + + private fun Collection.toStatusPayload(): String { + if (isEmpty()) { + return "[]" + } + val mapped = mapNotNull { state -> + when (state) { + MangaState.FINISHED -> "Completed" + MangaState.ONGOING -> "Ongoing" + else -> null + } + } + return mapped.toStringifiedPayload() + } + + private fun Collection.toTypePayload(): String { + if (isEmpty()) { + return "[]" + } + val mapped = mapNotNull { type -> + when (type) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Comics" + else -> null + } + } + return mapped.toStringifiedPayload() + } + + private fun Collection.toStringifiedPayload(): String { + if (isEmpty()) { + return "[]" + } + return joinToString(prefix = "[", postfix = "]", separator = ",") { value -> + "\"${value.escapeJson()}\"" + } + } + + private fun SortOrder.toApiSort(): String = when (this) { + SortOrder.POPULARITY -> "popular_desc" + SortOrder.UPDATED -> "post_desc" + SortOrder.NEWEST -> "release_desc" + SortOrder.ALPHABETICAL -> "title_asc" + SortOrder.ALPHABETICAL_DESC -> "title_desc" + else -> "post_desc" + } + + private fun String.escapeJson(): String { + return replace("\\", "\\\\") + .replace("\"", "\\\"") + } + + private fun Document.extractDescription(jsonLd: JSONObject?): String? { + val fromJson = jsonLd?.optString("description")?.nullIfEmpty() + if (fromJson != null) { + return fromJson + } + return selectFirst("div[data-description], div#description, div#synopsis, div:matchesOwn((?i)description)") + ?.nextElementSibling() + ?.text() + ?.nullIfEmpty() + ?: selectFirst("meta[name='description']")?.attr("content")?.nullIfEmpty() + ?: selectFirst("meta[property='og:description']")?.attr("content")?.nullIfEmpty() + } + + private fun Document.extractAltTitles(jsonLd: JSONObject?): Set { + val alt = mutableSetOf() + jsonLd?.opt("alternateName")?.let { value -> + when (value) { + is JSONArray -> { + repeat(value.length()) { index -> + value.optString(index).nullIfEmpty()?.let(alt::add) + } + } + is String -> value.nullIfEmpty()?.let(alt::add) + } + } + if (alt.isEmpty()) { + selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { alt.addAll(it) } + } + return alt + } + + private fun Document.extractTags(jsonLd: JSONObject?): Set { + val tags = LinkedHashSet() + jsonLd?.opt("genre")?.let { genre -> + when (genre) { + is JSONArray -> repeat(genre.length()) { index -> + genre.optString(index).nullIfEmpty()?.let { addTag(tags, it) } + } + is String -> genre.nullIfEmpty()?.let { addTag(tags, it) } + } + } + if (tags.isEmpty()) { + select("a[href*='/genre/']").mapNotNullToSet { anchor -> + anchor.text().nullIfEmpty()?.let { title -> + val href = anchor.attr("href").trimEnd('/') + val slug = href.substringAfterLast('/').substringBefore('?').nullIfEmpty() + ?: return@let null + MangaTag( + key = slug, + title = title, + source = source, + ) + } + } + .takeIf { it.isNotEmpty() } + ?.let(tags::addAll) + } + return tags + } + + private fun addTag(target: MutableSet, value: String) { + val normalized = value.trim().nullIfEmpty() ?: return + target.add( + MangaTag( + key = normalized.lowercase(Locale.US).replace(' ', '-'), + title = normalized, + source = source, + ), + ) + } + + private fun Document.extractAuthors(jsonLd: JSONObject?): Set { + val authors = LinkedHashSet() + fun addName(value: Any?) { + when (value) { + is JSONArray -> repeat(value.length()) { index -> addName(value.opt(index)) } + is JSONObject -> addName(value.opt("name")) + is String -> value.nullIfEmpty()?.let(authors::add) + } + } + addName(jsonLd?.opt("author")) + addName(jsonLd?.opt("creator")) + if (authors.isEmpty()) { + select("a[href*='/author/'], a[href*='/artist/']").forEach { anchor -> + anchor.text().nullIfEmpty()?.let(authors::add) + } + } + return authors + } + + private fun Document.extractStatus(): MangaState? { + val label = select("*").firstOrNull { element -> + element.ownText().trim().equals("Status", ignoreCase = true) + } + val statusText = label?.let { element -> + element.findSiblingText()?.nullIfEmpty() + } + ?: selectFirst("[data-status]")?.attr("data-status")?.nullIfEmpty() + ?: selectFirst("span.status, div.status")?.text()?.nullIfEmpty() + return statusText.toMangaState() + } + + private fun Document.extractRating(jsonLd: JSONObject?): Float { + jsonLd?.optJSONObject("aggregateRating")?.optDouble("ratingValue")?.takeIf { !it.isNaN() }?.let { + return it.toFloat() + } + return selectFirst("svg ~ span.font-semibold")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN + } + + private fun Document.findSeriesJsonLd(): JSONObject? { + for (script in select("script[type=application/ld+json]")) { + val raw = script.data().trim() + if (raw.isEmpty()) { + continue + } + val obj = raw.toJsonObjectOrNull() + if (obj != null) { + if (obj.matchesSeriesType()) { + return obj + } + continue + } + val array = raw.toJsonArrayOrNull() ?: continue + repeat(array.length()) { index -> + val candidate = array.optJSONObject(index) ?: return@repeat + if (candidate.matchesSeriesType()) { + return candidate + } + } + } + return null + } + + private fun String.toJsonObjectOrNull(): JSONObject? = runCatching { JSONObject(this) }.getOrNull() + + private fun String.toJsonArrayOrNull(): JSONArray? = runCatching { JSONArray(this) }.getOrNull() + + private fun JSONObject.matchesSeriesType(): Boolean { + val type = optString("@type") + if (type.isNullOrEmpty()) { + return false + } + return type.equals("ComicSeries", true) || + type.equals("ComicStory", true) || + type.equals("Book", true) || + type.equals("CreativeWorkSeries", true) + } + + private fun String?.withMangaId(id: String?): String { + val value = this?.nullIfEmpty() ?: return "/" + val cleanId = id?.nullIfEmpty() ?: return value + return if (value.contains('#')) { + if (value.contains("mid=")) value else "$value&mid=$cleanId" + } else { + "$value#mid=$cleanId" + } + } + + private fun extractMangaId(rawUrl: String): String? { + val fragment = rawUrl.substringAfter('#', "") + if (fragment.isNotEmpty()) { + fragment.split('&').firstOrNull { it.startsWith("mid=") }?.let { part -> + return part.substringAfter("mid=").nullIfEmpty() + } + } + val query = rawUrl.substringAfter('?', "") + if (query.isNotEmpty()) { + query.split('&').firstOrNull { it.startsWith("mid=") || it.startsWith("mangaId=") }?.let { part -> + return part.substringAfter('=').nullIfEmpty() + } + } + return null + } + + private fun String.urlWithoutFragment(): String = substringBefore('#') + + private suspend fun postJsonArray(url: String, payload: JSONObject): JSONArray { + val raw = webClient.httpPost(url, payload).parseRaw().trim() + if (raw.isEmpty()) { + return JSONArray() + } + return raw.toJsonArrayOrNull() ?: JSONArray() + } + + private suspend fun fetchDocument(url: String): Document = webClient.httpGet(url).parseHtml() + + private fun Collection.toGenrePayload(): String { + if (isEmpty()) { + return "[]" + } + val ids = LinkedHashSet(size) + for (tag in this) { + val numericKey = tag.key?.toIntOrNull()?.let { it.toString() } + ?: genreTags.firstOrNull { ref -> + ref.title.equals(tag.title, ignoreCase = true) + }?.key?.toIntOrNull()?.let { it.toString() } + if (numericKey != null) { + ids.add(numericKey) + } + } + if (ids.isEmpty()) { + return "[]" + } + return buildString(ids.size * 4) { + append('[') + ids.forEachIndexed { index, value -> + if (index > 0) { + append(',') + } + append(value) + } + append(']') + } + } + + private fun tag(id: String, title: String) = MangaTag(title = title, key = id, source = source) + + private fun Element.findSiblingText(): String? { + nextElementSibling()?.text()?.nullIfEmpty()?.let { return it } + val parent = parent() ?: return null + val children = parent.children() + val index = children.indexOf(this) + if (index == -1) { + return null + } + for (i in index + 1 until children.size) { + val text = children[i].text().nullIfEmpty() + if (text != null) { + return text + } + } + return null + } + + private companion object { + const val API_PAGES_PER_PAGE = 5 + const val API_PAGE_SIZE = 24 + const val CHAPTER_LIMIT = 9999 + + val ROLIA_TAGS = listOf( + "Action" to 5, + "Adaptation" to 49, + "Adapted to Manhua" to 717, + "Adult Cast" to 119, + "Adventure" to 19, + "Aliens" to 803, + "Animals" to 240, + "Award Winning" to 8, + "Childcare" to 1146, + "Combat Sports" to 358, + "Comedy" to 61, + "Cooking" to 266, + "Crime" to 248, + "Crossdressing" to 724, + "Delinquents" to 228, + "Demons" to 162, + "Detective" to 150, + "Drama" to 26, + "Ecchi" to 117, + "Erotica" to 202, + "Fantasy" to 17, + "Full Color" to 40, + "Gag Humor" to 1068, + "Game" to 1130, + "Gender Bender" to 1190, + "Ghosts" to 215, + "Gore" to 187, + "Gourmet" to 89, + "Harem" to 47, + "Historical" to 66, + "Horror" to 67, + "Isekai" to 55, + "Josei" to 1062, + "Light Novel" to 98, + "Long Strip" to 41, + "Love Status Quo" to 541, + "Mafia" to 356, + "Magic" to 45, + "Magical Sex Shift" to 551, + "Manga" to 97, + "Manhua" to 35, + "Manhwa" to 18, + "Martial Arts" to 56, + "Mature" to 404, + "Mecha" to 396, + "Medical" to 244, + "Military" to 131, + "Monster Girls" to 231, + "Monsters" to 46, + "Music" to 694, + "Mystery" to 34, + "Mythology" to 110, + "Ninja" to 163, + "Office Workers" to 505, + "Official Colored" to 866, + "Organized Crime" to 134, + "Otaku Culture" to 570, + "Parody" to 605, + "Philosophical" to 912, + "Post-Apocalyptic" to 241, + "Psychological" to 149, + "Regression" to 1131, + "Reincarnation" to 29, + "Revenge" to 964, + "Reverse Harem" to 1085, + "Romance" to 2, + "Romantic Subtext" to 486, + "School" to 14, + "School Life" to 27, + "Sci-Fi" to 33, + "Seinen" to 105, + "Self-Published" to 577, + "Sexual Violence" to 536, + "Shoujo" to 1071, + "Shounen" to 11, + "Showbiz" to 429, + "Slice of Life" to 93, + "Smut" to 742, + "Space" to 206, + "Sports" to 9, + "Streaming" to 1132, + "Suggestive" to 1116, + "Super Power" to 6, + "Superhero" to 865, + "Supernatural" to 65, + "Survival" to 236, + "Suspense" to 287, + "Team Sports" to 10, + "Thriller" to 184, + "Time Travel" to 37, + "Tragedy" to 316, + "Transmigiration" to 1133, + "Urban Fantasy" to 120, + "Vampire" to 209, + "Video Game" to 277, + "Video Games" to 616, + "Villainess" to 355, + "Virtual Reality" to 617, + "Web Comic" to 48, + "Webtoon" to 350, + "Workplace" to 138, + "Wuxia" to 68, + "Xianxia" to 718, + "Zombies" to 1115, + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/TopManhua.kt b/src/main/kotlin/tsuki/site/en/TopManhua.kt new file mode 100644 index 00000000..3ca840d1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/TopManhua.kt @@ -0,0 +1,15 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("TOPMANHUA", "ManhuaTop", "en") +internal class TopManhua(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.TOPMANHUA, "manhuatop.org") { + override val tagPrefix = "manhua-genre/" + override val listUrl = "manhua/" + override val datePattern = "MM/dd/yyyy" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/WeebCentral.kt b/src/main/kotlin/tsuki/site/en/WeebCentral.kt new file mode 100644 index 00000000..f5a106f8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/WeebCentral.kt @@ -0,0 +1,358 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser +import tsuki.MangaParserAuthProvider + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("WEEBCENTRAL", "Weeb Central", "en") +internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.WEEBCENTRAL), + MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain("weebcentral.com") + + override val authUrl: String + get() = "https://$domain" + + override suspend fun isAuthorized(): Boolean = + context.cookieJar.getCookies(domain).any { it.name == "access_token" } + + override suspend fun getUsername(): String { + return webClient.httpGet("https://$domain/users/me/profiles") + .parseHtml() + .selectFirstOrThrow("div:has(section > .avatar) .text-4xl") + .text() + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities = + MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val document = webClient.httpGet("https://$domain/search") + .parseHtml() + + val tags = document.select("section[x-show=show_filter] div:contains(tags) fieldset label").mapToSet { + MangaTag( + title = it.selectFirstOrThrow("span").text(), + key = it.selectFirstOrThrow("input[id$=value]").attr("value"), + source = source, + ) + } + + val states = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED, + ) + + val types = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS, + ) + + val rating = EnumSet.of( + ContentRating.SAFE, ContentRating.SUGGESTIVE, + ) + + return MangaListFilterOptions( + availableTags = tags, + availableStates = states, + availableContentTypes = types, + availableContentRating = rating, + ) + } + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { + val url = "https://$domain/search/data".toHttpUrl().newBuilder().apply { + addQueryParameter("limit", "32") + addQueryParameter("offset", offset.toString()) + filter.query?.let { + val query = it + .replace(Regex("""[^a-zA-Z0-9\s]"""), " ") + .replace(Regex("""\s+"""), " ") + .trim() + addQueryParameter("text", query) + } + addQueryParameter( + name = "sort", + value = when (order) { + SortOrder.RELEVANCE -> "Best Match" + SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "Alphabet" + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "Popularity" + SortOrder.RATING, SortOrder.RATING_ASC -> "Subscribers" + SortOrder.ADDED, SortOrder.ADDED_ASC -> "Recently Added" + SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "Latest Updates" + else -> throw UnsupportedOperationException("unsupported order: $order") + }, + ) + addQueryParameter( + name = "order", + value = when (order) { + SortOrder.RELEVANCE, SortOrder.ALPHABETICAL, SortOrder.POPULARITY_ASC, SortOrder.RATING_ASC, SortOrder.ADDED_ASC, SortOrder.UPDATED_ASC -> "Ascending" + SortOrder.ALPHABETICAL_DESC, SortOrder.POPULARITY, SortOrder.RATING, SortOrder.ADDED, SortOrder.UPDATED -> "Descending" + else -> throw UnsupportedOperationException("unsupported order: $order") + }, + ) + addQueryParameter("official", "Any") + addQueryParameter("anime", "Any") + with(filter.contentRating) { + addQueryParameter( + name = "adult", + value = when { + isEmpty() -> "Any" + ContentRating.SAFE in this && ContentRating.SUGGESTIVE in this -> "Any" + ContentRating.SAFE in this -> "False" + ContentRating.SUGGESTIVE in this -> "True" + else -> throw UnsupportedOperationException("unsupported content rating: $this") + }, + ) + } + filter.states.forEach { state -> + addQueryParameter( + name = "included_status", + value = when (state) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Complete" + MangaState.ABANDONED -> "Canceled" + MangaState.PAUSED -> "Hiatus" + else -> throw UnsupportedOperationException("unsupported state: $state") + }, + ) + } + filter.types.forEach { type -> + addQueryParameter( + name = "included_type", + value = when (type) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "OEL" + else -> throw UnsupportedOperationException("unsupported type: $type") + }, + ) + } + filter.tags.forEach { tag -> + addQueryParameter("included_tag", tag.key) + } + filter.tagsExclude.forEach { tag -> + addQueryParameter("excluded_tag", tag.key) + } + addQueryParameter("display_mode", "Full Display") + }.build() + + val document = webClient.httpGet(url).parseHtml() + + return document.select("article:has(section)").map { element -> + val mangaId = element.selectFirstOrThrow("a") + .attrAsAbsoluteUrl("href") + .toHttpUrl() + .pathSegments[1] + val author = document.select("div:contains(author) a").eachText().joinToString().nullIfEmpty() + val title = + element.selectFirst("div.text-ellipsis.truncate.text-white.text-center.text-lg.z-20.w-\\[90\\%\\]") + ?.text() + ?: "No name" + Manga( + id = generateUid(mangaId), + url = mangaId, + publicUrl = "https://$domain/series/$mangaId", + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = if (element.selectFirst("svg:has(style:containsData(ff0000))") == null) { + ContentRating.SAFE + } else { + ContentRating.SUGGESTIVE + }, + coverUrl = element.selectFirst("picture img")?.attrAsAbsoluteUrlOrNull("src"), + tags = element.selectFirst("div:contains(Tag(s): )")?.text() + ?.substringAfter("Tag(s): ") + ?.split(", ") + ?.mapToSet { + MangaTag( + title = it, + key = it, + source = source, + ) + } + .orEmpty(), + state = when (document.selectFirst("div:contains(status) span")?.text()) { + "Ongoing" -> MangaState.ONGOING + "Complete" -> MangaState.FINISHED + "Canceled" -> MangaState.ABANDONED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = setOfNotNull(author), + largeCoverUrl = null, + chapters = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val document = webClient.httpGet("https://$domain/series/${manga.url}") + .parseHtml() + + val chapters = async { getChapters(manga.url, document) } + + val sectionLeft = document.select("section[x-data] > section")[0] + val sectionRight = document.select("section[x-data] > section")[1] + val author = sectionLeft.select("ul > li:has(strong:contains(Author)) > span > a") + .eachText().joinToString() + + manga.copy( + title = sectionRight.selectFirstOrThrow("h1").text(), + altTitles = sectionRight.select("li:has(strong:contains(Associated Name)) li") + .eachText().toSet(), + publicUrl = "https://$domain/series/${manga.url}", + rating = RATING_UNKNOWN, + contentRating = if (sectionLeft.selectFirst("ul > li > strong:contains(Official Translation) + a:contains(Yes)") != null) { + ContentRating.SUGGESTIVE + } else { + ContentRating.SAFE + }, + coverUrl = sectionLeft.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + tags = sectionLeft.select("ul > li:has(strong:contains(Tag)) a").mapToSet { + MangaTag( + title = it.text(), + key = it.text(), + source = source, + ) + }, + state = when (sectionLeft.selectFirst("ul > li:has(strong:contains(Status)) > a")?.text()) { + "Ongoing" -> MangaState.ONGOING + "Complete" -> MangaState.FINISHED + "Canceled" -> MangaState.ABANDONED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = setOf(author), + description = Element("div").also { desc -> + sectionRight.selectFirst("li:has(strong:contains(Description)) > p")?.let { + desc.appendChild(it) + } + + val ul = Element("ul") + sectionLeft.select("ul > li:has(strong:contains(Track)) abbr").stream().forEach { abbr -> + abbr.selectFirst("a")?.attr("href")?.let { url -> + val a = Element("a") + .text( + abbr.attr("title"), + ) + .attr("href", url) + + ul.appendChild( + Element("li").appendChild(a), + ) + } + } + + if (ul.children().isNotEmpty()) { + desc.append("
Links:") + desc.appendChild(ul) + } + }.outerHtml(), + chapters = chapters.await(), + source = source, + ) + } + + private suspend fun getChapters(mangaId: String, mangaDocument: Document): List { + val document = if (mangaDocument.selectFirst("#chapter-list > button[hx-get*=full-chapter-list]") != null) { + webClient.httpGet("https://$domain/series/$mangaId/full-chapter-list").parseHtml() + } else { + mangaDocument + } + + return document.select("div[x-data] > a").mapChapters(reversed = true) { i, element -> + val chapterId = element.attrAsAbsoluteUrl("href") + .toHttpUrl() + .pathSegments[1] + val name = element.selectFirstOrThrow("span.flex > span").text() + + MangaChapter( + id = generateUid(chapterId), + url = chapterId, + title = name, + number = Regex("""(? "Official" + else -> null + }, + uploadDate = dateFormat.parseSafe( + element.selectFirst("time[datetime]")?.attr("datetime"), + ), + branch = null, + source = source, + ) + } + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + + override suspend fun getPages(chapter: MangaChapter): List { + val url = "https://$domain".toHttpUrl().newBuilder().apply { + addPathSegment("chapters") + addPathSegment(chapter.url) + addPathSegment("images") + addQueryParameter("is_prev", "False") + addQueryParameter("reading_style", "long_strip") + }.build() + + val document = webClient.httpGet(url).parseHtml() + + return document.select("section[x-data~=scroll] > img").map { element -> + val pageUrl = element.attrAsAbsoluteUrl("src") + + MangaPage( + id = generateUid(pageUrl), + url = pageUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val mangaId = link.pathSegments[1] + + return resolver.resolveManga(this, mangaId) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ZinManga.kt b/src/main/kotlin/tsuki/site/en/ZinManga.kt new file mode 100644 index 00000000..f2bf309c --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ZinManga.kt @@ -0,0 +1,62 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat + +@MangaSourceParser("ZINMANGA", "ZinManga.net", "en") +internal class Zinmanga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.ZINMANGA, "zinmanga.net") { + override val datePattern = "yyyy-MM-dd" + override val withoutAjax = true + + override val selectRequiredLogin = ".content-blocked" + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://www.zinmanga.net/") + .build() + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val slug = manga.url.removeSuffix("/").substringAfterLast('/') + val mangaUrl = manga.url.removeSuffix("/") + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val collected = ArrayList() + var page = 1 + while (true) { + val url = "https://$domain/api/comics/$slug/chapters?page=$page" + val data = webClient.httpGet(url).parseJson().getJSONObject("data") + data.getJSONArray("chapters").mapJSON { jo -> + val chapterSlug = jo.getString("chapter_slug") + val href = "$mangaUrl/$chapterSlug" + stylePage + collected += MangaChapter( + id = generateUid("$mangaUrl/$chapterSlug"), + title = jo.getStringOrNull("chapter_name"), + number = jo.getFloatOrDefault("chapter_num", 0f), + volume = 0, + url = href, + uploadDate = parseChapterDate( + dateFormat, + jo.getStringOrNull("updated_at")?.substringBefore('T'), + ), + source = source, + scanlator = null, + branch = null, + ) + } + val lastPage = data.getIntOrDefault("last_page", page) + if (page >= lastPage) break + page++ + } + return collected.sortedBy { it.number } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt b/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt new file mode 100644 index 00000000..d7ae523c --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt @@ -0,0 +1,18 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + + +@MangaSourceParser("ALLPORNCOMIC", "AllPornComic", "en", ContentType.HENTAI) +internal class AllPornComic(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.ALLPORNCOMIC, "allporncomic.com", pageSize = 24) { + override val withoutAjax = true + override val listUrl = "porncomic/" + override val tagPrefix = "porncomic-cat/" + override val datePattern = "MMMM dd, yyyy" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt b/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt new file mode 100644 index 00000000..2f2708b2 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt @@ -0,0 +1,287 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.util.* +import tsuki.model.* + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject + +@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) +internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( + context, MangaParserSource.COMICLAND, 20) { + + override val configKeyDomain = + ConfigKey.Domain("comicland.org") + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + val newRequest = request.newBuilder() + .header( + "Origin", + "https://comicland.org" + ) + .header( + "Referer", + "https://comicland.org/" + ) + .build() + + return chain.proceed(newRequest) + } + + override val availableSortOrders = + setOf( + SortOrder.UPDATED, // Recommended + SortOrder.POPULARITY, // Popular + SortOrder.NEWEST // Official/Ongoing equivalent + ) + + override val filterCapabilities = + MangaListFilterCapabilities( + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + isAuthorSearchSupported = false + ) + + private val api = + "https://api.comicland.org/api" + + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter + ): List { + + + val offset = page * 20 + val endpoint = + if (!filter.query.isNullOrBlank()) { + "$api/comic/search?q=${filter.query!!.urlEncoded()}" + } else { + val category = + filter.tags + .firstOrNull() + ?.key + when(category) { + "official" -> + "$api/comics/official" + "ongoing" -> + "$api/comics" + "popular" -> + "$api/comics/popular" + else -> + "$api/comics" + } + } + + val separator = + if (endpoint.contains("?")) "&" else "?" + + val response = + webClient.httpGet( + "$endpoint${separator}offset=$offset&limit=20" + ) + + val root = + JSONObject(response.body.string()) + + val data = + root.optJSONObject("data") + ?: return emptyList() + + val array = + data.optJSONArray("list") + ?: data.optJSONArray("items") + ?: return emptyList() + + val result = ArrayList() + for (i in 0 until array.length()) { + val comic = + array.getJSONObject(i) + val slug = + comic.optString("slug") + result.add( + Manga( + id = generateUid(slug), + title = comic.optString("title"), + url = slug, + publicUrl = "https://comicland.org/comic/$slug", + coverUrl = comic.optString("cover_url"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + contentRating = ContentRating.ADULT, + source = source + ) + ) + } + return result + } + + override suspend fun getDetails( + manga: Manga + ): Manga { + val response = + webClient.httpGet( + "$api/comic/detail?slug=${manga.url}" + ) + + val root = + JSONObject(response.body.string()) + + val comic = + root.optJSONObject("data") + ?: return manga + + val chapters = + ArrayList() + + val chapterArray = + comic.optJSONArray("chapters") + + if (chapterArray != null) { + for (i in 0 until chapterArray.length()) { + val chapter = + chapterArray.getJSONObject(i) + + val index = + chapter.optDouble( + "chapter_index", + 0.0 + ) + chapters.add( + MangaChapter( + id = generateUid("${comic.optString("slug")}/$index"), + title = chapter.optString("title"), + number = index.toFloat(), + volume = 0, + url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", + scanlator = null, + uploadDate = 0, + branch = null, + source = source + ) + ) + } + } + return manga.copy( + title = + comic.optString( + "title", + manga.title + ), + description = + comic.optString( + "description", + "" + ), + coverUrl = + comic.optString( + "cover_url", + manga.coverUrl + ), + authors = + parseNames( + comic.optJSONArray("authors") + ), + tags = emptySet(), + chapters = chapters + ) + } + + + private fun parseNames( + array: org.json.JSONArray? + ): Set { + if (array == null) + return emptySet() + return buildSet { + for (i in 0 until array.length()) { + add( + array.getJSONObject(i) + .optString("name") + ) + } + } + } + + + override suspend fun getPages( + chapter: MangaChapter + ): List { + + val slug = + chapter.url + .substringAfter("/comic/") + .substringBefore("/chapter/") + + val index = + chapter.url + .substringAfter("/chapter/") + + val response = + webClient.httpGet( + "$api/chapter/pages_by_index?slug=$slug&index=$index" + ) + + val root = + JSONObject(response.body.string()) + + val data = + root.optJSONObject("data") + ?: return emptyList() + + val pages = + data.optJSONArray("pages") + ?: return emptyList() + + return buildList { + for (i in 0 until pages.length()) { + val image = + pages.getString(i) + add( + MangaPage( + id = generateUid(image), + url = image, + preview = null, + source = source + ) + ) + } + } + } + + private val categories = mapOf( + "recommended" to "/comics", + "official" to "/comics/official", + "ongoing" to "/comics?status=ongoing", + "popular" to "/comics/popular" + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = categories.map { (key, _) -> + MangaTag( + title = key.replaceFirstChar { it.uppercase() }, + key = key, + source = source + ) + }.toSet(), + + availableContentRating = emptySet() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt new file mode 100644 index 00000000..b9996659 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt @@ -0,0 +1,379 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.GalleryAdultsParser + +import tsuki.util.* +import tsuki.model.* + +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Base64 + +private const val SERVER_PNG = "png" // old method uses now image_fallback +private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format + +@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) +internal class HentaiNexus(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { + override val selectGallery = "div.container div.columns div.column" + override val selectGalleryLink = "a" + override val selectGalleryTitle = ".card-header" + override val selectTitle = ".title" + override val selectTag = "tr:contains(Tags) td:nth-child(2) span.tag a" + override val selectAuthor = "tr:contains(Artist) td:nth-child(2) a" + override val selectLanguageChapter = "" + override val selectUrlChapter = "" + override val selectTotalPage = "div.column.is-2-fullhd" + + val selectReadUrl = "a:contains(Read Online)" + val selectPublisher = "tr:contains(Publisher) td:nth-child(2)" + val selectPublishedDate = "tr:contains(Published) td:nth-child(2)" + val selectDescription = "tr:contains(Description) td:nth-child(2)" + + var mangaInternalId: String = "" /* use as a flag for reloading data */ + var mangaPages: List = listOf() + + private val pageCache = object : LinkedHashMap>(10, 0.75f, true) { + override fun removeEldestEntry( + eldest: MutableMap.MutableEntry> + ): Boolean { + return size > 5 + } + } + + private val preferredImageModeKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG to "Old method (png)", + _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP to "New method (webp/avif)", + ), + defaultValue = _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG, + ) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(preferredImageModeKey) + } + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val document = webClient.httpGet("https://$domain/explore/categories/tag").parseHtml() + val tags = document.select("div.container div.columns div.column").mapToSet {div -> + val tag = div.selectFirstOrThrow("a").attr("href") + .substring(8) // href="/?q=tag:value" + .urlDecode() + .trim(' ', '\"') + MangaTag( + title = tag.replace(Regex("\\b[a-z]")) { it.value.uppercase(sourceLocale) }, + key = tag, + source = source, + ) + } + + return MangaListFilterOptions( + availableTags = tags + ) + } + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val url = buildString { + append("https://$domain/page/$page") + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("?q=") + append(query?.urlEncoded()) + } + + else -> { + val queries = mutableListOf() + + if (!filter.author.isNullOrEmpty()) { + queries.add("artist:${filter.author}") + } + + filter.tags.map { + val key = it.key + when { + key.split(" ").count() > 1 -> { + "\"${key.replace(Regex("\\s+")) { "+" }}\"" + } + else -> { + key + } + } + }.also { + if (it.count() > 0) { + queries.add("tag:${ it.joinToString("+") }") + } + } + + if (queries.count() > 0) { + append("?q=${queries.joinToString("+")}") + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(doc: Document): List { + return doc.select(selectGallery).map { div -> + val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(selectGalleryTitle).text().cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = div.selectFirst(selectGalleryImg)?.src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val tags = doc.select(selectTag).mapToSet { + val value = it.attr("href").substring(8).urlDecode().trim('\"') /* /?q=tag:"blow job" */ + MangaTag( + title = value.replace(Regex("\\b[a-z]")) { x -> x.value.uppercase(sourceLocale) }, + key = value, + source = source + ) + } + val authors = doc.select(selectAuthor).mapToSet { + it.attr("href").substring(11).urlDecode() /* /?q=artist:Danchino */ + } + + mangaPages = getPagesInternal(manga.url, doc) + mangaInternalId = manga.url.split("/").last() + + val format = SimpleDateFormat("dd MMMM yyyy") + + return manga.copy( + title = doc.select(selectTitle).text().cleanupTitle(), + tags = tags, + authors = authors, + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 0f, + volume = 0, + url = manga.url, + scanlator = doc.select(selectPublisher).text().replace(Regex(" \\([\\d,]+\\)")) { "" }, + uploadDate = format.parseSafe(doc.select(selectPublishedDate).text()), + branch = "English", + source = source, + ), + ), + description = doc.selectFirst(selectDescription)?.text() ?: "" + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { + mangaPages = getPagesInternal(chapter.url) + mangaInternalId = chapter.url.split("/").last() + + // preload image urls + val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") + if (reader != null && !pageCache.containsKey(reader)) { + pageCache[reader] = getPageUrlInternal(reader) + } + } + + return mangaPages + } + + override suspend fun getPageUrl(page: MangaPage): String { + val readerUrl = page.url.substringBefore("#") + + val pages = pageCache[readerUrl] + ?: getPageUrlInternal(readerUrl).also { + pageCache[readerUrl] = it + } + + val index = page.url.substringAfter("#").toInt() - 1 + + return pages[index] + } + + private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List { + val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() + val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") + + return document.select(selectTotalPage).mapIndexed { index, element -> + val url = "$readUrl#${index + 1}" + MangaPage( + id = generateUid(url), + url = url, + preview = element.select("img").attr("src"), + source = source, + ) + } + } + + private suspend fun getPageUrlInternal(pageUrl: String): List { + val doc = webClient + .httpGet(pageUrl.toAbsoluteUrl(domain)) + .parseHtml() + val script = doc.select("script") + .firstOrNull { it.data().contains("initReader") } + ?: throw Exception("Reader script not found") + val encryptedPagesData = script.data() + .substringAfter("initReader(\"") + .substringBefore("\",") + .replace("\\n", "") + .replace("\\r", "") + .trim() + val decryptedString = decrypt(encryptedPagesData) + .replace("\\/", "/") + val jsonArray = JSONArray(decryptedString) + val mode = config[preferredImageModeKey] ?: _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP + return buildList { + for (i in 0 until jsonArray.length()) { + val item = jsonArray.getJSONObject(i) + val url = when (mode) { + _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG -> { + val image = item.optString("image") + .takeIf { it.isNotBlank() } + val fallback = item.optString("image_fallback") + .takeIf { it.isNotBlank() } + fallback ?: image + } + _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP -> { + var found: String? = null + val keys = item.keys() + while (keys.hasNext()) { + val value = item.optString(keys.next()) + if ( + value.startsWith("http") && + ( + value.contains(".webp") || + value.contains(".avif") || + value.contains(".jpg") || + value.contains(".jpeg") || + value.contains(".png") + ) + ) { + found = value + break + } + } + found + } + else -> null + } + if (!url.isNullOrBlank()) { + add(normalizeImageUrl(url)) + } + } + } + } + + private fun normalizeImageUrl(url: String): String { + return when { + url.startsWith("http") -> url + + url.startsWith("//") -> + "https:$url" + + url.startsWith("/") -> + "https://$domain$url" + + else -> + "https://$domain/$url" + } + } + + private fun decrypt(encodedData: String): String { + val xorKey = listOf('h','e','n','t','a','i','n','e','x','u','s','.','c','o','m') + val keyLength = minOf(xorKey.size, 64) + + // Decode base64 string into characters (1 byte per char, unsigned) + val decodedBytes = Base64.getDecoder().decode(encodedData) + val decodedChars = decodedBytes.map { (it.toInt() and 0xFF).toChar() }.toMutableList() + + // XOR first 64 characters with the key + for (i in 0 until keyLength) { + decodedChars[i] = (decodedChars[i].code xor xorKey[i].code).toChar() + } + + val decodedString = decodedChars.joinToString("") + + // Prime sieve: first 16 primes + val sieve = BooleanArray(257) + val primeIndexes = mutableListOf() + var j = 2 + while (primeIndexes.size < 16) { + if (!sieve[j]) { + primeIndexes.add(j) + for (k in j * 2..256 step j) sieve[k] = true + } + j++ + } + + // Hash from first 64 chars + var hash = 0 + for (k in 0 until 64) { + hash = hash xor decodedString[k].code + repeat(8) { + hash = if ((hash and 1) == 1) { + (hash ushr 1) xor 12 + } else { + hash ushr 1 + } + } + } + hash = hash and 7 + + // RC4 key scheduling + val s = IntArray(256) { it } + j = 0 + for (i in 0 until 256) { + j = (j + s[i] + decodedString[i % 64].code) % 256 + s[i] = s[j].also { s[j] = s[i] } + } + + // Decrypt rest of the chars + val result = StringBuilder() + var i = 0 + j = 0 + var keyStream = 0 + var rnd = 0 + val step = primeIndexes[hash] + + for (n in 0 until decodedString.length - 64) { + i = (i + step) % 256 + j = (keyStream + s[(j + s[i]) % 256]) % 256 + keyStream = (keyStream + i + s[i]) % 256 + s[i] = s[j].also { s[j] = s[i] } + + rnd = s[(j + s[(i + s[(rnd + keyStream) % 256]) % 256]) % 256] + val decryptedChar = decodedString[n + 64].code xor rnd + result.append(decryptedChar.toChar()) + } + + return result.toString() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt b/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt new file mode 100644 index 00000000..d24318c4 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt @@ -0,0 +1,34 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import tsuki.util.parseHtml +import tsuki.util.toTitleCase + + + + +@MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) +internal class HentaixComic(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.HENTAIXCOMIC, "hentaixcomic.com", 16) { + + override suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/?s=&post_type=wp-manga").parseHtml() + val set = mutableSetOf() + val titles = mutableSetOf() + doc.select("div.checkbox-group input[type=checkbox]").forEach { input -> + val key = input.attr("value") + val title = input.nextElementSibling()?.text()?.toTitleCase() + if (key.isNotEmpty() && !title.isNullOrEmpty() && titles.add(title.lowercase())) { + set.add(MangaTag(key = key, title = title, source = source)) + } + } + return set + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt b/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt new file mode 100644 index 00000000..12f76972 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt @@ -0,0 +1,273 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON + +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.util.Calendar +import java.util.EnumSet + +@MangaSourceParser("HEYTOON", "HeyToon", "en", ContentType.HENTAI) +internal class HeyToon(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.HEYTOON, pageSize = 54, searchPageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("heytoon.net") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getAvailableGenres(), + ) + + private val headers = Headers.headersOf("X-Requested-With", "XMLHttpRequest") + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + return when { + !query.isNullOrEmpty() -> { + if (page > 1) return emptyList() + + val url = "https://$domain/api/complete-search?keyword=${query.urlEncoded()}" + val response = webClient.httpGet(url, headers).parseJsonArray() + + response.mapJSON { comic -> + val linkComic = comic.getString("linkComic") + val id = linkComic.substringAfterLast("/").substringBefore(".html") + Manga( + id = generateUid(id), + url = linkComic.toRelativeUrl(domain), + publicUrl = linkComic.toAbsoluteUrl(domain), + title = comic.getString("title"), + coverUrl = comic.getStringOrNull("raw_thumb"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + contentRating = null, + largeCoverUrl = null, + description = null, + chapters = null, + ) + } + } + + else -> { + val url = buildString { + append("https://") + append(domain) + append("/en/genres") + + filter.tags.firstOrNull()?.let { + append("/") + append(it.key) + } + + append("?orderBy=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + else -> append("latest") + } + + if (page > 1) { + append("&page=") + append(page) + } + } + + val doc = webClient.httpGet(url).parseHtml() + parseMangaList(doc) + } + } + } + + private fun parseMangaList(doc: Document): List { + return doc.select("div.comicItemCon a").map { element -> + val href = element.attrAsRelativeUrl("href") + val id = href.substringAfterLast("/").substringBefore(".html").substringBefore("-") + val img = element.selectFirstOrThrow("img[alt!=badge]") + Manga( + id = generateUid(id), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = img.attr("title").ifEmpty { img.attr("alt") }, + coverUrl = img.attr("data-src").toAbsoluteUrl(domain), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + contentRating = null, + largeCoverUrl = null, + description = null, + chapters = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirstOrThrow("#titleSubWrapper h1.titCon").text() + val description = doc.selectFirst("#modal_detail .cont_area p")?.text() + val genres = parseGenres(doc) + val status = parseStatus(doc) + val chapters = parseChapters(doc) + + return manga.copy( + title = title, + description = description, + tags = genres, + state = status, + coverUrl = doc.selectFirst("meta[property=og:image]")?.attr("content") ?: manga.coverUrl, + chapters = chapters, + ) + } + + private fun parseGenres(doc: Document): Set { + return doc.select("div.previewItemCon a[href*=/genres/]").mapNotNullToSet { a -> + val key = a.attr("href").substringAfterLast("/genres/").substringBefore("?") + if (key.isEmpty()) return@mapNotNullToSet null + MangaTag( + key = key, + title = key, + source = source, + ) + } + } + + private fun parseStatus(doc: Document): MangaState? { + return doc.select(".badgeArea span").eachText().let { badges -> + when { + badges.any { it.contains("Up") } -> MangaState.ONGOING + badges.any { it.contains("Completed") } -> MangaState.FINISHED + else -> null + } + } + } + + private fun parseChapters(doc: Document): List { + return doc.select(".episodeListConPC a#episodeItemCon").mapChapters(reversed = false) { i, element -> + val href = element.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = element.selectFirstOrThrow(".comicInfo p.episodeStitle").text(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + uploadDate = parseDateOrNull(element.selectFirst(".comicInfo .episodeDate")?.text()), + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + return doc.select("#comicContent img").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun getAvailableGenres(): Set { + val genres = listOf( + "Detective", + "Spin-Off", + "Mommy", + "Uncensored", + "New", + "In-Law", + "Cheating", + "MILF", + "Harem", + "College", + "Business", + "Supernatural", + "Thriller", + "Adventure", + "Romance", + "Drama", + ) + + return genres.mapToSet { genre -> + MangaTag( + key = genre, + title = genre, + source = source, + ) + } + } + + private fun parseDateOrNull(dateStr: String?): Long { + if (dateStr.isNullOrEmpty()) return 0 + return try { + val parts = dateStr.split(" ") + if (parts.size != 3) return 0 + + val month = when (parts[0].lowercase()) { + "jan" -> Calendar.JANUARY + "feb" -> Calendar.FEBRUARY + "mar" -> Calendar.MARCH + "apr" -> Calendar.APRIL + "may" -> Calendar.MAY + "jun" -> Calendar.JUNE + "jul" -> Calendar.JULY + "aug" -> Calendar.AUGUST + "sep" -> Calendar.SEPTEMBER + "oct" -> Calendar.OCTOBER + "nov" -> Calendar.NOVEMBER + "dec" -> Calendar.DECEMBER + else -> return 0 + } + + val day = parts[1].removeSuffix(",").toIntOrNull() ?: return 0 + val year = parts[2].toIntOrNull() ?: return 0 + + Calendar.getInstance().apply { + set(Calendar.YEAR, year) + set(Calendar.MONTH, month) + set(Calendar.DAY_OF_MONTH, day) + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } catch (e: Exception) { + 0 + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt b/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt new file mode 100644 index 00000000..0bb7f91b --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt @@ -0,0 +1,21 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser +import tsuki.Broken + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import okhttp3.Headers + +@Broken +@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) +internal class Hiperdex(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("x-svc-gate", "f5pabmx7sdek") + .build() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt b/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt new file mode 100644 index 00000000..2c6a29a8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt @@ -0,0 +1,185 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.CommonHeaders +import tsuki.network.UserAgents +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.insertCookies +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.jsoup.nodes.Document + +private const val F_URL = "fullUrl=" + +@MangaSourceParser("MADARADEX", "MadaraDex", "en", ContentType.HENTAI) +internal class MadaraDex(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MADARADEX, "madaradex.org") { + + init { + context.cookieJar.insertCookies(domain, "wpmanga-adault=1") + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders() + .newBuilder() + .set(CommonHeaders.USER_AGENT, UserAgents.CHROME_DESKTOP) + .set(CommonHeaders.REFERER, "https://$domain/") + .build() + + override val authUrl: String + get() = "https://$domain" + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name.contains("cm_uaid") + } + } + + override val listUrl = "title/" + override val tagPrefix = "genre/" + override val postReq = true + override val stylePage = "" + + + override suspend fun getPages(chapter: MangaChapter): List { + + val fullUrl = chapter.url.toAbsoluteUrl(domain) + + val doc = webClient + .httpGet(fullUrl) + .parseHtml() + + + val images = extractImages(doc) + + + if (images.isEmpty()) { + throw ParseException( + "No images found, try to log in", + fullUrl + ) + } + + + return images.map { image -> + + val imageUrl = image + .toRelativeUrl(domain) + .toHttpUrl() + .newBuilder() + .fragment(_root_ide_package_.tsuki.site.en.hentais.F_URL + fullUrl) + .build() + .toString() + + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + + private fun extractImages(document: Document): List { + + val pageBreakImages = + document + .select("div.page-break img") + .mapNotNull { + it.attr("data-src") + .ifBlank { + it.attr("src") + } + .nullIfEmpty() + } + + + if (pageBreakImages.isNotEmpty()) { + return pageBreakImages + } + + + return document + .body() + .select(selectBodyPage) + .flatMap { body -> + body.select(selectPage) + .flatMap { page -> + page.select("img") + .mapNotNull { + it.attr("src") + .nullIfEmpty() + } + } + } + } + + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + + val fullUrl = url.fragment + ?.substringAfter(_root_ide_package_.tsuki.site.en.hentais.F_URL, "") + + if (!fullUrl.isNullOrEmpty()) { + + copyCookies() + + val cleanUrl = url.newBuilder() + .fragment(null) + .build() + + val newRequest = request.newBuilder() + .header( + CommonHeaders.REFERER, + fullUrl + ) + .header( + CommonHeaders.USER_AGENT, + UserAgents.CHROME_DESKTOP + ) + .url(cleanUrl) + .build() + + return chain.proceed(newRequest) + } + + return super.intercept(chain) + } + + + private fun copyCookies() { + val cookies = context.cookieJar.getCookies(domain) + + cookies.forEach { + context.cookieJar.insertCookies( + "cdn.$domain", + "${it.name}=${it.value}" + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt b/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt new file mode 100644 index 00000000..03d27b50 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.Manga18Parser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) +internal class Manga18(context: MangaLoaderContext) : + Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt new file mode 100644 index 00000000..83ddbbcf --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt @@ -0,0 +1,103 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) +internal class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { + + override val tagPrefix = "publication-genre/" + override val withoutAjax: Boolean = true + override val datePattern: String = "MMMM d, yyyy" + override val stylePage: String = "?style=list" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override suspend fun getChapters(manga: Manga, doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + + val chapters = doc.body() + .select("li.wp-manga-chapter") + .mapNotNull { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + + val name = a.selectFirst("p")?.text() + ?: a.ownText() + + val dateText = + li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst("span.chapter-release-date i")?.text() + + MangaChapter( + id = generateUid(href), + title = name, + number = extractChapterNumber(name) ?: 0f, + volume = 0, + url = link, + uploadDate = parseChapterDate(dateFormat, dateText), + scanlator = null, + branch = null, + source = source, + ) + } + .sortedWith( + compareBy { it.number } + .thenBy { it.title } + ) + + return chapters + } + + private fun extractChapterNumber(name: String): Float? { + val match = Regex( + """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", + RegexOption.IGNORE_CASE + ).find(name) + + return match + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + } + + override suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/series/").parseHtml() + val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") + + return elements.mapNotNullToSet { a -> + val href = a.attr("href") + .removeSuffix("/") + .substringAfterLast(tagPrefix, "") + + if (href.isBlank()) return@mapNotNullToSet null + + MangaTag( + key = href, + title = a.text() + .replace(Regex("""\s*\(\d+\)"""), "") + .replace(Regex("""[^\x20-\x7E]"""), "") + .trim() + .toTitleCase(), + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt new file mode 100644 index 00000000..60e71e88 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt @@ -0,0 +1,144 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.util.HashSet +import java.util.Locale + +@MangaSourceParser("MANHWA18CC", "Manhwa18.cc", "en", ContentType.HENTAI) +internal class Manhwa18CC(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANHWA18CC, "manhwa18.cc", 24) { + + override val datePattern = "dd MMM yyyy" + override val sourceLocale: Locale = Locale.ENGLISH + override val listUrl = "webtoons/" + override val tagPrefix = "webtoon-genre/" + override val withoutAjax = true + override val selectTestAsync = "ul.row-content-chapter" + override val selectDate = "span.chapter-time" + override val selectChapter = "li.a-h" + override val selectBodyPage = "div.read-content" + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + @Volatile + private var tagsCache: Set? = null + private val tagsMutex = Mutex() + + override suspend fun fetchAvailableTags(): Set { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTagsFromSite().also { tagsCache = it } + } + } + + private suspend fun fetchTagsFromSite(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + val list = doc.body().selectFirstOrThrow("div.sub-menu").select("ul li") + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().ifEmpty { + a.selectFirst(".menu-image-title")?.text() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableStates = emptySet(), + availableContentRating = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + + when { + !query.isNullOrEmpty() -> { + append("/search?q=") + append(query.urlEncoded()) + append("&page=") + append(page.toString()) + } + else -> { + val tag = filter.tags.oneOrThrowIfMany() + if (filter.tags.isNotEmpty()) { + append("/$tagPrefix") + append(tag?.key.orEmpty()) + } else { + append("/$listUrl") + } + if (page > 1) { + append(page.toString()) + } + append("?orderby=") + when (order) { + SortOrder.POPULARITY -> append("trending") + SortOrder.UPDATED -> append("latest") + SortOrder.ALPHABETICAL -> append("alphabet") + SortOrder.RATING -> append("rating") + else -> append("latest") + } + } + } + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.manga-lists div.manga-item").map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src(), + title = div.selectFirstOrThrow("h3").text(), + altTitles = emptySet(), + rating = div.selectFirst(".item-rate span")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow(selectBodyPage) + return root.select("img").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt new file mode 100644 index 00000000..4357776c --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt @@ -0,0 +1,243 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) +internal class Manhwa18Com(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { + + override val configKeyDomain = ConfigKey.Domain("manhwa18.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, + SortOrder.NEWEST, SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) + + private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) + private val numericRegex = Regex("""\d+""") + + @Volatile + private var tagsCache: Set? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + ) + + private suspend fun getOrFetchTags(): Set { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTags().also { tagsCache = it } + } + } + + private suspend fun fetchTags(): Set { + val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() + val list = doc.select("[data-genre-id]") + if (list.isEmpty()) return emptySet() + return list.mapNotNull { item -> + val id = item.attr("data-genre-id") + val name = item.text().trim() + if (id.isNotEmpty() && name.isNotEmpty()) { + MangaTag(name.toTitleCase(Locale.ENGLISH), id, source) + } else null + }.toSet() + } + + private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { + doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } + doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } + val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") + ?: doc.selectFirst(".au-cover-art")?.attr("style") + if (styleUrl != null) { + val url = styleUrl.substringAfter("url('").substringBefore("')") + .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } + if (url.isNotEmpty()) { + return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) + } + } + return fallback + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + val url = buildString { + append("https://$domain/tim-kiem?page=$page") + if (query != null) { + append("&q=") + append(query.urlEncoded()) + } + append("&accept_genres=") + if (filter.tags.isNotEmpty()) { + append(filter.tags.joinToString(",") { it.key }) + } + append("&reject_genres=") + if (filter.tagsExclude.isNotEmpty()) { + append(filter.tagsExclude.joinToString(",") { it.key }) + } + append("&sort=") + append( + when (order) { + SortOrder.ALPHABETICAL -> "az" + SortOrder.POPULARITY -> "top" + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "like" + else -> "update" + } + ) + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "1" + MangaState.FINISHED -> "3" + MangaState.PAUSED -> "2" + else -> "" + } + ) + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".thumb-item-flow").map { element -> + val a = element.selectFirst("a") ?: return@map null + val absUrl = a.attrAsAbsoluteUrl("href") + val title = element.selectFirst(".series-title a")?.text() ?: a.text() + val cover = element.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg") ?: "" + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = title, + altTitles = emptySet(), + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title + val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) + val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() + + val statusText = doc.select(".au-stat") + .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } + ?.selectFirst(".v")?.text()?.trim() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "on hold" -> MangaState.PAUSED + else -> null + } + + val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> + val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } + ?: return@mapNotNull null + MangaTag(a.text().trim(), slug, source) + }.toSet() + + val chapters = parseChapters(doc) + + val fullManga = manga.copy( + title = title, + coverUrl = coverUrl, + description = description, + state = state, + tags = tags, + authors = emptySet(), + chapters = chapters, + ) + + detailsCache[manga.url] = fullManga + return fullManga + } + + private fun parseChapters(doc: org.jsoup.nodes.Document): List { + return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> + val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) + val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null + val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() + val uploadDate = dateStr?.let { + runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + } ?: 0L + val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() + ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() + ?: 0f + MangaChapter( + id = generateUid(chUrl), + title = chTitle, + number = number, + volume = 0, + url = chUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#chapter-content img.lazy").mapNotNull { img -> + val src = img.attrAsAbsoluteUrl("data-src") + .takeIf { it.isNotEmpty() } + ?: img.attrAsAbsoluteUrl("src").takeIf { it.isNotEmpty() } + ?: return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt new file mode 100644 index 00000000..134e9f21 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt @@ -0,0 +1,238 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) +internal class Manhwa210(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { + + override val configKeyDomain = ConfigKey.Domain("manhwa210.com") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + @Volatile + private var tagsCache: Set? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + private suspend fun getOrFetchTags(): Set { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTagsFromSite().also { tagsCache = it } + } + } + + private suspend fun fetchTagsFromSite(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + return doc.select("ul.grid.grid-cols-2 a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() + + val details = manga.copy( + altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), + state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + }, + tags = root.select("div.mt-2:contains(Genres) a.bg-gray-500").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + }, + authors = setOfNotNull(author), + description = root.selectFirst("meta[name=description]")?.attrOrNull("content"), + chapters = root.select("div.justify-between ul.overflow-y-auto.overflow-x-hidden a") + .mapNotNull { a -> + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() + val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() + val number = extractChapterNumber(name) + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + scanlator = null, + uploadDate = runCatching { chapterDateFormat.parse(dateText)?.time ?: 0L }.getOrDefault(0L), + branch = null, + source = source, + ) + } + .sortedBy { it.number } + ) + + detailsCache[manga.url] = details + return details + } + + private fun extractChapterNumber(title: String): Float { + val regex = Regex("""(?i)(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""") + regex.find(title)?.let { match -> + return match.groupValues[1].toFloatOrNull() ?: 0f + } + Regex("""(\d+(?:\.\d+)?)""").find(title)?.let { + return it.value.toFloatOrNull() ?: 0f + } + return 0f + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + val query = filter.query + when { + !query.isNullOrEmpty() -> { + append("/search") + append("?filter[name]=") + append(query.urlEncoded()) + if (page > 1) { + append("&page=") + append(page) + } + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + } + filter.tags.isNotEmpty() -> { + val tag = filter.tags.first() + append("/genre/") + append(tag.key) + append("?page=") + append(page) + } + else -> { + append("/list") + append("?sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + append("&page=") + append(page) + } + } + if (filter.states.isNotEmpty()) { + append("&filter[status]=") + filter.states.forEach { + append( + when (it) { + MangaState.ONGOING -> "2," + MangaState.FINISHED -> "1," + else -> "1,2" + }, + ) + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.grid div.relative").map { div -> + val href = div.selectFirst("a[href^=/manga/]")?.attrOrNull("href") + ?: div.parseFailed("Cant find manga image!") + val coverUrl = div.selectFirst("div.cover")?.attr("style") + ?.substringAfter("url('")?.substringBefore("')") + Manga( + id = generateUid(href), + title = div.select("div.p-2 a.text-ellipsis").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = coverUrl.orEmpty(), + tags = setOf(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.text-center img.lazy").mapNotNull { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt b/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt new file mode 100644 index 00000000..b26e0d9e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HeanCmsParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) +internal class OmegaScans(context: MangaLoaderContext) : + HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt b/src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt new file mode 100644 index 00000000..188deb6f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt @@ -0,0 +1,392 @@ +@file:Suppress("DEPRECATION") + +package tsuki.site.en.mangabox + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.parsers.MangaboxParser + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedHashSet +import java.util.Locale +import java.util.TimeZone +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +@MangaSourceParser("MANGABAT", "MangaBats", "en") +internal class Mangabats(context: MangaLoaderContext) : + MangaboxParser(context, MangaParserSource.MANGABAT) { + + override val configKeyDomain = ConfigKey.Domain("mangabats.com") + + override val webClient = OkHttpWebClient( + context.httpClient.newBuilder() + .build(), + source, + ) + + + override val listUrl = "/genre/all" + override val searchUrl = "/search/story" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(doc) } + + val stateText = doc.findInfo("Status")?.lowercase() + + val state = when { + stateText?.contains("ongoing") == true -> MangaState.ONGOING + stateText?.contains("completed") == true || + stateText?.contains("finish") == true -> MangaState.FINISHED + else -> null + } + + val alt = doc.findInfo("Alternative") + + manga.copy( + tags = doc.body() + .select(selectTag) + .mapToSet { a -> + val href = a.attr("href") + + MangaTag( + key = href.substringAfterLast("category=") + .substringBefore("&") + .takeIf { it != href } + ?: href.removeSuffix("/") + .substringAfterLast("/"), + + title = a.text().toTitleCase(), + source = source, + ) + }, + + description = "If empty Chapters, reload Manga!", + altTitles = setOfNotNull(alt), + state = state, + chapters = chaptersDeferred.await() + ) + } + + private fun Document.findInfo(label: String): String? { + return select("ul.manga-info-text li") + .firstOrNull { it.text().contains(label, ignoreCase = true) } + ?.text() + ?.substringAfter(":") + ?.trim() + ?.nullIfEmpty() + } + + @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) + override val searchQueryCapabilities: MangaSearchQueryCapabilities + get() = MangaSearchQueryCapabilities( + SearchCapability( + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + SearchCapability( + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), + isMultiple = false, + isExclusive = true, + ), + SearchCapability( + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + ) + + override suspend fun getListPage( + query: MangaSearchQuery, + page: Int + ): List { + + val titleQuery = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } + ?.value as? String + + val tagCriterion = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TAG } + + val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key + ?: tagCriterion?.values?.firstOrNull()?.toString() + + val state = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.STATE } + ?.values + ?.firstOrNull() as? MangaState + + val url = if (!titleQuery.isNullOrBlank()) { + + "https://$domain/search/story/${ + normalizeSearchQuery(titleQuery).replace('_', '-') + }?page=$page" + + } else { + + val base = if (!tagKey.isNullOrBlank()) { + "https://$domain/genre/$tagKey" + } else { + "https://$domain/genre/all" + } + + val filter = when { + state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 + state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 + + state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 + state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 + + query.order == SortOrder.POPULARITY -> 7 + query.order == SortOrder.NEWEST -> 1 + + else -> 4 // all + latest + } + + "$base?filter=$filter&page=$page" + } + + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + override suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") + .drop(1) + + val result = LinkedHashSet() + + for (a in tags) { + val key = a.attr("href") + .removeSuffix("/") + .substringAfter("/genre/") + .substringBefore("?") + .nullIfEmpty() + ?: continue + + if (result.any { it.key == key }) { + continue + } + + val title = a.text() + .replace(" Manga", "") + .toTitleCase(sourceLocale) + + result.add( + MangaTag( + key = key, + title = title, + source = source, + ), + ) + } + + return result + } + + override suspend fun getChapters(doc: Document): List { + val slug = doc.location() + .substringAfter("/manga/", "") + .substringBefore("/") + .trim() + .ifEmpty { return super.getChapters(doc) } + + val apiChapters = runCatching { + fetchChaptersApi(slug) + }.getOrDefault(emptyList()) + + return apiChapters.ifEmpty { + super.getChapters(doc) + } + } + + private suspend fun fetchChaptersApi(slug: String): List { + + suspend fun request(offset: Int): JSONObject { + val apiUrl = + "https://$domain/api/manga/$slug/chapters" + + "?limit=$CHAPTER_LIST_TAKE&offset=$offset" + + return webClient.httpGet(apiUrl).parseJson() + } + + fun extractChapters(json: JSONObject): List { + val array = json + .optJSONObject("data") + ?.optJSONArray("chapters") + ?: return emptyList() + + return buildList { + for (i in 0 until array.length()) { + array.optJSONObject(i)?.let(::add) + } + } + } + // first request + val first = request(0) + + val chapters = mutableListOf() + chapters += extractChapters(first) + + val pagination = + first.optJSONObject("data") + ?.optJSONObject("pagination") + + val hasMore = + pagination?.optBoolean("has_more", false) ?: false + + if (!hasMore) { + return mapChapters(slug, chapters) + } + + val total = + pagination.optInt("total", 0) + + val offsets = + (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) + .toList() + // parallel loading + coroutineScope { + val results = offsets.map { offset -> + async { + request(offset) + } + }.awaitAll() + results.forEach { + chapters += extractChapters(it) + } + } + return mapChapters(slug, chapters) + } + + private fun mapChapters( + slug: String, + chapters: List + ): List { + + return chapters.mapNotNull { chapter -> + val chapterSlug = + chapter.optString("chapter_slug") + .nullIfEmpty() + ?: return@mapNotNull null + + val chapterName = + chapter.optString("chapter_name") + .nullIfEmpty() + ?: "Chapter" + + val number = + chapter.optString("chapter_num") + .toFloatOrNull() + ?: 0f + + val url = "/manga/$slug/$chapterSlug" + + MangaChapter( + id = generateUid(url), + title = chapterName, + number = number, + volume = 0, + url = url, + uploadDate = parseApiDate( + chapter.optString("updated_at") + ), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { + it.number + } + } + + + private fun parseApiDate(date: String?): Long { + if (date.isNullOrBlank()) { + return 0L + } + + val formats = listOf( + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), + ).onEach { + it.timeZone = TimeZone.getTimeZone("UTC") + } + + for (format in formats) { + val parsed = format.parseSafe(date) + if (parsed != 0L) { + return parsed + } + } + + return 0L + } + + @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://www.mangabats.com/") + .build() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + if (request.method == "GET" || request.method == "POST") { + val newRequest = request.newBuilder() + .removeHeader("Content-Encoding") + .removeHeader("Accept-Encoding") + .build() + + return chain.proceed(newRequest) + } + + return chain.proceed(request) + } + + companion object { + + private const val CHAPTER_LIST_TAKE = 1000 + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt b/src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt new file mode 100644 index 00000000..4349a1ee --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt @@ -0,0 +1,392 @@ +@file:Suppress("DEPRECATION") + +package tsuki.site.en.mangabox + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.parsers.MangaboxParser + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedHashSet +import java.util.Locale +import java.util.TimeZone +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +@MangaSourceParser("MANGANATOGG", "Manganato.gg", "en") +internal class ManganatoGG(context: MangaLoaderContext) : + MangaboxParser(context, MangaParserSource.MANGANATOGG) { + + override val configKeyDomain = ConfigKey.Domain("manganato.gg") + + override val webClient = OkHttpWebClient( + context.httpClient.newBuilder() + .build(), + source, + ) + + + override val listUrl = "/genre/all" + override val searchUrl = "/search/story" + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(doc) } + + val stateText = doc.findInfo("Status")?.lowercase() + + val state = when { + stateText?.contains("ongoing") == true -> MangaState.ONGOING + stateText?.contains("completed") == true || + stateText?.contains("finish") == true -> MangaState.FINISHED + else -> null + } + + val alt = doc.findInfo("Alternative") + + manga.copy( + tags = doc.body() + .select(selectTag) + .mapToSet { a -> + val href = a.attr("href") + + MangaTag( + key = href.substringAfterLast("category=") + .substringBefore("&") + .takeIf { it != href } + ?: href.removeSuffix("/") + .substringAfterLast("/"), + + title = a.text().toTitleCase(), + source = source, + ) + }, + + description = "If empty Chapters, reload Manga!", + altTitles = setOfNotNull(alt), + state = state, + chapters = chaptersDeferred.await() + ) + } + + private fun Document.findInfo(label: String): String? { + return select("ul.manga-info-text li") + .firstOrNull { it.text().contains(label, ignoreCase = true) } + ?.text() + ?.substringAfter(":") + ?.trim() + ?.nullIfEmpty() + } + + @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) + override val searchQueryCapabilities: MangaSearchQueryCapabilities + get() = MangaSearchQueryCapabilities( + SearchCapability( + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + SearchCapability( + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), + isMultiple = false, + isExclusive = true, + ), + SearchCapability( + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + ) + + override suspend fun getListPage( + query: MangaSearchQuery, + page: Int + ): List { + + val titleQuery = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } + ?.value as? String + + val tagCriterion = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TAG } + + val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key + ?: tagCriterion?.values?.firstOrNull()?.toString() + + val state = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.STATE } + ?.values + ?.firstOrNull() as? MangaState + + val url = if (!titleQuery.isNullOrBlank()) { + + "https://$domain/search/story/${ + normalizeSearchQuery(titleQuery).replace('_', '-') + }?page=$page" + + } else { + + val base = if (!tagKey.isNullOrBlank()) { + "https://$domain/genre/$tagKey" + } else { + "https://$domain/genre/all" + } + + val filter = when { + state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 + state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 + + state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 + state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 + + query.order == SortOrder.POPULARITY -> 7 + query.order == SortOrder.NEWEST -> 1 + + else -> 4 // all + latest + } + + "$base?filter=$filter&page=$page" + } + + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + override suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") + .drop(1) + + val result = LinkedHashSet() + + for (a in tags) { + val key = a.attr("href") + .removeSuffix("/") + .substringAfter("/genre/") + .substringBefore("?") + .nullIfEmpty() + ?: continue + + if (result.any { it.key == key }) { + continue + } + + val title = a.text() + .replace(" Manga", "") + .toTitleCase(sourceLocale) + + result.add( + MangaTag( + key = key, + title = title, + source = source, + ), + ) + } + + return result + } + + override suspend fun getChapters(doc: Document): List { + val slug = doc.location() + .substringAfter("/manga/", "") + .substringBefore("/") + .trim() + .ifEmpty { return super.getChapters(doc) } + + val apiChapters = runCatching { + fetchChaptersApi(slug) + }.getOrDefault(emptyList()) + + return apiChapters.ifEmpty { + super.getChapters(doc) + } + } + + private suspend fun fetchChaptersApi(slug: String): List { + + suspend fun request(offset: Int): JSONObject { + val apiUrl = + "https://$domain/api/manga/$slug/chapters" + + "?limit=$CHAPTER_LIST_TAKE&offset=$offset" + + return webClient.httpGet(apiUrl).parseJson() + } + + fun extractChapters(json: JSONObject): List { + val array = json + .optJSONObject("data") + ?.optJSONArray("chapters") + ?: return emptyList() + + return buildList { + for (i in 0 until array.length()) { + array.optJSONObject(i)?.let(::add) + } + } + } + // first request + val first = request(0) + + val chapters = mutableListOf() + chapters += extractChapters(first) + + val pagination = + first.optJSONObject("data") + ?.optJSONObject("pagination") + + val hasMore = + pagination?.optBoolean("has_more", false) ?: false + + if (!hasMore) { + return mapChapters(slug, chapters) + } + + val total = + pagination.optInt("total", 0) + + val offsets = + (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) + .toList() + // parallel loading + coroutineScope { + val results = offsets.map { offset -> + async { + request(offset) + } + }.awaitAll() + results.forEach { + chapters += extractChapters(it) + } + } + return mapChapters(slug, chapters) + } + + private fun mapChapters( + slug: String, + chapters: List + ): List { + + return chapters.mapNotNull { chapter -> + val chapterSlug = + chapter.optString("chapter_slug") + .nullIfEmpty() + ?: return@mapNotNull null + + val chapterName = + chapter.optString("chapter_name") + .nullIfEmpty() + ?: "Chapter" + + val number = + chapter.optString("chapter_num") + .toFloatOrNull() + ?: 0f + + val url = "/manga/$slug/$chapterSlug" + + MangaChapter( + id = generateUid(url), + title = chapterName, + number = number, + volume = 0, + url = url, + uploadDate = parseApiDate( + chapter.optString("updated_at") + ), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { + it.number + } + } + + + private fun parseApiDate(date: String?): Long { + if (date.isNullOrBlank()) { + return 0L + } + + val formats = listOf( + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), + ).onEach { + it.timeZone = TimeZone.getTimeZone("UTC") + } + + for (format in formats) { + val parsed = format.parseSafe(date) + if (parsed != 0L) { + return parsed + } + } + + return 0L + } + + @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://www.mangabats.com/") + .build() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + if (request.method == "GET" || request.method == "POST") { + val newRequest = request.newBuilder() + .removeHeader("Content-Encoding") + .removeHeader("Accept-Encoding") + .build() + + return chain.proceed(newRequest) + } + + return chain.proceed(request) + } + + companion object { + + private const val CHAPTER_LIST_TAKE = 1000 + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/AinzScans.kt b/src/main/kotlin/tsuki/site/id/AinzScans.kt new file mode 100644 index 00000000..2628c074 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/AinzScans.kt @@ -0,0 +1,304 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale +import java.util.EnumSet +import java.util.TimeZone +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.Jsoup + +@MangaSourceParser("AINZSCANS", "AinzScans", "id") +internal class AinzScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.AINZSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("v2.ainzscans01.com") + + private val apiBase = "https://api.ainzscans01.com/api" + + override val sourceLocale: Locale = Locale.US + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + + @Volatile + private var genresCache: Map? = null + private val genresMutex = Mutex() + + private suspend fun getOrFetchGenres(): Map { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + val jsonStr = webClient.httpGet("$apiBase/genres").body?.string().orEmpty() + val map = mutableMapOf() + try { + val array = JSONArray(jsonStr) + for (i in 0 until array.length()) { + val obj = array.getJSONObject(i) + val name = obj.getString("name") + val slug = obj.getString("slug") + map[slug] = MangaTag(title = name, key = slug, source = source) + } + } catch (_: Exception) { } + genresCache = map + map + } + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append(apiBase) + append("/search?type=COMIC") + append("&limit=").append(pageSize) + append("&page=").append(page) + + filter.query?.takeIf { it.isNotBlank() }?.let { + append("&q=").append(it.urlEncoded()) + } + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "rate" + SortOrder.ALPHABETICAL -> "az" + else -> "latest" + } + ) + filter.states.oneOrThrowIfMany()?.let { state -> + append("&status=") + append( + when (state) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + else -> "" + } + ) + } + filter.tags.firstOrNull()?.let { tag -> + append("&genre=").append(tag.key) + } + filter.types.oneOrThrowIfMany()?.let { type -> + append("&comic_type=") + append( + when (type) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + } + ) + } + } + + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseListJson(json) + } + + private fun parseListJson(json: String): List { + val result = mutableListOf() + try { + val obj = JSONObject(json) + val dataArray = obj.optJSONArray("data") ?: return emptyList() + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val slug = item.optString("slug").ifEmpty { continue } + val relativeUrl = "/comic/$slug" + val title = item.optString("title").ifEmpty { slug } + val cover = item.optString("poster_image_url").nullIfEmpty() + result.add( + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + title = title, + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + tags = emptySet(), + state = null, + authors = emptySet(), + altTitles = emptySet(), + source = source, + ) + ) + } + } catch (_: Exception) { + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val slug = manga.url.substringAfter("/comic/") + val json = webClient.httpGet("$apiBase/series/comic/$slug").body?.string().orEmpty() + val result = parseDetailsJson(json, manga) + + detailsCache[manga.url] = result + return result + } + + private fun parseDetailsJson(json: String, manga: Manga): Manga { + try { + val obj = JSONObject(json) + val slug = obj.optString("slug") + val relativeUrl = "/comic/$slug" + + val title = obj.optString("title").ifEmpty { manga.title } + val desc = obj.optString("synopsis").nullIfEmpty()?.let { Jsoup.parse(it).text() } + val rating = obj.optString("rating_average").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN + val state = when (obj.optString("comic_status").uppercase()) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + else -> null + } + val author = obj.optString("author_name").takeIf { it.isNotEmpty() } + val artist = obj.optString("artist_name").takeIf { it.isNotEmpty() } + val genres = obj.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).mapNotNull { i -> + val g = arr.getJSONObject(i) + val gSlug = g.optString("slug") + val gName = g.optString("name") + if (gSlug.isNotEmpty() && gName.isNotEmpty()) MangaTag(gName, gSlug, source) else null + }.toSet() + } ?: emptySet() + + val chapters = parseUnitsJson(obj.optJSONArray("units"), slug) + + return manga.copy( + title = title, + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + description = desc, + rating = rating, + state = state, + authors = setOfNotNull(author, artist), + tags = genres, + chapters = chapters, + ) + } catch (_: Exception) { + return manga + } + } + + private fun parseUnitsJson(array: JSONArray?, seriesSlug: String): List { + if (array == null) return emptyList() + val chapters = mutableListOf() + for (i in 0 until array.length()) { + val obj = array.getJSONObject(i) + val chSlug = obj.optString("slug").ifEmpty { continue } + val number = obj.optString("number") + val date = obj.optString("created_at") + val uploadDate = parseDate(date) + chapters.add( + MangaChapter( + id = generateUid("/comic/$seriesSlug/chapter/$chSlug"), + url = "/comic/$seriesSlug/chapter/$chSlug", + title = "Chapter ${number.removeSuffix(".00")}", + number = number.toFloatOrNull() ?: (chapters.size + 1).toFloat(), + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val slug = chapter.url.substringAfter("/comic/").substringBefore("/chapter/") + val chSlug = chapter.url.substringAfterLast("/") + val json = webClient.httpGet("$apiBase/series/comic/$slug/chapter/$chSlug").body?.string().orEmpty() + try { + val obj = JSONObject(json) + val chapterObj = obj.optJSONObject("chapter") ?: return emptyList() + val pagesArray = chapterObj.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).mapNotNull { i -> + val pageObj = pagesArray.getJSONObject(i) + val rawUrl = pageObj.optString("image_url").nullIfEmpty() ?: return@mapNotNull null + val fixedUrl = fixImageUrl(rawUrl) + MangaPage(id = generateUid(fixedUrl), url = fixedUrl, preview = null, source = source) + } + } catch (_: Exception) { + return emptyList() + } + } + + private fun fixImageUrl(url: String): String { + var fixed = if (url.startsWith("http")) url else "https://api.ainzscans01.com$url" + + if (fixed.contains("googleusercontent.com") || fixed.contains("bp.blogspot.com")) { + fixed = fixed.replace(Regex("""=[swh]\d+[^/?]*($|\?)""", RegexOption.IGNORE_CASE), "=s0$1") + .replace(Regex("""/[swh]\d+[^/]*/""", RegexOption.IGNORE_CASE), "/s0/") + } + + val httpUrl = fixed.toHttpUrlOrNull() + if (httpUrl != null && httpUrl.queryParameterNames.any { it == "w" || it == "width" || it == "resize" }) { + fixed = httpUrl.newBuilder() + .removeAllQueryParameters("w") + .removeAllQueryParameters("width") + .removeAllQueryParameters("resize") + .build() + .toString() + } + return fixed + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private fun parseDate(dateStr: String?): Long { + if (dateStr.isNullOrEmpty()) return 0L + return try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + } + + private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/MGKomik.kt b/src/main/kotlin/tsuki/site/id/MGKomik.kt new file mode 100644 index 00000000..83d3cb54 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/MGKomik.kt @@ -0,0 +1,182 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.MangaParserAuthProvider +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Interceptor +import androidx.collection.arraySetOf +import okhttp3.Response + + +@MangaSourceParser("MGKOMIK", "MGKomik", "id") +internal class MGKomik(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MGKOMIK, "id.mgkomik.cc"), Interceptor, MangaParserAuthProvider { + + override val datePattern = "dd MMM yy" + override val tagPrefix = "genres/" + override val selectDesc = "div.description-summary div.summary__content p" + + override val withoutAjax = true + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val path = request.url.encodedPath + val isAjax = path.contains("admin-ajax.php") || + path.contains("wp-json") || + path.endsWith("/ajax/chapters") + + return if (isAjax) { + chain.proceed( + request.newBuilder() + .header("Sec-CH-UA-Model", "\"\"") + .header("X-Requested-With", "XMLHttpRequest") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Origin", "https://$domain") + .header("Priority", "u=1, i") + .removeHeader("Sec-Fetch-User") + .removeHeader("Upgrade-Insecure-Requests") + .build() + ) + } else { + chain.proceed( + request.newBuilder() + .header("Sec-CH-UA-Model", "\"\"") + .build() + ) + } + } + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + if (order == SortOrder.POPULARITY) { + val p = page + 1 + val url = buildString { + append("https://") + append(domain) + append("/komik") + if (p > 1) { + append("/page/") + append(p) + } + append("/?m_orderby=trending") + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + return super.getListPage(page, order, filter) + } + + override suspend fun fetchAvailableTags(): Set { + return super.fetchAvailableTags() + } + + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + + private fun availableTags() = arraySetOf( + MangaTag("2 hours ago", "2-hours-ago", source), + MangaTag("7 hours ago", "7-hours-ago", source), + MangaTag("8 hours ago", "8-hours-ago", source), + MangaTag("11 hours ago", "11-hours-ago", source), + MangaTag("13 hours ago", "13-hours-ago", source), + MangaTag("15 hours ago", "15-hours-ago", source), + MangaTag("1 day ago", "1-day-ago", source), + MangaTag("2 days ago", "2-days-ago", source), + MangaTag("Action", "action", source), + MangaTag("Adaptation", "adaptation", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Animals", "animals", source), + MangaTag("Apocalypse", "apocalypse", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Crime", "crime", source), + MangaTag("Demon", "demon", source), + MangaTag("Demons", "demons", source), + MangaTag("Drama", "drama", source), + MangaTag("Dungeons", "dungeons", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Fight", "fight", source), + MangaTag("Fighting", "fighting", source), + MangaTag("Full Color", "full-color", source), + MangaTag("Game", "game", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Josei(W)", "joseiw", source), + MangaTag("Kids", "kids", source), + MangaTag("Magic", "magic", source), + MangaTag("Manga", "manga", source), + MangaTag("Mangatoon", "mangatoon", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mature", "mature", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Military", "military", source), + MangaTag("Modern", "modern", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Murim", "murim", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Office Workers", "office-workers", source), + MangaTag("One-Shot", "one-shot", source), + MangaTag("Overpowered", "overpowered", source), + MangaTag("Police", "police", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Regression", "regression", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Revenge", "revenge", source), + MangaTag("Reverse Harem", "reverse-harem", source), + MangaTag("Rofan", "rofan", source), + MangaTag("Romance", "romance", source), + MangaTag("School", "school", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujoai", source), + MangaTag("Shoujo AI", "shoujo-ai", source), + MangaTag("Shoujo(G)", "shoujog", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Smut", "smut", source), + MangaTag("Sports", "sports", source), + MangaTag("Super Power", "super-power", source), + MangaTag("Superhero", "superhero", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Supranatural", "supranatural", source), + MangaTag("Survival", "survival", source), + MangaTag("System", "system", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Transmigration", "transmigration", source), + MangaTag("Vampire", "vampire", source), + MangaTag("Villainess", "villainess", source), + MangaTag("Violence", "violence", source), + MangaTag("War", "war", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Webtoons", "webtoons", source), + MangaTag("Wuxia", "wuxia", source), + MangaTag("Yuri", "yuri", source), + ) + +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/ReYume.kt b/src/main/kotlin/tsuki/site/id/ReYume.kt new file mode 100644 index 00000000..6c73e9b1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/ReYume.kt @@ -0,0 +1,179 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.ZeistMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.asTypedList + +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat + +@MangaSourceParser("REYUME", "ReYume", "id") +internal class ReYume(context: MangaLoaderContext) : + ZeistMangaParser(context, MangaParserSource.REYUME, "www.re-yume.my.id") { + + override val selectTags = "a[rel=tag]" + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val startIndex = maxMangaResults * (page - 1) + 1 + val query = filter.query + val url = buildString { + append("https://") + append(domain) + append("/feeds/posts/default/-/") + when { + !query.isNullOrEmpty() -> { + append("Series") + append("?alt=json&orderby=published&max-results=") + append((maxMangaResults + 1).toString()) + append("&start-index=") + append(startIndex.toString()) + append("&q=label:Series+") + append(query.urlEncoded()) + } + else -> { + if (filter.tags.isNotEmpty()) { + append(filter.tags.joinToString("/") { it.key.urlEncoded() }) + } else { + append("Series") + } + append("?alt=json&orderby=published&max-results=") + append((maxMangaResults + 1).toString()) + append("&start-index=") + append(startIndex.toString()) + } + } + } + + val json = webClient.httpGet(url).parseJson().getJSONObject("feed") + val list = if (json.toString().contains("\"entry\":")) { + parseMangaList(json.getJSONArray("entry")) + } else { + emptyList() + } + return if (list.size > maxMangaResults) list.dropLast(1) else list + } + + override suspend fun getDetails(manga: Manga):Manga { + val baseDetails = super.getDetails(manga) + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = if (baseDetails.title == "Unknown manga" || baseDetails.title.isEmpty()) { + doc.selectFirst("h1#post-title")?.text() ?: baseDetails.title + } else { + baseDetails.title + } + + var desc = baseDetails.description + if (desc.isNullOrEmpty()) { + desc = doc.getElementById("syn_bod")?.text() ?: "" + } + + var author = baseDetails.authors + if (author.isEmpty()) { + val authorText = doc.getElementById("tauther")?.text() ?: doc.getElementById("tauthers")?.text() + if (!authorText.isNullOrEmpty()) { + author = setOf(authorText) + } + } + + return baseDetails.copy( + title = title, + description = desc, + authors = author, + ) + } + + override fun parseMangaList(json: JSONArray): List { + return super.parseMangaList(json).map { + val cleanUrl = it.url.substringBefore("?m=1").substringBefore("&m=1") + val cleanPublic = it.publicUrl.substringBefore("?m=1").substringBefore("&m=1") + it.copy( + url = cleanUrl, + publicUrl = cleanPublic, + id = generateUid(cleanUrl) + ) + } + } + + override suspend fun loadChapters(mangaUrl: String, doc: Document): List { + val label = doc.selectFirst(".chapter_get")?.attr("data-labelchapter") + ?: doc.selectFirst(".libraryAdd")?.attr("data-title") + ?: return super.loadChapters(mangaUrl, doc) + + val url = buildString { + append("https://") + append(domain) + append("/feeds/posts/default/-/") + append(label.urlEncoded()) + append("?alt=json&orderby=published&max-results=9999") + } + + val json = webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList().reversed() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val mangaTitle = doc.selectFirst("h1#post-title")?.text().orEmpty() + + return json.mapIndexedNotNull { i, j -> + val name = j.getJSONObject("title").getString("\$t") + val chapterName = if (mangaTitle.isNotEmpty() && name.contains(mangaTitle, ignoreCase = true)) { + name.replace(mangaTitle, "", ignoreCase = true).trim().trim('-').trim().takeIf { it.isNotEmpty() } ?: name + } else { + name + } + val href = j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" }.getString("href") + val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") + val slug = mangaUrl.substringAfterLast('/') + val slugChapter = href.substringAfterLast('/') + if (slug == slugChapter) return@mapIndexedNotNull null + MangaChapter( + id = generateUid(href), + url = href, + title = chapterName, + number = i + 1f, + volume = 0, + branch = null, + uploadDate = dateFormat.parseSafe(dateText), + scanlator = null, + source = source, + ) + } + } + + override suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain").parseHtml() + val script = doc.selectFirst("script:containsData(filterGenre =)")?.data() + if (script != null) { + val genres = script.substringAfter("filterGenre = [").substringBefore("]") + .replace("'", "").replace("\"", "").split(",") + return genres.mapToSet { + val tag = it.trim() + MangaTag(key = tag, title = tag, source = source) + } + } + return super.fetchAvailableTags() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(selectPage) + .filter { el -> el.parents().any { it.className().contains("separator") } } + .map { img -> + MangaPage( + id = generateUid(img.requireSrc()), + url = img.requireSrc(), + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/Westmanga.kt b/src/main/kotlin/tsuki/site/id/Westmanga.kt new file mode 100644 index 00000000..7e413752 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Westmanga.kt @@ -0,0 +1,264 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.Jsoup +import java.util.EnumSet +import javax.crypto.Mac +import javax.crypto.spec.SecretKeySpec + +@MangaSourceParser("WESTMANGA", "WestManga", "id") +internal class WestmangaParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.WESTMANGA, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("v1.westmanga.cc") + + private val apiDomain: String + get() = "data.mantweh.online" + + private val accessKey = "WM_WEB_FRONT_END" + private val secretKey = "xxxoidj" + + private var genresCache: Set? = null + private val mutex = Mutex() + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.NEWEST, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchGenres(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = "https://$apiDomain/api/contents".toHttpUrl().newBuilder().apply { + if (!filter.query.isNullOrEmpty()) { + addQueryParameter("q", filter.query) + } + addQueryParameter("page", page.toString()) + addQueryParameter("per_page", "20") + addQueryParameter("type", "Comic") + + when (order) { + SortOrder.UPDATED -> addQueryParameter("sort", "updated_at") + SortOrder.POPULARITY -> addQueryParameter("sort", "views") + SortOrder.ALPHABETICAL -> addQueryParameter("sort", "title") + SortOrder.NEWEST -> addQueryParameter("sort", "created_at") + else -> {} + } + + filter.tags.forEach { tag -> + addQueryParameter("genre[]", tag.key) + } + + filter.states.oneOrThrowIfMany()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> return@let + }) + } + + filter.types.oneOrThrowIfMany()?.let { + addQueryParameter("country", when (it) { + ContentType.MANGA -> "JP" + ContentType.MANHWA -> "KR" + ContentType.MANHUA -> "CN" + else -> return@let + }) + } + }.build() + + val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() + val mangaArray = response.optJSONArray("data") ?: return emptyList() + + return mangaArray.mapJSON { jo -> + val slug = jo.getString("slug") + Manga( + id = generateUid(slug), + title = jo.getString("title"), + altTitles = emptySet(), + url = "/manga/$slug/", + publicUrl = "https://$domain/comic/$slug", + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = jo.getString("cover"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removeSurrounding("/manga/", "/") + val url = "https://$apiDomain/api/comic/$slug".toHttpUrl() + + val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() + val data = response.getJSONObject("data") + + val tags = buildSet { + when (data.getStringOrNull("country_id") ?: data.getStringOrNull("country")) { + "JP" -> add(MangaTag("Manga", "manga", source)) + "CN" -> add(MangaTag("Manhua", "manhua", source)) + "KR" -> add(MangaTag("Manhwa", "manhwa", source)) + } + if (data.optBoolean("color", false)) { + add(MangaTag("Colored", "colored", source)) + } + data.optJSONArray("genres")?.let { genres -> + for (i in 0 until genres.length()) { + val genre = genres.getJSONObject(i) + add(MangaTag( + title = genre.getString("name"), + key = genre.getInt("id").toString(), + source = source, + )) + } + } + } + + val description = buildString { + data.getStringOrNull("sinopsis")?.let { synopsis -> + append(Jsoup.parseBodyFragment(synopsis).wholeText().trim()) + } + data.getStringOrNull("alternative_name")?.let { alt -> + append("\n\nAlternative Name: ") + append(alt.trim()) + } + } + + val state = when (data.getStringOrNull("status")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + } + + val chapters = data.optJSONArray("chapters")?.mapJSON { chapterObj -> + val chapterSlug = chapterObj.getString("slug") + val updatedAt = chapterObj.optJSONObject("updated_at")?.getLongOrDefault("time", 0L) + ?: chapterObj.getLongOrDefault("updated_at", 0L) + MangaChapter( + id = generateUid(chapterSlug), + url = "/$chapterSlug/", + title = "Chapter ${chapterObj.getString("number")}", + number = chapterObj.getString("number").toFloatOrNull() ?: 0f, + volume = 0, + branch = null, + uploadDate = updatedAt * 1000, + scanlator = null, + source = source, + ) + } ?: emptyList() + + return manga.copy( + title = data.getString("title"), + coverUrl = data.getString("cover"), + tags = tags, + authors = setOfNotNull(data.getStringOrNull("author")), + description = description.takeIf { it.isNotEmpty() }, + state = state, + chapters = chapters.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val slug = chapter.url.removeSurrounding("/", "/") + val url = "https://$apiDomain/api/v/$slug".toHttpUrl() + + val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() + val images = response.optJSONObject("data")?.getJSONArray("images") + ?: response.getJSONArray("images") + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + val imageUrl = images.getString(i) + pages.add( + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + ) + } + return pages + } + + private suspend fun fetchGenres(): Set = mutex.withLock { + genresCache?.let { return@withLock it } + + val url = "https://$apiDomain/api/contents/genres".toHttpUrl() + val response = webClient.httpGet(url, createApiHeaders(url)).parseJson() + val genresArray = response.optJSONArray("data") ?: return@withLock emptySet() + + val genres = genresArray.mapJSONToSet { genreObj -> + MangaTag( + title = genreObj.getString("name"), + key = genreObj.getInt("id").toString(), + source = source, + ) + } + + genresCache = genres + return@withLock genres + } + + private fun createApiHeaders(url: HttpUrl): Headers { + val timestamp = (System.currentTimeMillis() / 1000).toString() + val message = "wm-api-request" + val key = timestamp + "GET" + url.encodedPath + accessKey + secretKey + + val mac = Mac.getInstance("HmacSHA256") + val secretKeySpec = SecretKeySpec(key.toByteArray(Charsets.UTF_8), "HmacSHA256") + mac.init(secretKeySpec) + val hash = mac.doFinal(message.toByteArray(Charsets.UTF_8)) + val signature = hash.joinToString("") { "%02x".format(it) } + + return Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .add("Referer", "https://$domain/") + .add("x-wm-request-time", timestamp) + .add("x-wm-accses-key", accessKey) + .add("x-wm-request-signature", signature) + .build() + } +} diff --git a/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt b/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt new file mode 100644 index 00000000..4f46ebbe --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt @@ -0,0 +1,186 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) +internal class BacaKomikpics(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance + + override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") + + override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .build() + + override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = false, // weird handling of website + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = when (order) { + SortOrder.POPULARITY -> "https://$domain/komik-populer/" + SortOrder.UPDATED -> { + if (page == 0) "https://$domain/?s=&post_type=komik_series" + else "https://$domain/page/${page + 1}/?s&post_type=komik_series" + } + else -> "https://$domain/komik-populer/" + } + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List { + return doc.select("a.card").mapNotNull { card -> + val href = card.attrAsAbsoluteUrl("href") + val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null + val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() + val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() + val status = card.selectFirst(".status-label")?.text()?.trim() + Manga( + id = generateUid(href), + url = href.toRelativeUrl(domain), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover ?: "", + tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), + state = when (status) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst(".post-title h1")?.text()?.trim() + ?: doc.selectFirst("h1.entry-title")?.text()?.trim() + ?: manga.title + + val description = doc.selectFirst(".sinopsis")?.text()?.trim() + ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" + + val cover = doc.selectFirst(".series-thumb img")?.attr("src") + ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl + + val tags = doc.select("a.genre-link").mapNotNull { a -> + val genre = a.text().trim() + val key = a.attr("href").substringAfter("/genre/").substringBefore("/") + if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null + }.toSet() + + val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() + ?.substringAfter("Status :")?.trim() + val state = when { + statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING + statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED + statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED + else -> null + } + + val chapters = extractChapters(doc) + + return manga.copy( + title = title, + description = description, + coverUrl = cover, + tags = tags, + state = state, + authors = emptySet(), + chapters = chapters, + ) + } + + private fun extractChapters(doc: Document): List { + val script = doc.select("script").find { it.data().contains("chapterData") } + ?: return emptyList() + val data = script.data() + + val prefixIndex = data.indexOf("chapterData") + if (prefixIndex == -1) return emptyList() + + val windowStart = (prefixIndex - 500).coerceAtLeast(0) + val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) + val window = data.substring(windowStart, windowEnd) + + val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) + val match = arrayRegex.find(window) ?: return emptyList() + val jsonStr = match.groupValues[1].replace("\\/", "/") + + return try { + val jsonArray = JSONArray(jsonStr) + (0 until jsonArray.length()).mapNotNull { i -> + val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null + val chTitle = obj.optString("title", "") + val chUrl = obj.optString("url", "") + val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) + .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) + MangaChapter( + id = generateUid(chUrl), + title = chTitle, + number = number, + volume = 0, + url = chUrl.toRelativeUrl(domain), + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ) + } + } catch (e: Exception) { + e.printStackTrace() + emptyList() + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> + var src = img.attr("src") + if (src.isBlank()) src = img.attr("data-src") + if (src.isBlank()) return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun String.nullIfEmpty() = ifBlank { null } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt b/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt new file mode 100644 index 00000000..36a8d1e6 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt @@ -0,0 +1,23 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.DoujinDesuParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.util.EnumSet + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) +internal class DoujinDesu(context: MangaLoaderContext) : + DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { + + override val defaultTypes: String = "doujinshi,manga,manhwa" + + override val availableContentTypes: Set = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.MANHWA + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/pt/HiperToon.kt b/src/main/kotlin/tsuki/site/pt/HiperToon.kt new file mode 100644 index 00000000..f709796e --- /dev/null +++ b/src/main/kotlin/tsuki/site/pt/HiperToon.kt @@ -0,0 +1,11 @@ +package tsuki.site.pt + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("HIPERTOON", "Hipertoon", "pt") +internal class HiperToon(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.HIPERTOON, "hipertoon.com") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/pt/Roxinha.kt b/src/main/kotlin/tsuki/site/pt/Roxinha.kt new file mode 100644 index 00000000..02a3d1b2 --- /dev/null +++ b/src/main/kotlin/tsuki/site/pt/Roxinha.kt @@ -0,0 +1,222 @@ +package tsuki.site.pt + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("ROXINHA", "Roxinha", "pt") +internal class Roxinha(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ROXINHA, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("roxinha.online") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + + ) + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS // Webtoon + ), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List { + val offset = (page - 1) * pageSize + val url = "https://$domain/api/manga/search/advanced".toHttpUrl().newBuilder().apply { + addQueryParameter("limit", pageSize.toString()) + addQueryParameter("offset", offset.toString()) + addQueryParameter("mode", "default") + + if (!filter.query.isNullOrBlank()) { + addQueryParameter("q", filter.query!!) + } + + val (sortField, sortOrder) = when (order) { + SortOrder.ALPHABETICAL -> "title" to "ASC" + SortOrder.UPDATED -> "updatedAt" to "DESC" + SortOrder.POPULARITY -> "views" to "DESC" + SortOrder.RATING -> "avgRating" to "DESC" + else -> "title" to "ASC" + } + addQueryParameter("sort", sortField) + addQueryParameter("order", sortOrder) + + if (filter.states.isNotEmpty()) { + val status = when (filter.states.first()) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> null + } + status?.let { addQueryParameter("status", it) } + } + + if (filter.types.isNotEmpty()) { + val type = when (filter.types.first()) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Webtoon" + else -> null + } + type?.let { addQueryParameter("type", it) } + } + }.build() + + val json = webClient.httpGet(url).parseJson() + val mangasArray = json.optJSONArray("mangas") ?: return emptyList() + + val list = mutableListOf() + for (i in 0 until mangasArray.length()) { + val item = mangasArray.getJSONObject(i) + list.add(item.toManga()) + } + return list + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + return Manga( + id = generateUid(id.toString()), + url = id.toString(), + publicUrl = "https://$domain/manga/$id", + title = getString("title"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } ?: "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val id = manga.url + val json = webClient.httpGet("https://$domain/api/manga/$id").parseJson() + + val title = json.getString("title") + val cover = json.optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } + val author = json.optString("author", null).nullIfEmpty() + val description = json.optString("description", null).nullIfEmpty() + val status = json.optString("status", null) + val genresRaw = json.optString("genres", null) + val tags = genresRaw?.split(",")?.mapNotNull { it -> it.trim().takeIf { it.isNotEmpty() } }?.map { + MangaTag(it, it.lowercase().replace(" ", "_"), source) + }.orEmpty().toSet() + + val state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val chaptersArray = json.optJSONArray("chapters") + val chapters = chaptersArray?.let { arr -> + (0 until arr.length()).map { i -> arr.getJSONObject(i).toChapter() } + } ?: emptyList() + + return manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + largeCoverUrl = cover ?: manga.largeCoverUrl, + authors = setOfNotNull(author), + description = description, + state = state, + tags = tags, + chapters = chapters, + ) + } + + private fun JSONObject.toChapter(): MangaChapter { + val id = getInt("id") + val number = optDouble("chapterNumber", Double.NaN).let { if (it.isNaN()) -1f else it.toFloat() } + val rawTitle = optString("title", null) + val createdAt = optString("createdAt", null) + + val title = if (!rawTitle.isNullOrBlank() && rawTitle.toDoubleOrNull() == null) { + rawTitle + } else { + "Capítulo ${number.toString().removeSuffix(".0")}" + } + + return MangaChapter( + id = generateUid(id.toString()), + title = title, + number = number, + volume = 0, + url = id.toString(), // chapter ID + uploadDate = createdAt?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = null, + branch = null, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chapterId = chapter.url + // access ticket + val ticketJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId/access").parseJson() + val ticket = ticketJson.getString("ticket") + + // fetch chapter pages with ticket header + val pagesHeaders = getRequestHeaders().newBuilder() + .add("x-chapter-access", ticket) + .build() + val pagesJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId", pagesHeaders).parseJson() + val pagesArray = pagesJson.optJSONArray("pages") ?: return emptyList() + + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + val fullUrl = if (url.startsWith("http")) url else "https://$domain$url" + MangaPage( + id = generateUid(fullUrl), + url = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + private fun String.nullIfEmpty() = ifBlank { null } + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt b/src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt new file mode 100644 index 00000000..c822f422 --- /dev/null +++ b/src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt @@ -0,0 +1,19 @@ +package tsuki.site.pt.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import okhttp3.Headers + +@MangaSourceParser("LERHENTAIS", "LerHentais", "pt", ContentType.HENTAI) +internal class LerHentais(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.LERHENTAIS, "lerhentais.com") { + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("x-flux-node", "G2ZsDdWhUwdU82Vw") + .build() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt b/src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt new file mode 100644 index 00000000..aa931403 --- /dev/null +++ b/src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt @@ -0,0 +1,14 @@ +package tsuki.site.vi + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("TRUYENTUOITHO", "Truyện Tuổi Thơ", "vi") +internal class TruyenTuoiTho(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.TRUYENTUOITHO, "truyentuoitho.com") { + override val datePattern = "dd/MM/yyyy" + override val withoutAjax = true +} diff --git a/src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt b/src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt new file mode 100644 index 00000000..1223b3e0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt @@ -0,0 +1,46 @@ +package tsuki.site.vi.hentai + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseFailed +import tsuki.util.parseHtml + +import org.jsoup.nodes.Document + +@MangaSourceParser("TRUYENTRANHDAMMYY", "Truyện Tranh Đam Mỹ", "vi", ContentType.HENTAI) +internal class TruyenTranhDamMyy(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.TRUYENTRANHDAMMYY, "truyentranhdammyy.site") { + override val postReq = true + override suspend fun loadChapters(mangaUrl: String, document: Document): List { + val mangaId = document.select("div#manga-chapters-holder").attr("data-id") + val url = "https://$domain/wp-admin/admin-ajax.php" + val postData = "action=manga_get_chapters&manga=$mangaId" + val doc = webClient.httpPost(url, postData).parseHtml() + return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> + val a = li.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: li.parseFailed("Link is missing") + val link = href + stylePage + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = i + 1f, + volume = 0, + branch = null, + uploadDate = 0, // Correct datePattern not found. + scanlator = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt b/src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt new file mode 100644 index 00000000..62811403 --- /dev/null +++ b/src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt @@ -0,0 +1,17 @@ +package tsuki.site.zh.hentai + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryParser + +import tsuki.model.ContentType +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource + +@MangaSourceParser("XIUTAKU", "Xiutaku", "zh", type = ContentType.OTHER) +internal class Xiutaku(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.XIUTAKU, "xiutaku.com") { + + override suspend fun getFilterOptions(): MangaListFilterOptions = + MangaListFilterOptions(availableTags = fetchTags()) +} \ No newline at end of file From 3cbe28a0d03ada8014df28bbbdf68cee20a30c9e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 22:18:33 +0200 Subject: [PATCH 009/434] Delete src/main/kotlin/tsuki/site/en/Novel24h.kt --- src/main/kotlin/tsuki/site/en/Novel24h.kt | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/en/Novel24h.kt diff --git a/src/main/kotlin/tsuki/site/en/Novel24h.kt b/src/main/kotlin/tsuki/site/en/Novel24h.kt deleted file mode 100644 index 479d4d3e..00000000 --- a/src/main/kotlin/tsuki/site/en/Novel24h.kt +++ /dev/null @@ -1,17 +0,0 @@ -package tsuki.site.en - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("TWENTYFOURHNOVEL", "24HNovel", "en") -internal class Novel24hParser(context: MangaLoaderContext) : - MadaraParser( - context = context, - source = MangaParserSource.TWENTYFOURHNOVEL, - domain = "24hnovel.com" - ) { - override val withoutAjax = true -} \ No newline at end of file From ad6f25dcd496b0b8a36af3bebce9335f2e4347f9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:43:20 +0200 Subject: [PATCH 010/434] fix for roliascan --- src/main/kotlin/tsuki/site/en/RoliaScan.kt | 782 +++++++-------------- 1 file changed, 260 insertions(+), 522 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/RoliaScan.kt b/src/main/kotlin/tsuki/site/en/RoliaScan.kt index 8f50d8c0..6348dba3 100644 --- a/src/main/kotlin/tsuki/site/en/RoliaScan.kt +++ b/src/main/kotlin/tsuki/site/en/RoliaScan.kt @@ -14,17 +14,17 @@ import okhttp3.HttpUrl import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document -import org.jsoup.nodes.Element +import org.jsoup.parser.Parser import java.security.MessageDigest import java.time.ZoneOffset import java.time.ZonedDateTime import java.time.format.DateTimeFormatter -import java.util.ArrayList -import java.util.Calendar import java.util.EnumSet -import java.util.HashSet -import java.util.LinkedHashSet import java.util.Locale +import java.util.Calendar + +// year filter broken +// filter does not work in other sortorder mode except updated @MangaSourceParser("ROLIASCAN", "Rolia Scan", "en") internal class RoliaScan(context: MangaLoaderContext) : @@ -32,15 +32,13 @@ internal class RoliaScan(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("roliascan.com") - private val chapterNumberRegex = Regex("""(\d+(?:\.\d+)?)""") - + private val numberRegex = Regex("""(\d+(?:\.\d+)?)""") + private val relativeDateRegex = Regex("""(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago""") private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) - private val genreTags by lazy { - ROLIA_TAGS.mapTo(LinkedHashSet(ROLIA_TAGS.size)) { (title, id) -> - tag(id.toString(), title) - } - } + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() override fun onCreateConfig(keys: MutableCollection>) { super.onCreateConfig(keys) @@ -55,386 +53,254 @@ internal class RoliaScan(context: MangaLoaderContext) : SortOrder.POPULARITY, ) - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isYearSupported = true, + ) + + private val genreTags by lazy { + ROLIA_TAGS.mapTo(LinkedHashSet(ROLIA_TAGS.size)) { (title, id) -> + MangaTag(title = title, key = id.toString(), source = source) + } + } override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = genreTags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.NOVEL ), ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val startApiPage = (page - 1) * API_PAGES_PER_PAGE + 1 - val endApiPage = startApiPage + API_PAGES_PER_PAGE - 1 + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val start = (page - 1) * API_PAGES_PER_PAGE + 1 + val end = start + API_PAGES_PER_PAGE - 1 val result = ArrayList(pageSize) - val seenIds = HashSet(pageSize * 2) + val seen = HashSet(pageSize * 2) - for (apiPage in startApiPage..endApiPage) { + for (apiPage in start..end) { + val yearsParam = if (filter.year > 0) "[${filter.year}]" else "[]" val payload = JSONObject().apply { put("page", apiPage) put("search", filter.query.orEmpty()) - put("years", "[]") + put("years", yearsParam) put("genres", filter.tags.toGenrePayload()) put("types", filter.types.toTypePayload()) put("statuses", filter.states.toStatusPayload()) put("sort", order.toApiSort()) put("genreMatchMode", "any") } - val pageJson = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) - val rawSize = pageJson.length() - if (rawSize == 0) { - break - } - repeat(rawSize) { index -> - val item = pageJson.optJSONObject(index) ?: return@repeat - if (item.optString("type").equals("Novel", ignoreCase = true)) { - return@repeat - } + val json = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) + if (json.length() == 0) break + repeat(json.length()) { i -> + val item = json.optJSONObject(i) ?: return@repeat + if (item.optString("type").equals("Novel", true)) return@repeat val rawUrl = item.optString("url").trim() - if (rawUrl.isEmpty()) { - return@repeat - } - val uniqueId = item.optString("id").nullIfEmpty() ?: rawUrl - if (!seenIds.add(uniqueId)) { - return@repeat - } - result.add(item.toManga()) - } - if (rawSize < API_PAGE_SIZE) { - break + if (rawUrl.isEmpty()) return@repeat + val uid = item.optString("id").nullIfEmpty() ?: rawUrl + if (seen.add(uid)) result.add(item.toManga()) } + if (json.length() < API_PAGE_SIZE) break } return result } + private fun JSONObject.toManga(): Manga { + val url = optString("url") + val id = optString("id") + val fullUrl = withMangaId(url, id) + return Manga( + id = generateUid(fullUrl), + title = optString("title").unescapeHtml(), + altTitles = emptySet(), + url = fullUrl, + publicUrl = url.nullIfEmpty()?.toAbsoluteUrl(domain) + ?: url.toRelativeUrl(domain).toAbsoluteUrl(domain), + rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover").nullIfEmpty(), + tags = emptySet(), + state = optString("status").toMangaState(), + authors = emptySet(), + description = optString("description").nullIfEmpty()?.unescapeHtml(), + source = source, + ) + } + override suspend fun getDetails(manga: Manga): Manga { - val detailUrl = manga.url.urlWithoutFragment().toAbsoluteUrl(domain) - val document = fetchDocument(detailUrl) - val jsonLd = document.findSeriesJsonLd() - val description = document.extractDescription(jsonLd) - val altTitles = document.extractAltTitles(jsonLd) - val tags = document.extractTags(jsonLd) - val authors = document.extractAuthors(jsonLd) - val state = document.extractStatus() - val rating = document.extractRating(jsonLd) + val doc = fetchDocument(manga.url.urlWithoutFragment().toAbsoluteUrl(domain)) + val ld = doc.findSeriesJsonLd() val mangaId = extractMangaId(manga.url) - ?: document.selectFirst("[data-manga-id]")?.attr("data-manga-id") - ?: jsonLd?.optString("identifier")?.nullIfEmpty() - ?: throw ParseException("Unable to determine manga id", detailUrl) - val chapters = loadChapters(mangaId, detailUrl) + ?: doc.selectFirst("[data-manga-id]")?.attr("data-manga-id") + ?: ld?.optString("identifier")?.nullIfEmpty() + ?: throw ParseException("Cannot find manga id", manga.url) + + val cleanTitle = doc.selectFirst("h1")?.ownText()?.trim()?.nullIfEmpty() + ?: ld?.optString("name")?.nullIfEmpty()?.unescapeHtml() + ?: manga.title + return manga.copy( - description = description, - tags = if (tags.isEmpty()) manga.tags else tags, - authors = if (authors.isEmpty()) manga.authors else authors, - altTitles = if (altTitles.isEmpty()) manga.altTitles else altTitles, - state = state, - rating = rating, - chapters = chapters, + title = cleanTitle, + description = doc.extractDescription(ld)?.unescapeHtml(), + altTitles = doc.extractAltTitles(ld).mapTo(mutableSetOf()) { it.unescapeHtml() }, + tags = doc.extractTags(ld).ifEmpty { manga.tags }, + authors = doc.extractAuthors(ld).ifEmpty { manga.authors }.mapTo(mutableSetOf()) { it.unescapeHtml() }, + state = doc.extractStatus(), + rating = doc.extractRating(ld), + chapters = loadChapters(mangaId, manga.url.toAbsoluteUrl(domain)), ) } private suspend fun loadChapters(mangaId: String, referer: String): List { val headers = Headers.headersOf("Referer", referer) val result = ArrayList() - val seenIds = HashSet() + val seen = HashSet() var offset = 0 val limit = CHAPTER_LIMIT - var expectedTotal = Int.MAX_VALUE - while (offset < expectedTotal) { - val (token, timestamp) = generateChapterToken() - val requestUrl = buildChapterUrl(mangaId, offset, limit, token, timestamp) - val raw = webClient.httpGet(requestUrl, headers).parseRaw().trim() - if (raw.isEmpty()) { - break - } + var total = Int.MAX_VALUE + while (offset < total) { + val (token, ts) = chapterToken() + val raw = webClient.httpGet( + buildChapterUrl(mangaId, offset, limit, token, ts), headers + ).parseRaw().trim() + if (raw.isEmpty()) break val json = runCatching { JSONObject(raw) }.getOrNull() ?: break - if (json.has("success") && !json.optBoolean("success", false)) { - break - } - expectedTotal = json.optInt("total", expectedTotal) - val chaptersArray = json.optJSONArray("chapters") ?: break - if (chaptersArray.length() == 0) { - break - } - for (i in 0 until chaptersArray.length()) { - val item = chaptersArray.getJSONObject(i) - val chapterUrl = item.optString("url").nullIfEmpty() ?: continue - val chapterLanguage = item.optString("language").nullIfEmpty() - if (chapterLanguage != null && !chapterLanguage.equals("en", ignoreCase = true)) { - continue - } - val key = item.optString("id").ifEmpty { chapterUrl } - if (!seenIds.add(key)) { - continue - } - val fallbackNumber = (offset + i + 1).toFloat() - result.add(item.toMangaChapter(chapterUrl, fallbackNumber)) - } - offset += chaptersArray.length() - if (json.has("has_more") && !json.optBoolean("has_more", false)) { - break - } - if (!json.has("has_more") && chaptersArray.length() < limit) { - break + if (!json.optBoolean("success", true)) break + total = json.optInt("total", total) + val arr = json.optJSONArray("chapters") ?: break + if (arr.length() == 0) break + for (i in 0 until arr.length()) { + val ch = arr.getJSONObject(i) + val url = ch.optString("url").nullIfEmpty() ?: continue + if (ch.optString("language").nullIfEmpty()?.equals("en", true) == false) continue + val key = ch.optString("id").ifEmpty { url } + if (!seen.add(key)) continue + result.add(ch.toChapter(url, offset + i + 1f)) } + offset += arr.length() + if (!json.optBoolean("has_more", true)) break } return result.reversed() } - private fun buildChapterUrl( - mangaId: String, - offset: Int, - limit: Int, - token: String, - timestamp: String, - ): String { - return HttpUrl.Builder() - .scheme("https") - .host(domain) - .addPathSegment("auth") - .addPathSegment("manga-chapters") - .addQueryParameter("manga_id", mangaId) - .addQueryParameter("offset", offset.toString()) - .addQueryParameter("limit", limit.toString()) - .addQueryParameter("order", "DESC") - .addQueryParameter("_t", token) - .addQueryParameter("_ts", timestamp) - .build() - .toString() - } - - private fun generateChapterToken(): Pair { - val timestamp = (System.currentTimeMillis() / 1000L).toString() - val hourKey = ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter) - val secret = "mng_ch_$hourKey" - val hashInput = timestamp + secret - val token = MessageDigest - .getInstance("MD5") - .digest(hashInput.toByteArray()) - .toHex() - .substring(0, 16) - return token to timestamp - } - - private fun ByteArray.toHex(): String { - if (isEmpty()) { - return "" - } - val chars = CharArray(size * 2) - var index = 0 - for (element in this) { - val unsigned = element.toInt() and 0xFF - chars[index++] = Character.forDigit((unsigned ushr 4) and 0xF, 16) - chars[index++] = Character.forDigit(unsigned and 0xF, 16) - } - return String(chars) - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterId = chapter.url - .toAbsoluteUrl(domain) - .substringBefore('?') - .substringAfterLast('/') - .substringAfterLast('-') - .nullIfEmpty() - ?: return emptyList() - - val requestUrl = HttpUrl.Builder() - .scheme("https") - .host(domain) - .addPathSegment("auth") - .addPathSegment("chapter-content") - .addQueryParameter("chapter_id", chapterId) - .build() - .toString() - - val raw = webClient.httpGet(requestUrl).parseRaw().trim() - if (raw.isEmpty()) { - return emptyList() - } - - val images = runCatching { - JSONObject(raw).optJSONArray("images") - ?: JSONArray(raw) - }.getOrNull() ?: return emptyList() - - val seen = HashSet(images.length()) - val result = ArrayList(images.length()) - repeat(images.length()) { index -> - val imageUrl = images.optString(index).nullIfEmpty() ?: return@repeat - val absoluteUrl = if (imageUrl.startsWith("http", ignoreCase = true)) { - imageUrl - } else { - imageUrl.toAbsoluteUrl(domain) - } - if (!seen.add(absoluteUrl)) { - return@repeat - } - result.add( - MangaPage( - id = generateUid(absoluteUrl), - url = absoluteUrl, - preview = null, - source = source, - ), - ) - } - return result - } - - private fun JSONObject.toManga(): Manga { - val id = optString("id") - val rawUrl = optString("url") - val relativeUrl = if (rawUrl.isBlank()) "/" else rawUrl.toRelativeUrl(domain) - val internalUrl = relativeUrl.withMangaId(id) - val publicUrl = rawUrl.nullIfEmpty() ?: relativeUrl.toAbsoluteUrl(domain) - return Manga( - id = generateUid(internalUrl), - title = optString("title"), - altTitles = emptySet(), - url = internalUrl, - publicUrl = publicUrl, - rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, - contentRating = null, - coverUrl = optString("cover").nullIfEmpty(), - tags = emptySet(), - state = optString("status").toMangaState(), - authors = emptySet(), - description = optString("description").nullIfEmpty(), - source = source, - ) - } - - private fun JSONObject.toMangaChapter(chapterUrl: String, fallbackNumber: Float): MangaChapter { - val rawChapter = optString("chapter") - val chapterLabel = rawChapter.nullIfEmpty() ?: fallbackNumber.toChapterLabel() + private fun JSONObject.toChapter(url: String, fallback: Float): MangaChapter { + val chapterNum = optString("chapter").nullIfEmpty() ?: fallback.toLabel() val rawTitle = optString("title").trim() - val title = when { - rawTitle.equals("N/A", ignoreCase = true) -> "Chapter $chapterLabel" - rawTitle == "—" -> "Chapter $chapterLabel" - rawTitle.isEmpty() -> null - else -> rawTitle + val number = numberRegex.find(chapterNum)?.value?.toFloatOrNull() ?: fallback + + val title = buildString { + append("Ch. ") + append(chapterNum) + val cleanTitle = rawTitle.takeUnless { + it.equals("N/A", true) || it == "—" || it.isEmpty() + }?.unescapeHtml() + if (!cleanTitle.isNullOrBlank()) { + append(" - ") + append(cleanTitle) + } } - val number = chapterNumberRegex.find(rawChapter)?.value?.toFloatOrNull() - val scanlator = optString("group_name") - .trim() - .nullIfEmpty() - ?.takeUnless { it.equals("N/A", ignoreCase = true) || it == "—" } return MangaChapter( - id = generateUid(chapterUrl), + id = generateUid(url), title = title, - number = number ?: fallbackNumber, + number = number, volume = 0, - url = chapterUrl.toRelativeUrl(domain), - scanlator = scanlator, - uploadDate = parseChapterDate(optString("date")), + url = url.toRelativeUrl(domain), + scanlator = null, + uploadDate = parseRelativeDate(optString("date")), branch = null, source = source, ) } - private fun Float.toChapterLabel(): String = if (this % 1f == 0f) { - toInt().toString() - } else { - replaceTrailingZeros() - } + private fun chapterToken() = Pair( + MessageDigest.getInstance("MD5") + .digest("${System.currentTimeMillis() / 1000L}mng_ch_${ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter)}".toByteArray()) + .toHex().substring(0, 16), + (System.currentTimeMillis() / 1000L).toString() + ) - private fun Float.replaceTrailingZeros(): String { - val text = toString() - val trimmed = text.trimEnd('0') - return if (trimmed.endsWith('.')) { - trimmed.dropLast(1) - } else { - trimmed - } - } + @Suppress("SameParameterValue") + private fun buildChapterUrl(mangaId: String, offset: Int, limit: Int, token: String, ts: String) = + HttpUrl.Builder().apply { + scheme("https").host(domain) + addPathSegment("auth").addPathSegment("manga-chapters") + addQueryParameter("manga_id", mangaId) + addQueryParameter("offset", offset.toString()) + addQueryParameter("limit", limit.toString()) + addQueryParameter("order", "DESC") + addQueryParameter("_t", token) + addQueryParameter("_ts", ts) + }.build().toString() - private fun String?.toMangaState(): MangaState? { - val normalized = this?.lowercase(Locale.US)?.trim() ?: return null - return when { - normalized.contains("ongoing") -> MangaState.ONGOING - normalized.contains("complete") || normalized.contains("completed") -> MangaState.FINISHED - normalized.contains("hiatus") -> MangaState.PAUSED - normalized.contains("canceled") || normalized.contains("cancelled") -> MangaState.ABANDONED - normalized.contains("upcoming") || normalized.contains("tba") -> MangaState.UPCOMING - else -> null + override suspend fun getPages(chapter: MangaChapter): List { + val id = chapter.url.toAbsoluteUrl(domain) + .removeSuffix("/").substringBefore('?').substringAfterLast('/').substringAfterLast('-') + .nullIfEmpty() ?: return emptyList() + val raw = webClient.httpGet( + "https://$domain/auth/chapter-content?chapter_id=$id", getRequestHeaders() + ).parseRaw().trim() + if (raw.isEmpty()) return emptyList() + val images = runCatching { JSONObject(raw).optJSONArray("images") ?: JSONArray(raw) }.getOrNull() ?: return emptyList() + val seen = HashSet(images.length()) + return (0 until images.length()).mapNotNull { i -> + val img = images.optString(i).nullIfEmpty() ?: return@mapNotNull null + val abs = if (img.startsWith("http", true)) img else img.toAbsoluteUrl(domain) + if (seen.add(abs)) MangaPage(id = generateUid(abs), url = abs, preview = null, source = source) else null } } - private fun parseChapterDate(raw: String?): Long { - val value = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L - if (!value.endsWith("ago")) { - return 0L - } - val amount = chapterNumberRegex.find(value)?.value?.toFloatOrNull() ?: return 0L - val calendar = Calendar.getInstance() - when { - value.contains("second") -> calendar.add(Calendar.SECOND, -amount.toInt()) - value.contains("minute") -> calendar.add(Calendar.MINUTE, -amount.toInt()) - value.contains("hour") -> calendar.add(Calendar.HOUR_OF_DAY, -amount.toInt()) - value.contains("day") -> calendar.add(Calendar.DAY_OF_YEAR, -amount.toInt()) - value.contains("week") -> calendar.add(Calendar.WEEK_OF_YEAR, -amount.toInt()) - value.contains("month") -> calendar.add(Calendar.MONTH, -amount.toInt()) - value.contains("year") -> calendar.add(Calendar.YEAR, -amount.toInt()) - else -> return 0L - } - return calendar.timeInMillis + private fun String?.toMangaState() = when { + this == null -> null + contains("ongoing", true) -> MangaState.ONGOING + contains("complete", true) -> MangaState.FINISHED + contains("hiatus", true) -> MangaState.PAUSED + contains("cancel", true) -> MangaState.ABANDONED + contains("upcoming", true) || contains("tba", true) -> MangaState.UPCOMING + else -> null } - private fun Collection.toStatusPayload(): String { - if (isEmpty()) { - return "[]" - } - val mapped = mapNotNull { state -> - when (state) { - MangaState.FINISHED -> "Completed" - MangaState.ONGOING -> "Ongoing" - else -> null - } + private fun parseRelativeDate(raw: String?): Long { + val v = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L + val m = relativeDateRegex.matchEntire(v) ?: return 0L + val (num, unit) = m.destructured + val cal = Calendar.getInstance() + val n = num.toIntOrNull() ?: return 0L + when (unit) { + "second" -> cal.add(Calendar.SECOND, -n) + "minute" -> cal.add(Calendar.MINUTE, -n) + "hour" -> cal.add(Calendar.HOUR_OF_DAY, -n) + "day" -> cal.add(Calendar.DAY_OF_YEAR, -n) + "week" -> cal.add(Calendar.WEEK_OF_YEAR, -n) + "month" -> cal.add(Calendar.MONTH, -n) + "year" -> cal.add(Calendar.YEAR, -n) } - return mapped.toStringifiedPayload() + return cal.timeInMillis } - private fun Collection.toTypePayload(): String { - if (isEmpty()) { - return "[]" - } - val mapped = mapNotNull { type -> - when (type) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Comics" - else -> null - } - } - return mapped.toStringifiedPayload() + private fun String.unescapeHtml(): String { + var d = this + var p: String + do { + p = d + d = Parser.unescapeEntities(d, false) + } while (d != p) + return d } - private fun Collection.toStringifiedPayload(): String { - if (isEmpty()) { - return "[]" - } - return joinToString(prefix = "[", postfix = "]", separator = ",") { value -> - "\"${value.escapeJson()}\"" - } - } + private fun Float.toLabel() = if (this % 1f == 0f) toInt().toString() else toString().trimEnd('0').removeSuffix(".") + + private fun Collection.toStatusPayload() = if (isEmpty()) "[]" else + mapNotNull { s -> when(s) { MangaState.FINISHED -> "Completed"; MangaState.ONGOING -> "Ongoing"; else -> null } }.toJsonArray() + + private fun Collection.toTypePayload() = if (isEmpty()) "[]" else + mapNotNull { t -> when(t) { ContentType.MANGA -> "Manga"; ContentType.MANHWA -> "Manhwa"; ContentType.MANHUA -> "Manhua"; ContentType.COMICS -> "Comics"; else -> null } }.toJsonArray() - private fun SortOrder.toApiSort(): String = when (this) { + private fun Collection.toJsonArray() = if (isEmpty()) "[]" else + joinToString(prefix = "[", postfix = "]", separator = ",") { "\"${it.escapeJson()}\"" } + + private fun SortOrder.toApiSort() = when(this) { SortOrder.POPULARITY -> "popular_desc" SortOrder.UPDATED -> "post_desc" SortOrder.NEWEST -> "release_desc" @@ -443,243 +309,115 @@ internal class RoliaScan(context: MangaLoaderContext) : else -> "post_desc" } - private fun String.escapeJson(): String { - return replace("\\", "\\\\") - .replace("\"", "\\\"") - } + private fun String.escapeJson() = replace("\\", "\\\\").replace("\"", "\\\"") - private fun Document.extractDescription(jsonLd: JSONObject?): String? { - val fromJson = jsonLd?.optString("description")?.nullIfEmpty() - if (fromJson != null) { - return fromJson + private fun Collection.toGenrePayload(): String { + if (isEmpty()) return "[]" + val ids = mapNotNull { tag -> + tag.key.toIntOrNull()?.toString() + ?: genreTags.find { it.title.equals(tag.title, true) }?.key?.toIntOrNull()?.toString() } - return selectFirst("div[data-description], div#description, div#synopsis, div:matchesOwn((?i)description)") - ?.nextElementSibling() - ?.text() - ?.nullIfEmpty() + return ids.toJsonArray() + } + + private fun Document.extractDescription(ld: JSONObject?) = + ld?.optString("description")?.nullIfEmpty() + ?: selectFirst("div[data-description], div#description, div#synopsis")?.nextElementSibling()?.text()?.nullIfEmpty() ?: selectFirst("meta[name='description']")?.attr("content")?.nullIfEmpty() ?: selectFirst("meta[property='og:description']")?.attr("content")?.nullIfEmpty() - } - private fun Document.extractAltTitles(jsonLd: JSONObject?): Set { - val alt = mutableSetOf() - jsonLd?.opt("alternateName")?.let { value -> - when (value) { - is JSONArray -> { - repeat(value.length()) { index -> - value.optString(index).nullIfEmpty()?.let(alt::add) - } - } - is String -> value.nullIfEmpty()?.let(alt::add) - } - } - if (alt.isEmpty()) { - selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { alt.addAll(it) } + private fun Document.extractAltTitles(ld: JSONObject?) = buildSet { + ld?.opt("alternateName")?.let { v -> + when(v) { is JSONArray -> repeat(v.length()) { it -> v.optString(it).nullIfEmpty()?.let { add(it) } }; is String -> v.nullIfEmpty()?.let { add(it) } } } - return alt + if (isEmpty()) selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { addAll(it) } } - private fun Document.extractTags(jsonLd: JSONObject?): Set { - val tags = LinkedHashSet() - jsonLd?.opt("genre")?.let { genre -> - when (genre) { - is JSONArray -> repeat(genre.length()) { index -> - genre.optString(index).nullIfEmpty()?.let { addTag(tags, it) } - } - is String -> genre.nullIfEmpty()?.let { addTag(tags, it) } - } + private fun Document.extractTags(ld: JSONObject?) = buildSet { + ld?.opt("genre")?.let { v -> + when(v) { is JSONArray -> repeat(v.length()) { it -> v.optString(it).nullIfEmpty()?.let { add(MangaTag(key = it.lowercase().replace(' ', '-'), title = it, source = source)) } }; is String -> v.nullIfEmpty()?.let { add(MangaTag(key = it.lowercase().replace(' ', '-'), title = it, source = source)) } } } - if (tags.isEmpty()) { - select("a[href*='/genre/']").mapNotNullToSet { anchor -> - anchor.text().nullIfEmpty()?.let { title -> - val href = anchor.attr("href").trimEnd('/') - val slug = href.substringAfterLast('/').substringBefore('?').nullIfEmpty() - ?: return@let null - MangaTag( - key = slug, - title = title, - source = source, - ) + if (isEmpty()) { + select("a[href*='/genre/']").mapNotNull { a -> + a.text().nullIfEmpty()?.let { title -> + val slug = a.attr("href").trimEnd('/').substringAfterLast('/').substringBefore('?').nullIfEmpty() ?: return@mapNotNull null + add(MangaTag(key = slug, title = title, source = source)) } } - .takeIf { it.isNotEmpty() } - ?.let(tags::addAll) } - return tags } - private fun addTag(target: MutableSet, value: String) { - val normalized = value.trim().nullIfEmpty() ?: return - target.add( - MangaTag( - key = normalized.lowercase(Locale.US).replace(' ', '-'), - title = normalized, - source = source, - ), - ) - } - - private fun Document.extractAuthors(jsonLd: JSONObject?): Set { - val authors = LinkedHashSet() - fun addName(value: Any?) { - when (value) { - is JSONArray -> repeat(value.length()) { index -> addName(value.opt(index)) } - is JSONObject -> addName(value.opt("name")) - is String -> value.nullIfEmpty()?.let(authors::add) - } + private fun Document.extractAuthors(ld: JSONObject?) = buildSet { + fun addName(v: Any?) { + when(v) { is JSONArray -> repeat(v.length()) { addName(v.opt(it)) }; is JSONObject -> addName(v.opt("name")); is String -> v.nullIfEmpty()?.let { add(it) } } } - addName(jsonLd?.opt("author")) - addName(jsonLd?.opt("creator")) - if (authors.isEmpty()) { - select("a[href*='/author/'], a[href*='/artist/']").forEach { anchor -> - anchor.text().nullIfEmpty()?.let(authors::add) - } - } - return authors + addName(ld?.opt("author")); addName(ld?.opt("creator")) + if (isEmpty()) select("a[href*='/author/'], a[href*='/artist/']").forEach { it -> it.text().nullIfEmpty()?.let { add(it) } } } private fun Document.extractStatus(): MangaState? { - val label = select("*").firstOrNull { element -> - element.ownText().trim().equals("Status", ignoreCase = true) - } - val statusText = label?.let { element -> - element.findSiblingText()?.nullIfEmpty() - } + val label = select("*").firstOrNull { it.ownText().trim().equals("Status", true) } + val text = label?.let { it.nextElementSibling()?.text()?.nullIfEmpty() } ?: selectFirst("[data-status]")?.attr("data-status")?.nullIfEmpty() ?: selectFirst("span.status, div.status")?.text()?.nullIfEmpty() - return statusText.toMangaState() + return text?.toMangaState() } - private fun Document.extractRating(jsonLd: JSONObject?): Float { - jsonLd?.optJSONObject("aggregateRating")?.optDouble("ratingValue")?.takeIf { !it.isNaN() }?.let { - return it.toFloat() - } - return selectFirst("svg ~ span.font-semibold")?.text()?.toFloatOrNull() ?: RATING_UNKNOWN + private fun Document.extractRating(ld: JSONObject?): Float { + val rawRating = ld?.optJSONObject("aggregateRating")?.optDouble("ratingValue") + ?.takeIf { !it.isNaN() } + ?: selectFirst("svg ~ span.font-semibold")?.text()?.toDoubleOrNull() + return if (rawRating != null && rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN } private fun Document.findSeriesJsonLd(): JSONObject? { - for (script in select("script[type=application/ld+json]")) { - val raw = script.data().trim() - if (raw.isEmpty()) { - continue - } - val obj = raw.toJsonObjectOrNull() - if (obj != null) { - if (obj.matchesSeriesType()) { - return obj - } - continue - } - val array = raw.toJsonArrayOrNull() ?: continue - repeat(array.length()) { index -> - val candidate = array.optJSONObject(index) ?: return@repeat - if (candidate.matchesSeriesType()) { - return candidate - } + for (s in select("script[type=application/ld+json]")) { + val raw = s.data().trim() + val obj = runCatching { JSONObject(raw) }.getOrNull() + if (obj != null && obj.matchesSeries()) return obj + runCatching { JSONArray(raw) }.getOrNull()?.let { arr -> + repeat(arr.length()) { it -> arr.optJSONObject(it)?.takeIf { it.matchesSeries() }?.let { return it } } } } return null } - private fun String.toJsonObjectOrNull(): JSONObject? = runCatching { JSONObject(this) }.getOrNull() + private fun JSONObject.matchesSeries() = optString("@type") in setOf("ComicSeries", "ComicStory", "Book", "CreativeWorkSeries") - private fun String.toJsonArrayOrNull(): JSONArray? = runCatching { JSONArray(this) }.getOrNull() - - private fun JSONObject.matchesSeriesType(): Boolean { - val type = optString("@type") - if (type.isNullOrEmpty()) { - return false - } - return type.equals("ComicSeries", true) || - type.equals("ComicStory", true) || - type.equals("Book", true) || - type.equals("CreativeWorkSeries", true) + private fun extractMangaId(url: String): String? { + url.substringAfter('#', "").split('&').firstOrNull { it.startsWith("mid=") }?.substringAfter("mid=")?.nullIfEmpty()?.let { return it } + url.substringAfter('?', "").split('&').firstOrNull { it.startsWith("mid=") }?.substringAfter("mid=")?.nullIfEmpty()?.let { return it } + return null } - private fun String?.withMangaId(id: String?): String { - val value = this?.nullIfEmpty() ?: return "/" - val cleanId = id?.nullIfEmpty() ?: return value - return if (value.contains('#')) { - if (value.contains("mid=")) value else "$value&mid=$cleanId" - } else { - "$value#mid=$cleanId" - } - } + private fun String.urlWithoutFragment() = substringBefore('#') - private fun extractMangaId(rawUrl: String): String? { - val fragment = rawUrl.substringAfter('#', "") - if (fragment.isNotEmpty()) { - fragment.split('&').firstOrNull { it.startsWith("mid=") }?.let { part -> - return part.substringAfter("mid=").nullIfEmpty() - } - } - val query = rawUrl.substringAfter('?', "") - if (query.isNotEmpty()) { - query.split('&').firstOrNull { it.startsWith("mid=") || it.startsWith("mangaId=") }?.let { part -> - return part.substringAfter('=').nullIfEmpty() - } + private fun withMangaId(rawUrl: String?, id: String?): String { + val url = rawUrl.nullIfEmpty() ?: return "/" + val cleanId = id?.nullIfEmpty() ?: return url + return if (url.contains('#')) { + if (url.contains("mid=")) url else "$url&mid=$cleanId" + } else { + "$url#mid=$cleanId" } - return null } - private fun String.urlWithoutFragment(): String = substringBefore('#') - private suspend fun postJsonArray(url: String, payload: JSONObject): JSONArray { val raw = webClient.httpPost(url, payload).parseRaw().trim() - if (raw.isEmpty()) { - return JSONArray() - } - return raw.toJsonArrayOrNull() ?: JSONArray() + return runCatching { JSONArray(raw) }.getOrDefault(JSONArray()) } - private suspend fun fetchDocument(url: String): Document = webClient.httpGet(url).parseHtml() + private suspend fun fetchDocument(url: String) = webClient.httpGet(url).parseHtml() - private fun Collection.toGenrePayload(): String { - if (isEmpty()) { - return "[]" - } - val ids = LinkedHashSet(size) - for (tag in this) { - val numericKey = tag.key?.toIntOrNull()?.let { it.toString() } - ?: genreTags.firstOrNull { ref -> - ref.title.equals(tag.title, ignoreCase = true) - }?.key?.toIntOrNull()?.let { it.toString() } - if (numericKey != null) { - ids.add(numericKey) - } - } - if (ids.isEmpty()) { - return "[]" - } - return buildString(ids.size * 4) { - append('[') - ids.forEachIndexed { index, value -> - if (index > 0) { - append(',') - } - append(value) - } - append(']') - } - } - - private fun tag(id: String, title: String) = MangaTag(title = title, key = id, source = source) - - private fun Element.findSiblingText(): String? { - nextElementSibling()?.text()?.nullIfEmpty()?.let { return it } - val parent = parent() ?: return null - val children = parent.children() - val index = children.indexOf(this) - if (index == -1) { - return null - } - for (i in index + 1 until children.size) { - val text = children[i].text().nullIfEmpty() - if (text != null) { - return text - } + private fun ByteArray.toHex(): String { + if (isEmpty()) return "" + val chars = CharArray(size * 2) + for ((i, b) in withIndex()) { + val v = b.toInt() and 0xFF + chars[i * 2] = Character.forDigit(v ushr 4, 16) + chars[i * 2 + 1] = Character.forDigit(v and 0xF, 16) } - return null + return String(chars) } private companion object { @@ -794,4 +532,4 @@ internal class RoliaScan(context: MangaLoaderContext) : "Zombies" to 1115, ) } -} \ No newline at end of file +} From 0895c5d866926dbfc8d3ddb106341649e23aab5e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 19 Jul 2026 23:44:55 +0200 Subject: [PATCH 011/434] fix for hiperdex --- src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt b/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt index 0bb7f91b..3086eae2 100644 --- a/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt +++ b/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt @@ -3,19 +3,17 @@ package tsuki.site.en.hentais import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.HiperParser -import tsuki.Broken import tsuki.model.ContentType import tsuki.model.MangaParserSource import okhttp3.Headers -@Broken @MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) internal class Hiperdex(context: MangaLoaderContext) : HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("x-svc-gate", "f5pabmx7sdek") + .set("x-cfg-auth", "yceqt7qgu004") .build() -} \ No newline at end of file +} From a3c3c2311b4c953414cd6f9fd0abac2f8ceee24d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 20 Jul 2026 00:06:40 +0200 Subject: [PATCH 012/434] Update HiperParser.kt --- src/main/kotlin/tsuki/parsers/HiperParser.kt | 105 +++++++++---------- 1 file changed, 50 insertions(+), 55 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HiperParser.kt b/src/main/kotlin/tsuki/parsers/HiperParser.kt index 756c5fa9..c499a2dd 100644 --- a/src/main/kotlin/tsuki/parsers/HiperParser.kt +++ b/src/main/kotlin/tsuki/parsers/HiperParser.kt @@ -4,24 +4,8 @@ import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient - -import tsuki.model.RATING_UNKNOWN -import tsuki.model.ContentRating -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaTag -import tsuki.model.SortOrder -import tsuki.model.ContentType -import tsuki.model.MangaState - -import tsuki.util.generateUid -import tsuki.util.parseHtml -import tsuki.util.urlEncoded +import tsuki.model.* +import tsuki.util.* import okhttp3.Headers import okhttp3.Request @@ -38,22 +22,19 @@ abstract class HiperParser( ) : PagedMangaParser(context, source, pageSize = 30) { protected open val availableContentTypes: Set = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA) - protected open val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED) - protected open val availableContentRating: Set = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) - protected open val mangaPath = "manga" override val configKeyDomain = ConfigKey.Domain(domainName) override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, SortOrder.POPULARITY, + SortOrder.RATING, SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.NEWEST_ASC, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.RELEVANCE + SortOrder.ALPHABETICAL ) override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, @@ -127,8 +108,8 @@ abstract class HiperParser( private fun searchPayload(query: String, sort: String, limit: Int, offset: Int, filter: MangaListFilter): String { val genres = filter.tags.map { it.key } - val types = filter.types.map { it.name.lowercase() } - val statusStr = filter.states.firstOrNull()?.let { + val typeValue = filter.types.firstOrNull()?.name?.lowercase() + val statusValue = filter.states.firstOrNull()?.let { when (it) { MangaState.ONGOING -> "ongoing" MangaState.FINISHED -> "completed" @@ -137,23 +118,25 @@ abstract class HiperParser( else -> null } } - val contentRatingStr = filter.contentRating.firstOrNull()?.let { + val ratingValue = filter.contentRating.firstOrNull()?.let { when (it) { ContentRating.SAFE -> "safe" ContentRating.SUGGESTIVE -> "suggestive" ContentRating.ADULT -> "pornographic" } } + return JSONObject().apply { put("0", JSONObject().apply { put("json", JSONObject().apply { put("q", query) put("sort", sort) put("filters", JSONObject().apply { - if (genres.isNotEmpty()) put("genres", JSONArray(genres)) else put("genres", JSONObject.NULL) - if (types.isNotEmpty()) put("type", types.first()) else put("type", JSONObject.NULL) - if (statusStr != null) put("status", statusStr) else put("status", JSONObject.NULL) - if (contentRatingStr != null) put("contentRating", contentRatingStr) else put("contentRating", JSONObject.NULL) + if (genres.isEmpty()) put("genres", JSONObject.NULL) + else put("genres", JSONArray(genres)) + put("type", typeValue) + put("status", statusValue) + put("contentRating", ratingValue) put("author", JSONObject.NULL) put("artist", JSONObject.NULL) put("year", JSONObject.NULL) @@ -165,9 +148,9 @@ abstract class HiperParser( put("meta", JSONObject().apply { put("values", JSONObject().apply { if (genres.isEmpty()) put("filters.genres", JSONArray().put("undefined")) - if (types.isEmpty()) put("filters.type", JSONArray().put("undefined")) - if (statusStr == null) put("filters.status", JSONArray().put("undefined")) - if (contentRatingStr == null) put("filters.contentRating", JSONArray().put("undefined")) + if (typeValue == null) put("filters.type", JSONArray().put("undefined")) + if (statusValue == null) put("filters.status", JSONArray().put("undefined")) + if (ratingValue == null) put("filters.contentRating", JSONArray().put("undefined")) put("filters.author", JSONArray().put("undefined")) put("filters.artist", JSONArray().put("undefined")) put("filters.year", JSONArray().put("undefined")) @@ -179,6 +162,7 @@ abstract class HiperParser( private fun detailsPayload(slug: String) = JSONObject().apply { put("0", JSONObject().apply { + put("json", JSONObject.NULL) put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) }) put("1", JSONObject().apply { @@ -193,9 +177,10 @@ abstract class HiperParser( put("1", JSONObject().apply { put("json", JSONObject().apply { put("seriesId", mangaId) + put("chapterId", JSONObject.NULL) put("sort", "best") put("page", 1) - put("limit", 20) + put("limit", 9999) }) put("meta", JSONObject().apply { put("values", JSONObject().apply { put("chapterId", JSONArray().put("undefined")) }) @@ -208,6 +193,7 @@ abstract class HiperParser( private fun pagesPayload(slug: String, number: Float) = JSONObject().apply { put("0", JSONObject().apply { + put("json", JSONObject.NULL) put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) }) put("1", JSONObject().apply { @@ -224,25 +210,17 @@ abstract class HiperParser( }) }.toString() - private fun parseLastItemJsonArray(body: String): JSONArray? = try { - val root = JSONArray(body) - root.optJSONObject(root.length() - 1) - ?.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") - } catch (_: Exception) { null } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { val q = filter.query?.trim().orEmpty() val sort = when (order) { + SortOrder.RELEVANCE -> "relevance" SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "score" SortOrder.UPDATED -> "recent" SortOrder.NEWEST -> "newest" SortOrder.NEWEST_ASC -> "oldest" - SortOrder.RATING -> "score" SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "relevance" - else -> "newest" + else -> "relevance" } val input = searchPayload(q, sort, pageSize, (page - 1) * pageSize, filter) val res = client.httpGet(apiUrl("search.query", input), getRequestHeaders()) @@ -263,10 +241,15 @@ abstract class HiperParser( id = generateUid("/$mangaPath/$slug#$id"), url = "/$mangaPath/$slug#$id", publicUrl = "https://$domainName/$mangaPath/$slug#$id", - title = title, coverUrl = cover, - altTitles = emptySet(), authors = emptySet(), tags = emptySet(), - rating = RATING_UNKNOWN, state = null, - contentRating = ContentRating.ADULT, source = source + title = title, + coverUrl = cover, + altTitles = emptySet(), + authors = emptySet(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = ContentRating.ADULT, + source = source ) } } @@ -313,18 +296,24 @@ abstract class HiperParser( private suspend fun loadChapters(manga: Manga): List { val mangaId = manga.url.substringAfterLast("#").toLongOrNull() ?: return emptyList() val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#").ifBlank { return emptyList() } - val res = client.httpGet(apiUrl("auth.me,comments.list,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) + val res = client.httpGet(apiUrl("auth.me,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) val body = res.body?.string() ?: return emptyList() val chaptersArray = parseLastItemJsonArray(body) ?: return emptyList() return (0 until chaptersArray.length()).mapNotNull { i -> chaptersArray.optJSONObject(i)?.let { obj -> val number = obj.optDouble("number", 0.0).toFloat() + val title = obj.optString("title", null) val createdAt = obj.optString("createdAt", null) val uploadDate = createdAt?.let { dateFormat.parse(it)?.time } ?: 0L + val displayTitle = when { + !title.isNullOrBlank() && title != "null" && !Regex("""\d+""").containsMatchIn(title) -> + "Chapter ${number.formatForTitle()} $title" + else -> "Chapter ${number.formatForTitle()}" + } MangaChapter( id = generateUid("/$mangaPath/$slug/$number"), url = "/$mangaPath/$slug/$number", - title = "Chapter ${number.formatForTitle()}", + title = displayTitle, number = number, volume = 0, uploadDate = uploadDate, scanlator = null, branch = null, source = source ) @@ -332,7 +321,6 @@ abstract class HiperParser( }.sortedBy { it.number } } - // triyng first tRPC API if it fails then chapter html scraping override suspend fun getPages(chapter: MangaChapter): List { val path = chapter.url.removePrefix("/$mangaPath/") val slug = path.substringBefore("/") @@ -347,7 +335,6 @@ abstract class HiperParser( val chapterUrl = "https://$domainName/$mangaPath/$slug/$num" val doc = client.httpGet(chapterUrl).parseHtml() - // search in json results image URLs val script = doc.select("script").find { it.data().contains("\"webpUrl\"") } ?: doc.select("script").find { it.data().contains("\"avifUrl\"") } if (script != null) { @@ -388,6 +375,14 @@ abstract class HiperParser( } } + private fun parseLastItemJsonArray(body: String): JSONArray? = try { + val root = JSONArray(body) + root.optJSONObject(root.length() - 1) + ?.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") + } catch (_: Exception) { null } + override suspend fun getPageUrl(page: MangaPage) = page.url private fun Float.formatForTitle() = if (this == toLong().toFloat()) toLong().toString() else toString() @@ -395,4 +390,4 @@ abstract class HiperParser( companion object { private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) } -} \ No newline at end of file +} From d360008c7edd33c874dc56808e1e473a1e34c780 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:09:53 +0200 Subject: [PATCH 013/434] Update Novelcrow.kt --- src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt b/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt index 10cfbc5b..1aa38146 100644 --- a/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt +++ b/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt @@ -1,4 +1,4 @@ -package tsuki.site.all.hentais +package tsuki.site.en.hentais import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -10,5 +10,7 @@ import tsuki.model.MangaParserSource @MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) internal class Novelcrow(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { - override val tagPrefix = "comic-genre/" -} \ No newline at end of file + override val withoutAjax = true + override val tagPrefix = "genres-2/" + override val listUrl = "" +} From 782d9a3596f019d4d945f14d17a62f4fd400b5b7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:12:31 +0200 Subject: [PATCH 014/434] Move Novelcrow.kt to the English directory --- src/main/kotlin/tsuki/site/{all => en}/hentais/Novelcrow.kt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/kotlin/tsuki/site/{all => en}/hentais/Novelcrow.kt (100%) diff --git a/src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt b/src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt similarity index 100% rename from src/main/kotlin/tsuki/site/all/hentais/Novelcrow.kt rename to src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt From efa29670534d61e463f4e2e8366786bbf5777cbb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 20 Jul 2026 14:23:24 +0200 Subject: [PATCH 015/434] Create Milftoon.kt --- src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt diff --git a/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt b/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt new file mode 100644 index 00000000..20e9bf21 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt @@ -0,0 +1,15 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MILFTOON", "MilfToon", "en", ContentType.HENTAI) +internal class MilfToon(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MILFTOON, "milftoon.xxx", 20) { + override val withoutAjax = true + override val listUrl = "comics/" +} From 6f028e341fea0b62985dede543d50ca020068f4a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 20 Jul 2026 15:27:10 +0200 Subject: [PATCH 016/434] Create ShadowManga.kt --- src/main/kotlin/tsuki/site/es/ShadowManga.kt | 310 +++++++++++++++++++ 1 file changed, 310 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/es/ShadowManga.kt diff --git a/src/main/kotlin/tsuki/site/es/ShadowManga.kt b/src/main/kotlin/tsuki/site/es/ShadowManga.kt new file mode 100644 index 00000000..1f61a251 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/ShadowManga.kt @@ -0,0 +1,310 @@ +package tsuki.site.es + +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.model.* +import tsuki.util.* +import java.text.SimpleDateFormat +import java.util.* +import kotlin.time.Duration.Companion.milliseconds + +@MangaSourceParser("SHADOWMANGA", "Shadow Manga", "es") +internal class ShadowManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SHADOWMANGA, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("shademanga.com") + private val baseApi = "https://$domain/api/series-locales" + + private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) + + private val cdnHosts = listOf("media.shademanga.com", "cdn.shademanga.com") + private val fallbackPrefix = "/api/media/" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", sourceLocale) + + private val rateLimiter = Mutex() + override val webClient by lazy { + tsuki.network.OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor { chain -> + if (chain.request().url.host == domain) { + runBlocking { rateLimiter.withLock { delay(333.milliseconds) } } + } + chain.proceed(chain.request()) + } + .build(), + source + ) + } + + override fun getRequestHeaders() = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showAdultKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + @Volatile private var genreCache: Set? = null + private val genreMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchGenres(), + ) + + private suspend fun fetchGenres(): Set { + genreCache?.let { return it } + return genreMutex.withLock { + genreCache?.let { return it } + try { + val bodyString = webClient.httpGet("$baseApi/tags", getRequestHeaders()).body?.string() + ?: return@withLock emptySet() + val arr = JSONArray(bodyString) + val tags = LinkedHashSet(arr.length()) + for (i in 0 until arr.length()) { + val name = arr.optString(i, "").trim() + if (name.isNotEmpty()) tags.add(MangaTag(title = name, key = name, source = source)) + } + genreCache = tags + tags + } catch (index: Exception) { + throw Exception(index) + } + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val adultParam = if (config[showAdultKey]) "true" else "false" + if (page > 1) return emptyList() + + val query = filter.query?.trim().orEmpty() + val includedTags = filter.tags.map { it.key } + val excludedTags = filter.tagsExclude.map { it.key } + + val url: String + val isSearch = query.isNotEmpty() || includedTags.isNotEmpty() || excludedTags.isNotEmpty() + + when { + isSearch -> { + val builder = "$baseApi/search-candidates".toHttpUrl().newBuilder() + .addQueryParameter("q", query) + .addQueryParameter("includeAdult", adultParam) + .addQueryParameter("showSinPortada", "false") + .addQueryParameter("take", "120") + includedTags.forEach { builder.addQueryParameter("tags", it) } + url = builder.build().toString() + } + order == SortOrder.POPULARITY -> url = "$baseApi/popular" + else -> url = "$baseApi/novedades" + } + + // Fetch and manually parse as JSONArray + val bodyString = webClient.httpGet(url, getRequestHeaders()).body?.string() + ?: return emptyList() + val rawJson = runCatching { JSONArray(bodyString) }.getOrNull() + ?: return emptyList() + + val allSeries = if (!isSearch) { + // Auto‑detect format for popular/novedades + val isGenreWrapped = rawJson.length() > 0 && rawJson.optJSONObject(0)?.has("genero") == true + if (isGenreWrapped) { + val seen = HashSet() + val seriesList = mutableListOf() + for (i in 0 until rawJson.length()) { + val group = rawJson.optJSONObject(i) ?: continue + val seriesArr = group.optJSONArray("series") ?: continue + for (j in 0 until seriesArr.length()) { + seriesArr.optJSONObject(j)?.let { series -> + val id = series.optInt("id", -1) + if (id > 0 && seen.add(id)) seriesList.add(series) + } + } + } + seriesList + } else { + (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } + } + } else { + // Search always returns a flat array + (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } + } + + val filtered = if (excludedTags.isNotEmpty() || includedTags.isNotEmpty()) { + allSeries.filter { series -> + val genres = series.optString("generos", "").split(",").map { it.trim() } + val includeOk = includedTags.isEmpty() || genres.any { it in includedTags } + val excludeOk = excludedTags.isEmpty() || genres.none { it in excludedTags } + includeOk && excludeOk + } + } else allSeries + + val sorted = if (isSearch) { + filtered.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.optString("titulo", "") }) + } else filtered + + return sorted.map { parseMangaFromSeries(it) } + } + + private fun parseMangaFromSeries(obj: JSONObject): Manga { + val id = obj.optInt("id") + val title = obj.optString("titulo", "") + val cover = obj.optString("portadaUrl", "").takeIf { it.isNotBlank() } + return Manga( + id = generateUid(id.toString()), + title = title, + altTitles = emptySet(), + url = id.toString(), + publicUrl = "https://$domain/serie/local/$id", + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val id = manga.url + val json = webClient.httpGet("$baseApi/$id", getRequestHeaders()).parseJson() + + val title = json.optString("titulo", manga.title) + val cover = json.optString("portadaUrl", "").takeIf { it.isNotBlank() } ?: manga.coverUrl + val description = json.optString("descripcion", "").takeIf { it.isNotBlank() } + val author = json.optString("autor", "").takeIf { it.isNotBlank() } + val statusText = json.optString("estado", "") + val state = when (statusText.lowercase()) { + "en curso" -> MangaState.ONGOING + "completado" -> MangaState.FINISHED + "pausada" -> MangaState.PAUSED + else -> null + } + val genres = json.optString("generos", "").split(",").map { it.trim() } + .filter { it.isNotEmpty() } + .map { MangaTag(title = it, key = it, source = source) } + .toSet() + + val chapters = json.optJSONArray("capitulos")?.let { parseChapters(it, id) } ?: emptyList() + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = setOfNotNull(author), + state = state, + tags = genres, + chapters = chapters, + ) + } + + private fun parseChapters(arr: JSONArray, mangaId: String): List { + val list = mutableListOf() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + val chId = ch.optInt("id") + val number = ch.optDouble("numeroCapitulo", 0.0).toFloat() + val title = ch.optString("titulo", "").trim().takeIf { it.isNotEmpty() } + val dateStr = ch.optString("fechaSubida", "") + val uploadDate = runCatching { dateFormat.parse(dateStr)?.time ?: 0L }.getOrDefault(0L) + + val displayNumber = if (number == number.toInt().toFloat()) number.toInt().toString() else number.toString() + val chapterTitle = buildString { + append("Cap. ") + append(displayNumber) + if (title != null) append(" - $title") + } + + list.add( + MangaChapter( + id = generateUid("$mangaId/$chId"), + title = chapterTitle, + url = "$mangaId/$chId", + number = number, + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val parts = chapter.url.split('/') + if (parts.size != 2) return emptyList() + val mangaId = parts[0] + val chapterId = parts[1] + + val json = webClient.httpGet("$baseApi/$mangaId/capitulos/$chapterId/paginas", getRequestHeaders()).parseJson() + val pagesArr = json.optJSONArray("paginas") ?: return emptyList() + + return (0 until pagesArr.length()).map { i -> + val originalUrl = pagesArr.optString(i) + MangaPage( + id = generateUid(originalUrl), + url = originalUrl, + preview = null, + source = source + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val originalUrl = page.url + val response = webClient.httpGet(originalUrl, getRequestHeaders()) + if (response.isSuccessful) { + response.close() + return originalUrl + } + response.close() + // CDN fallback + val url = originalUrl.toHttpUrl() + val key = when { + url.encodedPath.startsWith(fallbackPrefix) -> url.encodedPath.removePrefix(fallbackPrefix) + else -> url.encodedPath.removePrefix("/") + } + // Try other CDN hosts + for (host in cdnHosts) { + if (host == url.host) continue + val cdnUrl = "https://$host/$key" + val cdnResponse = webClient.httpGet(cdnUrl, getRequestHeaders()) + if (cdnResponse.isSuccessful) { + cdnResponse.close() + return cdnUrl + } + cdnResponse.close() + } + // Fallback to main domain API + val fallbackUrl = "https://$domain$fallbackPrefix$key" + return fallbackUrl + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() +} From fbfd0ecd09593d0736a6bc9a37cfe556b6fe68ed Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 21 Jul 2026 16:07:49 +0200 Subject: [PATCH 017/434] Update NHentaiParser.kt --- .../tsuki/site/all/hentais/NHentaiParser.kt | 50 +++++++++++-------- 1 file changed, 28 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt index fdc94f7f..932d94b8 100644 --- a/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt +++ b/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt @@ -46,11 +46,11 @@ internal class NHentaiParser(context: MangaLoaderContext) : private val preferredServerKey = ConfigKey.PreferredImageServer( presetValues = mapOf( - _root_ide_package_.tsuki.site.all.hentais.TWO_IMG_SERVER to "First server", - _root_ide_package_.tsuki.site.all.hentais.THREE_IMG_SERVER to "Second server", - _root_ide_package_.tsuki.site.all.hentais.FOUR_IMG_SERVER to "Third server", + TWO_IMG_SERVER to "First server", + THREE_IMG_SERVER to "Second server", + FOUR_IMG_SERVER to "Third server", ), - defaultValue = _root_ide_package_.tsuki.site.all.hentais.TWO_IMG_SERVER, + defaultValue = TWO_IMG_SERVER, ) override fun onCreateConfig(keys: MutableCollection>) { @@ -842,7 +842,6 @@ internal class NHentaiParser(context: MangaLoaderContext) : ): List { ensureNhConfig() - // Direct ID search val directId = extractGalleryId(filter.query) if (directId != null) { return try { @@ -860,31 +859,38 @@ internal class NHentaiParser(context: MangaLoaderContext) : return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } } - val lang = filter.locale?.let { loc -> - LANG_MAP[loc.language] + val lang = filter.locale?.let { LANG_MAP[it.language] } + val searchQuery = buildString { + if (!filter.query.isNullOrBlank()) { + append("title:\"${filter.query}\"") + } + if (lang != null) { + if (isNotEmpty()) append(" ") + append("language:$lang") + } } - val url = when { - // Search with language - lang != null -> { - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - else -> "date" + val url: String + when (order) { + SortOrder.UPDATED -> { + url = if (searchQuery.isEmpty()) { + "https://$domain/api/v2/galleries?page=$page" + } else { + "https://$domain/api/v2/search?query=${searchQuery.urlEncoded()}&sort=date&page=$page" } - "https://$domain/api/v2/search?query=language:$lang&sort=$sort&page=$page" } - // Latest - order == SortOrder.UPDATED -> "https://$domain/api/v2/galleries?page=$page" - // Popular (all / today / week / month) else -> { val sort = when (order) { + SortOrder.POPULARITY -> "popular" SortOrder.POPULARITY_TODAY -> "popular-today" - SortOrder.POPULARITY_WEEK -> "popular-week" + SortOrder.POPULARITY_WEEK -> "popular-week" SortOrder.POPULARITY_MONTH -> "popular-month" - SortOrder.POPULARITY -> "popular" else -> "popular" } - "https://$domain/api/v2/search?query=%22%22&sort=$sort&page=$page" + val finalQuery = searchQuery.ifBlank { + lang?.let { "language:$it" } ?: "language:english" + } + url = "https://$domain/api/v2/search?query=${finalQuery.urlEncoded()}&sort=$sort&page=$page" } } @@ -1026,7 +1032,7 @@ internal class NHentaiParser(context: MangaLoaderContext) : publicUrl = "https://$domain/g/$id/", rating = RATING_UNKNOWN, contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$thumbPath", + coverUrl = "$thumbServer/$coverPath", largeCoverUrl = "$imageServer/$coverPath", tags = tags, state = null, @@ -1076,4 +1082,4 @@ internal class NHentaiParser(context: MangaLoaderContext) : override val authUrl: String get() = "https://$domain" override suspend fun isAuthorized(): Boolean = true override suspend fun getUsername(): String = "" -} \ No newline at end of file +} From 9de6a85153c88a0b0bf8d064c240e08230c573c4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 21 Jul 2026 23:59:40 +0200 Subject: [PATCH 018/434] Create RavenScans.kt --- src/main/kotlin/tsuki/site/en/RavenScans.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/RavenScans.kt diff --git a/src/main/kotlin/tsuki/site/en/RavenScans.kt b/src/main/kotlin/tsuki/site/en/RavenScans.kt new file mode 100644 index 00000000..568a6219 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/RavenScans.kt @@ -0,0 +1,13 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("RAVENSCANS", "RavenScans", "en") +internal class RavensScan(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.org") { + override val mangaDirectory = "series" +} From 4f04b802bb9da3a9892870ef9dbf663f08fb316e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:03:17 +0200 Subject: [PATCH 019/434] mangageko fixxed descrption, alttitles and relatedmanga --- src/main/kotlin/tsuki/site/en/MangaGeko.kt | 40 ++++++++++++---------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaGeko.kt b/src/main/kotlin/tsuki/site/en/MangaGeko.kt index 92a25d2f..bfe960a3 100644 --- a/src/main/kotlin/tsuki/site/en/MangaGeko.kt +++ b/src/main/kotlin/tsuki/site/en/MangaGeko.kt @@ -145,6 +145,8 @@ internal class MangaGeko(context: MangaLoaderContext) : val cover = header.selectFirst(".cover img")?.let { it.attr("data-src").ifBlank { it.attr("src") } } ?: manga.coverUrl + val altTitle = doc.selectFirst(".alternative-title")?.textOrNull() + val description = doc.selectFirst(".description")?.html() val author = header.selectFirst(".author a")?.attr("title")?.trim()?.takeIf { it.lowercase() != "updating" @@ -167,12 +169,12 @@ internal class MangaGeko(context: MangaLoaderContext) : return manga.copy( title = title, - altTitles = emptySet(), + altTitles = setOfNotNull(altTitle), coverUrl = cover, largeCoverUrl = cover, state = state, authors = setOfNotNull(author), - description = null, + description = description, tags = tags, chapters = chapters, ) @@ -203,17 +205,20 @@ internal class MangaGeko(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-reader img").mapNotNull { img -> - val src = img.absUrl("src") - if (src.isEmpty()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("center img") + .mapNotNull { it.attr("src").takeIf { src -> src.isNotBlank() } } + .filterNot { it.startsWith("data:image") || it.contains("credits-mgeko.png") } + .distinct().map { url -> + val finalUrl = url.toRelativeUrl(domain) + MangaPage( + id = generateUid(finalUrl), + url = finalUrl, + preview = null, + source = source, + ) + } } private fun Element.toManga(): Manga { @@ -223,6 +228,7 @@ internal class MangaGeko(context: MangaLoaderContext) : val cover = selectFirst(".comic-card__cover img, .novel-cover img")?.let { it.attr("data-src").ifBlank { it.attr("src") } } ?: "" + val coverUrl = cover.takeIf { it.isNotEmpty() }?.toAbsoluteUrl(domain) return Manga( id = generateUid(href), url = href, @@ -231,15 +237,13 @@ internal class MangaGeko(context: MangaLoaderContext) : altTitles = emptySet(), rating = RATING_UNKNOWN, contentRating = null, - coverUrl = cover, + coverUrl = coverUrl, tags = emptySet(), state = null, authors = emptySet(), source = source, ) } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} \ No newline at end of file +} From 5918562baaa23d63493153132aed4e8ceb59ab9c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:03:47 +0200 Subject: [PATCH 020/434] Add AquaReader.kt file --- .../kotlin/tsuki/site/en/{AquaManga.kt => AquaReader.kt} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename src/main/kotlin/tsuki/site/en/{AquaManga.kt => AquaReader.kt} (96%) diff --git a/src/main/kotlin/tsuki/site/en/AquaManga.kt b/src/main/kotlin/tsuki/site/en/AquaReader.kt similarity index 96% rename from src/main/kotlin/tsuki/site/en/AquaManga.kt rename to src/main/kotlin/tsuki/site/en/AquaReader.kt index 307f99a7..d1a9bf21 100644 --- a/src/main/kotlin/tsuki/site/en/AquaManga.kt +++ b/src/main/kotlin/tsuki/site/en/AquaReader.kt @@ -14,8 +14,8 @@ import java.util.EnumSet import java.util.Locale import java.util.TimeZone -@MangaSourceParser("AQUAMANGA", "AquaManga", "en") -internal class AquaManga(context: MangaLoaderContext) : +@MangaSourceParser("AQUAMANGA", "AquaReader", "en") +internal class AquaReader(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.AQUAMANGA, "aquareader.org", 20) { override val withoutAjax = true @@ -197,4 +197,4 @@ internal class AquaManga(context: MangaLoaderContext) : return 0L } -} \ No newline at end of file +} From 12ed12057c7d765d4168929855dd1fc66015ac8d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:07:30 +0200 Subject: [PATCH 021/434] Create MangaSusuku.kt --- .../tsuki/site/id/hentais/MangaSusuku.kt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt diff --git a/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt b/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt new file mode 100644 index 00000000..85493684 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt @@ -0,0 +1,19 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaReaderParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.util.Locale + +@MangaSourceParser("MANGASUSUKU", "MangaSusuku", "id", ContentType.HENTAI) +internal class MangaSusuku(context: MangaLoaderContext) : + MangaReaderParser(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com", pageSize = 20, searchPageSize = 20) { + override val listUrl = "/komik" + override val datePattern = "MMM d, yyyy" + override val sourceLocale: Locale = Locale.ENGLISH + override val isNetShieldProtected = true +} From cec979e9b6ae3b6440d4463d5e9be9a9bc2e6047 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:46:50 +0200 Subject: [PATCH 022/434] Create Komiku.kt --- src/main/kotlin/tsuki/site/id/Komiku.kt | 235 ++++++++++++++++++++++++ 1 file changed, 235 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Komiku.kt diff --git a/src/main/kotlin/tsuki/site/id/Komiku.kt b/src/main/kotlin/tsuki/site/id/Komiku.kt new file mode 100644 index 00000000..2633ba70 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Komiku.kt @@ -0,0 +1,235 @@ +package org.koitharu.kotatsu.parsers.site.mangareader.id + +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities +import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser +import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.mapChapters +import org.koitharu.kotatsu.parsers.util.mapNotNullToSet +import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.parseSafe +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toRelativeUrl +import org.koitharu.kotatsu.parsers.util.toTitleCase +import org.koitharu.kotatsu.parsers.util.urlEncoded +import java.text.SimpleDateFormat +import java.util.EnumSet + +/* Need some refactors */ +@MangaSourceParser("KOMIKU", "Komiku", "id") +internal class Komiku(context: MangaLoaderContext) : + MangaReaderParser(context, MangaParserSource.KOMIKU, "komiku.org", 10, 10) { + + private val apiDomain = "api.komiku.org" + override val datePattern = "dd/MM/yyyy" + override val selectPage = "#Baca_Komik img" + override val selectTestScript = "script:containsData(thisIsNeverFound)" + override val listUrl = "/manga/" + override val selectMangaList = "div.bge" + override val selectMangaListImg = "div.bgei img" + override val selectMangaListTitle = "div.kan h3" + override val selectChapter = "#Daftar_Chapter tr:has(td.judulseries)" + override val detailsDescriptionSelector = "#Sinopsis > p" + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query?.trim().orEmpty() + val url = buildString { + append("https://") + append(apiDomain) + if (query.isNotEmpty()) { + append("/?post_type=manga&s=") + append(query.urlEncoded()) + if (page > 1) { + append("&page=") + append(page) + } + } else { + append("/manga/page/") + append(page) + append("/") + val params = mutableListOf() + val genres = filter.tags.map { it.key } + genres.getOrNull(0)?.let { params += "genre=$it" } + genres.getOrNull(1)?.let { params += "genre2=$it" } + val orderParam = when (order) { + SortOrder.UPDATED -> "modified" + SortOrder.NEWEST -> "date" + SortOrder.POPULARITY -> "meta_value_num" + SortOrder.ALPHABETICAL -> "title" + else -> null + } + if (orderParam != null) params += "orderby=$orderParam" + filter.types.oneOrThrowIfMany()?.let { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> null + } + }?.let { params += "tipe=$it" } + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "end" + else -> null + } + }?.let { params += "status=$it" } + if (params.isNotEmpty()) { + append("?") + append(params.joinToString("&")) + } + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(docs: Document): List { + return docs.select(selectMangaList).mapNotNull { element -> + val a = element.selectFirst("div.bgei a[href*=/manga/]") + ?: element.selectFirst("a[href*=/manga/]") + ?: return@mapNotNull null + + val href = a.attr("href") + val relativeUrl = href.toRelativeUrl(domain) + + val thumbnailUrl = element.selectFirst(selectMangaListImg)?.let { img -> + img.attr("data-src").ifBlank { img.attr("src") } + }?.substringBeforeLast("?") + + val title = element.selectFirst(selectMangaListTitle)?.text() + ?: return@mapNotNull null + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = title, + altTitles = emptySet(), + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = thumbnailUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + description = element.selectFirst("div.kan p")?.text(), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> + val a = element.selectFirst("td.judulseries a") ?: return@mapChapters null + val url = a.attrAsRelativeUrl("href") + val dateText = element.selectFirst("td.tanggalseries")?.text() + MangaChapter( + id = generateUid(url), + title = a.selectFirst("span")?.text() ?: a.text(), + url = url, + number = index + 1f, + volume = 0, + scanlator = null, + uploadDate = dateFormat.parseSafe(dateText), + branch = null, + source = source, + ) + } + + return parseInfo(docs, manga, chapters) + } + + override suspend fun parseInfo(docs: Document, manga: Manga, chapters: List): Manga { + val tags = docs.select("ul.genre li.genre a, table.inftable a[href*=/genre/]").mapNotNullToSet { element -> + val href = element.attr("href") + val genreKey = href.substringAfter("/genre/").substringBefore("/").ifBlank { return@mapNotNullToSet null } + val genreTitle = element.selectFirst("span[itemprop='genre']")?.text() ?: element.text() + MangaTag( + key = genreKey, + title = genreTitle.toTitleCase(sourceLocale), + source = source, + ) + } + + val statusText = docs.selectFirst("table.inftable tr > td:contains(Status) + td")?.text().orEmpty() + val state = when { + statusText.contains("Ongoing", true) -> MangaState.ONGOING + statusText.contains("Completed", true) -> MangaState.FINISHED + statusText.contains("Tamat", true) -> MangaState.FINISHED + statusText.contains("End", true) -> MangaState.FINISHED + else -> null + } + + val author = docs.selectFirst("table.inftable tr:has(td:contains(Pengarang)) td:last-child")?.text() + val altTitle = docs.selectFirst("table.inftable tr:has(td:contains(Judul Indonesia)) td:last-child")?.text() + + val thumbnail = docs.selectFirst("div.ims > img")?.let { img -> + img.attr("data-src").ifBlank { img.attr("src") } + }?.substringBeforeLast("?") + + return manga.copy( + altTitles = setOfNotNull(altTitle), + description = docs.selectFirst(detailsDescriptionSelector)?.text() ?: manga.description, + state = state ?: manga.state, + authors = setOfNotNull(author), + tags = tags, + chapters = chapters, + coverUrl = thumbnail ?: manga.coverUrl, + ) + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/pustaka/").parseHtml() + val tags = mutableSetOf() + + doc.select("select[name='genre'] option").forEach { option -> + val value = option.attr("value").trim() + val title = option.text().substringBefore("(").trim() + + if (value.isNotBlank() && !title.equals("Genre", true)) { + tags.add( + MangaTag( + key = value, + title = title, + source = source, + ), + ) + } + } + return tags + } +} From db57cc0bc6be28064f715a786ac1b658813a47de Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:47:11 +0200 Subject: [PATCH 023/434] Update Komiku.kt --- src/main/kotlin/tsuki/site/id/Komiku.kt | 165 ++++++++++++++---------- 1 file changed, 97 insertions(+), 68 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Komiku.kt b/src/main/kotlin/tsuki/site/id/Komiku.kt index 2633ba70..5f7e5bb9 100644 --- a/src/main/kotlin/tsuki/site/id/Komiku.kt +++ b/src/main/kotlin/tsuki/site/id/Komiku.kt @@ -1,35 +1,17 @@ -package org.koitharu.kotatsu.parsers.site.mangareader.id +package tsuki.site.id + +import okhttp3.Headers +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaReaderParser +import tsuki.model.* +import tsuki.util.* import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.site.mangareader.MangaReaderParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat +import java.util.Calendar import java.util.EnumSet -/* Need some refactors */ @MangaSourceParser("KOMIKU", "Komiku", "id") internal class Komiku(context: MangaLoaderContext) : MangaReaderParser(context, MangaParserSource.KOMIKU, "komiku.org", 10, 10) { @@ -45,23 +27,57 @@ internal class Komiku(context: MangaLoaderContext) : override val selectChapter = "#Daftar_Chapter tr:has(td.judulseries)" override val detailsDescriptionSelector = "#Sinopsis > p" - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - ) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val allGenres = listOf( + "action" to "Action", "adult" to "Adult", "adventure" to "Adventure", + "comedy" to "Comedy", "cooking" to "Cooking", "drama" to "Drama", + "ecchi" to "Ecchi", "fantasy" to "Fantasy", "gender-bender" to "Gender Bender", + "harem" to "Harem", "historical" to "Historical", "horror" to "Horror", + "isekai" to "Isekai", "josei" to "Josei", "magic" to "Magic", + "manga" to "Manga", "manhwa" to "Manhwa", "martial-arts" to "Martial Arts", + "mature" to "Mature", "mecha" to "Mecha", "medical" to "Medical", + "military" to "Military", "mystery" to "Mystery", "one-shot" to "One Shot", + "psychological" to "Psychological", "romance" to "Romance", + "school-life" to "School Life", "sci-fi" to "Sci-fi", "seinen" to "Seinen", + "shoujo" to "Shoujo", "shounen" to "Shounen", "slice-of-life" to "Slice of Life", + "sports" to "Sports", "supernatural" to "Supernatural", "tragedy" to "Tragedy", + "yaoi" to "Yaoi", "yuri" to "Yuri", + "academy" to "Academy", "adaptation" to "Adaptation", "apocalypse" to "Apocalypse", + "beasts" to "Beasts", "blacksmith" to "Blacksmith", "comic" to "Comic", + "crime" to "Crime", "crossdressing" to "Crossdressing", + "dark-fantasy" to "Dark Fantasy", "demons" to "Demons", + "doujinshi" to "Doujinshi", "entertainment" to "Entertainment", + "game" to "Game", "genderswap" to "Genderswap", "genius" to "Genius", + "ghosts" to "Ghosts", "gore" to "Gore", "gyaru" to "Gyaru", + "hentai" to "Hentai", "knight" to "Knight", "long-strip" to "Long Strip", + "magical-girls" to "Magical Girls", "mangatoon" to "Mangatoon", + "martial-art" to "Martial Art", "mc-rebirth" to "MC Rebirth", + "monster" to "Monster", "monster-girls" to "Monster Girls", + "monsters" to "Monsters", "murim" to "Murim", "music" to "Music", + "office-workers" to "Office Workers", "oneshot" to "Oneshot", + "police" to "Police", "regression" to "Regression", + "reincarnation" to "Reincarnation", "revenge" to "Revenge", + "school" to "School", "sexual-violence" to "Sexual Violence", + "shotacon" to "Shotacon", "shoujo-ai" to "Shoujo Ai", + "shounen-ai" to "Shounen Ai", "slow-life" to "Slow Life", + "smut" to "Smut", "sport" to "Sport", "strategy" to "Strategy", + "super-power" to "Super Power", "survival" to "Survival", + "sword-fight" to "Sword Fight", "sword-master" to "Sword Master", + "swormanship" to "Swormanship", "system" to "System", + "thriller" to "Thriller", "trauma" to "Trauma", "vampire" to "Vampire", + "villainess" to "Villainess", "violence" to "Violence", + "web-comic" to "Web Comic", "webtoon" to "Webtoon", + "webtoons" to "Webtoons", "xianxia" to "Xianxia", "xuanhuan" to "Xuanhuan" + ) override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ), + availableTags = allGenres.map { (title, key) -> MangaTag(key, title, source) }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { @@ -106,7 +122,7 @@ internal class Komiku(context: MangaLoaderContext) : MangaState.FINISHED -> "end" else -> null } - }?.let { params += "status=$it" } + }?.let { params += "statusmanga=$it" } if (params.isNotEmpty()) { append("?") append(params.joinToString("&")) @@ -127,7 +143,7 @@ internal class Komiku(context: MangaLoaderContext) : val thumbnailUrl = element.selectFirst(selectMangaListImg)?.let { img -> img.attr("data-src").ifBlank { img.attr("src") } - }?.substringBeforeLast("?") + }?.substringBeforeLast("?")?.toAbsoluteUrl(domain) val title = element.selectFirst(selectMangaListTitle)?.text() ?: return@mapNotNull null @@ -152,11 +168,15 @@ internal class Komiku(context: MangaLoaderContext) : override suspend fun getDetails(manga: Manga): Manga { val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> val a = element.selectFirst("td.judulseries a") ?: return@mapChapters null val url = a.attrAsRelativeUrl("href") val dateText = element.selectFirst("td.tanggalseries")?.text() + val date = if (dateText != null && dateText.contains("lalu")) { + parseRelativeDate(dateText) + } else { + SimpleDateFormat(datePattern, sourceLocale).parseSafe(dateText) + } MangaChapter( id = generateUid(url), title = a.selectFirst("span")?.text() ?: a.text(), @@ -164,7 +184,7 @@ internal class Komiku(context: MangaLoaderContext) : number = index + 1f, volume = 0, scanlator = null, - uploadDate = dateFormat.parseSafe(dateText), + uploadDate = date, branch = null, source = source, ) @@ -188,9 +208,7 @@ internal class Komiku(context: MangaLoaderContext) : val statusText = docs.selectFirst("table.inftable tr > td:contains(Status) + td")?.text().orEmpty() val state = when { statusText.contains("Ongoing", true) -> MangaState.ONGOING - statusText.contains("Completed", true) -> MangaState.FINISHED - statusText.contains("Tamat", true) -> MangaState.FINISHED - statusText.contains("End", true) -> MangaState.FINISHED + statusText.contains("Completed", true) || statusText.contains("Tamat", true) || statusText.contains("End", true) -> MangaState.FINISHED else -> null } @@ -199,7 +217,7 @@ internal class Komiku(context: MangaLoaderContext) : val thumbnail = docs.selectFirst("div.ims > img")?.let { img -> img.attr("data-src").ifBlank { img.attr("src") } - }?.substringBeforeLast("?") + }?.substringBeforeLast("?")?.toAbsoluteUrl(domain) return manga.copy( altTitles = setOfNotNull(altTitle), @@ -212,24 +230,35 @@ internal class Komiku(context: MangaLoaderContext) : ) } - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/pustaka/").parseHtml() - val tags = mutableSetOf() - - doc.select("select[name='genre'] option").forEach { option -> - val value = option.attr("value").trim() - val title = option.text().substringBefore("(").trim() - - if (value.isNotBlank() && !title.equals("Genre", true)) { - tags.add( - MangaTag( - key = value, - title = title, - source = source, - ), - ) - } + private fun parseRelativeDate(date: String): Long { + val trimmedDate = date.substringBefore(" lalu").removeSuffix("s").split(" ") + val calendar = Calendar.getInstance() + when (trimmedDate[1]) { + "jam" -> calendar.add(Calendar.HOUR_OF_DAY, -trimmedDate[0].toInt()) + "menit" -> calendar.add(Calendar.MINUTE, -trimmedDate[0].toInt()) + "detik" -> calendar.add(Calendar.SECOND, 0) } - return tags + return calendar.timeInMillis } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select(selectPage).mapIndexed { _, img -> + val src = img.attr("abs:src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .build() } From 6ada00f6ea6922ee4bfc5a4479e75db1b00fe95f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 01:49:18 +0200 Subject: [PATCH 024/434] Create Shinigami.kt --- src/main/kotlin/tsuki/site/id/Shinigami.kt | 278 +++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Shinigami.kt diff --git a/src/main/kotlin/tsuki/site/id/Shinigami.kt b/src/main/kotlin/tsuki/site/id/Shinigami.kt new file mode 100644 index 00000000..2f80aaf7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Shinigami.kt @@ -0,0 +1,278 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.* + +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("SHINIGAMI", "Shinigami", "id") +internal class Shinigami(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SHINIGAMI, 30) { + + override val configKeyDomain = ConfigKey.Domain("g.shinigami.asia") + private val apiSuffix = "api.shngm.io/v1" + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + @Volatile + private var apiHeaders: Headers? = null + + private fun getApiHeaders(): Headers { + if (apiHeaders == null) { + apiHeaders = getRequestHeaders().newBuilder() + .add("Accept", "application/json") + .add("DNT", "1") + .add("Origin", "https://$domain") + .add("Sec-GPC", "1") + .removeAll("X-Requested-With") + .build() + } + return apiHeaders!! + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .add("sec-fetch-dest", "empty") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + } + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(apiSuffix) + append("/manga/list") + append("?page=").append(page) + append("&page_size=").append(pageSize) + + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popularity" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "latest" + SortOrder.RATING, SortOrder.RATING_ASC -> "rating" + else -> "latest" + } + ) + append("&sort_order=") + append( + when (order) { + SortOrder.POPULARITY_ASC, SortOrder.NEWEST_ASC, SortOrder.RATING_ASC -> "asc" + else -> "desc" + } + ) + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.FINISHED -> "completed" + MangaState.ONGOING -> "ongoing" + MangaState.PAUSED -> "hiatus" + else -> "" + } + ) + } + + if (filter.types.isNotEmpty()) { + append("&format=") + append( + filter.types.joinToString(",") { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + } + ) + } + + if (filter.tags.isNotEmpty()) { + append("&genre_include=") + filter.tags.joinTo(this, separator = ",") { it.key } + append("&genre_include_mode=and") + } + + if (filter.tagsExclude.isNotEmpty()) { + append("&genre_exclude=") + filter.tagsExclude.joinTo(this, separator = ",") { it.key } + append("&genre_exclude_mode=and") + } + + if (!filter.author.isNullOrEmpty()) { + append("&author=").append(filter.author.urlEncoded()) + } + + if (!filter.query.isNullOrEmpty()) { + append("&q=").append(filter.query.urlEncoded()) + } + } + + val json = webClient.httpGet(url, getApiHeaders()).parseJson() + val data = json.getJSONArray("data") + + return data.mapJSON { jo -> + val id = jo.getString("manga_id") + Manga( + id = generateUid(id), + url = "/manga/detail/$id", + publicUrl = "https://$domain/series/$id", + title = jo.getString("title"), + altTitles = setOfNotNull(jo.getStringOrNull("alternative_title")), + coverUrl = jo.getStringOrNull("cover_image_url"), + largeCoverUrl = jo.getStringOrNull("cover_portrait_url"), + authors = jo.optJSONObject("taxonomy") + ?.optJSONArray("Author") + ?.mapJSONToSet { it.getString("name") } + .orEmpty(), + tags = jo.optJSONObject("taxonomy") + ?.optJSONArray("Genre") + ?.mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name").toTitleCase(sourceLocale), + source = source, + ) + }.orEmpty(), + state = when (jo.getIntOrDefault("status", 0)) { + 1 -> MangaState.ONGOING + 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED + else -> null + }, + description = jo.getStringOrNull("description"), + contentRating = null, + source = source, + rating = RATING_UNKNOWN, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = webClient.httpGet("https://$apiSuffix${manga.url}", getApiHeaders()).parseJson() + val jo = json.getJSONObject("data") + + return manga.copy( + tags = jo.optJSONObject("taxonomy") + ?.optJSONArray("Genre") + ?.mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name"), + source = source, + ) + } ?: manga.tags, + authors = jo.optJSONObject("taxonomy") + ?.optJSONArray("Author") + ?.mapJSONToSet { it.getString("name") } + ?: manga.authors, + state = when (jo.getIntOrDefault("status", 0)) { + 1 -> MangaState.ONGOING + 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED + else -> null + }, + description = jo.getStringOrNull("description"), + largeCoverUrl = jo.getStringOrNull("cover_portrait_url"), + chapters = getChapters(jo.getString("manga_id")), + ) + } + + private suspend fun getChapters(mangaId: String): List { + val url = "https://$apiSuffix/chapter/$mangaId/list?page_size=3000" + val json = webClient.httpGet(url, getApiHeaders()).parseJson() + val data = json.getJSONArray("data") + + return data.mapJSON { jo -> + val chapterId = jo.getString("chapter_id") + val number = jo.getDouble("chapter_number") + + MangaChapter( + id = generateUid(chapterId), + title = "Chapter ${number.toFloat().toString().removeSuffix(".0")}", + number = number.toFloat(), + url = "/chapter/detail/$chapterId", + scanlator = null, + uploadDate = jo.getString("release_date").parseDate(), + branch = null, + source = source, + volume = 0, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val json = webClient.httpGet("https://$apiSuffix${chapter.url}", getApiHeaders()).parseJson() + val data = json.getJSONObject("data") + val chapterData = data.getJSONObject("chapter") + val baseUrl = data.getString("base_url") + val basePath = chapterData.getString("path") + val pagesArray = chapterData.getJSONArray("data") + + return List(pagesArray.length()) { i -> + val imgExt = pagesArray.getString(i) + val imageUrl = "$baseUrl$basePath$imgExt" + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set { + val json = webClient.httpGet("https://$apiSuffix/genre/list", getApiHeaders()).parseJson() + return json.getJSONArray("data").mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name").toTitleCase(sourceLocale), + source = source, + ) + } + } + + private fun String.parseDate(): Long { + return try { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US) + .apply { timeZone = TimeZone.getTimeZone("UTC") } + .parse(this)?.time ?: 0L + } catch (_: Exception) { 0L } + } +} From a52dacadfb2e5726f17636943edf815e3c42c675 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:10:57 +0200 Subject: [PATCH 025/434] Create KomikCast.kt --- src/main/kotlin/tsuki/site/KomikCast.kt | 329 ++++++++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/KomikCast.kt diff --git a/src/main/kotlin/tsuki/site/KomikCast.kt b/src/main/kotlin/tsuki/site/KomikCast.kt new file mode 100644 index 00000000..276c8bd7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/KomikCast.kt @@ -0,0 +1,329 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany + +import org.json.JSONObject +import java.text.DecimalFormat +import java.text.DecimalFormatSymbols +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("KOMIKCAST", "KomikCast", "id") +internal class Komikcast(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KOMIKCAST, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain("v3.komikcast.fit") + + private val apiBase = "https://be.komikcast.cc" + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept", "application/json") + .add("Accept-language", "en-US,en;q=0.9,id;q=0.8") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genreMap = fetchGenreMap() + return MangaListFilterOptions( + availableTags = genreMap.values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append(apiBase) + append("/series?includeMeta=true") + append("&take=").append(pageSize) + append("&page=").append(page) + + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY -> "popularity" + SortOrder.UPDATED -> "latest" + SortOrder.ALPHABETICAL -> "alphabetical" + SortOrder.RATING -> "rating" + else -> "latest" + } + ) + append("&sortOrder=desc") + + if (!filter.query.isNullOrEmpty()) { + val escaped = filter.query!!.replace("\"", "\\\"") + append("&filter=title=like=\"$escaped\",nativeTitle=like=\"$escaped\"") + } + + filter.states.oneOrThrowIfMany()?.let { state -> + append("&status=") + append( + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> "" + } + ) + } + + filter.types.oneOrThrowIfMany()?.let { type -> + append("&format=") + append( + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + ) + } + + if (filter.tags.isNotEmpty()) { + append("&genreIds=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: String): List { + val result = mutableListOf() + try { + val root = JSONObject(json) + val dataArray = root.getJSONArray("data") + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val data = item.getJSONObject("data") + val slug = data.getString("slug") + val relativeUrl = "/series/$slug" + + result.add( + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + title = data.getString("title"), + altTitles = emptySet(), + rating = data.optDouble("rating", -1.0).let { + if (it >= 0.0) (it / 10.0).toFloat() else RATING_UNKNOWN + }, + contentRating = null, + coverUrl = data.optString("coverImage", null), + tags = emptySet(), + state = when (data.optString("status").lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + }, + authors = setOfNotNull(data.optString("author").takeIf { it.isNotEmpty() }), + source = source, + ) + ) + } + } catch (_: Exception) { } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removeSuffix("/").substringAfterLast("/") + val url = "$apiBase/series/$slug" + val json = webClient.httpGet(url).body?.string().orEmpty() + val seriesData = parseSeriesDetail(json) + + val chapters = fetchChapters(slug) + + return manga.copy( + title = seriesData.title, + description = seriesData.synopsis, + state = when (seriesData.status.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + }, + authors = setOfNotNull(seriesData.author), + tags = seriesData.genres.map { + MangaTag(title = it.name, key = it.id.toString(), source = source) + }.toSet(), + coverUrl = seriesData.coverImage, + rating = if (seriesData.rating > 0) seriesData.rating / 10f else RATING_UNKNOWN, + chapters = chapters, + ) + } + + private fun parseSeriesDetail(json: String): SeriesData { + val root = JSONObject(json) + val data = root.getJSONObject("data").getJSONObject("data") + + val genres = mutableListOf() + val genresArray = data.optJSONArray("genres") + if (genresArray != null) { + for (i in 0 until genresArray.length()) { + val g = genresArray.getJSONObject(i) + genres.add( + GenreData( + id = g.getInt("id"), + name = g.getJSONObject("data").getString("name") + ) + ) + } + } + + return SeriesData( + title = data.getString("title"), + synopsis = data.optString("synopsis", ""), + status = data.getString("status"), + author = data.optString("author").takeIf { it.isNotEmpty() }, + coverImage = data.optString("coverImage", ""), + rating = data.optDouble("rating", 0.0).toFloat(), + genres = genres, + ) + } + + private suspend fun fetchChapters(slug: String): List { + val url = "$apiBase/series/$slug/chapters" + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseChaptersJson(json, slug) + } + + private fun parseChaptersJson(json: String, slug: String): List { + val result = mutableListOf() + try { + val root = JSONObject(json) + val dataArray = root.getJSONArray("data") + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val data = item.getJSONObject("data") + val index = data.getDouble("index") + val formattedIndex = chapterNumberFormatter.format(index) + + val chapterUrl = "/series/$slug/chapter/$formattedIndex" + val title = data.optString("title").takeIf { it.isNotEmpty() && it != "null" } + ?: "Chapter $formattedIndex" + + result.add( + MangaChapter( + id = generateUid(chapterUrl), + title = title, + url = chapterUrl, + number = index.toFloat(), + volume = 0, + scanlator = null, + uploadDate = parseChapterDate(item.optString("createdAt", "")), + branch = null, + source = source, + ) + ) + } + } catch (_: Exception) { } + return result.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List { + // chapter.url = /series/{slug}/chapter/{formattedIndex} + val parts = chapter.url.split("/") + val slug = parts[2] + val chapterIndex = parts[4] + val url = "$apiBase/series/$slug/chapters/$chapterIndex" + + val json = webClient.httpGet(url).body?.string().orEmpty() + try { + val root = JSONObject(json) + val data = root.getJSONObject("data").getJSONObject("data") + val imagesArray = data.getJSONArray("images") + return (0 until imagesArray.length()).mapNotNull { i -> + val imgUrl = imagesArray.getString(i) + if (imgUrl.isBlank()) return@mapNotNull null + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } catch (_: Exception) { + return emptyList() + } + } + + @Volatile + private var genreCache: Map? = null + + private suspend fun fetchGenreMap(): Map { + genreCache?.let { return it } + val url = "$apiBase/genres" + val jsonStr = webClient.httpGet(url).body?.string().orEmpty() + val map = mutableMapOf() + try { + val dataArray = JSONObject(jsonStr).getJSONArray("data") + for (i in 0 until dataArray.length()) { + val genreObj = dataArray.getJSONObject(i) + val name = genreObj.getJSONObject("data").getString("name") + val id = genreObj.getInt("id") + map[name] = MangaTag(title = name, key = id.toString(), source = source) + } + } catch (_: Exception) { } + genreCache = map + return map + } + + private fun parseChapterDate(dateStr: String): Long { + if (dateStr.isBlank()) return 0L + return try { + isoDateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + } + + private data class SeriesData( + val title: String, + val synopsis: String, + val status: String, + val author: String?, + val coverImage: String, + val rating: Float, + val genres: List, + ) + + private data class GenreData(val id: Int, val name: String) + + companion object { + private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT) + private val chapterNumberFormatter = DecimalFormat( + "#.##", + DecimalFormatSymbols.getInstance(Locale.US) + ) + } +} From a622933313788f60af967401938bd5a4de47162f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 15:11:08 +0200 Subject: [PATCH 026/434] Rename src/main/kotlin/tsuki/site/KomikCast.kt to src/main/kotlin/tsuki/site/id/KomikCast.kt --- src/main/kotlin/tsuki/site/{ => id}/KomikCast.kt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/kotlin/tsuki/site/{ => id}/KomikCast.kt (100%) diff --git a/src/main/kotlin/tsuki/site/KomikCast.kt b/src/main/kotlin/tsuki/site/id/KomikCast.kt similarity index 100% rename from src/main/kotlin/tsuki/site/KomikCast.kt rename to src/main/kotlin/tsuki/site/id/KomikCast.kt From 9b20cd9ad81ac194af0c05dcf3a866b6d9929981 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:07:32 +0200 Subject: [PATCH 027/434] Create ManhwaRead.kt --- .../tsuki/site/en/hentais/ManhwaRead.kt | 294 ++++++++++++++++++ 1 file changed, 294 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt diff --git a/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt b/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt new file mode 100644 index 00000000..faecb6fa --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt @@ -0,0 +1,294 @@ +package tsuki.site.en.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import java.util.Base64 +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.nio.charset.StandardCharsets +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANHWAREAD", "ManhwaRead", "en", ContentType.HENTAI) +internal class ManhwaRead(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWAREAD, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain("manhwaread.com") + private val baseUrl = "https://$domain" + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + private val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, +// SortOrder.ALPHABETICAL, // A-Z +// SortOrder.RATING, // rating +// SortOrder.POPULARITY, // all time +// SortOrder.POPULARITY_TODAY, // daily +// SortOrder.POPULARITY_WEEK, // weekly +// SortOrder.POPULARITY_MONTH, // monthly +// SortOrder.POPULARITY_YEAR, // yearly + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + companion object { + val GENRE_ID_MAP = mapOf( + "Action" to "650", "Adventure" to "645", "Comedy" to "536", + "Drama" to "530", "Ecchi" to "537", "Fantasy" to "646", + "Hentai" to "531", "Horror" to "590", "Isekai" to "2735", + "Mahou Shoujo" to "2696", "Mystery" to "626", "Psychological" to "591", + "Romance" to "538", "Sci-Fi" to "688", "Slice of Life" to "532", + "Sports" to "677", "Supernatural" to "544", "Thriller" to "580" + ) + val allTags: Set = run { + val genreTags = GENRE_ID_MAP.keys.map { name -> + MangaTag(name, "genre:${GENRE_ID_MAP[name]}", MangaParserSource.MANHWAREAD) + } + genreTags.toSet() + } + } + + private val tags by lazy { allTags } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val sortBy = when (order) { + SortOrder.UPDATED -> "release" + SortOrder.NEWEST -> "new" +// SortOrder.ALPHABETICAL -> "alphabet" +// SortOrder.RATING -> "rating" +// SortOrder.POPULARITY -> "all_top" +// SortOrder.POPULARITY_TODAY -> "daily_top" +// SortOrder.POPULARITY_WEEK -> "weekly_top" +// SortOrder.POPULARITY_MONTH -> "monthly_top" +// SortOrder.POPULARITY_YEAR -> "yearly_top" + // doenst work idk why? search resulsts are different too + else -> "release" + } + + val url = buildString { + append(baseUrl) + if (page > 1) { + append("/page/") + append(page) + append("/") + } + + val params = linkedMapOf() + params["s"] = filter.query ?: "" + params["sortby"] = sortBy + + if (!filter.query.isNullOrBlank()) { + params["keyword_mode"] = "contain" + params["s_mode"] = "AND" + } + + filter.states.oneOrThrowIfMany()?.let { state -> + params["status"] = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on-hold" + MangaState.ABANDONED -> "canceled" + else -> null + } + } + + append(params.toQueryString()) + + val includedGenres = mutableListOf() + val includedTags = mutableListOf() + val excludedTags = mutableListOf() + + filter.tags.forEach { tag -> + val key = tag.key + when { + key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) + } + } + filter.tagsExclude.forEach { tag -> + val key = tag.key + when { + key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + } + } + + includedGenres.forEach { append("&genres[]=$it") } + includedTags.forEach { append("&including[]=$it") } + excludedTags.forEach { append("&excluding[]=$it") } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List { + return doc.select(".main-container .manga-item").mapNotNull { card -> + val link = card.selectFirst("a.manga-item__link") ?: return@mapNotNull null + val href = link.absUrl("href") + val title = link.text().trim() + val cover = card.selectFirst(".manga-item__img img")?.absUrl("src") + Manga( + id = generateUid(href), + url = href.removePrefix(baseUrl), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("#mangaSummary .manga-titles h1")?.text() ?: manga.title + val cover = doc.selectFirst("head meta[property=og:image]")?.absUrl("content") ?: manga.coverUrl + + val author = doc.select("#mangaSummary .text-primary:contains(Author:) + .flex a span:first-child").text().trim() + val artist = doc.select("#mangaSummary .text-primary:contains(Artist:) + .flex a span:first-child").text().trim() + val publisher = doc.select("#mangaSummary .text-primary:contains(Publisher:) + .flex a span:first-child").text().trim() + + val statusText = doc.selectFirst("#mangaSummary .manga-status")?.attr("data-status") + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "canceled" -> MangaState.ABANDONED + "on-hold" -> MangaState.PAUSED + else -> null + } + + val genres = doc.select("#mangaSummary .manga-genres a").map { it.text() } + val tags = doc.select("#mangaSummary .text-primary:contains(Tags:) + .flex a span:first-child").map { it.text() } + val allTags = (genres + tags).map { name -> + val genreId = GENRE_ID_MAP[name] + if (genreId != null) MangaTag(name, "genre:$genreId", source) + else MangaTag(name, "tag:$name", source) + }.toSet() + + val descBuilder = StringBuilder() + if (publisher.isNotEmpty()) descBuilder.append("Publisher: $publisher\n") + doc.selectFirst("#mangaDesc > .manga-desc__content")?.text()?.let { descBuilder.append("\n$it") } + doc.selectFirst("#mangaSummary .manga-titles h2")?.text()?.split("|")?.joinToString("\n") { "- " + it.trim() }?.let { altTitles -> + if (altTitles.isNotEmpty()) descBuilder.append("\n\nAlternative titles:\n$altTitles") + } + + val chapters = parseChapters(doc) + + val result = manga.copy( + title = title, + coverUrl = cover, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + state = state, + tags = allTags, + description = descBuilder.toString().trim(), + chapters = chapters, + ) + + detailsCache[manga.url] = result + return result + } + + private fun parseChapters(doc: Document): List { + return doc.select("#chaptersList > a.chapter-item").mapNotNull { el -> + val href = el.absUrl("href") + val name = el.selectFirst("span.chapter-item__name")?.text() ?: return@mapNotNull null + val dateStr = el.selectFirst("span.chapter-item__date")?.text() + val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L + MangaChapter( + id = generateUid(href), + title = name, + number = 0f, + volume = 0, + url = href.removePrefix(baseUrl), + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val script = doc.selectFirst("script:containsData(chapterData)") + ?: throw Exception("Chapter data not found") + val dataMatch = CHAPTER_DATA_REGEX.find(script.data()) ?: throw Exception("Chapter data regex failed") + val jsonStr = dataMatch.groupValues[1] + val chapterData = JSONObject(jsonStr) + val base = chapterData.getString("base") + val encodedData = chapterData.getString("data") + val decodedBytes = Base64.getDecoder().decode(encodedData) + val decodedStr = String(decodedBytes, StandardCharsets.UTF_8) + val imagesArray = JSONArray(decodedStr) + + return (0 until imagesArray.length()).map { i -> + val imgObj = imagesArray.getJSONObject(i) + val imgSrc = imgObj.getString("src") + val imgUrl = "$base/$imgSrc" + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private fun SimpleDateFormat.parseSafe(date: String): Long { + return try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } + } + + private fun Map.toQueryString(): String { + val sb = StringBuilder() + for ((key, value) in this) { + if (!value.isNullOrEmpty()) { + if (sb.isNotEmpty()) sb.append("&") + sb.append(key).append("=").append(value.urlEncoded()) + } + } + return if (sb.isNotEmpty()) "?$sb" else "" + } + private val CHAPTER_DATA_REGEX = Regex("""var\s+chapterData\s*=\s*(\{.*\})""") +} From 55208a71d7ec9337b4e3fb7bd51ab1dd12fc15cf Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:28:32 +0200 Subject: [PATCH 028/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/en/Mangadotnet.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt index 393d01a4..b6b488b1 100644 --- a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt @@ -30,7 +30,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) private val genreNames = setOf( - "Josei", "Seinen", "Shoujo", "Shounen", "Action", "Adventure", "Comedy", "Drama", "Fantasy", "Historical", "Horror", "Mecha", "Mystery", "Psychological", "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", @@ -43,8 +42,8 @@ internal class Mangadotnet(context: MangaLoaderContext) : "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", - "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", - "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", + "Magical Girls", "Mahou Shoujo", "Mature", "Medieval Area", + "Monster Girls", "Monsters", "Ninja", "Nobility", "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", @@ -471,4 +470,4 @@ internal class Mangadotnet(context: MangaLoaderContext) : @Suppress("UNCHECKED_CAST") private fun Map.asMap(key: String): Map? = this[key] as? Map private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this -} \ No newline at end of file +} From 90458f7472a68c2475dd80bbc386a89773a608a9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:51:09 +0200 Subject: [PATCH 029/434] Create NatsuParser.kt --- src/main/kotlin/tsuki/parsers/NatsuParser.kt | 374 +++++++++++++++++++ 1 file changed, 374 insertions(+) create mode 100644 src/main/kotlin/tsuki/parsers/NatsuParser.kt diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt new file mode 100644 index 00000000..36a64c04 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -0,0 +1,374 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.await +import tsuki.util.parseJsonArray +import tsuki.util.toRelativeUrl +import tsuki.util.urlBuilder + +import okhttp3.Request +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.MultipartBody +import okhttp3.OkHttpClient +import org.json.JSONArray +import org.jsoup.HttpStatusException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import org.jsoup.nodes.Document + +internal abstract class NatsuParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 24, +) : PagedMangaParser(context, source, pageSize, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override val sourceLocale: Locale = Locale.ENGLISH + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + @Volatile + private var genreCache: Set? = null + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, + ContentType.COMICS, ContentType.NOVEL, + ), + ) + + private suspend fun getOrFetchGenres(): Set { + genreCache?.let { return it } + val tags = fetchAvailableTags() + genreCache = tags + return tags + } + + private var nonce: String? = null + + private suspend fun getNonce(): String { + if (nonce == null) { + val doc = webClient.httpGet( + "https://${domain}/wp-admin/admin-ajax.php?type=search_form&action=get_nonce" + ).parseHtml() + nonce = doc.selectFirst("input[name=search_nonce]")?.attr("value") + ?: throw Exception("Unable to get nonce") + } + return nonce!! + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val query = filter.query + val author = filter.author + + val formParts = mutableMapOf() + formParts["nonce"] = getNonce() + + formParts["inclusion"] = "OR" + formParts["genre"] = if (filter.tags.isNotEmpty()) + JSONArray(filter.tags.map { it.key }).toString() + else "[]" + + formParts["exclusion"] = "OR" + formParts["genre_exclude"] = if (filter.tagsExclude.isNotEmpty()) + JSONArray(filter.tagsExclude.map { it.key }).toString() + else "[]" + + formParts["page"] = page.toString() + + formParts["author"] = if (!author.isNullOrEmpty()) + JSONArray(author).toString() + else "[]" + + formParts["artist"] = "[]" + formParts["project"] = "0" + + formParts["type"] = if (filter.types.isNotEmpty()) + JSONArray(filter.types.mapNotNull { type -> + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> null + } + }).toString() + else "[]" + + formParts["status"] = if (filter.states.isNotEmpty()) + JSONArray(filter.states.mapNotNull { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on-hiatus" + else -> null + } + }).toString() + else "[]" + + formParts["order"] = "desc" + formParts["orderby"] = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.RATING -> "rating" + else -> "popular" + } + + if (!query.isNullOrEmpty()) formParts["query"] = query + + val doc = httpPost("https://${domain}/wp-admin/admin-ajax.php?action=advanced_search", formParts) + return parseMangaList(doc) + } + + protected open suspend fun parseMangaList(doc: Document): List { + val slugs = doc.select("div > a[href*=/manga/]:has(> img)").mapNotNull { a -> + a.absUrl("href").toHttpUrlOrNull()?.pathSegments?.getOrNull(1) + }.filter { it.isNotEmpty() } + + if (slugs.isEmpty()) return emptyList() + + val url = "https://$domain/wp-json/wp/v2/manga".toHttpUrl().newBuilder().apply { + slugs.forEach { addQueryParameter("slug[]", it) } + addQueryParameter("per_page", "${slugs.size + 1}") + addQueryParameter("_embed", null) + }.build() + + val jsonArray = webClient.httpGet(url).parseJsonArray() + val mangaMap = mutableMapOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getJSONObject("title").getString("rendered") + val cover = obj.optJSONObject("_embedded") + ?.optJSONArray("wp:featuredmedia") + ?.optJSONObject(0) + ?.optString("source_url", null) + mangaMap[slug] = Manga( + id = generateUid("/manga/$slug"), + url = "/manga/$slug", + publicUrl = "https://$domain/manga/$slug/", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + return slugs.mapNotNull { mangaMap[it] } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val slug = manga.url.removePrefix("/manga/").removeSuffix("/") + val id = getMangaIdFromSlug(slug) ?: return manga + + val url = "https://$domain/wp-json/wp/v2/manga/$id?_embed" + val obj = webClient.httpGet(url).parseJson() + val title = obj.getJSONObject("title").getString("rendered") + val description = obj.getJSONObject("content").getString("rendered") + .let { org.jsoup.Jsoup.parseBodyFragment(it).wholeText().trim() } + val cover = obj.optJSONObject("_embedded") + ?.optJSONArray("wp:featuredmedia") + ?.optJSONObject(0) + ?.optString("source_url", null) + + val embedded = obj.optJSONObject("_embedded") + val terms = embedded?.optJSONArray("wp:term") + + fun getTermNames(taxonomy: String): List { + if (terms == null) return emptyList() + for (i in 0 until terms.length()) { + val termArray = terms.optJSONArray(i) ?: continue + if (termArray.length() > 0 && termArray.getJSONObject(0).optString("taxonomy") == taxonomy) + return (0 until termArray.length()).map { j -> termArray.getJSONObject(j).getString("name") } + } + return emptyList() + } + + val genres = getTermNames("genre") + getTermNames("type") + val statusList = getTermNames("status") + val state = when { + statusList.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING + statusList.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED + statusList.any { it.equals("Cancelled", ignoreCase = true) } -> MangaState.ABANDONED + statusList.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED + else -> null + } + val authors = getTermNames("series-author") + val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() + val chapters = loadChapters(id.toString(), "https://$domain/manga/$slug/") + + val result = manga.copy( + title = title, + description = description, + coverUrl = cover ?: manga.coverUrl, + state = state, + authors = authors.toSet(), + tags = tags, + chapters = chapters, + ) + + detailsCache[manga.url] = result + return result + } + + private suspend fun getMangaIdFromSlug(slug: String): Int? { + val url = "https://$domain/wp-json/wp/v2/manga?slug=$slug" + val jsonArray = webClient.httpGet(url).parseJsonArray() + return if (jsonArray.length() > 0) jsonArray.getJSONObject(0).getInt("id") else null + } + + protected open suspend fun loadChapters(mangaId: String, mangaAbsoluteUrl: String): List { + val headers = Headers.headersOf( + "HX-Request", "true", + "HX-Target", "chapter-list", + "HX-Trigger", "chapter-list", + "HX-Current-URL", mangaAbsoluteUrl, + "Referer", mangaAbsoluteUrl, + ) + return buildList { + for (page in 1..50) { + val url = urlBuilder() + .addPathSegment("wp-admin") + .addPathSegment("admin-ajax.php") + .addQueryParameter("manga_id", mangaId) + .addQueryParameter("page", page.toString()) + .addQueryParameter("action", "chapter_list") + val doc = try { + webClient.httpGet(url.build(), headers).parseHtml() + } catch (e: HttpStatusException) { + if (e.statusCode == 520) break else throw e + } + val elements = doc.select("div#chapter-list > div[data-chapter-number]") + if (elements.isEmpty()) break + elements.mapNotNullTo(this) { el -> + val a = el.selectFirst("a") ?: return@mapNotNullTo null + val href = a.attrAsRelativeUrl("href").takeIf { it.isNotBlank() } ?: return@mapNotNullTo null + val name = el.selectFirst("div.font-medium span")?.text() ?: "" + val number = el.attr("data-chapter-number").toFloatOrNull() ?: -1f + val dateText = el.selectFirst("time")?.attr("datetime") + val uploadDate = dateFormat.parseSafe(dateText) + MangaChapter( + id = generateUid(href), + title = name, + url = href, + number = number, + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + }.reversed() + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US) + private fun SimpleDateFormat.parseSafe(date: String?): Long = + date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("main .relative section > img").map { img -> + val url = img.attrAsAbsoluteUrl("src").toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + protected open suspend fun fetchAvailableTags(): Set { + val jsonArray = webClient.httpGet( + "https://${domain}/wp-json/wp/v2/genre?per_page=100&page=1&orderby=count&order=desc" + ).parseJsonArray() + val tags = mutableSetOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.getJSONObject(i) + val name = obj.optString("name").takeIf { it.isNotBlank() } ?: continue + val slug = obj.optString("slug").takeIf { it.isNotBlank() } ?: continue + tags += MangaTag(name, slug, source) + } + return tags + } + + private val multipartHttpClient by lazy { OkHttpClient.Builder().build() } + protected open suspend fun httpPost(url: String, form: Map): Document { + val body = MultipartBody.Builder().setType(MultipartBody.FORM) + form.forEach { (k, v) -> body.addFormDataPart(k, v) } + val request = Request.Builder().url(url).post(body.build()) + .header("User-Agent", config[userAgentKey]) + .header("Referer", "https://${domain}/advanced-search/") + .header("Origin", "https://${domain}") + .build() + val response = multipartHttpClient.newCall(request).await() + return response.parseHtml() + } +} From 30fd5d2f876fa5c39b902bf5e6faf9e166470879 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 22:51:31 +0200 Subject: [PATCH 030/434] Create Kiryuu.kt --- src/main/kotlin/tsuki/site/id/Kiryuu.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Kiryuu.kt diff --git a/src/main/kotlin/tsuki/site/id/Kiryuu.kt b/src/main/kotlin/tsuki/site/id/Kiryuu.kt new file mode 100644 index 00000000..b1703213 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Kiryuu.kt @@ -0,0 +1,10 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.MangaParserSource +import tsuki.parsers.NatsuParser + +@MangaSourceParser("KIRYUU", "Kiryuu", "id") +internal class Kiryuu(context: MangaLoaderContext) : + NatsuParser(context, MangaParserSource.KIRYUU, "v7.kiryuu.to") From 6ff7c130a69f4c59a8bbec41dc6021a028f53f2d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 22 Jul 2026 23:11:11 +0200 Subject: [PATCH 031/434] Update NatsuParser.kt --- src/main/kotlin/tsuki/parsers/NatsuParser.kt | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt index 36a64c04..7e10e681 100644 --- a/src/main/kotlin/tsuki/parsers/NatsuParser.kt +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -49,7 +49,7 @@ internal abstract class NatsuParser( ) : PagedMangaParser(context, source, pageSize, pageSize) { override val configKeyDomain = ConfigKey.Domain(domain) - + override val sourceLocale: Locale = Locale.ENGLISH override fun onCreateConfig(keys: MutableCollection>) { @@ -261,6 +261,9 @@ internal abstract class NatsuParser( val authors = getTermNames("series-author") val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() val chapters = loadChapters(id.toString(), "https://$domain/manga/$slug/") + val rawRating = fetchRatingFromPage("https://$domain/manga/$slug/") + val rating = if (rawRating >= 0.0f) rawRating else RATING_UNKNOWN + val altTitles = fetchAltTitlesFromPage("https://$domain/manga/$slug/") val result = manga.copy( title = title, @@ -270,6 +273,8 @@ internal abstract class NatsuParser( authors = authors.toSet(), tags = tags, chapters = chapters, + rating = rating, + altTitles = altTitles, ) detailsCache[manga.url] = result @@ -282,6 +287,22 @@ internal abstract class NatsuParser( return if (jsonArray.length() > 0) jsonArray.getJSONObject(0).getInt("id") else null } + private suspend fun fetchRatingFromPage(pageUrl: String): Float { + val doc = webClient.httpGet(pageUrl).parseHtml() + val ratingEl = doc.selectFirst("meta[itemprop=ratingValue]") + ?: doc.selectFirst("div[itemprop=ratingValue]") + val ratingStr = ratingEl?.attr("content")?.ifEmpty { ratingEl.text() } + val rawRating = ratingStr?.toFloatOrNull() ?: -1f + return if (rawRating >= 0.0f) rawRating else -1f + } + + private suspend fun fetchAltTitlesFromPage(pageUrl: String): Set { + val doc = webClient.httpGet(pageUrl).parseHtml() + val titleElement = doc.selectFirst("h1[itemprop=name]") + val altText = titleElement?.nextElementSibling()?.text() + return altText?.split(',')?.mapNotNull { it.trim().takeIf(String::isNotBlank) }?.toSet() ?: emptySet() + } + protected open suspend fun loadChapters(mangaId: String, mangaAbsoluteUrl: String): List { val headers = Headers.headersOf( "HX-Request", "true", From 596514e2f0ebd2e08dfdef8c5b2a555f953589ec Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 14:38:35 +0200 Subject: [PATCH 032/434] Update NatsuParser.kt --- src/main/kotlin/tsuki/parsers/NatsuParser.kt | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt index 7e10e681..fcaac8e2 100644 --- a/src/main/kotlin/tsuki/parsers/NatsuParser.kt +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -261,8 +261,6 @@ internal abstract class NatsuParser( val authors = getTermNames("series-author") val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() val chapters = loadChapters(id.toString(), "https://$domain/manga/$slug/") - val rawRating = fetchRatingFromPage("https://$domain/manga/$slug/") - val rating = if (rawRating >= 0.0f) rawRating else RATING_UNKNOWN val altTitles = fetchAltTitlesFromPage("https://$domain/manga/$slug/") val result = manga.copy( @@ -273,7 +271,6 @@ internal abstract class NatsuParser( authors = authors.toSet(), tags = tags, chapters = chapters, - rating = rating, altTitles = altTitles, ) @@ -287,15 +284,6 @@ internal abstract class NatsuParser( return if (jsonArray.length() > 0) jsonArray.getJSONObject(0).getInt("id") else null } - private suspend fun fetchRatingFromPage(pageUrl: String): Float { - val doc = webClient.httpGet(pageUrl).parseHtml() - val ratingEl = doc.selectFirst("meta[itemprop=ratingValue]") - ?: doc.selectFirst("div[itemprop=ratingValue]") - val ratingStr = ratingEl?.attr("content")?.ifEmpty { ratingEl.text() } - val rawRating = ratingStr?.toFloatOrNull() ?: -1f - return if (rawRating >= 0.0f) rawRating else -1f - } - private suspend fun fetchAltTitlesFromPage(pageUrl: String): Set { val doc = webClient.httpGet(pageUrl).parseHtml() val titleElement = doc.selectFirst("h1[itemprop=name]") From 81cadc23ef44d58f66a72f73af75a8c509fc0039 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:19:06 +0200 Subject: [PATCH 033/434] Update Mangaball.kt --- src/main/kotlin/tsuki/site/all/Mangaball.kt | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt index 8986773d..fda064c3 100644 --- a/src/main/kotlin/tsuki/site/all/Mangaball.kt +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -33,7 +33,7 @@ internal abstract class MangaBallParser( private val siteLanguages: Set, ) : PagedMangaParser(context, source, pageSize = 20), Interceptor { - override val configKeyDomain = ConfigKey.Domain(_root_ide_package_.tsuki.site.all.DOMAIN) + override val configKeyDomain = ConfigKey.Domain(DOMAIN) private val showSuspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } @@ -527,10 +527,7 @@ internal abstract class MangaBallParser( if (csrfToken.isNullOrEmpty()) { refreshCsrf() if (csrfToken.isNullOrEmpty()) { - refreshCsrf() - } - if (csrfToken.isNullOrEmpty()) { - error("CSRF token not found after two attempts") + error("CSRF token not found") } } return csrfToken!! @@ -888,4 +885,4 @@ internal abstract class MangaBallParser( MangaParserSource.MANGABALL_ZH, setOf("zh", "zh-cn", "zh-hk", "zh-mo", "zh-sg", "zh-tw"), ) -} \ No newline at end of file +} From 25e3e8cd1b404c317d556fac861b2f61ec4e851b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:39:44 +0200 Subject: [PATCH 034/434] Update Batcave.kt --- src/main/kotlin/tsuki/site/en/Batcave.kt | 88 +++++++++++++++++------- 1 file changed, 63 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 3ab015a0..5d29ce2a 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -4,6 +4,8 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient import tsuki.model.* import tsuki.util.* @@ -15,8 +17,6 @@ import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import org.json.JSONObject -import tsuki.exception.ParseException -import tsuki.network.OkHttpWebClient import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -27,21 +27,24 @@ internal class Batcave(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("batcave.biz") - private var publishers: List> = emptyList() - private var genres: List> = emptyList() - private var filterParseFailed = false - private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.ENGLISH) - private val apiClient: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() + @Volatile + private var genreList: List>? = null + @Volatile + private var publisherList: List>? = null + private var filterFetchFailed = false + + private val rawHttpClient: OkHttpClient by lazy { + context.httpClient.newBuilder() .addInterceptor(::refererInterceptor) .addInterceptor(::dleGuardInterceptor) .build() - OkHttpWebClient(httpClient, source) } - private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } + private val apiClient: OkHttpWebClient by lazy { + OkHttpWebClient(rawHttpClient, source) + } private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { val request = chain.request() @@ -73,17 +76,38 @@ internal class Batcave(context: MangaLoaderContext) : keys.add(userAgentKey) } - override val availableSortOrders: Set = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + override val availableSortOrders: Set = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, isMultipleTagsSupported = false, + isYearSupported = false, ) - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = emptySet(), - ) + private suspend fun fetchFilters() { + if (genreList != null && publisherList != null) return + if (filterFetchFailed) return + + try { + val doc = apiClient.httpGet("https://$domain/comix").parseHtml() + val script = doc.selectFirst("script:containsData(window.__XFILTER__)")?.data() + ?: throw ParseException("Filter data not found", "https://$domain/comix") + + val rawJson = script + .substringAfter("window.__XFILTER__ = ") + .substringBeforeLast(";") + .trim() + val root = JSONObject(rawJson) + val filterItems = root.getJSONObject("filter_items") + + genreList = parseFilterValues(filterItems, "g") + publisherList = parseFilterValues(filterItems, "p") + } catch (_: Exception) { + filterFetchFailed = true + } } private fun parseFilterValues(filterItems: JSONObject, key: String): List> { @@ -97,14 +121,26 @@ internal class Batcave(context: MangaLoaderContext) : return result } + override suspend fun getFilterOptions(): MangaListFilterOptions { + fetchFilters() + val tags = mutableSetOf() + genreList?.forEach { (name, id) -> + tags += MangaTag(name, "g_$id", source) + } + publisherList?.forEach { (name, id) -> + tags += MangaTag(name, "p_$id", source) + } + return MangaListFilterOptions( + availableTags = tags, + ) + } + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } if (query != null) return searchManga(query, page) val urlBuilder = StringBuilder().apply { append("https://$domain/ComicList/") - if (filter.yearFrom != YEAR_UNKNOWN) append("y[from]=${filter.yearFrom}/") - if (filter.yearTo != YEAR_UNKNOWN) append("y[to]=${filter.yearTo}/") val pIds = filter.tags.filter { it.key.startsWith("p_") }.map { it.key.removePrefix("p_") } val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } if (pIds.isNotEmpty()) append("p=${pIds.joinToString(",")}/") @@ -176,11 +212,16 @@ internal class Batcave(context: MangaLoaderContext) : val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") val description = doc.selectFirst("div.page__text")?.text() - val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() + + val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer)) a")?.text() + ?: doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { "Ongoing" -> MangaState.ONGOING else -> MangaState.FINISHED } + val tags = doc.select("div.page__tags a").map { a -> + MangaTag(a.text(), a.text().lowercase().replace(" ", "-"), source) + }.toSet() + MangaTag("Comic", "comic", source) val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() ?: throw ParseException("Chapter data script not found", manga.url) @@ -210,7 +251,7 @@ internal class Batcave(context: MangaLoaderContext) : description = description, authors = setOfNotNull(author), state = state, - tags = emptySet(), + tags = tags, chapters = chapters, ) detailsCache[manga.url] = result @@ -229,16 +270,13 @@ internal class Batcave(context: MangaLoaderContext) : .url("https://$domain/engine/ajax/controller.php?mod=api&action=reader/getChapterData") .header("Referer", "https://$domain/") .header("Content-Type", "application/json") - .post( - jsonBody - .toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) - ) + .post(jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())) .build() val response = rawHttpClient.newCall(request).execute() val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) val data = json.getJSONObject("data") - val images = data.getJSONArray("images") + val images = data.optJSONArray("images") ?: throw ParseException("No images found", chapter.url) return (0 until images.length()).map { i -> var img = images.getString(i).trim() @@ -251,4 +289,4 @@ internal class Batcave(context: MangaLoaderContext) : private fun SimpleDateFormat.parseSafe(date: String?): Long = if (date.isNullOrBlank()) 0L else runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} \ No newline at end of file +} From fa4632e88e8bc71227bd07d6fd20115065ef639b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 15:51:07 +0200 Subject: [PATCH 035/434] Update ComicLand.kt --- .../kotlin/tsuki/site/en/hentais/ComicLand.kt | 180 +++++------------- 1 file changed, 43 insertions(+), 137 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt b/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt index 2f2708b2..3923793f 100644 --- a/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt +++ b/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt @@ -16,35 +16,20 @@ import org.json.JSONObject internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( context, MangaParserSource.COMICLAND, 20) { - override val configKeyDomain = - ConfigKey.Domain("comicland.org") + override val configKeyDomain = ConfigKey.Domain("comicland.org") override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() - val newRequest = request.newBuilder() - .header( - "Origin", - "https://comicland.org" - ) - .header( - "Referer", - "https://comicland.org/" - ) + .header("Referer", "https://$domain/") + .header("Origin", "https://$domain") .build() - return chain.proceed(newRequest) } - override val availableSortOrders = - setOf( - SortOrder.UPDATED, // Recommended - SortOrder.POPULARITY, // Popular - SortOrder.NEWEST // Official/Ongoing equivalent - ) + override val availableSortOrders = setOf(SortOrder.UPDATED) - override val filterCapabilities = - MangaListFilterCapabilities( + override val filterCapabilities = MangaListFilterCapabilities( isMultipleTagsSupported = false, isTagsExclusionSupported = false, isSearchSupported = true, @@ -53,55 +38,31 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( isAuthorSearchSupported = false ) - private val api = - "https://api.comicland.org/api" - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter - ): List { - - - val offset = page * 20 + private val api = "https://api.comicland.org/api" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val offset = (page - 1) * 20 val endpoint = if (!filter.query.isNullOrBlank()) { - "$api/comic/search?q=${filter.query!!.urlEncoded()}" + "$api/comic/search?q=${filter.query.urlEncoded()}" } else { - val category = - filter.tags - .firstOrNull() - ?.key - when(category) { - "official" -> - "$api/comics/official" - "ongoing" -> - "$api/comics" - "popular" -> - "$api/comics/popular" - else -> - "$api/comics" + val category = filter.tags.firstOrNull()?.key + when (category) { + "official" -> "$api/comics/official" + "ongoing" -> "$api/comics" + "popular" -> "$api/comics/popular" + else -> "$api/comics" } } - val separator = - if (endpoint.contains("?")) "&" else "?" - - val response = - webClient.httpGet( - "$endpoint${separator}offset=$offset&limit=20" - ) + val separator = if (endpoint.contains("?")) "&" else "?" + val response = webClient.httpGet("$endpoint${separator}offset=$offset&limit=20") + val root = JSONObject(response.body.string()) - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") + val data = root.optJSONObject("data") ?: return emptyList() - val array = - data.optJSONArray("list") + val array = data.optJSONArray("list") ?: data.optJSONArray("items") ?: return emptyList() @@ -131,37 +92,19 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( return result } - override suspend fun getDetails( - manga: Manga - ): Manga { - val response = - webClient.httpGet( - "$api/comic/detail?slug=${manga.url}" - ) - - val root = - JSONObject(response.body.string()) - - val comic = - root.optJSONObject("data") + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet("$api/comic/detail?slug=${manga.url}") + val root = JSONObject(response.body.string()) + val comic = root.optJSONObject("data") ?: return manga - - val chapters = - ArrayList() - - val chapterArray = - comic.optJSONArray("chapters") + val chapters = ArrayList() + val chapterArray = comic.optJSONArray("chapters") if (chapterArray != null) { for (i in 0 until chapterArray.length()) { - val chapter = - chapterArray.getJSONObject(i) + val chapter = chapterArray.getJSONObject(i) - val index = - chapter.optDouble( - "chapter_index", - 0.0 - ) + val index = chapter.optDouble("chapter_index", 0.0) chapters.add( MangaChapter( id = generateUid("${comic.optString("slug")}/$index"), @@ -178,34 +121,16 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( } } return manga.copy( - title = - comic.optString( - "title", - manga.title - ), - description = - comic.optString( - "description", - "" - ), - coverUrl = - comic.optString( - "cover_url", - manga.coverUrl - ), - authors = - parseNames( - comic.optJSONArray("authors") - ), + title = comic.optString("title", manga.title), + description = comic.optString("description", ""), + coverUrl = comic.optString("cover_url", manga.coverUrl), + authors = parseNames(comic.optJSONArray("authors")), tags = emptySet(), chapters = chapters ) } - - private fun parseNames( - array: org.json.JSONArray? - ): Set { + private fun parseNames(array: org.json.JSONArray?): Set { if (array == null) return emptySet() return buildSet { @@ -219,39 +144,21 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( } - override suspend fun getPages( - chapter: MangaChapter - ): List { - - val slug = - chapter.url + override suspend fun getPages(chapter: MangaChapter): List { + val slug = chapter.url .substringAfter("/comic/") .substringBefore("/chapter/") - - val index = - chapter.url + val index = chapter.url .substringAfter("/chapter/") - - val response = - webClient.httpGet( - "$api/chapter/pages_by_index?slug=$slug&index=$index" - ) - - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") + val response = webClient.httpGet("$api/chapter/pages_by_index?slug=$slug&index=$index") + val root = JSONObject(response.body.string()) + val data = root.optJSONObject("data") ?: return emptyList() - - val pages = - data.optJSONArray("pages") + val pages = data.optJSONArray("pages") ?: return emptyList() - return buildList { for (i in 0 until pages.length()) { - val image = - pages.getString(i) + val image = pages.getString(i) add( MangaPage( id = generateUid(image), @@ -280,8 +187,7 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( source = source ) }.toSet(), - availableContentRating = emptySet() ) } -} \ No newline at end of file +} From 6b02be6f7d5315bdcef71ed00a275328a2008ec9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 16:25:09 +0200 Subject: [PATCH 036/434] Update FlameComics.kt --- src/main/kotlin/tsuki/site/en/FlameComics.kt | 131 ++++++++++++------- 1 file changed, 85 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/FlameComics.kt b/src/main/kotlin/tsuki/site/en/FlameComics.kt index 651fa5bc..349ccd51 100644 --- a/src/main/kotlin/tsuki/site/en/FlameComics.kt +++ b/src/main/kotlin/tsuki/site/en/FlameComics.kt @@ -17,9 +17,9 @@ import tsuki.util.json.mapJSONNotNull import tsuki.util.json.toStringSet import tsuki.util.suspendlazy.suspendLazy -import androidx.collection.ArraySet import okhttp3.HttpUrl import org.json.JSONArray +import org.json.JSONException import org.json.JSONObject import java.util.EnumSet import java.util.concurrent.TimeUnit @@ -32,56 +32,90 @@ internal class FlameComics(context: MangaLoaderContext) : private val removeSpecialCharsRegex = Regex("[^A-Za-z0-9 ]") override val availableSortOrders: Set = EnumSet.of( - SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, ) override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( isMultipleTagsSupported = true, - isTagsExclusionSupported = true, + isTagsExclusionSupported = false, isSearchSupported = true, - isSearchWithFiltersSupported = true, + isSearchWithFiltersSupported = false, ) override val configKeyDomain = ConfigKey.Domain("flamecomics.xyz") + private fun JSONObject.getJSONArraySafe(key: String): JSONArray? { + return try { + val value = get(key) + value as? JSONArray + } catch (_: JSONException) { + null + } + } + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("browse.json") - .build() + val query = filter.query + val hasSearchQuery = !query.isNullOrEmpty() + val hasTagFilter = filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() - val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") + val rawMangas = if (hasSearchQuery || hasTagFilter || order != SortOrder.UPDATED) { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") - val allManga = json.mapJSONNotNull { jo -> - parseManga(jo).takeIf { it.tags.matches(filter) } + json.mapJSONNotNull { jo -> + val manga = parseManga(jo) + val lastEdit = jo.getLongOrDefault("last_edit", 0L) + val views = jo.getLongOrDefault("views", 0L) + Triple(manga, lastEdit, views) + } + } else { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("index.json") + .build() + val json = webClient.httpGet(url).parseJson() + val seriesArray = json + .getJSONObject("pageProps") + .getJSONObject("latestEntries") + .getJSONArray("blocks") + .getJSONObject(0) + .getJSONArray("series") + + seriesArray.mapJSONNotNull { jo -> + val manga = parseManga(jo) + Triple(manga, 0L, 0L) + } } - // Filter by search if provided - val query = filter.query - val filteredManga = if (!query.isNullOrEmpty()) { + val filteredByTags = rawMangas.filter { (manga, _, _) -> manga.tags.matches(filter) } + + val filteredManga = if (hasSearchQuery) { val normalizedQuery = removeSpecialCharsRegex.replace(query.lowercase(), "") - allManga.filter { manga -> + filteredByTags.filter { (manga, _, _) -> val titles = mutableListOf(manga.title) titles.addAll(manga.altTitles) - titles.any { title -> normalizedQuery in removeSpecialCharsRegex.replace(title.lowercase(), "") } } } else { - allManga + filteredByTags } - - return filteredManga - .let { list -> - when (order) { - SortOrder.ALPHABETICAL -> list.sortedBy { it.title } - else -> list - } - } + val sorted = when (order) { + SortOrder.UPDATED -> filteredManga.sortedByDescending { it.second } + SortOrder.POPULARITY -> filteredManga.sortedByDescending { it.third } + else -> filteredManga + } + return sorted.map { it.first } } override suspend fun getDetails(manga: Manga): Manga = getDetailsImpl(manga.url.toLong()) @@ -156,21 +190,26 @@ internal class FlameComics(context: MangaLoaderContext) : .addPathSegment(commonPrefix.get()) .addPathSegment("browse.json") .build() - return webClient.httpGet(url).parseJson() + val seriesArray = webClient.httpGet(url).parseJson() .getJSONObject("pageProps") .getJSONArray("series") - .mapJSONNotNull { it.getStringOrNull("categories") } - .flatMapTo(ArraySet()) { - JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } + + val tags = mutableSetOf() + for (i in 0 until seriesArray.length()) { + val jo = seriesArray.getJSONObject(i) + val categoriesArray = jo.getJSONArraySafe("categories") + categoriesArray?.asTypedList()?.forEach { tagName -> + tags += tagName.toMangaTag() } + } + return tags } private fun parseManga(jo: JSONObject): Manga { - // Try different possible field names for series ID, including string conversion val seriesId = jo.getLongOrDefault("series_id", -1L).takeIf { it != -1L } - ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } // Handle novels + ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } ?: jo.getStringOrNull("series_id")?.toLongOrNull() - ?: jo.getStringOrNull("novel_id")?.toLongOrNull() // Handle novels as string + ?: jo.getStringOrNull("novel_id")?.toLongOrNull() ?: jo.getLongOrDefault("id", -1L).takeIf { it != -1L } ?: jo.getStringOrNull("id")?.toLongOrNull() ?: jo.getLongOrDefault("seriesId", -1L).takeIf { it != -1L } @@ -184,7 +223,13 @@ internal class FlameComics(context: MangaLoaderContext) : "JSON sample: ${jo.toString().take(500)}" ) val cover = jo.getStringOrNull("cover") - val author = jo.getStringOrNull("author") + + val authorArray = jo.getJSONArraySafe("author") + val authors = authorArray?.asTypedList()?.toSet() ?: emptySet() + + val categoriesArray = jo.getJSONArraySafe("categories") + val tags = categoriesArray?.asTypedList()?.mapToSet { it.toMangaTag() } ?: emptySet() + return Manga( id = generateUid(seriesId), title = jo.getString("title"), @@ -197,12 +242,8 @@ internal class FlameComics(context: MangaLoaderContext) : contentRating = null, coverUrl = if (cover != null) { imageUrl(seriesId, cover, 384) - } else { - null - }, - tags = jo.getStringOrNull("categories")?.let { - JSONArray(it).asTypedList().mapToSet { tagName -> tagName.toMangaTag() } - }.orEmpty(), + } else null, + tags = tags, state = when (jo.getStringOrNull("status")) { "Dropped" -> MangaState.ABANDONED "Completed" -> MangaState.FINISHED @@ -210,12 +251,10 @@ internal class FlameComics(context: MangaLoaderContext) : "Ongoing" -> MangaState.ONGOING else -> null }, - authors = setOfNotNull(author), + authors = authors, largeCoverUrl = if (cover != null) { imageUrl(seriesId, cover, 720) - } else { - null - }, + } else null, description = jo.getStringOrNull("description"), source = source, ) @@ -269,4 +308,4 @@ internal class FlameComics(context: MangaLoaderContext) : key = this.lowercase().trim(), source = source, ) -} \ No newline at end of file +} From 1fe6b79c11feec1a2fa00cde85b436ba17a93850 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:43:21 +0200 Subject: [PATCH 037/434] Update MangaKatana.kt --- src/main/kotlin/tsuki/site/en/MangaKatana.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaKatana.kt b/src/main/kotlin/tsuki/site/en/MangaKatana.kt index 898d646e..73af0fa4 100644 --- a/src/main/kotlin/tsuki/site/en/MangaKatana.kt +++ b/src/main/kotlin/tsuki/site/en/MangaKatana.kt @@ -31,10 +31,10 @@ internal class MangaKatana(context: MangaLoaderContext): private val preferredServerKey = ConfigKey.PreferredImageServer( presetValues = mapOf( - _root_ide_package_.tsuki.site.en.two to "First server", - _root_ide_package_.tsuki.site.en.three to "Second server", + two to "First server", + three to "Second server", ), - defaultValue = _root_ide_package_.tsuki.site.en.two, + defaultValue = two, ) override fun onCreateConfig(keys: MutableCollection>) { @@ -395,4 +395,4 @@ internal class MangaKatana(context: MangaLoaderContext): return pages.ifEmpty { null } } -} \ No newline at end of file +} From 88567518386e0f79b759f7a9e3550c6f3b766a45 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 22:59:53 +0200 Subject: [PATCH 038/434] Update Batcave.kt --- src/main/kotlin/tsuki/site/en/Batcave.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 5d29ce2a..6dcff078 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -199,13 +199,16 @@ internal class Batcave(context: MangaLoaderContext) : }.filterNotNull() } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() @@ -254,7 +257,9 @@ internal class Batcave(context: MangaLoaderContext) : tags = tags, chapters = chapters, ) - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } From f2cda61498c1e1ab9b489f6aec5fd1099206bbef Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:05:12 +0200 Subject: [PATCH 039/434] Update Atsumaru.kt --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index 52817815..d8a150f2 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -255,7 +255,9 @@ internal class Atsumaru(context: MangaLoaderContext) : // limit tags to 20 override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val result = coroutineScope { val mangaId = manga.url.substringAfterLast("/") @@ -368,16 +370,16 @@ internal class Atsumaru(context: MangaLoaderContext) : altTitles = altTitles, ) } - - detailsCache.put(manga.url, result) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { - return size > 10 - } + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } override suspend fun getRelatedManga(seed: Manga): List { @@ -456,4 +458,4 @@ internal class Atsumaru(context: MangaLoaderContext) : private data class Genre(val name: String, val id: String) private data class Tag(val name: String, val id: String) -} \ No newline at end of file +} From 473c3a2cade599e656293dc472e52754379ca28c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:05:27 +0200 Subject: [PATCH 040/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/en/Mangadotnet.kt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt index b6b488b1..4003469b 100644 --- a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt @@ -229,14 +229,17 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } @Suppress("UNCHECKED_CAST") override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return manga @@ -314,8 +317,9 @@ internal class Mangadotnet(context: MangaLoaderContext) : authors = setOfNotNull(author).filterTo(mutableSetOf()) { it.isNotBlank() }, chapters = chapters, ) - - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } From eb715d70b02b9d2087c9cdf9cba175e523f2f1b0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:05:40 +0200 Subject: [PATCH 041/434] Update Manhwa210.kt --- .../kotlin/tsuki/site/en/hentais/Manhwa210.kt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt index 134e9f21..2eaec65d 100644 --- a/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt +++ b/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt @@ -70,7 +70,8 @@ internal class Manhwa210(context: MangaLoaderContext) : } } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } @@ -78,12 +79,14 @@ internal class Manhwa210(context: MangaLoaderContext) : private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() - val details = manga.copy( + val result = manga.copy( altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { "Ongoing" -> MangaState.ONGOING @@ -119,9 +122,10 @@ internal class Manhwa210(context: MangaLoaderContext) : } .sortedBy { it.number } ) - - detailsCache[manga.url] = details - return details + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result } private fun extractChapterNumber(title: String): Float { @@ -235,4 +239,4 @@ internal class Manhwa210(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 9c5cda5334bd11d48489b6e8c5d6dee043ad5215 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:05:53 +0200 Subject: [PATCH 042/434] Update Mangaball.kt --- src/main/kotlin/tsuki/site/all/Mangaball.kt | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt index fda064c3..93f71060 100644 --- a/src/main/kotlin/tsuki/site/all/Mangaball.kt +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -113,7 +113,9 @@ internal abstract class MangaBallParser( } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } ensureAdultCookie() val doc = client.httpGet(getMangaUrl(manga.url)).parseHtml() @@ -159,7 +161,9 @@ internal abstract class MangaBallParser( chapters = getChapterList(manga.url), contentRating = contentRating, ) - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } @@ -558,7 +562,8 @@ internal abstract class MangaBallParser( } } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } From 7f47990a71a0c2778ac2b1c8a545d42c221d3f34 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:06:12 +0200 Subject: [PATCH 043/434] Update Manhwa18Com.kt --- .../tsuki/site/en/hentais/Manhwa18Com.kt | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt index 4357776c..380c07bb 100644 --- a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt +++ b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt @@ -154,13 +154,16 @@ internal class Manhwa18Com(context: MangaLoaderContext) : }.filterNotNull() } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() @@ -186,7 +189,7 @@ internal class Manhwa18Com(context: MangaLoaderContext) : val chapters = parseChapters(doc) - val fullManga = manga.copy( + val result = manga.copy( title = title, coverUrl = coverUrl, description = description, @@ -195,9 +198,10 @@ internal class Manhwa18Com(context: MangaLoaderContext) : authors = emptySet(), chapters = chapters, ) - - detailsCache[manga.url] = fullManga - return fullManga + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result } private fun parseChapters(doc: org.jsoup.nodes.Document): List { @@ -240,4 +244,4 @@ internal class Manhwa18Com(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 84d7ca1f9f460ef4740180ebddce79440eb5ba0b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:06:22 +0200 Subject: [PATCH 044/434] Update DoujinDesuParser.kt --- .../kotlin/tsuki/parsers/DoujinDesuParser.kt | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt index 2bbdb13e..af409a2c 100644 --- a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -58,7 +58,8 @@ internal abstract class DoujinDesuParser( private var genresCache: Set? = null private val genresMutex = Mutex() - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } @@ -204,7 +205,9 @@ internal abstract class DoujinDesuParser( } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val slug = manga.url.removePrefix("/manga/").removeSuffix("/") val url = "/api/manga/$slug".toAbsoluteUrl(domain) @@ -279,7 +282,7 @@ internal abstract class DoujinDesuParser( if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) } - val fullManga = manga.copy( + val result = manga.copy( authors = setOfNotNull(author), description = cleanDesc, state = state, @@ -289,8 +292,10 @@ internal abstract class DoujinDesuParser( chapters = chapters.reversed() ) - detailsCache[manga.url] = fullManga - return fullManga + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result } override suspend fun getPages(chapter: MangaChapter): List { @@ -394,4 +399,4 @@ internal abstract class DoujinDesuParser( page.replace("/upload/", "/storage/upload/") else -> page } -} \ No newline at end of file +} From 185a6d3cba96c48da6a6c4d99dc3d076a992cc53 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:06:35 +0200 Subject: [PATCH 045/434] Update AinzScans.kt --- src/main/kotlin/tsuki/site/id/AinzScans.kt | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/AinzScans.kt b/src/main/kotlin/tsuki/site/id/AinzScans.kt index 2628c074..75ba679c 100644 --- a/src/main/kotlin/tsuki/site/id/AinzScans.kt +++ b/src/main/kotlin/tsuki/site/id/AinzScans.kt @@ -74,7 +74,8 @@ internal class AinzScans(context: MangaLoaderContext) : } } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 } @@ -170,13 +171,17 @@ internal class AinzScans(context: MangaLoaderContext) : } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val slug = manga.url.substringAfter("/comic/") val json = webClient.httpGet("$apiBase/series/comic/$slug").body?.string().orEmpty() val result = parseDetailsJson(json, manga) - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } @@ -301,4 +306,4 @@ internal class AinzScans(context: MangaLoaderContext) : } private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this -} \ No newline at end of file +} From 51e206523748b89156234d69e2ae062e8c65d01c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:06:50 +0200 Subject: [PATCH 046/434] fix for MangaPlus --- src/main/kotlin/tsuki/site/all/MangaPlus.kt | 894 +++++++++++++++----- 1 file changed, 689 insertions(+), 205 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaPlus.kt b/src/main/kotlin/tsuki/site/all/MangaPlus.kt index 0d150371..512770f2 100644 --- a/src/main/kotlin/tsuki/site/all/MangaPlus.kt +++ b/src/main/kotlin/tsuki/site/all/MangaPlus.kt @@ -4,217 +4,698 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.SinglePageMangaParser - import tsuki.model.* import tsuki.util.* -import tsuki.util.json.asTypedList -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.json.mapJSONNotNull -import tsuki.util.suspendlazy.suspendLazy - import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Interceptor import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.Response import okhttp3.ResponseBody.Companion.toResponseBody -import org.json.JSONArray -import org.json.JSONObject - +import okio.Buffer +import okio.IOException import java.util.EnumSet import java.util.UUID +import java.util.concurrent.Semaphore + +// created very small protobuf reader via Okio +@Throws(IOException::class) +private fun Buffer.readProtobufVarint(): Long { + var result = 0L + var shift = 0 + while (true) { + val b = readByte().toInt() and 0xFF + result = result or ((b and 0x7F).toLong() shl shift) + if (b and 0x80 == 0) break + shift += 7 + if (shift > 63) throw IOException("Malformed varint") + } + return result +} + +@Throws(IOException::class) +private fun Buffer.readProtobufString(): String = readByteArray(readProtobufVarint()).toString(Charsets.UTF_8) + +@Throws(IOException::class) +private fun Buffer.readProtobufBytes(): ByteArray = readByteArray(readProtobufVarint()) + +data class MPResponse(val success: SuccessResult?, val error: String?) + +data class SuccessResult( + val titleRankingView: TitleRankingView?, + val webHomeView: WebHomeView?, + val allTitlesView: AllTitlesView?, + val titleDetailView: TitleDetailView?, + val mangaViewer: MangaViewer? +) + +data class TitleRankingView(val rankedTitles: List) +data class RankedTitle(val titles: List) +data class WebHomeView(val groups: List<UpdatedTitleGroup>) +data class UpdatedTitleGroup(val titles: List<UpdatedTitle>) +data class UpdatedTitle(val title: Title?) // can be null + +data class AllTitlesView(val allTitlesGroup: List<AllTitlesGroup>) +data class AllTitlesGroup(val titles: List<Title>) + +data class Title( + val titleId: Int, + val name: String, + val author: String?, + val portraitImageUrl: String, + val language: Int +) + +data class TitleDetailView( + val title: Title, + val overview: String, + val viewingPeriodDescription: String, + val nonAppearanceInfo: String, + val chapterListGroup: List<ChapterListGroup> +) + +data class ChapterListGroup( + val firstChapterList: List<Chapter>, + val lastChapterList: List<Chapter> +) + +data class Chapter( + val chapterId: Int, + val name: String, + val subTitle: String?, + val startTimeStamp: Int +) + +data class MangaViewer( + val pages: List<MangaPlusPage>, + val titleId: Int?, + val viewToken: String? +) + +data class MangaPlusPage(val mangaPage: MPImagePage?) +data class MPImagePage(val imageUrl: String, val encryptionKey: String?) + +// field numbers +private const val FIELD_SUCCESS = 1 +private const val FIELD_ERROR = 2 + +// SuccessResult sub‑fields +private const val FIELD_TITLE_DETAIL_VIEW = 8 +private const val FIELD_MANGA_VIEWER = 10 +private const val FIELD_ALL_TITLES_VIEW = 25 +private const val FIELD_TITLE_RANKING_VIEW= 37 +private const val FIELD_WEB_HOME_VIEW = 38 + +// TitleDetailView fields +private const val FIELD_TITLE = 1 +private const val FIELD_OVERVIEW = 3 +private const val FIELD_VIEWING_PERIOD = 7 +private const val FIELD_NON_APPEARANCE = 8 +private const val FIELD_CHAPTER_LIST_GROUP = 28 + +// Title fields +private const val FIELD_TITLE_ID = 1 +private const val FIELD_TITLE_NAME = 2 +private const val FIELD_AUTHOR = 3 +private const val FIELD_PORTRAIT_IMAGE = 4 +private const val FIELD_LANGUAGE = 7 + +// ChapterListGroup fields +private const val FIELD_FIRST_CHAPTER_LIST = 2 +private const val FIELD_LAST_CHAPTER_LIST = 4 + +// Chapter fields +private const val FIELD_CHAPTER_ID = 2 +private const val FIELD_CHAPTER_NAME = 3 +private const val FIELD_SUBTITLE = 4 +private const val FIELD_START_TIMESTAMP = 6 + +// MangaViewer fields +private const val FIELD_PAGES = 1 +private const val FIELD_TITLE_ID_MV = 9 +private const val FIELD_VIEW_TOKEN = 19 + +// MangaPlusPage fields +private const val FIELD_MANGA_PAGE = 1 + +// MangaPage fields +private const val FIELD_IMAGE_URL = 1 +private const val FIELD_ENCRYPTION_KEY = 5 + +// TitleRankingView fields +private const val FIELD_RANKED_TITLES = 3 +// RankedTitle fields +private const val FIELD_RANKED_TITLE_TITLES = 2 + +// WebHomeView fields +private const val FIELD_GROUPS = 2 +// UpdatedTitleGroup fields +private const val FIELD_GROUP_TITLES = 2 +// UpdatedTitle fields +private const val FIELD_LATEST_CHAPTER = 3 +// LatestChapter fields +private const val FIELD_LATEST_CHAPTER_TITLE = 1 + +// AllTitlesView fields +private const val FIELD_ALL_TITLES_GROUP = 1 +// AllTitlesGroup fields +private const val FIELD_GROUP_TITLES_ALL = 2 + + +private const val one = "low" +private const val two = "high" +private const val three = "super_high" + +@Throws(IOException::class) +private fun Buffer.parseMPResponse(): MPResponse { + var success: SuccessResult? = null + var error: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fieldNumber = (tag shr 3).toInt() + val wireType = (tag and 0x07).toInt() + when (fieldNumber) { + FIELD_SUCCESS -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + success = Buffer().apply { write(bytes) }.parseSuccessResult() + } else skipField(wireType) + } + FIELD_ERROR -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + // parse simple error message (first string body) + val buf = Buffer().apply { write(bytes) } + while (!buf.exhausted()) { + val t = buf.readProtobufVarint() + val fn = (t shr 3).toInt() + val wt = (t and 0x07).toInt() + if (fn == 2 && wt == 2) { + val nested = buf.readProtobufBytes() + val nestedBuf = Buffer().apply { write(nested) } + // Try to extract body field + while (!nestedBuf.exhausted()) { + val nt = nestedBuf.readProtobufVarint() + val nfn = (nt shr 3).toInt() + val nwt = (nt and 0x07).toInt() + if (nfn == 2 && nwt == 2) { + error = nestedBuf.readProtobufString() + break + } else nestedBuf.skipField(nwt) + } + } else buf.skipField(wt) + } + } else skipField(wireType) + } + else -> skipField(wireType) + } + } + return MPResponse(success, error) +} + +@Throws(IOException::class) +private fun Buffer.skipField(wireType: Int) { + when (wireType) { + 0 -> readProtobufVarint() + 1 -> skip(8) + 2 -> skip(readProtobufVarint()) + 5 -> skip(4) + else -> throw IOException("Unknown wire type $wireType") + } +} + +@Throws(IOException::class) +private fun Buffer.parseSuccessResult(): SuccessResult { + var titleDetailView: TitleDetailView? = null + var mangaViewer: MangaViewer? = null + var allTitlesView: AllTitlesView? = null + var titleRankingView: TitleRankingView? = null + var webHomeView: WebHomeView? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fieldNumber = (tag shr 3).toInt() + val wireType = (tag and 0x07).toInt() + when (fieldNumber) { + FIELD_TITLE_DETAIL_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + titleDetailView = Buffer().apply { write(bytes) }.parseTitleDetailView() + } else skipField(wireType) + } + FIELD_MANGA_VIEWER -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + mangaViewer = Buffer().apply { write(bytes) }.parseMangaViewer() + } else skipField(wireType) + } + FIELD_ALL_TITLES_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + allTitlesView = Buffer().apply { write(bytes) }.parseAllTitlesView() + } else skipField(wireType) + } + FIELD_TITLE_RANKING_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + titleRankingView = Buffer().apply { write(bytes) }.parseTitleRankingView() + } else skipField(wireType) + } + FIELD_WEB_HOME_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + webHomeView = Buffer().apply { write(bytes) }.parseWebHomeView() + } else skipField(wireType) + } + else -> skipField(wireType) + } + } + return SuccessResult(titleRankingView, webHomeView, allTitlesView, titleDetailView, mangaViewer) +} + +@Throws(IOException::class) +private fun Buffer.parseTitle(): Title { + var titleId = 0 + var name = "" + var author: String? = null + var portraitImageUrl = "" + var language = 0 + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_TITLE_ID -> if (wt == 0) titleId = readProtobufVarint().toInt() + FIELD_TITLE_NAME -> if (wt == 2) name = readProtobufString() + FIELD_AUTHOR -> if (wt == 2) author = readProtobufString() + FIELD_PORTRAIT_IMAGE -> if (wt == 2) portraitImageUrl = readProtobufString() + FIELD_LANGUAGE -> if (wt == 0) language = readProtobufVarint().toInt() + else -> skipField(wt) + } + } + return Title(titleId, name, author, portraitImageUrl, language) +} + +@Throws(IOException::class) +private fun Buffer.parseTitleDetailView(): TitleDetailView { + var title: Title? = null + var overview = "" + var viewingPeriodDescription = "" + var nonAppearanceInfo = "" + val chapterListGroup = mutableListOf<ChapterListGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_TITLE -> if (wt == 2) { + val bytes = readProtobufBytes() + title = Buffer().apply { write(bytes) }.parseTitle() + } + FIELD_OVERVIEW -> if (wt == 2) overview = readProtobufString() + FIELD_VIEWING_PERIOD -> if (wt == 2) viewingPeriodDescription = readProtobufString() + FIELD_NON_APPEARANCE -> if (wt == 2) nonAppearanceInfo = readProtobufString() + FIELD_CHAPTER_LIST_GROUP -> if (wt == 2) { + val bytes = readProtobufBytes() + chapterListGroup += Buffer().apply { write(bytes) }.parseChapterListGroup() + } + else -> skipField(wt) + } + } + return TitleDetailView(title!!, overview, viewingPeriodDescription, nonAppearanceInfo, chapterListGroup) +} + +@Throws(IOException::class) +private fun Buffer.parseChapterListGroup(): ChapterListGroup { + val first = mutableListOf<Chapter>() + val last = mutableListOf<Chapter>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_FIRST_CHAPTER_LIST -> if (wt == 2) { + val bytes = readProtobufBytes() + first += Buffer().apply { write(bytes) }.parseChapter() + } + FIELD_LAST_CHAPTER_LIST -> if (wt == 2) { + val bytes = readProtobufBytes() + last += Buffer().apply { write(bytes) }.parseChapter() + } + else -> skipField(wt) + } + } + return ChapterListGroup(first, last) +} + +@Throws(IOException::class) +private fun Buffer.parseChapter(): Chapter { + var chapterId = 0 + var name = "" + var subTitle: String? = null + var startTimeStamp = 0 + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_CHAPTER_ID -> if (wt == 0) chapterId = readProtobufVarint().toInt() + FIELD_CHAPTER_NAME -> if (wt == 2) name = readProtobufString() + FIELD_SUBTITLE -> if (wt == 2) subTitle = readProtobufString() + FIELD_START_TIMESTAMP -> if (wt == 0) startTimeStamp = readProtobufVarint().toInt() + else -> skipField(wt) + } + } + return Chapter(chapterId, name, subTitle, startTimeStamp) +} + +@Throws(IOException::class) +private fun Buffer.parseMangaViewer(): MangaViewer { + val pages = mutableListOf<MangaPlusPage>() + var titleId: Int? = null + var viewToken: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_PAGES -> if (wt == 2) { + val bytes = readProtobufBytes() + pages += Buffer().apply { write(bytes) }.parseMangaPlusPage() + } + FIELD_TITLE_ID_MV -> if (wt == 0) titleId = readProtobufVarint().toInt() + FIELD_VIEW_TOKEN -> if (wt == 2) viewToken = readProtobufString() + else -> skipField(wt) + } + } + return MangaViewer(pages, titleId, viewToken) +} + +@Throws(IOException::class) +private fun Buffer.parseMangaPlusPage(): MangaPlusPage { + var mangaPage: MPImagePage? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_MANGA_PAGE && wt == 2) { + val bytes = readProtobufBytes() + mangaPage = Buffer().apply { write(bytes) }.parseMPImagePage() + } else skipField(wt) + } + return MangaPlusPage(mangaPage) +} + +@Throws(IOException::class) +private fun Buffer.parseMPImagePage(): MPImagePage { + var imageUrl = "" + var encryptionKey: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_IMAGE_URL -> if (wt == 2) imageUrl = readProtobufString() + FIELD_ENCRYPTION_KEY -> if (wt == 2) encryptionKey = readProtobufString() + else -> skipField(wt) + } + } + return MPImagePage(imageUrl, encryptionKey) +} + +@Throws(IOException::class) +private fun Buffer.parseTitleRankingView(): TitleRankingView { + val ranked = mutableListOf<RankedTitle>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_RANKED_TITLES && wt == 2) { + val bytes = readProtobufBytes() + ranked += Buffer().apply { write(bytes) }.parseRankedTitle() + } else skipField(wt) + } + return TitleRankingView(ranked) +} + +@Throws(IOException::class) +private fun Buffer.parseRankedTitle(): RankedTitle { + val titles = mutableListOf<Title>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_RANKED_TITLE_TITLES && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseTitle() + } else skipField(wt) + } + return RankedTitle(titles) +} + +@Throws(IOException::class) +private fun Buffer.parseWebHomeView(): WebHomeView { + val groups = mutableListOf<UpdatedTitleGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUPS && wt == 2) { + val bytes = readProtobufBytes() + groups += Buffer().apply { write(bytes) }.parseUpdatedTitleGroup() + } else skipField(wt) + } + return WebHomeView(groups) +} + +@Throws(IOException::class) +private fun Buffer.parseUpdatedTitleGroup(): UpdatedTitleGroup { + val titles = mutableListOf<UpdatedTitle>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUP_TITLES && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseUpdatedTitle() + } else skipField(wt) + } + return UpdatedTitleGroup(titles) +} + +@Throws(IOException::class) +private fun Buffer.parseUpdatedTitle(): UpdatedTitle { + var title: Title? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_LATEST_CHAPTER && wt == 2) { + val bytes = readProtobufBytes() + val nested = Buffer().apply { write(bytes) } + while (!nested.exhausted()) { + val ntag = nested.readProtobufVarint() + val nfn = (ntag shr 3).toInt() + val nwt = (ntag and 0x07).toInt() + if (nfn == FIELD_LATEST_CHAPTER_TITLE && nwt == 2) { + val titleBytes = nested.readProtobufBytes() + title = Buffer().apply { write(titleBytes) }.parseTitle() + } else nested.skipField(nwt) + } + } else skipField(wt) + } + return UpdatedTitle(title) +} + +@Throws(IOException::class) +private fun Buffer.parseAllTitlesView(): AllTitlesView { + val groups = mutableListOf<AllTitlesGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_ALL_TITLES_GROUP && wt == 2) { + val bytes = readProtobufBytes() + groups += Buffer().apply { write(bytes) }.parseAllTitlesGroup() + } else skipField(wt) + } + return AllTitlesView(groups) +} + +@Throws(IOException::class) +private fun Buffer.parseAllTitlesGroup(): AllTitlesGroup { + val titles = mutableListOf<Title>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUP_TITLES_ALL && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseTitle() + } else skipField(wt) + } + return AllTitlesGroup(titles) +} internal abstract class MangaPlusParser( context: MangaLoaderContext, source: MangaParserSource, - private val sourceLang: String -): - SinglePageMangaParser(context, source), Interceptor { + private val sourceLang: String, + private val internalLangName: String, + private val internalLangCode: Int +) : SinglePageMangaParser(context, source), Interceptor { - private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") + private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" + private val session = UUID.randomUUID().toString() + private val imageSemaphore = Semaphore(1) + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + one to "Low quality", + two to "High quality", + three to "Super high quality", + ), + defaultValue = two, + ) + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { super.onCreateConfig(keys) keys.add(userAgentKey) + keys.add(preferredServerKey) } override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.ALPHABETICAL ) - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - + override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) override suspend fun getFilterOptions() = MangaListFilterOptions() - private val extraHeaders = Headers.headersOf("Session-Token", UUID.randomUUID().toString()) + private suspend fun apiCall(url: String): MPResponse { + val fullUrl = "$apiUrl$url".toHttpUrl().newBuilder().build() + val headers = Headers.headersOf("SESSION-TOKEN", session) + val response = webClient.httpGet(fullUrl, headers) + val body = response.body + return Buffer().apply { write(body.bytes()) }.parseMPResponse() + } override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { - return when { - filter.query.isNullOrEmpty() -> { - when (order) { - SortOrder.POPULARITY -> getPopularList() - SortOrder.UPDATED -> getLatestList() - else -> getAllTitleList() - } - } - else -> getAllTitleList(filter.query) + val query = filter.query + if (!query.isNullOrEmpty()) { + val resp = apiCall("/title_list/allV2") + val titles = resp.success?.allTitlesView?.allTitlesGroup?.flatMap { it.titles } ?: emptyList() + return titles.filterByLang().filter { + it.name.contains(query, true) || it.author?.contains(query, true) ?: false + }.map { it.toManga() } } - } - - private suspend fun getPopularList(): List<Manga> { - val json = apiCall("/title_list/ranking") - - return json.getJSONObject("titleRankingView") - .getJSONArray("titles") - .asTypedList<JSONObject>() - .toMangaList() - } - - private suspend fun getLatestList(): List<Manga> { - val json = apiCall("/title_list/updated") - - return json.getJSONObject("titleUpdatedView") - .getJSONArray("latestTitle") - .mapJSON { it.getJSONObject("title") } - .toMangaList() - } - - private val allTitleCache = suspendLazy { - apiCall("/title_list/allV2") - .getJSONObject("allTitlesViewV2") - .getJSONArray("AllTitlesGroup") - .mapJSON { it.getJSONArray("titles").asTypedList<JSONObject>() } - .flatten() - } - private suspend fun getAllTitleList(query: String? = null): List<Manga> { - return allTitleCache.get().toMangaList(query) - } - - private fun List<JSONObject>.toMangaList(query: String? = null): List<Manga> { - return mapNotNull { - val language = it.getStringOrNull("language") ?: "ENGLISH" - - if (language != sourceLang) { - return@mapNotNull null + return when (order) { + SortOrder.POPULARITY -> { + val resp = apiCall("/title_list/rankingV2?lang=$internalLangName&type=hottest&clang=$internalLangName") + resp.success?.titleRankingView?.rankedTitles?.flatMap { it.titles } + .orEmpty().filterByLang().map { it.toManga() } } - - val name = it.getString("name") - val author = it.getString("author") - .split('/') - .joinToString(transform = String::trim) - - if (query != null && !(name.contains(query, true) || author.contains(query, true))) { - return@mapNotNull null + SortOrder.UPDATED -> { + val resp = apiCall("/web/web_homeV4?lang=$internalLangName&clang=$internalLangName") + resp.success?.webHomeView?.groups?.flatMap { it.titles }?.mapNotNull { it.title } + .orEmpty().filterByLang().map { it.toManga() } } - - val titleId = it.getInt("titleId").toString() - - Manga( - id = generateUid(titleId), - url = titleId, - publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), - title = name, - coverUrl = it.getString("portraitImageUrl"), - altTitles = emptySet(), - authors = setOf(author), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - source = source, - tags = emptySet(), - ) + SortOrder.ALPHABETICAL -> { + val resp = apiCall("/title_list/allV2") + resp.success?.allTitlesView?.allTitlesGroup?.flatMap { it.titles } + .orEmpty().filterByLang().sortedBy { it.name.lowercase() }.map { it.toManga() } + } + else -> emptyList() } } - override suspend fun getDetails(manga: Manga): Manga { - val json = apiCall("/title_detailV3?title_id=${manga.url}") - .getJSONObject("titleDetailView") - val title = json.getJSONObject("title") + private fun Title.toManga() = Manga( + id = generateUid(titleId.toString()), + url = titleId.toString(), + publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), + title = name, + coverUrl = portraitImageUrl, + altTitles = emptySet(), + authors = author?.split("/")?.joinToString(", ") { it.trim() }?.let { setOf(it) } ?: emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + source = source, + tags = emptySet(), + ) - val completed = json.getJSONObject("titleLabels") - .getString("releaseSchedule").let { - it == "DISABLED" || it == "COMPLETED" - } + private fun List<Title>.filterByLang() = filter { it.titleId != 0 && it.language == internalLangCode }.distinctBy { it.titleId } - val hiatus = json.getStringOrNull("nonAppearanceInfo")?.contains("on a hiatus") == true - val author = title.getString("author") - .split("/").joinToString(transform = String::trim) - - return manga.copy( - title = title.getString("name"), - publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain), - coverUrl = title.getString("portraitImageUrl"), - authors = setOf(author), - description = buildString { - json.getString("overview").let(::append) - json.getStringOrNull("viewingPeriodDescription") - ?.takeIf { !completed } - ?.let { append("<br><br>", it) } - }, - chapters = parseChapters( - json.getJSONArray("chapterListGroup"), - title.getStringOrNull("language") ?: "ENGLISH", - ), + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val resp = apiCall("/title_detailV3?title_id=${manga.url}&clang=$internalLangName") + val detail = resp.success?.titleDetailView ?: throw IOException("No detail") + val completed = detail.nonAppearanceInfo.contains(Regex("completado|complete|completo", RegexOption.IGNORE_CASE)) + val hiatus = detail.nonAppearanceInfo.contains(Regex("on a hiatus", RegexOption.IGNORE_CASE)) + val result = manga.copy( + title = detail.title.name, + coverUrl = detail.title.portraitImageUrl, + authors = detail.title.author?.split("/")?.joinToString(", ") { it.trim() }?.let { setOf(it) } ?: emptySet(), + description = listOfNotNull(detail.overview.takeIf { it.isNotEmpty() }, detail.viewingPeriodDescription.takeIf { it.isNotEmpty() }).joinToString("\n\n"), state = when { completed -> MangaState.FINISHED hiatus -> MangaState.PAUSED else -> MangaState.ONGOING }, + chapters = detail.chapterListGroup + .flatMap { it.firstChapterList + it.lastChapterList } + .filter { it.subTitle != null } + .map { chapter -> + MangaChapter( + id = generateUid(chapter.chapterId.toString()), + url = chapter.chapterId.toString(), + title = chapter.subTitle ?: chapter.name, + number = chapter.name.substringAfter("#").toFloatOrNull() ?: -1f, + volume = 0, + uploadDate = chapter.startTimeStamp * 1000L, + branch = when (sourceLang) { + "PORTUGUESE_BR" -> "Portuguese (Brazil)" + else -> sourceLang.lowercase().toTitleCase() + }, + scanlator = null, + source = source, + ) + } ) - } - private fun parseChapters(chapterListGroup: JSONArray, language: String): List<MangaChapter> { - val chapterList = chapterListGroup - .asTypedList<JSONObject>() - .flatMap { - it.optJSONArray("firstChapterList")?.asTypedList<JSONObject>().orEmpty() + - it.optJSONArray("lastChapterList")?.asTypedList<JSONObject>().orEmpty() - } - - return chapterList.mapChapters { _, chapter -> - val chapterId = chapter.getInt("chapterId").toString() - val subtitle = chapter.getStringOrNull("subTitle") ?: return@mapChapters null - - MangaChapter( - id = generateUid(chapterId), - url = chapterId, - title = subtitle, - number = chapter.getString("name") - .substringAfter("#") - .toFloatOrNull() ?: -1f, - volume = 0, - uploadDate = chapter.getInt("startTimeStamp") * 1000L, - branch = when (language) { - "PORTUGUESE_BR" -> "Portuguese (Brazil)" - else -> language.lowercase().toTitleCase() - }, - scanlator = null, - source = source, - ) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result } + return result } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high") - .getJSONObject("mangaViewer") - .getJSONArray("pages") - - return pages.mapJSONNotNull { - val mangaPage = it.optJSONObject("mangaPage") - ?: return@mapJSONNotNull null - val url = mangaPage.getString("imageUrl") - val encryptionKey = mangaPage.getStringOrNull("encryptionKey") + val quality = config[preferredServerKey] + val resp = apiCall("/manga_viewer_v3?chapter_id=${chapter.url}&split=yes&img_quality=$quality&clang=$internalLangName") + val viewer = resp.success?.mangaViewer ?: return emptyList() + val viewToken = viewer.viewToken ?: "" + return viewer.pages.mapNotNull { it.mangaPage }.map { mpImage -> + val baseUrl = mpImage.imageUrl + val encryptionKey = mpImage.encryptionKey + val separator = if (baseUrl.contains("?")) "&" else "?" + val fullUrl = buildString { + append(baseUrl) + append(separator).append("vtoken=").append(viewToken) + if (encryptionKey != null) append("#").append(encryptionKey) + } + MangaPage( - id = generateUid(url), - url = url + if (encryptionKey == null) "" else "#$encryptionKey", + id = generateUid(fullUrl), + url = fullUrl, preview = null, source = source, ) @@ -223,75 +704,78 @@ internal abstract class MangaPlusParser( override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() - val response = chain.proceed(request) - val encryptionKey = request.url.fragment - - if (encryptionKey.isNullOrEmpty()) { - return response - } - - return response.map { responseBody -> - val contentType = response.headers["Content-Type"] ?: "image/jpeg" - val image = responseBody.bytes().decodeXorCipher(encryptionKey) - image.toResponseBody(contentType.toMediaTypeOrNull()) + val url = request.url + val host = url.host + + return if (host.startsWith("jumpg-assets")) { + imageSemaphore.acquire() + try { + processImageRequest(chain, request, url) + } finally { + imageSemaphore.release() + } + } else { + processImageRequest(chain, request, url) } } - private fun ByteArray.decodeXorCipher(key: String): ByteArray { - val keyStream = key.chunked(2) - .map { it.toInt(16) } + private fun processImageRequest(chain: Interceptor.Chain, request: okhttp3.Request, url: okhttp3.HttpUrl): Response { + val encryptionKey = url.fragment + val vtoken = url.queryParameter("vtoken") - return mapIndexed { i, byte -> byte.toInt() xor keyStream[i % keyStream.size] } - .map(Int::toByte) - .toByteArray() - } + val newUrl = url.newBuilder() + .removeAllQueryParameters("vtoken") + .build() - private suspend fun apiCall(url: String): JSONObject { - val newUrl = "$apiUrl$url".toHttpUrl().newBuilder() - .addQueryParameter("format", "json") + val newRequest = request.newBuilder() + .url(newUrl) + .apply { + if (vtoken != null) { + header("Plus-Vw-Token", vtoken) + } + } .build() - val response = webClient.httpGet(newUrl, extraHeaders).parseJson() - val success = response.optJSONObject("success") + val response = chain.proceed(newRequest) - return checkNotNull(success) { - val error = response.getJSONObject("error") - val reason = error.getJSONArray("popups") - .asTypedList<JSONObject>() - .firstOrNull { it.getStringOrNull("language") == null } + if (encryptionKey.isNullOrEmpty()) return response - if (reason?.getStringOrNull("subject") == "Not Found" && url.contains("manga_viewer")) { - "This chapter has expired" - } else { - reason?.getStringOrNull("body") ?: "Unknown Error" - } - } + val keyStream = encryptionKey.chunked(2).map { it.toInt(16) } + val contentType = response.header("Content-Type") ?: "image/jpeg" + val bytes = response.body.bytes() + val decrypted = bytes.mapIndexed { i, byte -> + (byte.toInt() xor keyStream[i % keyStream.size]).toByte() + }.toByteArray() + + return response.newBuilder() + .body(decrypted.toResponseBody(contentType.toMediaTypeOrNull())) + .build() } @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") - class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH") + class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH", "eng", 0) @MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH") + class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH", "esp", 1) @MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus (French)", "fr") - class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH") + class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH", "fra", 2) @MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus (Indonesian)", "id") - class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN") + class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN", "ind", 3) @MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR") + class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR", "ptb", 4) @MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus (Russian)", "ru") - class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN") + class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN", "rus", 5) @MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus (Thai)", "th") - class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI") + class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI", "tha", 6) @MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus (Vietnamese)", "vi") - class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE") + class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE", "vie", 9) @MangaSourceParser("MANGAPLUSPARSER_DE", "MANGA Plus (German)", "de") - class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN") -} \ No newline at end of file + class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN", "deu", 7) +} From f210f91620c3fe6d4a793ba9356b5ec88510e543 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:16:24 +0200 Subject: [PATCH 047/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 014d5b46..f3f9cd3a 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -217,6 +217,13 @@ abstract class MangaThemesia( doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() } } + val altnames = if (hasTable) { + table.selectFirst("tr:has(td:contains(Alternative)) td:last-child") + ?.text() + ?.trim() + ?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } + ?.let { setOf(it) } + } else emptySet() val state = parseStatus(statusText) val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() @@ -234,6 +241,7 @@ abstract class MangaThemesia( state = state, rating = normalizedRating, chapters = chapters, + altTitles = altnames ?: emptySet() ) } @@ -411,4 +419,4 @@ abstract class MangaThemesia( "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено", ) } -} \ No newline at end of file +} From ee498c41e3523cb92def092ce730593e555e12dc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:50:39 +0200 Subject: [PATCH 048/434] Create Komiktap.kt --- .../kotlin/tsuki/site/id/hentais/Komiktap.kt | 97 +++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt diff --git a/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt b/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt new file mode 100644 index 00000000..56a1e770 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt @@ -0,0 +1,97 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.MangaParserAuthProvider +import tsuki.parsers.MangaThemesia +import tsuki.exception.ParseException + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.Response +import okhttp3.ResponseBody.Companion.asResponseBody +import org.json.JSONArray +import org.json.JSONObject +import java.util.Base64 + +@MangaSourceParser("KOMIKTAP", "Komiktap", "id", ContentType.HENTAI) +internal class Komiktap(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KOMIKTAP, "komiktap.info", pageSize = 25), MangaParserAuthProvider, Interceptor { + + private val selectScript = "script[src^='data:text/javascript;base64,']" + private val selectTestScript = "script:containsData(ts_reader.run)" + private val selectPage = "div#readerarea img" + + override val authUrl: String get() = "https://$domain/" + + override suspend fun isAuthorized(): Boolean { + val response = webClient.httpGet("https://$domain/") + return response.headers["x-sucuri-cache"] != null || + response.headers["x-sucuri-id"] == null + } + + override suspend fun getUsername(): String = "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + + val test = doc.select(selectTestScript) + if (test.isEmpty()) { + return doc.select(selectPage).mapNotNull { img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + val images = try { + val base64Script = doc.select(selectScript) + var decode = "" + for (elem in base64Script) { + val src = elem.attr("src") + if (src.startsWith("data:text/javascript;base64,")) { + decode = Base64.getDecoder() + .decode(src.removePrefix("data:text/javascript;base64,")) + .toString(Charsets.UTF_8) + if (decode.startsWith("ts_reader.run")) break + } + } + + if (decode.startsWith("ts_reader.run")) { + extractImagesFromScript(decode) + } else { + val scriptElem = doc.selectFirstOrThrow(selectTestScript) + extractImagesFromScript(scriptElem.data()) + } + } catch (e: Exception) { + throw ParseException("Failed to extract images: ${e.message}", chapterUrl) + } + + return (0 until images.length()).map { i -> + val url = images.getString(i) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + private fun extractImagesFromScript(scriptBody: String): JSONArray { + val json = scriptBody.substringAfter('(').substringBeforeLast(')') + return JSONObject(json) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val mime = response.headers["Content-Type"] + if (response.isSuccessful && mime == "application/octet-stream") { + val newBody = response.body.source().asResponseBody("image/jpeg".toMediaTypeOrNull()!!) + return response.newBuilder().body(newBody).build() + } + return response + } +} From 1de84dcc59617c9a52cbc5ea765584c9493b8b1e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:50:51 +0200 Subject: [PATCH 049/434] Create Kanzenin.kt --- src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt diff --git a/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt b/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt new file mode 100644 index 00000000..08aa259a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("KANZENIN", "Kanzenin", "id", ContentType.HENTAI) +internal class Kanzenin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KANZENIN, "kanzenin.info") From 32795acff892456baa44cf30ae1386276761ab77 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:51:05 +0200 Subject: [PATCH 050/434] Create SekteDoujin.kt --- src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt diff --git a/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt b/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt new file mode 100644 index 00000000..698f9fad --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.hentais + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("SEKTEDOUJIN", "SekteDoujin", "id", ContentType.HENTAI) +internal class SekteDoujin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SEKTEDOUJIN, "sektedoujin.cc") From 853e201b34dce9aea49e99c3bc1a5d973a07228b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:55:24 +0200 Subject: [PATCH 051/434] Update MangaSusuku.kt --- src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt b/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt index 85493684..bb10e48f 100644 --- a/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt +++ b/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt @@ -2,18 +2,17 @@ package tsuki.site.id.hentais import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.parsers.MangaReaderParser +import tsuki.parsers.MangaThemesia import tsuki.model.ContentType import tsuki.model.MangaParserSource +import java.text.SimpleDateFormat import java.util.Locale @MangaSourceParser("MANGASUSUKU", "MangaSusuku", "id", ContentType.HENTAI) internal class MangaSusuku(context: MangaLoaderContext) : - MangaReaderParser(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com", pageSize = 20, searchPageSize = 20) { - override val listUrl = "/komik" - override val datePattern = "MMM d, yyyy" - override val sourceLocale: Locale = Locale.ENGLISH - override val isNetShieldProtected = true + MangaThemesia(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com") { + override val mangaDirectory = "komik" + override val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) } From 49befedee2bb87f4a9b8a76dcb8e0b19358dde42 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 23 Jul 2026 23:56:38 +0200 Subject: [PATCH 052/434] Update Komiktap.kt --- src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt b/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt index 56a1e770..6f4ec9a7 100644 --- a/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt +++ b/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt @@ -6,8 +6,15 @@ import tsuki.MangaParserAuthProvider import tsuki.parsers.MangaThemesia import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl import okhttp3.Interceptor import okhttp3.MediaType.Companion.toMediaTypeOrNull From bbf132a7632df0e6665e1d21ae68f77038ac017e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:06:39 +0200 Subject: [PATCH 053/434] Update EnThunderScans.kt --- .../kotlin/tsuki/site/en/EnThunderScans.kt | 34 +++++++++++++------ 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt index 105bd162..bfa29cd5 100644 --- a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -2,24 +2,38 @@ package tsuki.site.en import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.parsers.MangaReaderParser +import tsuki.parsers.MangaThemesia +import tsuki.model.Manga import tsuki.model.MangaListFilterCapabilities import tsuki.model.MangaParserSource +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + @MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") internal class EnThunderScans(context: MangaLoaderContext) : - MangaReaderParser( - context, - MangaParserSource.ENTHUNDERSCANS, - "en-thunderscans.com", - pageSize = 30, - searchPageSize = 10, - ) { - override val listUrl = "/comics" + MangaThemesia(context, MangaParserSource.ENTHUNDERSCANS, "en-thunderscans.com", pageSize = 30) { + + override val mangaDirectory = "comics" + + override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( isTagsExclusionSupported = false, ) -} \ No newline at end of file + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val base = super.getDetails(manga) + val altTitle = doc.selectFirst(".alternative .desktop-titles, .alternative .mobile-titles.first") + ?.text() + ?.trim() + ?.takeIf { it.isNotBlank() } + return base.copy( + altTitles = if (altTitle != null) setOf(altTitle) else base.altTitles + ) + } +} From e40e2bd32fdaf19f619bc908807a9d036d61981f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:06:57 +0200 Subject: [PATCH 054/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index f3f9cd3a..eecc1a0a 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -60,7 +60,7 @@ abstract class MangaThemesia( protected open val withoutAjax = false protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" - protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num), div.bxcl li:not(a[data-bs-target='#lockedChapterModal'])" + protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num)" protected open val pageSelector = "div#readerarea img" override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( From 82110f1c547fd999f224e79c2036d363e198e7e1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 24 Jul 2026 00:20:57 +0200 Subject: [PATCH 055/434] Update MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 120 +++++++++++++++++--- 1 file changed, 105 insertions(+), 15 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 9c169c50..3898e29d 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -15,7 +15,87 @@ import org.jsoup.Jsoup import java.util.EnumSet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import java.util.Base64 + +internal class VrfSigner { + + fun interceptor() = Interceptor { chain -> + val request = chain.request() + val url = request.url + + if (url.encodedPath.startsWith("/api/")) { + val params = url.queryParameterNames + .flatMap { key -> url.queryParameterValues(key).map { key to it } } + .sortedBy { it.first } + + val sortedQueryUrl = buildString { + append(url.encodedPath.removePrefix("/api")) + if (params.isNotEmpty()) { + append('?') + var lastKey = "" + var index = 0 + append( + params.joinToString("&") { (key, value) -> + val newKey = if (key.endsWith("[]")) { + if (lastKey != key) index = 0 + lastKey = key + key.replace("[]", "[${index++}]") + } else { + key + } + "$newKey=$value" + }, + ) + } + } + + val builder = url.newBuilder().query(null) + params.forEach { (k, v) -> builder.addQueryParameter(k, v) } + builder.addQueryParameter("vrf", sign(sortedQueryUrl)) + val newRequest = request.newBuilder().url(builder.build()).build() + chain.proceed(newRequest) + } else { + chain.proceed(request) + } + } + + private fun sign(path: String): String { + var data = path.toByteArray(Charsets.UTF_8) + for ((table, key, iv) in STAGES) { + data = encryptStage(data, table, key, iv) + } + return Base64.getUrlEncoder().withoutPadding().encodeToString(data) + } + + private fun encryptStage(data: ByteArray, table: ByteArray, key: ByteArray, iv: Int): ByteArray { + val out = ByteArray(data.size) + var prev = iv + val keySize = key.size + for (i in data.indices) { + prev = table[(data[i].toInt() xor key[i % keySize].toInt() xor prev) and 0xFF].toInt() and 0xFF + out[i] = prev.toByte() + } + return out + } + + companion object { + private val decoder = Base64.getDecoder() + + private const val TABLE_1 = "yINlmUNho8VYJT+ibTIP+9ESiULpVEtMOoD6U6lRE0R/xwXo/Xp9NrUgC4cw/Lmo33vUyjUE40kUoEWIr/fxfNNcq2s79ShQ5NhNrFnJ4hXPwOu/SuXzIbuTQKGFvfm08E9jvCfqAtoDqvQq3dVWPQFmJjgvkISBeXY3BgANR+yVnjGbcxZ47d6kLNfZPIayTq3/YGySb1KuVZodWp/WGNAO5pfMcpaK53Hhs0allBszaMaxuouOwdxbwgxIw6YunSsXjI05Yi0j9j4eHKfSXR8Ifo/Od+8iamRfCXTyvm7NGRGYdcQ0ywcK/u6RXhrbcCm4t2eCtrDgQVecJGkQ+A==" + private const val KEY_1 = "0Ec58JOY3uBzJK9m3zqIOpdlF7UFiax9DmA=" + private const val TABLE_2 = "IUFltCxD3Oc2cwCgkJffthaOg9cgPUb0LgW6H/VtfcF0kc5F25t+aWj6JH9VOhOaY0rAFdUxlDnl5BLNvwEJvQtP5qcw7vdb/K+chnbwnspSHT8mz5lqwz41TezG0hkO06FTjJZhsyNuFLDpD2ZZxQj/QIRcF90zpmQ7Byu483WsQqUE0C342HL+JXngRB6fRzxRyVTaKu83h7UYTJ0QMt6ixFh6S3F8gqkKwrGTL3jHNBsD45UnifK8+RGtishQV2K3rujLKEkiZxpr2dYcudFW4oFsDKhad3CLBvuyTqsCo4B7mL5IKQ1vXo/MOOvq1I1d8ar9X6Ttu5KF4fZgiA==" + private const val KEY_2 = "AAdjb1iPY8CiDmq9H34tKTBF8a3oDQ==" + private const val TABLE_3 = "NQHlu1/wVO5EmkwQymF810qqY2xG1k2obcas4Z9mCsPEIFl9pRIjFxbJ7ybMHbBckT5Ton85E0FOeHezbh/mjlEYpmpnlXOS8dgrqeq2KfxImTh1YK9y0PeMNhzA1OQzSY9brYOJq/l2QnE/hwOeZIhPixVSKIUlDb5vLcH6RWKxkIEMuP0bDwIqQ71AJJaEaMJL7A6YtyIwoRT+L5v4aZzodN/0+3nOGsfblFjgxSfPzVDjNFeNl5P26+kEC/8AHgdrpAbt3hHz3HrRN1Y6e+JHgF7ncFWnoF0y3THL1S71WgWGCa6KtSzTCCG58n68nTyj2T3Sshk7utqCtMi/ZQ==" + private const val KEY_3 = "DELOJgPsVaCcblDtTGMdHzM=" + + private val STAGES: List<Triple<ByteArray, ByteArray, Int>> = listOf( + Triple(decoder.decode(TABLE_1), decoder.decode(KEY_1), 0x5A), + Triple(decoder.decode(TABLE_2), decoder.decode(KEY_2), 0x35), + Triple(decoder.decode(TABLE_3), decoder.decode(KEY_3), 0xBA), + ) + } +} internal abstract class MangaFireParser( context: MangaLoaderContext, @@ -38,6 +118,7 @@ internal abstract class MangaFireParser( private val apiClient by lazy { val newHttpClient = context.httpClient.newBuilder() + .addInterceptor(VrfSigner().interceptor()) .addInterceptor { chain -> val request = chain.request().newBuilder() .header("Referer", "https://$domain/") @@ -130,13 +211,11 @@ internal abstract class MangaFireParser( override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = tags, availableStates = EnumSet.of( - MangaState.ONGOING, MangaState.FINISHED, - MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER, + MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, ), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), availableDemographics = EnumSet.of(Demographic.SHOUNEN, Demographic.SHOUJO, Demographic.SEINEN, Demographic.JOSEI), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) ) override suspend fun getListPage( @@ -162,6 +241,15 @@ internal abstract class MangaFireParser( urlBuilder.addQueryParameter("year_to", filter.yearTo.toString()) } + filter.contentRating.forEach { rating -> + val value = when (rating) { + ContentRating.SAFE -> "safe" + ContentRating.SUGGESTIVE -> "suggestive" + ContentRating.ADULT -> "pornographic" + } + value.let { urlBuilder.addQueryParameter("content_rating[]", it) } + } + filter.types.forEach { type -> val value = when (type) { ContentType.MANGA -> "manga" @@ -251,8 +339,15 @@ internal abstract class MangaFireParser( return mangas } + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val result = coroutineScope { val hid = extractHid(manga.url) @@ -323,17 +418,12 @@ internal abstract class MangaFireParser( chapters = chapters, ) } - - detailsCache.put(manga.url, result) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean { - return size > 10 - } - } private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List<MangaChapter> { val chapters = mutableListOf<MangaChapter>() @@ -491,4 +581,4 @@ internal abstract class MangaFireParser( @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire Portuguese (Brazil)", "pt") class PortugueseBR(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") -} \ No newline at end of file +} From 144e124163fe67c4fb04dcba93f20ff21405c28c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 25 Jul 2026 13:49:37 +0200 Subject: [PATCH 056/434] Refractor MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 439 +++++++++++--------- 1 file changed, 235 insertions(+), 204 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 3898e29d..89b74704 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -14,20 +14,22 @@ import okhttp3.Interceptor import org.jsoup.Jsoup import java.util.EnumSet import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.supervisorScope import java.util.Base64 +import java.util.concurrent.TimeUnit +import java.util.concurrent.ScheduledThreadPoolExecutor +import java.util.concurrent.Semaphore internal class VrfSigner { - fun interceptor() = Interceptor { chain -> val request = chain.request() val url = request.url - if (url.encodedPath.startsWith("/api/")) { val params = url.queryParameterNames .flatMap { key -> url.queryParameterValues(key).map { key to it } } .sortedBy { it.first } - val sortedQueryUrl = buildString { append(url.encodedPath.removePrefix("/api")) if (params.isNotEmpty()) { @@ -48,11 +50,9 @@ internal class VrfSigner { ) } } - val builder = url.newBuilder().query(null) params.forEach { (k, v) -> builder.addQueryParameter(k, v) } builder.addQueryParameter("vrf", sign(sortedQueryUrl)) - val newRequest = request.newBuilder().url(builder.build()).build() chain.proceed(newRequest) } else { @@ -97,6 +97,26 @@ internal class VrfSigner { } } +private class RateLimitInterceptor( + maxRequestsPerSecond: Int = 2 +) : Interceptor { + + private val semaphore = Semaphore(maxRequestsPerSecond) + private val scheduler = ScheduledThreadPoolExecutor(1).apply { + removeOnCancelPolicy = true + } + private val releaseDelayMs = 1000L / maxRequestsPerSecond + + override fun intercept(chain: Interceptor.Chain): okhttp3.Response { + semaphore.acquire() + try { + return chain.proceed(chain.request()) + } finally { + scheduler.schedule({ semaphore.release() }, releaseDelayMs, TimeUnit.MILLISECONDS) + } + } +} + internal abstract class MangaFireParser( context: MangaLoaderContext, source: MangaParserSource, @@ -106,25 +126,27 @@ internal abstract class MangaFireParser( override val configKeyDomain = ConfigKey.Domain("mangafire.to") override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, // chapter update - SortOrder.POPULARITY, // most views - SortOrder.RATING, // rating score - SortOrder.NEWEST, // created manga - SortOrder.ALPHABETICAL, // title asc - SortOrder.RELEVANCE, // relevance sc + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.RELEVANCE, SortOrder.POPULARITY_WEEK, SortOrder.POPULARITY_MONTH, ) private val apiClient by lazy { val newHttpClient = context.httpClient.newBuilder() +// .addInterceptor(RateLimitInterceptor(2)) future feature? .addInterceptor(VrfSigner().interceptor()) .addInterceptor { chain -> - val request = chain.request().newBuilder() - .header("Referer", "https://$domain/") - .header("Accept", "application/json") - .build() - chain.proceed(request) + chain.proceed( + chain.request().newBuilder() + .header("Referer", "https://$domain/") + .header("Accept", "application/json") + .build() + ) } .build() OkHttpWebClient(newHttpClient, source) @@ -222,7 +244,7 @@ internal abstract class MangaFireParser( page: Int, order: SortOrder, filter: MangaListFilter, - ): List<Manga> { + ): List<Manga> = try { val urlBuilder = okhttp3.HttpUrl.Builder() .scheme("https") .host(domain) @@ -233,7 +255,6 @@ internal abstract class MangaFireParser( if (!filter.query.isNullOrBlank()) { urlBuilder.addQueryParameter("keyword", filter.query) } - if (filter.yearFrom > 0) { urlBuilder.addQueryParameter("year_from", filter.yearFrom.toString()) } @@ -242,52 +263,50 @@ internal abstract class MangaFireParser( } filter.contentRating.forEach { rating -> - val value = when (rating) { + urlBuilder.addQueryParameter("content_rating[]", when (rating) { ContentRating.SAFE -> "safe" ContentRating.SUGGESTIVE -> "suggestive" ContentRating.ADULT -> "pornographic" - } - value.let { urlBuilder.addQueryParameter("content_rating[]", it) } + }) } filter.types.forEach { type -> - val value = when (type) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.OTHER -> "other" - else -> null + when (type) { + ContentType.MANGA -> urlBuilder.addQueryParameter("types[]", "manga") + ContentType.MANHWA -> urlBuilder.addQueryParameter("types[]", "manhwa") + ContentType.MANHUA -> urlBuilder.addQueryParameter("types[]", "manhua") + ContentType.OTHER -> urlBuilder.addQueryParameter("types[]", "other") + else -> {} } - value?.let { urlBuilder.addQueryParameter("types[]", it) } } filter.demographics.forEach { demo -> - val id = when (demo) { - Demographic.JOSEI -> "268919" - Demographic.SEINEN -> "268920" - Demographic.SHOUJO -> "268917" - Demographic.SHOUNEN -> "268918" - else -> null + when (demo) { + Demographic.JOSEI -> urlBuilder.addQueryParameter("demographics[]", "268919") + Demographic.SEINEN -> urlBuilder.addQueryParameter("demographics[]", "268920") + Demographic.SHOUJO -> urlBuilder.addQueryParameter("demographics[]", "268917") + Demographic.SHOUNEN -> urlBuilder.addQueryParameter("demographics[]", "268918") + else -> {} } - id?.let { urlBuilder.addQueryParameter("demographics[]", it) } } filter.tags.forEach { urlBuilder.addQueryParameter("genres_in[]", it.key) } filter.tagsExclude.forEach { urlBuilder.addQueryParameter("genres_ex[]", it.key) } filter.states.forEach { state -> - val apiState = when (state) { - MangaState.ONGOING -> "releasing" - MangaState.FINISHED -> "finished" - MangaState.ABANDONED -> "discontinued" - MangaState.PAUSED -> "on_hiatus" - MangaState.UPCOMING -> "not_yet_released" - else -> null + when (state) { + MangaState.ONGOING -> urlBuilder.addQueryParameter("statuses[]", "releasing") + MangaState.FINISHED -> urlBuilder.addQueryParameter("statuses[]", "finished") + MangaState.ABANDONED -> urlBuilder.addQueryParameter("statuses[]", "discontinued") + MangaState.PAUSED -> urlBuilder.addQueryParameter("statuses[]", "on_hiatus") + MangaState.UPCOMING -> urlBuilder.addQueryParameter("statuses[]", "not_yet_released") + else -> {} } - apiState?.let { urlBuilder.addQueryParameter("statuses[]", it) } } - val sortParam = when (order) { + val sortParam = if (!filter.query.isNullOrBlank()) { + "relevance" to "desc" + } else when (order) { SortOrder.UPDATED -> "chapter_updated_at" to "desc" SortOrder.POPULARITY -> "views_total" to "desc" SortOrder.RATING -> "score" to "desc" @@ -298,125 +317,126 @@ internal abstract class MangaFireParser( SortOrder.POPULARITY_MONTH -> "views_30d" to "desc" else -> null } - sortParam?.let { (field, dir) -> urlBuilder.addQueryParameter("order[$field]", dir) } val url = urlBuilder.build().toString() - val response = apiClient.httpGet(url).parseJson() - val items = response.getJSONArray("items") + val items = response.optJSONArray("items") ?: return emptyList() + val mangas = mutableListOf<Manga>() for (i in 0 until items.length()) { - val obj = items.getJSONObject(i) - val hid = obj.getString("hid") - val slug = obj.optString("slug", null) - val title = obj.getString("title") - val poster = obj.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") + try { + val obj = items.getJSONObject(i) + val hid = obj.getString("hid") + val title = obj.getString("title") + val slug = obj.optString("slug", null) + val poster = obj.optJSONObject("poster") + val cover = poster?.optString("large") ?: poster?.optString("medium") ?: poster?.optString("small") ?: "" - val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" - mangas.add( - Manga( - id = generateUid(urlPath), - url = urlPath, - publicUrl = "https://$domain$urlPath", - title = title, - coverUrl = cover, - source = source, - altTitles = emptySet(), - largeCoverUrl = null, - authors = emptySet(), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - tags = emptySet(), + val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" + mangas.add( + Manga( + id = generateUid(urlPath), + url = urlPath, + publicUrl = "https://$domain$urlPath", + title = title, + coverUrl = cover, + source = source, + altTitles = emptySet(), + largeCoverUrl = null, + authors = emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + tags = emptySet(), + ) ) - ) + } catch (_: Exception) { + } } - return mangas + mangas + } catch (_: Exception) { + emptyList() } private val detailsCacheLock = Any() - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } + override suspend fun getDetails(manga: Manga): Manga { synchronized(detailsCacheLock) { detailsCache[manga.url]?.let { return it } } - - val result = coroutineScope { - val hid = extractHid(manga.url) - - val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() - val data = detailsJson.getJSONObject("data") - - val hasVolumes = data.optBoolean("hasVolumes", false) - - val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } - - val title = data.getString("title") - val poster = data.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") + val result = try { + coroutineScope { + val hid = extractHid(manga.url) + val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() + val data = detailsJson.getJSONObject("data") + val hasVolumes = data.optBoolean("hasVolumes", false) + + val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } + + val title = data.getString("title") + val poster = data.optJSONObject("poster") + val cover = poster?.optString("large") ?: poster?.optString("medium") ?: poster?.optString("small") - val synopsisHtml = data.optString("synopsisHtml", null) - val status = data.optString("status", null) - val type = data.optString("type", null) - val authorsList = data.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val artistsList = data.optJSONArray("artists")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val genres = data.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val themes = data.optJSONArray("themes")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - - val rawRating = data.optDouble("rating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + val synopsisHtml = data.optString("synopsisHtml", null) + val status = data.optString("status", null) + val type = data.optString("type", null) + val authorsList = data.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val artistsList = data.optJSONArray("artists")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val genres = data.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val themes = data.optJSONArray("themes")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val rawRating = data.optDouble("rating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" + + val genreList = buildList { + type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } + genres?.let { addAll(it) } + themes?.let { addAll(it) } + } + val genreTags = genreList.mapNotNull { name -> + tags.find { it.title == name } + }.toSet() - val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" + val chapters = chaptersDeferred.await() - val genreList = buildList { - type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } - genres?.let { addAll(it) } - themes?.let { addAll(it) } + manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + authors = (authorsList + artistsList).toSet(), + description = synopsisText.trim(), + rating = rating, + state = when (status?.lowercase()) { + "releasing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + "discontinued" -> MangaState.ABANDONED + "on_hiatus" -> MangaState.PAUSED + "not_yet_released" -> MangaState.UPCOMING + else -> null + }, + tags = genreTags, + altTitles = altTitlesArray.toSet(), + chapters = chapters, + ) } - val genreTags = genreList.mapNotNull { name -> - tags.find { it.title == name } - }.toSet() - - val chapters = chaptersDeferred.await() - - manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - authors = (authorsList + artistsList).toSet(), - description = synopsisText.trim(), - rating = rating, - state = when (status?.lowercase()) { - "releasing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - "discontinued" -> MangaState.ABANDONED - "on_hiatus" -> MangaState.PAUSED - "not_yet_released" -> MangaState.UPCOMING - else -> null - }, - tags = genreTags, - altTitles = altTitlesArray.toSet(), - chapters = chapters, - ) + } catch (_: Exception) { + manga } synchronized(detailsCacheLock) { detailsCache[manga.url] = result @@ -424,73 +444,57 @@ internal abstract class MangaFireParser( return result } - private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List<MangaChapter> { - val chapters = mutableListOf<MangaChapter>() val base = "https://$domain/api/titles/$hid" + val chapters = mutableListOf<MangaChapter>() - var page = 1 - var lastPage: Int - do { - val json = apiClient.httpGet( - "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" - ).parseJson() - val items = json.getJSONArray("items") - val meta = json.optJSONObject("meta") - lastPage = meta?.optInt("lastPage", 1) ?: 1 - - for (i in 0 until items.length()) { - val ch = items.getJSONObject(i) - if (ch.getString("language") != siteLang) continue - - val id = ch.getInt("id") - val number = ch.getDouble("number").toFloat() - val name = ch.optString("name", null) - val createdAt = ch.optLong("createdAt", 0L) * 1000L - val type = ch.getString("type") - val chapterUrl = "/title/$hid/$id" - val displayName = buildString { - append("Ch. ") - append(number.toString().removeSuffix(".0")) - if (!name.isNullOrBlank()) append(" - $name") + try { + val firstUrl = "$base/chapters?language=$siteLang&sort=number&order=desc&page=1&limit=200" + val firstJson = apiClient.httpGet(firstUrl).parseJson() + val firstItems = firstJson.optJSONArray("items") ?: org.json.JSONArray() + val meta = firstJson.optJSONObject("meta") + val lastPage = meta?.optInt("lastPage", 1) ?: 1 + + addChapters(firstItems, hid, chapters) + + if (lastPage > 1) { + supervisorScope { + val deferred = (2..lastPage).map { page -> + async { + try { + val json = apiClient.httpGet( + "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" + ).parseJson() + json.optJSONArray("items") ?: org.json.JSONArray() + } catch (_: Exception) { + org.json.JSONArray() + } + } + } + deferred.awaitAll().forEach { items -> + addChapters(items, hid, chapters) + } } - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = displayName, - number = number, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = createdAt, - branch = type, - source = source, - ) - ) } - page++ - } while (page <= lastPage) + } catch (_: Exception) { + } if (hasVolumes) { try { val volJson = apiClient.httpGet("$base/volumes?language=$siteLang").parseJson() - val volItems = volJson.getJSONArray("items") + val volItems = volJson.optJSONArray("items") ?: org.json.JSONArray() for (i in 0 until volItems.length()) { val vol = volItems.getJSONObject(i) - if (vol.getString("language") != siteLang) continue - + if (vol.optString("language", "") != siteLang) continue val volId = vol.getInt("id") val volNumber = vol.getDouble("number").toFloat() val volName = vol.optString("name", "").takeIf { it.isNotBlank() } val chapterCount = vol.optInt("chapterCount", 0) - val title = buildString { append("Vol. ") append(volNumber.toString().removeSuffix(".0")) if (volName != null) append(" - $volName") } - val name = if (chapterCount > 0) "$chapterCount chapters" else "" - chapters.add( MangaChapter( id = generateUid("/title/$hid/vol/$volId"), @@ -498,20 +502,18 @@ internal abstract class MangaFireParser( number = volNumber, volume = 0, url = "/title/$hid/vol/$volId", - scanlator = name, + scanlator = if (chapterCount > 0) "$chapterCount chapters" else "", uploadDate = 0L, branch = "Volume", source = source, ) ) } - } catch (_: Exception) { - } + } catch (_: Exception) {} } val distinctBranches = chapters.map { it.branch }.distinct() val useGroups = distinctBranches.size > 1 - return chapters .map { chapter -> chapter.copy( @@ -521,13 +523,48 @@ internal abstract class MangaFireParser( .sortedBy { it.number } } - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val chapterId = chapter.url.substringAfterLast("/") // numeric ID + private fun addChapters(items: org.json.JSONArray, hid: String, list: MutableList<MangaChapter>) { + for (i in 0 until items.length()) { + try { + val ch = items.getJSONObject(i) + if (ch.optString("language", "") != siteLang) continue + val id = ch.getInt("id") + val number = ch.getDouble("number").toFloat() + val name = ch.optString("name", null) + val createdAt = ch.optLong("createdAt", 0L) * 1000L + val type = ch.optString("type", "Unknown") + val chapterUrl = "/title/$hid/$id" + val displayName = buildString { + append("Ch. ") + append(number.toString().removeSuffix(".0")) + if (!name.isNullOrBlank()) append(" - $name") + } + list.add( + MangaChapter( + id = generateUid(chapterUrl), + title = displayName, + number = number, + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = createdAt, + branch = type, + source = source, + ) + ) + } catch (_: Exception) { + } + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { + val chapterId = chapter.url.substringAfterLast("/") val isVolume = chapter.url.contains("/vol/") val endpoint = if (isVolume) "volumes" else "chapters" val response = apiClient.httpGet("https://$domain/api/$endpoint/$chapterId").parseJson() - val pagesArray = response.getJSONObject("data").getJSONArray("pages") + val data = response.optJSONObject("data") ?: return emptyList() + val pagesArray = data.optJSONArray("pages") ?: return emptyList() val pages = ArrayList<MangaPage>(pagesArray.length()) for (i in 0 until pagesArray.length()) { val pageObj = pagesArray.getJSONObject(i) @@ -541,7 +578,9 @@ internal abstract class MangaFireParser( ) ) } - return pages + pages + } catch (_: Exception) { + emptyList() } override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() @@ -550,7 +589,6 @@ internal abstract class MangaFireParser( override suspend fun isAuthorized(): Boolean = true override suspend fun getUsername(): String = "" - private fun extractHid(url: String): String { val lastPart = url.removeSuffix("/").substringAfterLast("/") return when { @@ -563,10 +601,7 @@ internal abstract class MangaFireParser( @MangaSourceParser("MANGAFIRE_EN", "MangaFire (English)", "en") class English(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_EN, "en") - @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") - - @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire Spanish (Latin)", "es") + @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire (Spanish)", "es") class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") @@ -575,10 +610,6 @@ internal abstract class MangaFireParser( @MangaSourceParser("MANGAFIRE_JA", "MangaFire (Japanese)", "ja") class Japanese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_JA, "ja") - @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") - - @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire Portuguese (Brazil)", "pt") - class PortugueseBR(context: MangaLoaderContext) : - MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") + @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire (Portuguese)", "pt") + class PortugueseBR(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") } From 6d318fca350328fde7c07f2cbd2a08bc892032ff Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:04:36 +0200 Subject: [PATCH 057/434] Update libs.versions.toml --- gradle/libs.versions.toml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index d781b981..9553a389 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,8 +12,10 @@ json = "20251224" jsoup = "1.17.2" quickjs = "1.1.0" korte = "4.0.10" +#noinspection NewerVersionAvailable +serialization = "1.7.3" simplexml = "2.7.1" -tsuki = "1.0.1" +tsuki = "1.0.3" [plugins] kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } @@ -25,6 +27,7 @@ tsuki = { module = "com.github.UsagiApp:Tsuki", version.ref = "tsuki"} ksp-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp" } kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines" } +kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" } junit-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } From 3ded2f0bfe063e300ed24d2c72118bfcf2e15852 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 25 Jul 2026 22:07:02 +0200 Subject: [PATCH 058/434] Update build.gradle.kts --- build.gradle.kts | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index c21b32c1..b0df5d12 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import tasks.ReportGenerateTask -import tasks.DexPluginTask plugins { `java-library` @@ -10,7 +9,7 @@ plugins { } group = "org.usagi" -version = "1.0.2" +version = "1.0.3" tasks.test { useJUnitPlatform() @@ -21,7 +20,8 @@ ksp { } tasks.jar { - archiveFileName.set("raw.jar") + // remember to dex it with d8 before sideloading + archiveFileName.set("uma.jar") exclude("android/**") exclude("androidx/annotation/**") exclude("androidx/preference/**") @@ -54,6 +54,7 @@ publishing { dependencies { implementation(libs.kotlinx.coroutines.core) + implementation(libs.kotlinx.serialization.json) implementation(libs.okhttp) implementation(libs.okio) implementation(libs.json) @@ -72,19 +73,7 @@ dependencies { testImplementation(libs.quickjs) } -tasks.register("buildJar") { - description = "Build all sources to a JAR file" - dependsOn("dexJar") -} - -tasks.register<DexPluginTask>("dexJar") { - description = "Dex classes after build" - dependsOn(tasks.jar) - inputJar.set(tasks.jar.flatMap { it.archiveFile }) - outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) - classpath.from(configurations.runtimeClasspath) -} - tasks.register<ReportGenerateTask>("generateTestsReport") { description = "Generate a HTML file to get tests report" } + From 0c2a70459d07de185bb10cb31e68c5998db53e94 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:40:39 +0200 Subject: [PATCH 059/434] Update build.gradle.kts --- build.gradle.kts | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/build.gradle.kts b/build.gradle.kts index b0df5d12..9bab032b 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ import tasks.ReportGenerateTask +import tasks.DexPluginTask plugins { `java-library` @@ -27,6 +28,20 @@ tasks.jar { exclude("androidx/preference/**") } +tasks.register("buildJar") { + description = "Build all sources to a JAR file" + dependsOn("dexJar") +} + +tasks.register<DexPluginTask>("dexJar") { + description = "Dex classes after build" + dependsOn(tasks.jar) + inputJar.set(tasks.jar.flatMap { it.archiveFile }) + outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) + classpath.from(configurations.runtimeClasspath) +} + + tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach { compilerOptions { freeCompilerArgs.addAll( From bf448d552d84a9ad60f8e8ac4bff3e98471860fd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 25 Jul 2026 23:59:24 +0200 Subject: [PATCH 060/434] Update buildJar.xml --- .idea/runConfigurations/buildJar.xml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.idea/runConfigurations/buildJar.xml b/.idea/runConfigurations/buildJar.xml index 09571555..01bdc1e1 100644 --- a/.idea/runConfigurations/buildJar.xml +++ b/.idea/runConfigurations/buildJar.xml @@ -1,5 +1,5 @@ <component name="ProjectRunConfigurationManager"> - <configuration default="false" name="buildJar" type="GradleRunConfiguration" factoryName="Gradle"> + <configuration default="false" name="compileKotlin" type="GradleRunConfiguration" factoryName="Gradle" activateToolWindowBeforeRun="false"> <ExternalSystemSettings> <option name="executionName" /> <option name="externalProjectPath" value="$PROJECT_DIR$" /> @@ -10,15 +10,15 @@ </option> <option name="taskNames"> <list> - <option value="buildJar" /> + <option value="compileKotlin" /> </list> </option> <option name="vmOptions" /> </ExternalSystemSettings> - <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess> + <ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess> <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess> <DebugAllEnabled>false</DebugAllEnabled> <RunAsTest>false</RunAsTest> <method v="2" /> </configuration> -</component> \ No newline at end of file +</component> From 756f52ca0b86c707f0b8865577f8e765760adfb9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:00:02 +0200 Subject: [PATCH 061/434] Update libs.versions.toml From 26af706fabadbc0c28b9898462012543cc2a6140 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:00:22 +0200 Subject: [PATCH 062/434] Update build.gradle.kts --- build.gradle.kts | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 9bab032b..b0df5d12 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,5 +1,4 @@ import tasks.ReportGenerateTask -import tasks.DexPluginTask plugins { `java-library` @@ -28,20 +27,6 @@ tasks.jar { exclude("androidx/preference/**") } -tasks.register("buildJar") { - description = "Build all sources to a JAR file" - dependsOn("dexJar") -} - -tasks.register<DexPluginTask>("dexJar") { - description = "Dex classes after build" - dependsOn(tasks.jar) - inputJar.set(tasks.jar.flatMap { it.archiveFile }) - outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) - classpath.from(configurations.runtimeClasspath) -} - - tasks.withType<org.jetbrains.kotlin.gradle.tasks.KotlinCompile>().configureEach { compilerOptions { freeCompilerArgs.addAll( From 1d813bb32f77ed9f202a2ba92ca1d4907dac0ed5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:09:17 +0200 Subject: [PATCH 063/434] Update MangaKatana.kt --- src/main/kotlin/tsuki/site/en/MangaKatana.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaKatana.kt b/src/main/kotlin/tsuki/site/en/MangaKatana.kt index 73af0fa4..109c04b7 100644 --- a/src/main/kotlin/tsuki/site/en/MangaKatana.kt +++ b/src/main/kotlin/tsuki/site/en/MangaKatana.kt @@ -24,10 +24,10 @@ internal class MangaKatana(context: MangaLoaderContext): override val configKeyDomain = ConfigKey.Domain("mangakatana.com") - override val availableSortOrders = EnumSet.of( + override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.UPDATED, SortOrder.NEWEST, - )!! + ) private val preferredServerKey = ConfigKey.PreferredImageServer( presetValues = mapOf( From b99b031a08ddf53c7885076a1607dc272f96096a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:22:45 +0200 Subject: [PATCH 064/434] Update buildJar.xml --- .idea/runConfigurations/buildJar.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.idea/runConfigurations/buildJar.xml b/.idea/runConfigurations/buildJar.xml index 01bdc1e1..3eb34e73 100644 --- a/.idea/runConfigurations/buildJar.xml +++ b/.idea/runConfigurations/buildJar.xml @@ -1,5 +1,5 @@ <component name="ProjectRunConfigurationManager"> - <configuration default="false" name="compileKotlin" type="GradleRunConfiguration" factoryName="Gradle" activateToolWindowBeforeRun="false"> + <configuration default="false" name="buildJar" type="GradleRunConfiguration" factoryName="Gradle"> <ExternalSystemSettings> <option name="executionName" /> <option name="externalProjectPath" value="$PROJECT_DIR$" /> @@ -10,12 +10,12 @@ </option> <option name="taskNames"> <list> - <option value="compileKotlin" /> + <option value="buildJar" /> </list> </option> <option name="vmOptions" /> </ExternalSystemSettings> - <ExternalSystemDebugServerProcess>false</ExternalSystemDebugServerProcess> + <ExternalSystemDebugServerProcess>true</ExternalSystemDebugServerProcess> <ExternalSystemReattachDebugProcess>true</ExternalSystemReattachDebugProcess> <DebugAllEnabled>false</DebugAllEnabled> <RunAsTest>false</RunAsTest> From be4a924ede750415e593c74715479ba220085a51 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:23:06 +0200 Subject: [PATCH 065/434] Update build.gradle.kts --- build.gradle.kts | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index b0df5d12..4cf02d16 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,4 +1,5 @@ import tasks.ReportGenerateTask +import tasks.DexPluginTask plugins { `java-library` @@ -20,8 +21,7 @@ ksp { } tasks.jar { - // remember to dex it with d8 before sideloading - archiveFileName.set("uma.jar") + archiveFileName.set("raw.jar") exclude("android/**") exclude("androidx/annotation/**") exclude("androidx/preference/**") @@ -73,7 +73,19 @@ dependencies { testImplementation(libs.quickjs) } +tasks.register("buildJar") { + description = "Build all sources to a JAR file" + dependsOn("dexJar") +} + +tasks.register<DexPluginTask>("dexJar") { + description = "Dex classes after build" + dependsOn(tasks.jar) + inputJar.set(tasks.jar.flatMap { it.archiveFile }) + outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) + classpath.from(configurations.runtimeClasspath) +} + tasks.register<ReportGenerateTask>("generateTestsReport") { description = "Generate a HTML file to get tests report" } - From 16780634ebf0d538c6c489cda04d39108257bf05 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:23:17 +0200 Subject: [PATCH 066/434] Update libs.versions.toml From f4b7029f3fe917974d317c5ce332f061f7ffd7d4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 00:59:29 +0200 Subject: [PATCH 067/434] fix webtoons --- src/main/kotlin/tsuki/site/all/Webtoons.kt | 417 +++++++++++++-------- 1 file changed, 259 insertions(+), 158 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt index 00526da8..55a627ab 100644 --- a/src/main/kotlin/tsuki/site/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -1,20 +1,24 @@ package tsuki.site.all -import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException +import tsuki.MangaLoaderContext import tsuki.model.* import tsuki.util.* -import tsuki.util.json.getStringOrNull -import androidx.collection.arraySetOf import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import kotlin.math.max import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject import org.jsoup.nodes.Element +import org.jsoup.parser.Parser +import java.net.SocketException +import java.text.DecimalFormat +import java.util.Calendar import java.util.EnumSet private const val ID_SEARCH_PREFIX = "id:" @@ -34,39 +38,32 @@ internal abstract class WebtoonsParser( private val contentKey = ConfigKey.PreferredImageServer( presetValues = mapOf( - _root_ide_package_.tsuki.site.all.webtoons to "Webtoons", - _root_ide_package_.tsuki.site.all.canvas to "Canvas", + webtoons to "Webtoons", + canvas to "Canvas", ), - defaultValue = _root_ide_package_.tsuki.site.all.webtoons, + defaultValue = webtoons, ) + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(contentKey) + } + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( SortOrder.POPULARITY, - SortOrder.RATING, + SortOrder.RELEVANCE, SortOrder.UPDATED, + SortOrder.RATING, ) override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36") + get() = MangaListFilterCapabilities(isSearchSupported = true) - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), + override val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36" ) - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(contentKey) - } - - override suspend fun getPageUrl(page: MangaPage): String { - return page.url.toAbsoluteUrl(staticDomain) - } - private val languageCode: String get() = when (val tag = sourceLocale.toLanguageTag()) { "in" -> "id" @@ -74,6 +71,42 @@ internal abstract class WebtoonsParser( else -> tag } + private val localeForCookie: String + get() = if (languageCode == "zh-hant") "zh_TW" else languageCode + + private val customClient by lazy { + context.httpClient.newBuilder() + .addInterceptor { chain -> + val original = chain.request() + val url = original.url + if (url.host.endsWith("webtoons.com")) { + val newRequest = original.newBuilder() + .header("Cookie", "ageGatePass=true; locale=$localeForCookie; needGDPR=false") + .build() + try { + chain.proceed(newRequest) + } catch (_: SocketException) { + chain.proceed(newRequest) + } + } else { + chain.proceed(original) + } + } + .build() + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + + override val webClient by lazy { + tsuki.network.OkHttpWebClient(customClient, source) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + private suspend fun fetchEpisodes(titleNo: Long, type: String): List<MangaChapter> { val baseUrl = "https://$mobileApiDomain/api/v1/$type/$titleNo/episodes" val url = if (type == "canvas") { @@ -82,27 +115,72 @@ internal abstract class WebtoonsParser( "$baseUrl?pageSize=99999" } val json = webClient.httpGet(url).parseJson() - val episodeList = json.optJSONObject("result")?.optJSONArray("episodeList") ?: throw ParseException("No episodes found for title $titleNo", url) - return episodeList.mapChapters { _, jo -> - val episodeTitle = jo.getStringOrNull("episodeTitle") ?: "" - val episodeNo = jo.getInt("episodeNo") + val numberFormatter = DecimalFormat("#.##") + data class RawEpisode( + val jo: JSONObject, + var chapterNumber: Float = -1f, + var seasonNumber: Int = 1, + ) + + val rawList = (0 until episodeList.length()).map { i -> + RawEpisode(episodeList.getJSONObject(i)) + } + + for (raw in rawList) { + val title = raw.jo.optString("episodeTitle") ?: "" + val match = episodeNoRegex.find(title)?.groupValues + if (match != null && match.getOrNull(6).isNullOrBlank()) { + raw.chapterNumber = match.getOrNull(11)?.toFloatOrNull() ?: -1f + raw.seasonNumber = match.getOrNull(4)?.takeIf(String::isNotBlank)?.toInt() ?: 1 + } + } + + var maxChapterNumber = 0f + var currentSeason = 1 + var seasonOffset = 0f + + for ((idx, raw) in rawList.withIndex()) { + if (raw.chapterNumber != -1f) { + val original = raw.chapterNumber + if (raw.seasonNumber > currentSeason) { + currentSeason = raw.seasonNumber + if (original <= maxChapterNumber) { + seasonOffset = maxChapterNumber + } + } + raw.chapterNumber = seasonOffset + original + maxChapterNumber = max(maxChapterNumber, raw.chapterNumber) + } else { + val prev = rawList.getOrNull(idx - 1) + if (prev != null) { + raw.chapterNumber = prev.chapterNumber + 0.01f + } else { + raw.chapterNumber = 0f + } + } + } + + return rawList.mapIndexed { idx, raw -> + val jo = raw.jo + val episodeTitle = jo.optString("episodeTitle", "") val viewerLink = jo.getString("viewerLink") val hasBgm = jo.optBoolean("hasBgm", false) + val chapterNumber = raw.chapterNumber MangaChapter( - id = generateUid("$titleNo-$episodeNo"), + id = generateUid("$titleNo-$idx"), title = buildString { - append(org.jsoup.parser.Parser.unescapeEntities(episodeTitle, false)) - append(" (ch. $episodeNo)") + append(Parser.unescapeEntities(episodeTitle, false)) + append(" (ch. ", numberFormatter.format(chapterNumber), ")") if (hasBgm) append(" ♫") }, - number = episodeNo.toFloat(), + number = chapterNumber, volume = 0, url = viewerLink, - uploadDate = jo.getLong("exposureDateMillis"), + uploadDate = jo.optLong("exposureDateMillis"), branch = null, scanlator = null, source = source, @@ -110,6 +188,11 @@ internal abstract class WebtoonsParser( }.sortedBy { it.number } } + private val episodeNoRegex = Regex( + """(?:(s(eason)?|saison|part|vol(ume)?)\s*\.?\s*(\d+).*?)?(.*?(mini|bonus|special).*?)?(e(p(isode)?)?|ch(apter)?)\s*\.?\s*(\d+(\.\d+)?)""", + RegexOption.IGNORE_CASE, + ) + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() val detailsUrl = manga.publicUrl.ifBlank { @@ -178,114 +261,19 @@ internal abstract class WebtoonsParser( ) } - private fun getSortOrderParam(order: SortOrder): String { - return when (order) { - SortOrder.POPULARITY -> "MANA" - SortOrder.RATING -> "LIKEIT" - SortOrder.UPDATED -> "UPDATE" - else -> "MANA" - } - } - - private fun availableTags() = arraySetOf( - MangaTag("Action", "action", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Drama", "drama", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Horror", "horror", source), - MangaTag("Romance", "romance", source), - MangaTag("Sci-Fi", "sf", source), - MangaTag("Slice of Life", "slice_of_life", source), - MangaTag("Sports", "sports", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Historical", "historical", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Superhero", "super_hero", source), - MangaTag("Heartwarming", "heartwarming", source), - MangaTag("Graphic Novel", "graphic_novel", source), - MangaTag("Informative", "tiptoon", source), - ) - - private val genreUrlMap: Map<String, String> = availableTags().associate { - it.title.lowercase() to it.key - } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query.orEmpty() - val currentContent = config[contentKey] ?: _root_ide_package_.tsuki.site.all.webtoons - - var directTitleNo: Long? = null - var directType: String? = null - - when { - query.startsWith("https://") -> { - val url = query.toHttpUrl() - if (!url.host.endsWith("webtoons.com")) { - throw Exception("Unsupported domain: ${url.host}") - } - directTitleNo = url.queryParameter("title_no")?.toLongOrNull() - ?: throw Exception("Missing 'title_no' parameter in URL") - val path = url.pathSegments.filter(String::isNotEmpty) - if (path.size < 2) throw Exception("Invalid URL path") - if (path[0] != languageCode) { - throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") - } - directType = if (path.getOrNull(1) == "canvas") _root_ide_package_.tsuki.site.all.canvas else _root_ide_package_.tsuki.site.all.webtoons - } + val currentContent = config[contentKey] ?: webtoons - query.startsWith(_root_ide_package_.tsuki.site.all.ID_SEARCH_PREFIX) -> { - val idPart = query.removePrefix(_root_ide_package_.tsuki.site.all.ID_SEARCH_PREFIX) - val parts = idPart.split(":") - val (type, lang, titleNo) = when (parts.size) { - 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[0]}")) - 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[1]}")) - 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[2]}")) - else -> throw Exception("Invalid id search format") - } - if (lang != languageCode) return emptyList() - if (type !in listOf( - _root_ide_package_.tsuki.site.all.webtoons, - _root_ide_package_.tsuki.site.all.canvas - )) { - throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") - } - directTitleNo = titleNo - directType = type - } - - query.toLongOrNull() != null -> { - directTitleNo = query.toLong() - directType = currentContent - } - } - - if (directTitleNo != null && directType != null) { - val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" - val manga = getDetails(Manga( - id = generateUid(directTitleNo), - title = "", - altTitles = emptySet(), - url = directTitleNo.toString(), - publicUrl = detailsUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - )) - return listOf(manga) + if (query.startsWith("https://") || query.startsWith(ID_SEARCH_PREFIX)) { + return handleDirectSearch(query, currentContent) } val selectedGenre = filter.tags.firstOrNull() val url = buildString { append("https://$domain/$languageCode") + when { query.isNotBlank() -> { append("/search") @@ -303,36 +291,58 @@ internal abstract class WebtoonsParser( } if (searchType == null) { searchType = when (currentContent) { - _root_ide_package_.tsuki.site.all.webtoons -> "originals" - _root_ide_package_.tsuki.site.all.canvas -> "canvas" + webtoons -> "originals" + canvas -> "canvas" else -> null } } if (searchType != null) append("/$searchType") append("?keyword=${cleanQuery.urlEncoded()}") - if (page > 1) append("&page=$page") - append("&mode=$currentContent") + if (page > 1 && searchType != null) append("&page=$page") } + selectedGenre != null -> { val genreUrlPath = genreUrlMap[selectedGenre.key] ?: selectedGenre.key append("/genres/$genreUrlPath") append("?sortOrder=${getSortOrderParam(order)}") if (page > 1) append("&page=$page") - append("&mode=$currentContent") } + else -> { - val rankingType = when { - currentContent == _root_ide_package_.tsuki.site.all.canvas -> "canvas" - else -> when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "trending" - SortOrder.UPDATED -> "originals" - else -> "popular" + if (currentContent == canvas) { + append("/ranking/canvas") + if (page > 1) append("?page=$page") + } else { + when (order) { + SortOrder.POPULARITY -> { + append("/ranking/popular") + if (page > 1) append("?page=$page") + } + SortOrder.RATING -> { + append("/ranking/trending") + if (page > 1) append("?page=$page") + } + SortOrder.RELEVANCE -> { + append("/ranking/originals") + if (page > 1) append("?page=$page") + } + SortOrder.UPDATED -> { + val day = Calendar.getInstance().get(Calendar.DAY_OF_WEEK) + val dayName = when (day) { + Calendar.MONDAY -> "monday" + Calendar.TUESDAY -> "tuesday" + Calendar.WEDNESDAY -> "wednesday" + Calendar.THURSDAY -> "thursday" + Calendar.FRIDAY -> "friday" + Calendar.SATURDAY -> "saturday" + Calendar.SUNDAY -> "sunday" + else -> "sunday" + } + append("/originals/$dayName?sortOrder=UPDATE") + } + else -> throw Exception("Unknown sort order") } } - append("/ranking/$rankingType") - if (page > 1) append("?page=$page") - append("&mode=$currentContent") } } } @@ -342,6 +352,11 @@ internal abstract class WebtoonsParser( .map { element -> createMangaFromElement(element, source as MangaParserSource, selectedGenre) } } + private fun extractTitleNoFromUrl(url: String): Long { + return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() + ?: throw ParseException("Could not extract title_no from URL: $url", url) + } + private fun createMangaFromElement( element: Element, source: MangaParserSource, @@ -370,14 +385,80 @@ internal abstract class WebtoonsParser( ) } - private fun extractTitleNoFromUrl(url: String): Long { - return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() - ?: throw ParseException("Could not extract title_no from URL: $url", url) + private fun getSortOrderParam(order: SortOrder): String = when (order) { + SortOrder.POPULARITY -> "MANA" + SortOrder.RELEVANCE -> "LIKEIT" + SortOrder.RATING -> "LIKEIT" + SortOrder.UPDATED -> "UPDATE" + else -> "MANA" + } + + private suspend fun handleDirectSearch(query: String, currentContent: String): List<Manga> { + var directTitleNo: Long? = null + var directType: String? = null + + when { + query.startsWith("https://") -> { + val url = query.toHttpUrl() + if (!url.host.endsWith("webtoons.com")) throw Exception("Unsupported domain: ${url.host}") + directTitleNo = url.queryParameter("title_no")?.toLongOrNull() + ?: throw Exception("Missing 'title_no' parameter in URL") + val path = url.pathSegments.filter(String::isNotEmpty) + if (path.size < 2) throw Exception("Invalid URL path") + if (path[0] != languageCode) { + throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") + } + directType = if (path.getOrNull(1) == "canvas") "canvas" else "webtoon" + } + query.startsWith(ID_SEARCH_PREFIX) -> { + val idPart = query.removePrefix(ID_SEARCH_PREFIX) + val parts = idPart.split(":") + val (type, lang, titleNo) = when (parts.size) { + 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[0]}")) + 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[1]}")) + 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[2]}")) + else -> throw Exception("Invalid id search format") + } + if (lang != languageCode) return emptyList() + if (type !in listOf("webtoons", "canvas")) throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") + directTitleNo = titleNo + directType = type + } + query.toLongOrNull() != null -> { + directTitleNo = query.toLong() + directType = currentContent + } + } + + if (directTitleNo != null && directType != null) { + val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" + val manga = getDetails( + Manga( + id = generateUid(directTitleNo), + title = "", + altTitles = emptySet(), + url = directTitleNo.toString(), + publicUrl = detailsUrl, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + return listOf(manga) + } + return emptyList() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val absUrl = chapter.url.toAbsoluteUrl(domain) val doc = try { - val absUrl = chapter.url.toAbsoluteUrl(domain) webClient.httpGet(absUrl).parseHtml() } catch (e: Exception) { throw ParseException("Failed to get pages for chapter: ${chapter.title}", chapter.url, e) @@ -398,14 +479,10 @@ internal abstract class WebtoonsParser( if (images.isEmpty()) { val motionPages = fetchMotionToonPages(doc) - if (motionPages.isNotEmpty()) { - images.addAll(motionPages) - } + if (motionPages.isNotEmpty()) images.addAll(motionPages) } - if (images.isEmpty()) { - throw ParseException("No images found in chapter.", chapter.url) - } + if (images.isEmpty()) throw ParseException("No images found in chapter.", chapter.url) return images } @@ -436,6 +513,30 @@ internal abstract class WebtoonsParser( } } + private fun availableTags() = setOf( + MangaTag("Action", "action", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Drama", "drama", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Horror", "horror", source), + MangaTag("Romance", "romance", source), + MangaTag("Sci-Fi", "sf", source), + MangaTag("Slice of Life", "slice_of_life", source), + MangaTag("Sports", "sports", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Historical", "historical", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Superhero", "super_hero", source), + MangaTag("Heartwarming", "heartwarming", source), + MangaTag("Graphic Novel", "graphic_novel", source), + MangaTag("Informative", "tiptoon", source), + ) + + private val genreUrlMap: Map<String, String> = availableTags().associate { + it.title.lowercase() to it.key + } + @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", type = ContentType.MANGA) class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_EN) @@ -456,4 +557,4 @@ internal abstract class WebtoonsParser( @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", type = ContentType.MANGA) class German(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_DE) -} \ No newline at end of file +} From 6a7f9d8e660d70620355092fdc17990b105ea4b3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:51:13 +0200 Subject: [PATCH 068/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index eecc1a0a..3e2a18fd 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -74,7 +74,7 @@ abstract class MangaThemesia( override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, isMultipleTagsSupported = true, - isTagsExclusionSupported = true, + isTagsExclusionSupported = false, ) @Volatile @@ -148,7 +148,6 @@ abstract class MangaThemesia( }) } filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } - filter.tagsExclude.forEach { append("&genre[]=-${it.key.urlEncoded()}") } append("&order=") append(when (order) { SortOrder.UPDATED -> "update" From 758844c8ba15d2dd54f0bf4bbff198dbe745f6db Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 12:51:26 +0200 Subject: [PATCH 069/434] Update EnThunderScans.kt --- src/main/kotlin/tsuki/site/en/EnThunderScans.kt | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt index bfa29cd5..db4ba9b6 100644 --- a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -19,11 +19,6 @@ internal class EnThunderScans(context: MangaLoaderContext) : override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isTagsExclusionSupported = false, - ) - override suspend fun getDetails(manga: Manga): Manga { val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() From fc7c3f831c0b9bf2be00f87375e43ce92054a9d1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:00:56 +0200 Subject: [PATCH 070/434] Create ScytheScans.kt --- src/main/kotlin/tsuki/site/en/ScytheScans.kt | 45 ++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/ScytheScans.kt diff --git a/src/main/kotlin/tsuki/site/en/ScytheScans.kt b/src/main/kotlin/tsuki/site/en/ScytheScans.kt new file mode 100644 index 00000000..5829ce10 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ScytheScans.kt @@ -0,0 +1,45 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import java.util.Base64 + +@MangaSourceParser("SYCTHESCANS", "Scythe Scans", "en") +internal class ScytheScans(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SYCTHESCANS, "erosscans.com", pageSize = 20) { + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val script = doc.selectFirst("script[src^=data:text/javascript;base64,dHNfcmVhZGVyLnJ1bih7]") + if (script != null) { + val base64 = script.attr("src").substringAfter("base64,") + val decoded = String(Base64.getDecoder().decode(base64), Charsets.UTF_8) + val imageListJson = Regex("""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + .find(decoded) + ?.groupValues + ?.getOrNull(1) + .orEmpty() + + val jsonArray = org.json.JSONArray(imageListJson) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return super.getPages(chapter) + } +} From 11845043e8141064aae1b621e85977924e1612e6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:01:50 +0200 Subject: [PATCH 071/434] Delete src/main/kotlin/tsuki/site/all/hentais directory --- .../tsuki/site/all/hentais/AsmHentai.kt | 46 - .../kotlin/tsuki/site/all/hentais/BuonDua.kt | 19 - .../tsuki/site/all/hentais/HentaiEnvy.kt | 90 -- .../tsuki/site/all/hentais/HentaiFox.kt | 136 --- .../tsuki/site/all/hentais/HitomiLaParser.kt | 798 ------------ .../kotlin/tsuki/site/all/hentais/IMHentai.kt | 248 ---- .../kotlin/tsuki/site/all/hentais/Kiutaku.kt | 35 - .../tsuki/site/all/hentais/MangaCrazy.kt | 15 - .../kotlin/tsuki/site/all/hentais/Misskon.kt | 152 --- .../kotlin/tsuki/site/all/hentais/Multporn.kt | 235 ---- .../tsuki/site/all/hentais/NHentaiParser.kt | 1085 ----------------- .../site/all/hentais/NHentaiXXXParser.kt | 187 --- 12 files changed, 3046 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/Misskon.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/Multporn.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt delete mode 100644 src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt diff --git a/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt b/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt deleted file mode 100644 index 77be444c..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/AsmHentai.kt +++ /dev/null @@ -1,46 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.GalleryAdultsParser - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType -import tsuki.model.MangaTag -import tsuki.util.mapToSet -import tsuki.util.removeSuffix - -import org.jsoup.nodes.Element -import java.util.Locale - -@MangaSourceParser("ASMHENTAI", "AsmHentai", type = ContentType.HENTAI) -internal class AsmHentai(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.ASMHENTAI, "asmhentai.com") { - - override val selectGallery = ".preview_item" - override val selectGalleryLink = ".image a" - override val selectGalleryImg = ".image img" - override val pathTagUrl = "/tags/?page=" - override val selectTags = ".tags_page ul.tags" - override val selectAuthor = "div.tags:contains(Artists:) .tag_list a span.tag" - override val idImg = "fimg" - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale.CHINESE, - Locale("tr"), - ), - ) - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".tag")?.html()?.substringBefore("<") ?: it.html().substringBefore("<") - MangaTag( - key = key, - title = name, - source = source, - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt b/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt deleted file mode 100644 index 68619e29..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/BuonDua.kt +++ /dev/null @@ -1,19 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.parsers.GalleryParser - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType - -@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) -internal class BuonDua(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { - - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( - "buondua.com", - "buondua.us", - ) -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt b/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt deleted file mode 100644 index dbc978ff..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/HentaiEnvy.kt +++ /dev/null @@ -1,90 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.ErrorMessages -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.GalleryAdultsParser - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaListFilter -import tsuki.model.SortOrder - -import tsuki.util.oneOrThrowIfMany -import tsuki.util.parseHtml -import tsuki.util.urlEncoded - -import java.util.EnumSet -import java.util.Locale - - -@MangaSourceParser("HENTAIENVY", "HentaiEnvy", type = ContentType.HENTAI) -internal class HentaiEnvy(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAIENVY, "hentaienvy.com", pageSize = 24) { - override val selectGalleryLink = "a" - override val selectGalleryTitle = "div.title" - override val selectTags = ".tags_items" - override val selectTag = ".gt_right_tags ul:contains(Tags:)" - override val selectAuthor = ".gt_right_tags ul:contains(Artists:) a" - override val selectLanguageChapter = ".gt_right_tags ul:contains(Languages:) a" - override val idImg = "fimg" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf( - Locale.ENGLISH, - Locale.FRENCH, - Locale.JAPANESE, - Locale.CHINESE, - Locale("es"), - Locale("ru"), - Locale("ko"), - Locale.GERMAN, - Locale("pt"), - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("/search/?s_key=") - append(query?.urlEncoded()) - append("&") - } - - else -> { - if (filter.tags.isNotEmpty()) { - if (filter.locale != null) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) - } - filter.tags.oneOrThrowIfMany()?.let { - append("/tag/") - append(it.key) - if (order == SortOrder.POPULARITY) { - append("/popular") - } - append("/?") - } - } else if (filter.locale != null) { - val locale = filter.locale - append("/language/") - append(locale?.toLanguagePath()) - append("/?") - } else { - append("/?") - } - } - } - append("page=") - append(page) - - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt deleted file mode 100644 index 0d25ec78..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/HentaiFox.kt +++ /dev/null @@ -1,136 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.GalleryAdultsParser - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType -import tsuki.model.MangaTag -import tsuki.util.mapToSet -import tsuki.util.removeSuffix -import tsuki.util.oneOrThrowIfMany -import tsuki.util.parseHtml -import tsuki.util.urlEncoded - -import tsuki.model.Manga -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.SortOrder - -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Element -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("HENTAIFOX", "HentaiFox", type = ContentType.HENTAI) -internal class HentaiFox(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAIFOX, "hentaifox.com") { - override val selectGallery = ".lc_galleries .thumb, .related_galleries .thumb" - override val pathTagUrl = "/tags/popular/pag/" - override val selectTags = ".list_tags" - override val selectTag = "ul.tags" - override val selectLanguageChapter = "ul.languages a.tag_btn" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - ) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("/search/?q=") - append(query?.urlEncoded()) - if (page > 1) { - append("&page=") - append(page.toString()) - } - } - - else -> { - if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { - append("/search/?q=") - append(buildQuery(filter.tags, filter.locale)) - if (page > 1) { - append("&page=") - append(page.toString()) - } - - if (order == SortOrder.POPULARITY) { - append("&sort=popular") - } - } else if (filter.tags.isNotEmpty()) { - filter.tags.oneOrThrowIfMany()?.let { - append("/tag/") - append(it.key) - } - append("/") - if (order == SortOrder.POPULARITY) { - append("popular/") - } - - if (page > 1) { - append("/pag/") - append(page.toString()) - append("/") - } - } else if (filter.locale != null) { - val locale = filter.locale - append("/language/") - append(locale?.toLanguagePath()) - append("/") - if (order == SortOrder.POPULARITY) { - append("popular/") - } - - if (page > 1) { - append("/pag/") - append(page.toString()) - append("/") - } - } else { - if (page > 2) { - append("/pag/") - append(page.toString()) - append("/") - } else if (page > 1) { - append("/page/") - append(page.toString()) - append("/") - } - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".list_tag")?.text() ?: it.html().substringBefore("<") - MangaTag( - key = key, - title = name, - source = source, - ) - } - - private fun buildQuery(tags: Collection<MangaTag>, language: Locale?): String { - val joiner = StringUtil.StringJoiner(" ") - tags.forEach { tag -> - joiner.add(tag.key) - } - language?.let { lc -> - joiner.add(lc.toLanguagePath()) - } - return joiner.complete() - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt deleted file mode 100644 index fadc5a10..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/HitomiLaParser.kt +++ /dev/null @@ -1,798 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.AbstractMangaParser - -import tsuki.model.ContentRating -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder - -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.suspendlazy.suspendLazy -import tsuki.util.attrAsRelativeUrl -import tsuki.util.generateUid -import tsuki.util.parseBytes -import tsuki.util.parseHtml -import tsuki.util.parseRaw -import tsuki.util.parseSafe -import tsuki.util.runCatchingCancellable -import tsuki.util.selectFirstOrThrow -import tsuki.util.splitByWhitespace -import tsuki.util.toAbsoluteUrl -import tsuki.util.toCamelCase -import tsuki.util.toTitleCase -import tsuki.util.urlDecode - -import androidx.collection.ArraySet -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.Jsoup -import java.nio.ByteBuffer -import java.nio.ByteOrder -import java.security.MessageDigest -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.LinkedList -import java.util.Locale -import kotlin.math.min - -@OptIn(ExperimentalUnsignedTypes::class) -@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) -internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) { - override val configKeyDomain = ConfigKey.Domain("hitomi.la") - - private val cdnDomain = "gold-usergeneratedcontent.net" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - private val ltnBaseUrl get() = "https://ltn.$cdnDomain" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY_TODAY, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - SortOrder.POPULARITY_YEAR, - ) - - private val localeMap: Map<Locale, String> = mapOf( - Locale.forLanguageTag("id") to "indonesian", - Locale.forLanguageTag("jv") to "javanese", - Locale.forLanguageTag("ca") to "catalan", - Locale.forLanguageTag("ceb") to "cebuano", - Locale.forLanguageTag("cs") to "czech", - Locale.forLanguageTag("da") to "danish", - Locale.forLanguageTag("de") to "german", - Locale.forLanguageTag("et") to "estonian", - Locale.ENGLISH to "english", - Locale.forLanguageTag("es") to "spanish", - Locale.forLanguageTag("eo") to "esperanto", - Locale.forLanguageTag("fr") to "french", - Locale.forLanguageTag("it") to "italian", - Locale.forLanguageTag("hi") to "hindi", - Locale.forLanguageTag("hu") to "hungarian", - Locale.forLanguageTag("pl") to "polish", - Locale.forLanguageTag("pt") to "portuguese", - Locale.forLanguageTag("vi") to "vietnamese", - Locale.forLanguageTag("tr") to "turkish", - Locale.forLanguageTag("ru") to "russian", - Locale.forLanguageTag("uk") to "ukrainian", - Locale.forLanguageTag("ar") to "arabic", - Locale.KOREAN to "korean", - Locale.CHINESE to "chinese", - Locale.JAPANESE to "japanese", - ) - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = localeMap.keys, - ) - - private fun Locale?.getSiteLang(): String = when (this) { - null -> "all" - else -> localeMap[this] ?: "all" - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { - ('a'..'z').map { alphabet -> - async { - val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() - - doc.select(".posts > li").mapNotNull { element -> - val num = - element.ownText().let { - Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 - } - - if (num > 100) { - val url = element.selectFirst("a") - val href = - url?.attrAsRelativeUrl("href") - ?: return@mapNotNull null - - MangaTag( - title = url.ownText().toTagTitle(), - key = href.tagUrlToTag(), - source = source, - ) - } else { - null - } - } - } - }.awaitAll().flatten().toSet() - } - - private var cachedSearchIds: List<Int> = emptyList() - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> = when { - filter.query.isNullOrEmpty() -> { - if (filter.tags.isEmpty()) { - when (order) { - SortOrder.POPULARITY_TODAY -> { - getGalleryIDsFromNozomi( - "popular", - "today", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_WEEK -> { - getGalleryIDsFromNozomi( - "popular", - "week", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_MONTH -> { - getGalleryIDsFromNozomi( - "popular", - "month", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - SortOrder.POPULARITY_YEAR -> { - getGalleryIDsFromNozomi( - "popular", - "year", - filter.locale.getSiteLang(), - offset.nextOffsetRange(), - ) - } - - else -> { - getGalleryIDsFromNozomi(null, "index", filter.locale.getSiteLang(), offset.nextOffsetRange()) - } - } - } else { - if (offset == 0) { - cachedSearchIds = - hitomiSearch( - filter.tags.joinToString(" ") { it.key }, - order, - filter.locale.getSiteLang(), - ).toList() - } - cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) - } - } - - else -> { - if (offset == 0) { - cachedSearchIds = hitomiSearch(filter.query ?: "", order).toList() - } - cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) - } - }.toMangaList() - - private fun Int.nextOffsetRange(): LongRange { - val bytes = this * 4L - return bytes.until(bytes + 100L) - } - - private suspend fun hitomiSearch( - query: String, - sortByPopularity: SortOrder = SortOrder.UPDATED, - language: String = "all", - ): Set<Int> = - coroutineScope { - val terms = query - .trim() - .replace(Regex("""^\?"""), "") - .lowercase() - .splitByWhitespace() - .map { - it.replace('_', ' ') - } - - val positiveTerms = LinkedList<String>() - val negativeTerms = LinkedList<String>() - - for (term in terms) { - if (term.startsWith("-")) { - negativeTerms.push(term.removePrefix("-")) - } else if (term.isNotBlank()) { - positiveTerms.push(term) - } - } - - val positiveResults = positiveTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val negativeResults = negativeTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val results = when { - sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) - sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) - sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) - sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) - sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) - positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) - else -> emptySet() - }.toMutableSet() - - fun filterPositive(newResults: Set<Int>) { - when { - results.isEmpty() -> results.addAll(newResults) - else -> results.retainAll(newResults) - } - } - - fun filterNegative(newResults: Set<Int>) { - results.removeAll(newResults) - } - - // positive results - positiveResults.forEach { - filterPositive(it.await()) - } - - // negative results - negativeResults.forEach { - filterNegative(it.await()) - } - - results - } - - // search.js - private suspend fun getGalleryIDsForQuery( - query: String, - language: String = "all", - ): Set<Int> { - query.replace("_", " ").let { - if (it.indexOf(':') > -1) { - val sides = it.split(":") - val ns = sides[0] - var tag = sides[1] - - var area: String? = ns - var lang = language - when (ns) { - "female", "male" -> { - area = "tag" - tag = it - } - - "language" -> { - area = null - lang = tag - tag = "index" - } - } - - return getGalleryIDsFromNozomi(area, tag, lang) - } - - val key = hashTerm(it) - val node = getGalleryNodeAtAddress(0) - val data = bSearch(key, node) ?: return emptySet() - - return getGalleryIDsFromData(data) - } - } - - private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> { - val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" - val (offset, length) = data - require(length in 1..100000000) { - "Length $length is too long" - } - - val inbuf = getRangedResponse(url, offset.until(offset + length)) - - val galleryIDs = mutableSetOf<Int>() - - val buffer = - ByteBuffer - .wrap(inbuf) - .order(ByteOrder.BIG_ENDIAN) - - val numberOfGalleryIDs = buffer.int - - val expectedLength = numberOfGalleryIDs * 4 + 4 - - require(numberOfGalleryIDs in 1..10000000) { - "number_of_galleryids $numberOfGalleryIDs is too long" - } - require(inbuf.size == expectedLength) { - "inbuf.byteLength ${inbuf.size} != expected_length $expectedLength" - } - - for (i in 0.until(numberOfGalleryIDs)) - galleryIDs.add(buffer.int) - - return galleryIDs - } - - private suspend fun bSearch( - key: UByteArray, - node: Node, - ): Pair<Long, Int>? { - fun compareArrayBuffers( - dv1: UByteArray, - dv2: UByteArray, - ): Int { - val top = min(dv1.size, dv2.size) - - for (i in 0.until(top)) { - if (dv1[i] < dv2[i]) { - return -1 - } else if (dv1[i] > dv2[i]) { - return 1 - } - } - - return 0 - } - - fun locateKey( - key: UByteArray, - node: Node, - ): Pair<Boolean, Int> { - for (i in node.keys.indices) { - val cmpResult = compareArrayBuffers(key, node.keys[i]) - - if (cmpResult <= 0) { - return Pair(cmpResult == 0, i) - } - } - - return Pair(false, node.keys.size) - } - - fun isLeaf(node: Node): Boolean { - for (subnode in node.subNodeAddresses) - if (subnode != 0L) { - return false - } - - return true - } - - if (node.keys.isEmpty()) { - return null - } - - val (there, where) = locateKey(key, node) - if (there) { - return node.datas[where] - } else if (isLeaf(node)) { - return null - } - - val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) - return bSearch(key, nextNode) - } - - private suspend fun getGalleryIDsFromNozomi( - area: String?, - tag: String, - language: String, - range: LongRange? = null, - ): Set<Int> { - val nozomiAddress = when (area) { - null -> "$ltnBaseUrl/$tag-$language.nozomi" - else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" - } - - val bytes = getRangedResponse(nozomiAddress, range) - val nozomi = mutableSetOf<Int>() - - val arrayBuffer = ByteBuffer - .wrap(bytes) - .order(ByteOrder.BIG_ENDIAN) - - while (arrayBuffer.hasRemaining()) - nozomi.add(arrayBuffer.int) - - return nozomi - } - - private val galleriesIndexVersion = suspendLazy { - webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() - } - - private data class Node( - val keys: List<UByteArray>, - val datas: List<Pair<Long, Int>>, - val subNodeAddresses: List<Long>, - ) - - private fun decodeNode(data: ByteArray): Node { - val buffer = ByteBuffer - .wrap(data) - .order(ByteOrder.BIG_ENDIAN) - - val uData = data.toUByteArray() - - val numberOfKeys = buffer.int - val keys = ArrayList<UByteArray>() - - for (i in 0.until(numberOfKeys)) { - val keySize = buffer.int - - check(keySize in 1..32) { "Invalid key size $keySize" } - - keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) - buffer.position(buffer.position() + keySize) - } - - val numberOfDatas = buffer.int - val datas = ArrayList<Pair<Long, Int>>() - - for (i in 0.until(numberOfDatas)) { - val offset = buffer.long - val length = buffer.int - - datas.add(Pair(offset, length)) - } - - val numberOfSubNodeAddresses = 16 + 1 - val subNodeAddresses = ArrayList<Long>() - - for (i in 0.until(numberOfSubNodeAddresses)) { - val subNodeAddress = buffer.long - subNodeAddresses.add(subNodeAddress) - } - - return Node(keys, datas, subNodeAddresses) - } - - private suspend fun getGalleryNodeAtAddress(address: Long): Node { - val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index" - - val nodedata = getRangedResponse(url, address.until(address + 464)) - - return decodeNode(nodedata) - } - - private suspend fun getRangedResponse( - url: String, - range: LongRange? = null, - ): ByteArray { - val rangeHeaders = when (range) { - null -> Headers.headersOf() - else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") - } - - return webClient.httpGet(url, rangeHeaders).parseBytes() - } - - private fun hashTerm(term: String): UByteArray { - return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray() - } - - private fun sha256(data: ByteArray): ByteArray { - return MessageDigest.getInstance("SHA-256").digest(data) - } - - private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope { - map { id -> - async { - runCatchingCancellable { - val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { - val baseUri = it.request.url.toString() - val html = it.parseRaw() - Jsoup.parse(rewriteTnPaths(html), baseUri) - } - - Manga( - id = generateUid(id.toString()), - title = doc.selectFirstOrThrow("h1").text(), - url = id.toString(), - coverUrl = - "https:" + - doc.selectFirstOrThrow("picture > img") - .attr("data-src"), - publicUrl = - doc.selectFirstOrThrow("h1 > a") - .attrAsRelativeUrl("href") - .toAbsoluteUrl(domain), - authors = emptySet(), - tags = emptySet(), - contentRating = ContentRating.ADULT, - rating = RATING_UNKNOWN, - altTitles = emptySet(), - state = null, - source = source, - ) - }.getOrNull() - } - }.awaitAll().filterNotNull() - } - - override suspend fun getDetails(manga: Manga): Manga { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - val author = - json.optJSONArray("artists") - ?.mapJSON { it.getString("artist").toCamelCase() } - ?.joinToString() - - return manga.copy( - title = json.getString("title"), - largeCoverUrl = - json.getJSONArray("files").getJSONObject(0).let { - val hash = it.getString("hash") - val imageId = imageIdFromHash(hash) - val subDomain = 'a' + subdomainOffset(imageId) - - "https://${subDomain}tn.$cdnDomain/webpbigtn/${thumbPathFromHash(hash)}/$hash.webp" - }, - authors = setOfNotNull(author), - publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), - tags = - buildSet - { - json.optJSONArray("characters") - ?.mapToTags("character") - ?.let(::addAll) - json.optJSONArray("tags") - ?.mapToTags("tag") - ?.let(::addAll) - json.optJSONArray("artists") - ?.mapToTags("artist") - ?.let(::addAll) - json.optJSONArray("parodys") - ?.mapToTags("parody") - ?.let(::addAll) - json.optJSONArray("groups") - ?.mapToTags("group") - ?.let(::addAll) - }, - chapters = listOf( - MangaChapter( - id = generateUid(manga.url), - url = manga.url, - title = json.getStringOrNull("title"), - scanlator = json.getString("type").toTitleCase(), - number = 1f, - volume = 0, - branch = json.getString("language_localname"), - source = source, - uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), - ), - ), - ) - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) - - private fun JSONArray.mapToTags(key: String): Set<MangaTag> { - val tags = ArraySet<MangaTag>(length()) - mapJSON { - MangaTag( - title = - it.getString(key).toCamelCase().let { title -> - if (it.getStringOrNull("female")?.toIntOrNull() == 1) { - "$title ♀" - } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { - "$title ♂" - } else { - title - } - }, - key = it.getString("url").tagUrlToTag(), - source = source, - ).let(tags::add) - } - return tags - } - - private fun String.tagUrlToTag(): String { - val urlContent = this.split("/") - val ns = urlContent[1] - val tag = - urlContent[2] - .substringBeforeLast("-") - .urlDecode() - .replace(" ", "_") - - return if (tag.split(":")[0] in listOf("female", "male")) { - tag - } else { - "$ns:$tag" - } - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - - // any better way to get List<Int> from this json? - return json.getJSONArray("related").let { - 0.until(it.length()).map { i -> it.getInt(i) } - }.toMangaList() - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") - .parseRaw() - .substringAfter("var galleryinfo = ") - .let(::JSONObject) - - return json.getJSONArray("files").mapJSON { image -> - val hash = image.getString("hash") - val commonId = commonImageId() - val imageId = imageIdFromHash(hash) - val subDomain = subdomainOffset(imageId) + 1 - val thumbSubdomain = 'a' + subdomainOffset(imageId) - MangaPage( - id = generateUid(hash), - url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", - preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", - source = source, - ) - } - } - -// / ---> - - private var scriptLastRetrieval: Long = -1L - private val mutex = Mutex() - private var subdomainOffsetDefault = 0 - private val subdomainOffsetMap = mutableMapOf<Int, Int>() - private var commonImageId = "" - - private suspend fun refreshScript() = mutex.withLock { - if (scriptLastRetrieval == -1L || (scriptLastRetrieval + 60000) < System.currentTimeMillis()) { - val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() - - subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt() - val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt() - - subdomainOffsetMap.clear() - Regex("case (\\d+):").findAll(ggScript).forEach { - val case = it.groupValues[1].toInt() - subdomainOffsetMap[case] = o - } - - commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1] - - scriptLastRetrieval = System.currentTimeMillis() - } - } - - // m <-- gg.js - private suspend fun subdomainOffset(imageId: Int): Int { - refreshScript() - return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault - } - - // b <-- gg.js - private suspend fun commonImageId(): String { - refreshScript() - return commonImageId - } - - // s <-- gg.js - private fun imageIdFromHash(hash: String): Int { - val match = Regex("(..)(.)$").find(hash) - return match!!.groupValues.let { it[2] + it[1] }.toInt(16) - } - - // real_full_path_from_hash <-- common.js - private fun thumbPathFromHash(hash: String): String { - return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") - } - - // rewrite_tn_paths <-- common.js - private suspend fun rewriteTnPaths(html: String): String { - val thumbUrlRegex = Regex( - """(?<protocol>//)(?<host>[a-z0-9.-]+\.(?:hitomi\.la|${Regex.escape(cdnDomain)}))/(?<pathAfterHost>(?:avif|webp)?(?:small)?(?:big|small|medium)tn/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}\.(?:webp|avif|gif|png|jpe?g))""", - ) - - var resultHtml = html - thumbUrlRegex.findAll(html).forEach { matchResult -> - val originalUrl = matchResult.value - val groups = matchResult.groups - - val pathAfterHost = groups["pathAfterHost"]?.value ?: return@forEach - val newTnSubdomain = subdomainFromURL(originalUrl, "tn") - val correctedUrl = "${groups["protocol"]!!.value}$newTnSubdomain.$cdnDomain/$pathAfterHost" - - if (originalUrl != correctedUrl) { - resultHtml = resultHtml.replace(originalUrl, correctedUrl) - } - } - return resultHtml - } - - private suspend fun subdomainFromURL(url: String, base: String?): String { - val resultSubdomain = base ?: "b" - - // This regex extracts the last 3 hex characters from the hash in the URL - // The hash is 64 characters, so we look for the 61st character onward - val hashRegex = Regex("""/([0-9a-f]{61}[0-9a-f]{3})[./]""") - val fullHashMatch = hashRegex.find(url) - ?: // If no hash is found, default to "a" + base (typically "atn") - return "a$resultSubdomain" - - val fullHash = fullHashMatch.groupValues[1] - - val lastThreeChars = fullHash.takeLast(3) - val lastDigit = lastThreeChars.last() - val lastTwoDigits = lastThreeChars.take(2) - - val imageId = "$lastDigit$lastTwoDigits".toIntOrNull(16) - - return if (imageId != null) { - ('a' + subdomainOffset(imageId)).toString() + resultSubdomain - } else { - "a$resultSubdomain" - } - } - - private fun String.toTagTitle(): String { - return toCamelCase() - .replace("♂", "(male)") - .replace("♀", "(female)") - } -} diff --git a/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt b/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt deleted file mode 100644 index 1f41ff03..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/IMHentai.kt +++ /dev/null @@ -1,248 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import androidx.collection.ArraySet -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.jsoup.nodes.Element -import java.util.ArrayList -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI) -internal class ImHentai(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.IMHENTAI, 20) { - - override val availableSortOrders: Set<SortOrder> = - EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING) - - override val configKeyDomain = ConfigKey.Domain("imhentai.xxx") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale("es"), - Locale.FRENCH, - Locale("kr"), - Locale.GERMAN, - Locale("ru"), - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.COMICS, - ContentType.IMAGE_SET, - ContentType.ARTIST_CG, - ContentType.GAME_CG, - ), - ) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - val url = buildString { - append("https://") - append(domain) - append("/search/?page=") - append(page.toString()) - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("&key=") - append(query?.urlEncoded()) - } - - else -> { - - if (filter.tags.isNotEmpty()) { - append("&key=") - filter.tags.joinTo(this, separator = ",") { it.key } - } - - var types = "&m=1&d=1&w=1&i=1&a=1&g=1" - if (filter.types.isNotEmpty()) { - types = "&m=0&d=0&w=0&i=0&a=0&g=0" - filter.types.forEach { - when (it) { - ContentType.MANGA -> types = types.replace("&m=0", "&m=1") - ContentType.DOUJINSHI -> types = types.replace("&d=0", "&d=1") - ContentType.COMICS -> types = types.replace("&w=0", "&w=1") - ContentType.IMAGE_SET -> types = types.replace("&i=0", "&i=1") - ContentType.ARTIST_CG -> types = types.replace("&a=0", "&a=1") - ContentType.GAME_CG -> types = types.replace("&g=0", "&g=1") - else -> {} - } - } - } - append(types) - - - var lang = "&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1" - filter.locale?.let { - lang = "&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0" - lang = lang.replace("${it.language}=0", "${it.language}=1") - } - append(lang) - - when (order) { - SortOrder.UPDATED -> append("<=1&pp=0") - SortOrder.POPULARITY -> append("<=0&pp=1") - SortOrder.RATING -> append("<=0&pp=0") - else -> append("<=1&pp=0") - } - } - } - - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.galleries div.thumb").map { div -> - val a = div.selectFirstOrThrow(".inner_thumb a") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = a.selectFirst("img")?.src(), - title = div.selectFirst(".caption")?.text().orEmpty(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. - private suspend fun fetchAvailableTags(): Set<MangaTag> { - return coroutineScope { - (1..3).map { page -> - async { fetchTagsPage(page) } - } - }.awaitAll().flattenTo(ArraySet(360)) - } - - private suspend fun fetchTagsPage(page: Int): Set<MangaTag> { - val url = "https://$domain/tags/popular/?page=$page" - val root = webClient.httpGet(url).parseHtml() - return root.parseTags() - } - - private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet { - val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') - MangaTag( - key = href, - title = it.selectFirstOrThrow("h3.list_tag").text().toTitleCase(sourceLocale), - source = source, - ) - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val author = doc.selectFirst("li:contains(Artists) a.tag")?.ownTextOrNull() - manga.copy( - tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet { - val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') - MangaTag( - key = href, - title = it.ownText().toTitleCase(sourceLocale), - source = source, - ) - }, - authors = setOfNotNull(author), - chapters = listOf( - MangaChapter( - id = manga.id, - title = null, - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0, - branch = doc.selectFirst("li:contains(Language) a.tag")?.ownTextOrNull()?.toTitleCase(sourceLocale), - source = source, - ), - ), - ) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().selectFirstOrThrow("div.related") - return root.select("div.thumb").mapNotNull { div -> - val a = div.selectFirstOrThrow(".inner_thumb a") - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = a.selectFirst("img")?.src(), - title = div.selectFirst(".caption")?.text().orEmpty(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() - val baseImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img") - val baseUrl = baseImg.selectFirstParentOrThrow("a").attrAsRelativeUrl("href").replace("/1/", - "/$/" - ) - val baseThumbUrl = baseImg.src()?.replace("/1t.", $$"/$t.") - val pages = ArrayList<MangaPage>(totalPages) - repeat(totalPages) { i -> - val url = baseUrl.replace("$", (i + 1).toString()) - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = baseThumbUrl?.replace("$", (i + 1).toString()), - source = source, - ), - ) - } - return pages - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val img = doc.body().requireElementById("gimg") - return img.requireSrc() - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt b/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt deleted file mode 100644 index 01df3f33..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/Kiutaku.kt +++ /dev/null @@ -1,35 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.network.CommonHeaders -import tsuki.parsers.GalleryParser - -import okhttp3.Interceptor -import okhttp3.Response - -@MangaSourceParser("KIUTAKU", "Kiutaku", type = ContentType.OTHER) -internal class Kiutaku(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.KIUTAKU, "kiutaku.com") { - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val url = request.url.toString() - - val headers = if (url.contains("wp-content")) { - request.headers.newBuilder() - .removeAll(CommonHeaders.REFERER) - .build() - } else { - request.headers - } - - val newRequest = request.newBuilder() - .headers(headers) - .build() - - return chain.proceed(newRequest) - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt b/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt deleted file mode 100644 index da7863a5..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/MangaCrazy.kt +++ /dev/null @@ -1,15 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -import java.util.Locale - -@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) -internal class MangaCrazy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { - override val sourceLocale: Locale = Locale.ENGLISH -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt b/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt deleted file mode 100644 index a0b4eb28..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/Misskon.kt +++ /dev/null @@ -1,152 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import java.util.EnumSet - -@MangaSourceParser("MISSKON", "MissKon", type = ContentType.HENTAI) -internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MISSKON, 24) { - - override val configKeyDomain = ConfigKey.Domain("misskon.com") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities(isSearchSupported = true) - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - append("/page/$page/") - append("?s=") - append((filter.query?.urlEncoded() ?: "")) - } - order == SortOrder.POPULARITY -> { - append("/top3/") - } - else -> { - append("/page/$page") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select("article.item-list").map { article -> - val titleEl = article.selectFirst(".post-box-title")!! - val href = titleEl.selectFirst("a")?.attrAsRelativeUrl("href") - ?: article.parseFailed("Cannot find manga link") - - Manga( - id = generateUid(href), - title = titleEl.text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = article.selectFirst(".post-thumbnail img")?.absUrl("data-src").orEmpty(), - tags = setOf(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val postInnerEl = doc.selectFirst("article > .post-inner")!! - - return manga.copy( - tags = postInnerEl.select(".post-tag > a").mapToSet { a -> - MangaTag( - key = a.text().lowercase(), - title = a.text(), - source = source - ) - }, - chapters = listOf( - MangaChapter( - id = manga.id, - title = "Oneshot", // 1 album, idk - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source - ) - ) - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val basePageUrl = doc.selectFirst("link[rel=canonical]")?.absUrl("href") - ?: chapter.url.toAbsoluteUrl(domain) - - val pages = mutableListOf<MangaPage>() - val pageLinks = doc.select("div.post-inner div.page-link:nth-child(1) .post-page-numbers") - - if (pageLinks.isEmpty()) { - // Single page gallery - return doc.select("div.post-inner > div.entry > p > img") - .mapNotNull { img -> img.absUrl("data-src") } - .mapIndexed { i, url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } - - // Multi-page gallery - pageLinks.forEachIndexed { index, pageEl -> - val pageDoc = when (index) { - 0 -> doc - else -> { - val url = "$basePageUrl${pageEl.text()}/" - webClient.httpGet(url).parseHtml() - } - } - - pages.addAll( - pageDoc.select("div.post-inner > div.entry > p > img") - .mapNotNull { img -> img.absUrl("data-src") } - .map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - ) - } - return pages - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt b/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt deleted file mode 100644 index 2f4ccaf8..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/Multporn.kt +++ /dev/null @@ -1,235 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.network.CommonHeaders - -import tsuki.model.* -import tsuki.util.* - -import okhttp3.Headers -import org.jsoup.nodes.Document -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MULTPORN", "Multporn", type = ContentType.HENTAI) -internal class Multporn(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MULTPORN, 42) { - - override val configKeyDomain = ConfigKey.Domain("multporn.net") - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add(CommonHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36") - .add(CommonHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=UTF-8") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.UPDATED, - SortOrder.UPDATED_ASC, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - init { - setFirstPage(0) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableLocales = setOf( - Locale("en"), - Locale("de"), - Locale("ru"), - Locale("zh"), - Locale("es"), - ), - availableContentTypes = EnumSet.of( - ContentType.COMICS, - ContentType.HENTAI, - ), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - when { - !query.isNullOrEmpty() -> { - append("/search?search_api_views_fulltext=") - val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> - part.urlEncoded() - } - append(encodedQuery) - append("&undefined=Search") - append("&page=$page") - } - - filter.tags.isNotEmpty() -> { - val tag = filter.tags.first() - append("/category/") - append(tag.key) - - append("?sort_by=") - append( - when (order) { - SortOrder.NEWEST -> "created" - else -> "title" // default - } - ) - - append("&page=0,") - append(page) - } - - else -> { - append("/new") - append("?type=") - if (filter.types.isNotEmpty()) { - filter.types.oneOrThrowIfMany()?.let { - append( - when (it) { - ContentType.COMICS -> "1" - ContentType.HENTAI -> "2" - else -> "All" // all - }, - ) - } - } else append("All") - - - filter.locale?.let { - append("&language=") - append( - when (it) { - Locale("en") -> "1" - Locale("de") -> "2" - Locale("ru") -> "3" - Locale("zh") -> "4" - Locale("es") -> "5" - else -> "All" - } - ) - } - - append("&field_user_discription_value=All") - - append("&sort_by=") - append( - when (order) { - SortOrder.NEWEST -> "created&sort_order=DESC" - SortOrder.NEWEST_ASC -> "created&sort_order=ASC" - SortOrder.UPDATED -> "changed&sort_order=DESC" - SortOrder.UPDATED_ASC -> "changed&sort_order=ASC" - else -> "created&sort_order=DESC" // default - } - ) - - append("&undefined=Apply") - append("&page=$page") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".masonry-item").map { div -> - val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") - val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() - Manga( - id = generateUid(href), - title = div.select(".views-field-title").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = coverUrl, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val authors = (doc.select(".field:has(.field-label:contains(Author:)) .links a").map { it.text() } + - parseUnlabelledAuthorNames(doc)).distinct() - - val tags = listOf("Tags", "Section", "Characters") - .flatMap { type -> - doc.select(".field:has(.field-label:contains($type:)) .links a").map { it.text() } - } - .distinct() - .map { tag -> - MangaTag( - title = tag, - key = tag.lowercase().replace(" ", "_"), - source = source, - ) - }.toSet() - - val isOngoing = doc.select(".field .links a").any { it.text() == "Ongoings" } - - return manga.copy( - authors = authors.toSet(), - tags = tags, - description = buildString { - append("Pages: ") - append(doc.select(".jb-image img").size) - append("\n\n") - doc.select(".field:has(.field-label:contains(Section:)) .links a").joinTo(this, prefix = "Section: ") { it.text() } - doc.select(".field:has(.field-label:contains(Characters:)) .links a").joinTo(this, prefix = "\n\nCharacters: ") { it.text() } - }, - state = if (isOngoing) MangaState.ONGOING else MangaState.FINISHED, - chapters = listOf( - MangaChapter( - id = generateUid(manga.url), - title = "Oneshot", - number = 1f, - volume = 0, - url = manga.url, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ) - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(".jb-image img").map { img -> - val url = img.attrAsAbsoluteUrl("src") - .replace("/styles/juicebox_2k/public", "") - .substringBefore("?") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun parseUnlabelledAuthorNames(document: Document): List<String> { - val authorClasses = listOf( - "field-name-field-author", - "field-name-field-authors-gr", - "field-name-field-img-group", - "field-name-field-hentai-img-group", - "field-name-field-rule-63-section" - ) - return authorClasses.flatMap { className -> - document.select(".$className a").map { it.text().trim() } - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt deleted file mode 100644 index 932d94b8..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/NHentaiParser.kt +++ /dev/null @@ -1,1085 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.MangaParserAuthProvider - -import tsuki.model.* -import tsuki.util.* - -import org.json.JSONObject -import java.util.EnumSet -import java.util.Locale - - -// removed first server, it has always problems -private const val TWO_IMG_SERVER = "2" -private const val THREE_IMG_SERVER = "3" -private const val FOUR_IMG_SERVER = "4" - -@MangaSourceParser("NHENTAI", "NHentai.net", type = ContentType.HENTAI) -internal class NHentaiParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.NHENTAI, 25), - MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain("nhentai.net") - - private var imageServer: String = "" - private var thumbServer: String = "" - - private var displayFullTitle: Boolean = true - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, // latest (default) - SortOrder.POPULARITY, // popular (all time) - SortOrder.POPULARITY_TODAY, - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - TWO_IMG_SERVER to "First server", - THREE_IMG_SERVER to "Second server", - FOUR_IMG_SERVER to "Third server", - ), - defaultValue = TWO_IMG_SERVER, - ) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(preferredServerKey) - } - - private val selectedServer: String - get() = config[preferredServerKey].toString() - - companion object { - val LANG_MAP = mapOf( - "en" to "english", - "ja" to "japanese", - "zh" to "chinese", - ) - } - - private val nhTagMap: Map<String, String> by lazy { - val raw = """ -2937=big breasts -35762=sole female -35763=sole male -8010=group -14283=anal -19440=lolicon -24201=stockings -10314=schoolgirl uniform -13720=nakadashi -29859=blowjob -8378=glasses -20905=full color -32341=shotacon -27553=rape -15658=bondage -23895=yaoi -27473=mosaic censorship -13989=ahegao -22942=incest -21712=males only -1207=milf -19018=dark skin -22945=double penetration -25614=paizuri -20035=x-ray -779=futanari -23237=tankoubon -21572=multi-work series -20525=defloration -14971=sex toys -8653=netorare -3735=swimsuit -19954=yuri -15348=ffm threesome -8368=full censorship -15408=femdom -29224=impregnation -29013=dilf -85295=twintails -31044=collar -85288=ponytail -24380=pantyhose -9260=cheating -28031=sister -16828=hairy -31880=bbm -30555=big penis -15782=crossdressing -31775=tentacles -27384=mind break -19175=bikini -8739=story arc -30473=muscle -24102=lactation -7752=schoolboy uniform -20617=mind control -9083=big ass -29023=tomgirl -81774=kemonomimi -1590=sweating -9162=masturbation -7256=mmf threesome -28550=teacher -190=maid -8693=uncensored -19899=exhibitionism -6343=pregnant -8050=females only -6817=unusual pupils -25871=lingerie -10988=anthology -20282=footjob -15853=mother -15785=harem -14072=huge breasts -30035=gender bender -1643=kissing -130025=anal intercourse -1033=handjob -12824=condom -31386=catgirl -10476=urination -3666=garter belt -26130=fingering -81707=beauty mark -22079=drugs -105833=gloves -4435=gag -25601=small breasts -5820=piercing -12695=prostitution -16228=demon girl -7155=cunnilingus -22950=tanlines -832=elf -31012=blindfold -17773=kimono -2820=scat -29182=blackmail -23132=bunny girl -32484=stomach deformation -2515=virginity -27063=filming -7142=bbw -21989=inflation -88846=horns -104227=tail -26953=bukkake -28800=bloomers -25050=gyaru -24676=rimjob -23632=big areolae -16533=sleeping -73750=bald -18567=monster -35972=sole dickgirl -18328=thigh high boots -5810=strap-on -29565=school swimsuit -32996=deepthroat -370=business suit -7550=monster girl -1067=inseki -50585=webtoon -12523=bestiality -27697=leotard -30645=dick growth -29631=inverted nipples -29366=tomboy -24412=bodysuit -15492=scanmark -9406=enema -35970=dickgirl on dickgirl -29399=daughter -18613=military -11941=replaced -6525=nurse -9661=cervix penetration -33129=slave -4573=corruption -5529=urethra insertion -10542=snuff -683=squirting -51399=crotch tattoo -122908=very long hair -7838=magical girl -24726=apron -23183=breast expansion -20074=latex -28426=hairy armpits -27217=guro -31285=fox girl -106119=no penetration -24764=drunk -9990=prostate massage -35968=dickgirl on male -2956=old man -32752=shibari -6900=miko -2153=wings -706=birth -10794=breast feeding -14069=ryona -25822=smell -5357=humiliation -5962=spanking -2531=transformation -21538=bike shorts -31101=incomplete -32745=chikan -16236=shemale -36957=bisexual -26952=tall girl -25663=oppai loli -7995=big nipples -32602=fisting -106733=hair buns -1088=bdsm -21283=masked face -15225=blowjob face -2633=leg lock -27378=artbook -35971=male on dickgirl -27112=tiara -107705=facial hair -24933=eyepatch -4549=torture -30206=tribadism -1037=oni -89056=hidden sex -13136=facesitting -3391=nun -25766=gokkun -5200=pegging -17531=cosplaying -28521=voyeurism -19479=nipple fuck -17349=tracksuit -22221=blood -50505=oyakodon -50486=tail plug -560=twins -23965=chloroform -15425=vore -25457=possession -129668=eye-covering bang -24984=orgasm denial -144644=extraneous ads -28589=hotpants -17752=foot licking -32282=piss drinking -19390=cousin -32589=feminization -11376=body modification -20362=gyaru-oh -28778=large insertions -27720=smegma -10811=double vaginal -3614=triple penetration -3455=chastity belt -2452=scar -31319=yandere -7354=amputee -28335=giantess -26848=waitress -28349=cbt -24967=sumata -104893=vtuber -8516=emotionless sex -26380=demon -17591=robot -17801=solo action -13640=frottage -25996=gaping -23035=aunt -23967=huge penis -31846=body writing -25744=cheerleader -24708=cowgirl -25085=swinging -18322=brother -101724=leash -10354=milking -97795=pixie cut -11089=body swap -32224=eggs -10606=pasties -3947=onahole -14573=tall man -10604=dog -14362=low lolicon -15242=lab coat -4935=farting -13468=shimapan -5620=double anal -14138=freckles -50390=josou seme -15119=dog girl -93324=fishnets -22025=prolapse -15471=asphyxiation -21774=human pet -31337=kunoichi -15712=eyemask -30126=big clit -92409=thick eyebrows -109360=cumflation -7208=catboy -31687=randoseru -24529=bride -19561=big balls -24450=chinese dress -121738=focus anal -22967=diaper -29347=miniguy -29001=parasite -25296=armpit licking -6220=orc -7546=witch -30895=sunglasses -7372=corset -28119=nose hook -8429=machine -7684=armpit sex -14516=wolf girl -15045=niece -13882=tutor -8391=public use -30811=christmas -104245=small penis -266=sundress -17501=phimosis -17800=tickling -25794=widow -7288=vomit -1215=unusual teeth -72471=dickgirls only -107503=soushuuhen -138044=exposed clothing -1352=slime -31986=age regression -23917=long tongue -24115=angel -114993=shimaidon -13722=moral degeneration -26898=age progression -27120=selfcest -7577=vampire -17676=ghost -88103=clothed female nude male -13515=coach -141098=nipple stimulation -9116=unbirth -5936=time stop -18420=all the way through -72139=clothed paizuri -27530=ball sucking -16518=coprophagia -28869=stuck in wall -2527=bandages -24621=insect -11399=metal armor -106006=large tattoo -3843=fundoshi -20120=multiple paizuri -8400=goblin -129321=mesuiki -124610=mouth mask -10693=dougi -31371=mecha girl -21450=minigirl -10685=double blowjob -118056=petplay -20789=policewoman -3031=underwater -31173=first person perspective -78262=shaved head -19064=pubic stubble -14280=bunny boy -25949=gothic lolita -23463=wrestling -16947=horse -11247=skinsuit -11073=living clothes -30786=watermarked -23073=assjob -52826=dark sclera -107478=drill hair -23225=non-h -109930=domination loss -20170=poor grammar -138200=gender change -16759=artistcg -80978=nudity only -15749=oil -30176=petrification -25848=human cattle -559=ttf threesome -14010=snake girl -11276=multiple penises -90671=original -18024=touhou project -1841=kantai collection -35605=fate grand order -20925=the idolmaster -972=granblue fantasy -78245=azur lane -17137=neon genesis evangelion -3185=love live -391=girls und panzer -11219=pokemon -15021=sailor moon -4505=mahou shoujo lyrical nanoha -128408=blue archive -10222=fate stay night -27431=to love-ru -13159=naruto -123503=genshin impact -3984=sword art online -3603=street fighter -22174=one piece -16285=puella magi madoka magica -91195=princess connect -12232=my hero academia -3163=king of fighters -26172=k-on -7259=touken ranbu -19080=code geass -37544=love live sunshine -17077=cardcaptor sakura -27547=the melancholy of haruhi suzumiya -13508=final fantasy vii -10954=shingeki no kyojin -25430=vocaloid -32687=free -4577=toheart2 -22146=dead or alive -20025=gochuumon wa usagi desu ka -8485=dragon ball z -5037=bleach -3218=bakemonogatari -12624=ore no imouto ga konna ni kawaii wake ga nai -37109=kono subarashii sekai ni syukufuku o -4369=monster hunter -127065=hololive -74788=girls frontline -24886=fate kaleid liner prisma illya -6999=toaru kagaku no railgun -22032=boku wa tomodachi ga sukunai -18350=ragnarok online -21674=dragon quest iii -14345=ojamajo doremi -7832=darkstalkers -24135=ah my goddess -32394=samurai spirits -1283=queens blade -16639=haikyuu -13924=yu-gi-oh -79467=kimetsu no yaiba -18238=danganronpa -26336=yu-gi-oh zexal -16984=persona 4 -18569=kuroko no basuke -1910=smile precure -30587=sakura taisen -16166=mahou sensei negima -12285=ranma 12 -8470=infinite stratos -32363=toaru majutsu no index -22708=saki -8708=to heart -108082=arknights -16707=detective conan -22210=guilty gear -947=gundam seed destiny -22677=tenchi muyo -23429=pretty cure -18512=strike witches -31027=lucky star -7408=league of legends -394=love hina -23201=kanon -27704=amagami -127052=nijisanji -70802=kemono friends -52098=persona 5 -22215=super robot wars -27567=hayate no gotoku -35251=osomatsu-san -7633=pripara -34823=ensemble stars -37914=re zero kara hajimeru isekai seikatsu -74918=bang dream -15041=martian successor nadesico -24783=dragon ball -120519=love live nijigasaki high school idol club -2803=love plus -5085=senki zesshou symphogear -28474=zero no tsukaima -15197=gundam build fighters -15427=dragon quest iv -1163=rozen maiden -23859=yu-gi-oh arc-v -75023=dragon quest xi -2112=dungeon ni deai o motomeru no wa machigatteiru darou ka -36418=voiceroid -28281=mitsudomoe -11624=the legend of zelda -14694=fullmetal alchemist -16847=dragon quest v -2497=urusei yatsura -5671=tengen toppa gurren lagann -22754=amagi brilliant park -20606=tsukihime -5165=gundam build fighters try -4114=macross frontier -20763=inazuma eleven -14550=sister princess -19083=jojos bizarre adventure -21052=fate hollow ataraxia -29922=teitoku -51810=gudao -16643=producer -13848=reimu hakurei -25125=asuka langley soryu -17279=sakuya izayoi -10496=patchouli knowledge -37739=shielder -3206=shinji ikari -38068=gran -4675=sanae kochiya -21779=rei ayanami -14040=fate testarossa -3870=flandre scarlet -23902=remilia scarlet -21688=atago -11373=marisa kirisame -35128=kashima -17154=sakura kinomoto -31462=satori komeiji -30080=kaga -10802=alice margatroid -17017=aya shameimaru -17862=yukari yakumo -5340=shimakaze -18935=nanoha takamachi -18896=shirou emiya -16555=rin tosaka -16130=rito yuuki -15890=reisen udongein inaba -7724=takao -27060=jeanne darc -78989=jeanne alter -7718=naruto uzumaki -5337=nami -22975=chun-li -17502=illyasviel von einzbern -20111=tifa lockhart -21131=youmu konpaku -18026=kazuto kirigaya -92923=shikikan -29856=saber -71442=minamoto no raikou -1843=asuna yuuki -51419=gudako -7488=mai shiranui -9835=koishi komeiji -16916=kasumi -30026=maki nishikino -143975=sensei -26906=izuku midoriya -37275=scathach -7696=momiji inubashiri -38039=astolfo -27794=mikoto misaka -20062=hamakaze -78285=artoria pendragon -34860=katsuki bakugou -3328=homura akemi -37687=djeeta -32200=suzuya -21108=rin shibuya -35964=nico yazawa -27494=levi ackerman -609=eren jaeger -11920=sakura haruno -20427=sailor mercury -24714=chino kafuu -31456=mikan yuuki -866=koyomi araragi -12149=kyousuke kousaka -277=haruka nanase -19926=haruna -3763=haruhi suzumiya -26427=mio akiyama -25439=hinata hyuga -17811=ran yakumo -14857=kongou -18548=kotori minami -32364=rider -15641=madoka kaname -2613=hong meiling -491=makoto tachibana -20702=koakuma -15315=tomoyo daidouji -10730=shigure -14265=touma kamijou -80311=bb -4203=mami tomoe -37706=kazuma satou -33070=umi sonoda -27172=yuyuko saigyouji -3353=yuuka kazami -2078=nagato -6311=arisu tachibana -647=belldandy -9274=maya -24889=sena kashiwazaki -15125=golden darkness -6555=sailor jupiter -25695=mika jougasaki -50929=shuten douji -33077=sailor mars -8293=minami nitta -7451=lelouch vi britannia -389=rika jougasaki -22469=prinz eugen -16108=azusa nakano -12812=tenryuu -7311=ami mizuno -6642=byakuren hijiri -7097=suwako moriya -19172=miki hoshii -9657=ayane -29433=c.c. -25220=sakura matou -14499=tsunade -10665=tenshi hinanai -16564=miku hatsune -29190=kallen stadtfeld -3312=kirino kousaka -1234=yuki nagato -26261=ranma saotome -19002=rin kaenbyou -12748=nico robin -32765=rin matsuoka -4241=fumika sagisawa -1729=tamaki kousaka -23997=ruri gokou -29684=sailor venus -19160=nitori kawashiro -27302=uzuki shimamura -23216=android 18 -8489=hibiki -7333=suguha kirigaya -1267=kodaka hasegawa -2345=morrigan aensland -9371=yamato -26087=inazuma -27532=archer -26587=miho nishizumi -12346=utsuho reiuji -37108=megumin -22407=takane shijou -15914=sasuke uchiha -2774=kyouko sakura -80930=abigail williams -81288=gudao | ritsuka fujimaru -73756=nightingale -6109=eri ayase -27492=akagi -17899=sakura kasugano -32137=cirno -11760=yui kotegawa -75029=eli ayase -11740=sailor moon -49158=narmaya -29693=ikazuchi -126586=aether -20918=iori minase -24832=misato katsuragi -2883=kasen ibara -6932=souji okita -28555=tamamo-no-mae -14428=kokoa hoto -26783=taihou -12763=rumia -401=nakoruru -72475=musashi miyamoto -23122=maho nishizumi -29188=eirin yagokoro -466=usagi tsukino -29638=kyon -15995=makoto kino -11744=amatsukaze -6175=cammy white -30331=ichika orimura -23473=mikuru asahina -28807=ruri hoshino -2572=hatate himekaidou -15291=chen -23386=fujiwara no mokou -9237=shoukaku -28763=tewi inaba -23851=gilgamesh -10672=aqua -9702=ro-500 -31074=keine kamishirasawa -32443=charlotte dunois -2196=sayaka miki -1645=zuikaku -5925=akatsuki -4196=hestia -33171=shiho nishizumi -19534=hayate yagami -79507=belfast -12433=kaede takagaki -12872=warrior -8170=len kagamine -50415=rem -14409=momoka sakurai -2211=mari illustrious makinami -99075=kokkoro -1907=rei hino -15651=miyu edelfelt -26169=musashi -8053=lum -50596=you watanabe -9883=kagami hiiragi -24509=darjeeling -11992=lala satalin deviluke -32683=hachiman hikigaya -31076=kuroko shirai -20836=red saber -12902=isuzu sento -10379=bianca whitaker -16181=nozomi toujou -27774=bismarck -28219=yui hirasawa -1271=momo velia deviluke -49852=subaru natsuki -5918=shinobu oshino -28056=link -25605=rangiku matsumoto -35313=cagliostro -18453=hero -75102=nozomi tojo -20722=mutsu -29170=yuma tsukumo -9486=nue houjuu -33049=ritsuko akizuki -23626=murakumo -20323=tsumugi kotobuki -16566=ritsu tainaka -14016=yuu narukami -11609=yoko ritona -107011=chloe von einzbern -52132=riko sakurauchi -32114=onpu segawa -11924=kagerou imaizumi - """.trimIndent() - raw.lineSequence() - .mapNotNull { line -> - val parts = line.split('=') - if (parts.size == 2 && parts[0].isNotBlank() && parts[1].isNotBlank()) { - parts[0] to parts[1] - } else null - }.toMap() - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = nhTagMap.map { (id, name) -> - MangaTag( - title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, - key = id, // numeric tag ID - source = source, - ) - }.toSet(), - availableLocales = setOf( - Locale.ENGLISH, - Locale.JAPANESE, - Locale.CHINESE, - ), - ) - - private suspend fun ensureNhConfig() { - if (imageServer.isBlank() || thumbServer.isBlank()) { - try { - val json = webClient.httpGet("https://$domain/api/v2/config").parseJson() - val imgServers = json.getJSONArray("image_servers") - .let { (0 until it.length()).map { i -> it.getString(i) } } - val thbServers = json.getJSONArray("thumb_servers") - .let { (0 until it.length()).map { i -> it.getString(i) } } - imageServer = imgServers[selectedServer.toInt() - 1] - thumbServer = thbServers[selectedServer.toInt() - 1] - } catch (_: Exception) { - imageServer = "https://i$selectedServer.nhentai.net" - thumbServer = "https://t$selectedServer.nhentai.net" - } - } - } - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - ensureNhConfig() - - val directId = extractGalleryId(filter.query) - if (directId != null) { - return try { - listOf(fetchMangaById(directId)) - } catch (_: Exception) { - emptyList() - } - } - - val selectedTag = filter.tags.firstOrNull() - if (selectedTag != null) { - val url = "https://$domain/api/v2/galleries/tagged?tag_id=${selectedTag.key}&page=$page" - val json = webClient.httpGet(url).parseJson() - val items = json.getJSONArray("result") - return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } - } - - val lang = filter.locale?.let { LANG_MAP[it.language] } - val searchQuery = buildString { - if (!filter.query.isNullOrBlank()) { - append("title:\"${filter.query}\"") - } - if (lang != null) { - if (isNotEmpty()) append(" ") - append("language:$lang") - } - } - - val url: String - when (order) { - SortOrder.UPDATED -> { - url = if (searchQuery.isEmpty()) { - "https://$domain/api/v2/galleries?page=$page" - } else { - "https://$domain/api/v2/search?query=${searchQuery.urlEncoded()}&sort=date&page=$page" - } - } - else -> { - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.POPULARITY_TODAY -> "popular-today" - SortOrder.POPULARITY_WEEK -> "popular-week" - SortOrder.POPULARITY_MONTH -> "popular-month" - else -> "popular" - } - val finalQuery = searchQuery.ifBlank { - lang?.let { "language:$it" } ?: "language:english" - } - url = "https://$domain/api/v2/search?query=${finalQuery.urlEncoded()}&sort=$sort&page=$page" - } - } - - val json = webClient.httpGet(url).parseJson() - val items = json.getJSONArray("result") - return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } - } - - private fun extractGalleryId(query: String?): Int? { - if (query.isNullOrBlank()) return null - val trimmed = query.trim() - return when { - trimmed.startsWith("id:") -> trimmed.removePrefix("id:").toIntOrNull() - trimmed.all(Char::isDigit) -> trimmed.toIntOrNull() - else -> null - } - } - - private suspend fun fetchMangaById(id: Int): Manga { - ensureNhConfig() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - return json.toMangaDetails() - } - - override suspend fun getDetails(manga: Manga): Manga { - ensureNhConfig() - val id = manga.url.removeSurrounding("/g/", "/").toInt() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - return json.toMangaDetails().copy( - coverUrl = manga.coverUrl?.ifBlank { - "${thumbServer}/${json.getJSONObject("cover").optString("path")}" - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - ensureNhConfig() - val id = chapter.url.removeSurrounding("/g/", "/").toInt() - val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() - val pagesArray = json.getJSONArray("pages") - return (0 until pagesArray.length()).map { i -> - val pageObj = pagesArray.getJSONObject(i) - val path = pageObj.getString("path") - MangaPage( - id = generateUid(path), - url = "$imageServer/$path", - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - private fun JSONObject.toManga(): Manga { - val id = getInt("id") - val titleObj = optJSONObject("title") - val rawTitle = titleObj?.optString("english") - ?: titleObj?.optString("pretty") - ?: optString("english_title") - ?: optString("japanese_title") - ?: "Gallery $id" - - val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() - val thumbPath = optJSONObject("thumbnail")?.optString("path") - ?: optString("thumbnail").takeIf { !it.isNullOrBlank() } - ?: "galleries/$id/thumb.webp" - - return Manga( - id = generateUid("/g/$id/"), - title = title, - altTitles = emptySet(), - url = "/g/$id/", - publicUrl = "https://$domain/g/$id/", - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$thumbPath", - tags = extractTags(), - state = null, - authors = emptySet(), - source = source, - ) - } - - private fun JSONObject.toMangaDetails(): Manga { - val id = getInt("id") - val titleObj = optJSONObject("title") - val rawTitle = titleObj?.optString("english") - ?: titleObj?.optString("japanese") - ?: titleObj?.optString("pretty") - ?: "Gallery $id" - val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() - - val thumbPath = getJSONObject("thumbnail").getString("path") - val coverPath = getJSONObject("cover").optString("path", thumbPath) - val numPages = optInt("num_pages") - val uploadDate = optLong("upload_date") * 1000 - - val tags = extractTags() - val artists = getTagsOfType("artist") - val groups = getTagsOfType("group") - val author = artists.joinToString(", ").ifBlank { groups.joinToString(", ") } - - var langName: String? - var translated = false - val languageCandidates = mutableListOf<String>() - - for (tag in tags) { - val parts = tag.key.split(":", limit = 2) - if (parts.size == 2 && parts[0] == "language") { - val name = parts[1] - when { - name.equals("translated", ignoreCase = true) -> translated = true - else -> languageCandidates.add(name) - } - } - } - - langName = languageCandidates.firstOrNull { it.equals("chinese", ignoreCase = true) } - ?: languageCandidates.firstOrNull { it.equals("japanese", ignoreCase = true) } - ?: languageCandidates.firstOrNull { it.equals("english", ignoreCase = true) } - ?: languageCandidates.firstOrNull() - - val description = buildString { - langName?.let { - val displayLang = it.replaceFirstChar { c -> c.uppercase() } - append(if (translated) "$displayLang (translated)" else displayLang) - append("\n") - } - append("Pages: $numPages") - } - - return Manga( - id = generateUid("/g/$id/"), - title = title, - altTitles = listOfNotNull(titleObj?.optString("japanese")).toSet(), - url = "/g/$id/", - publicUrl = "https://$domain/g/$id/", - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$coverPath", - largeCoverUrl = "$imageServer/$coverPath", - tags = tags, - state = null, - authors = setOfNotNull(author.ifBlank { null }), - description = description, - chapters = listOf( - MangaChapter( - id = generateUid("/g/$id/"), - title = title, - number = 1f, - volume = 0, - url = "/g/$id/", - scanlator = groups.joinToString(", ").ifBlank { null }, - uploadDate = uploadDate, - branch = null, - source = source, - ) - ), - source = source, - ) - } - - private fun JSONObject.extractTags(): Set<MangaTag> { - val tagsArray = optJSONArray("tags") ?: return emptySet() - return (0 until tagsArray.length()).map { i -> - val tag = tagsArray.getJSONObject(i) - val name = tag.getString("name") - MangaTag( - title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, - key = "${tag.getString("type")}:$name", - source = source, - ) - }.toSet() - } - - private fun JSONObject.getTagsOfType(type: String): List<String> { - val tagsArray = optJSONArray("tags") ?: return emptyList() - return (0 until tagsArray.length()).mapNotNull { i -> - val tag = tagsArray.getJSONObject(i) - if (tag.getString("type") == type) tag.getString("name") else null - } - } - - private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") - private fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() - - override val authUrl: String get() = "https://$domain" - override suspend fun isAuthorized(): Boolean = true - override suspend fun getUsername(): String = "" -} diff --git a/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt deleted file mode 100644 index 7c1d4a8b..00000000 --- a/src/main/kotlin/tsuki/site/all/hentais/NHentaiXXXParser.kt +++ /dev/null @@ -1,187 +0,0 @@ -package tsuki.site.all.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.ErrorMessages -import tsuki.parsers.GalleryAdultsParser - -import tsuki.model.* -import tsuki.util.* - -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("NHENTAIXXX", "NHentai.xxx", type = ContentType.HENTAI) -internal class NHentaiXXXParser(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.NHENTAIXXX, "nhentai.xxx", 25) { - - val supportedImageFormats = setOf("jpg", "webp", "jpeg", "png", "bmp") - - override val selectGallery = "div.galleries_box .gallery_item, #related-container .gallery_item" - override val selectGalleryLink = "a" - override val selectGalleryTitle = ".caption" - override val pathTagUrl = "/tags/popular?page=" - override val selectTitle = "h1" - override val selectTags = "div.tags_items" - override val selectTag = ".tags:contains(Tags)" - override val selectAuthor = ".tags:contains(Artists) span.tag_name" - override val selectLanguageChapter = - ".tags:contains(Languages) a:not([href=\"/language/translated/\"]) span.tag_name" - override val idImg = "fimg" - - override val availableSortOrders: Set<SortOrder> = - EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableLocales = setOf(Locale.ENGLISH, Locale.JAPANESE, Locale.CHINESE), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - val tags = filter.tags - val lang = filter.locale - if (tags.isNotEmpty() && lang != null) { - throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) - } - - when { - lang != null -> { - append("/language/") - append(lang.toLanguagePath()) - append("/?") - } - - else -> { - append("/search/?key=") - - val joiner = StringUtil.StringJoiner("+") - tags.forEach { tag -> - joiner.add(tag.title) - } - - if (!query.isNullOrEmpty()) { - joiner.add(query.urlEncoded()) - } - append(joiner.complete()) - } - } - - when (order) { - SortOrder.POPULARITY, SortOrder.RELEVANCE -> append("&sort=popular") - SortOrder.UPDATED -> {} - else -> {} - } - - if (page > 1) { - append("&page=") - append(page.toString()) - } - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun parseMangaList(doc: Document): List<Manga> { - return doc.select(selectGallery).map { div -> - val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(selectGalleryTitle).text().cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = div.selectFirstOrThrow(selectGalleryImg).src(), - tags = emptySet(), - state = null, - authors = emptySet(), - largeCoverUrl = null, - description = null, - chapters = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() - val firstPageUrl = doc.requireElementById(idImg).requireSrc() - return (1..totalPages).map { - val url = replacePageNumber(firstPageUrl, it) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - return if (hasError(page.url)) { - findAlternativeUrl(page.url) ?: page.url - } else { - page.url - } - } - - private suspend fun hasError(url: String): Boolean { - return try { - webClient.httpHead(url) - false - } catch (e: Exception) { - true - } - } - - private suspend fun findAlternativeUrl(originalUrl: String): String? { - val lastSegment = originalUrl.substringAfterLast("/") - val oldFormat = lastSegment.substringAfterLast(".", "") - - return supportedImageFormats.firstNotNullOfOrNull { format -> - val newUrl = originalUrl.replace(".$oldFormat", ".$format") - try { - webClient.httpHead(newUrl) - newUrl - } catch (e: Exception) { - null // Skip errors and continue checking other formats - } - } - } - - override fun Element.parseTags() = select("a").mapToSet { - val key = it.attr("href").removeSuffix('/').substringAfterLast('/') - val name = it.selectFirst(".tag_name")?.text() ?: it.text() - MangaTag( - key = key, - title = name.toTitleCase(sourceLocale), - source = source, - ) - } - - private fun replacePageNumber(url: String, newPageNumber: Int): String { - val lastSegment = url.substringAfterLast("/") - val extension = lastSegment.substringAfterLast(".", "") - - return if (extension.isNotEmpty()) { - url.substringBeforeLast("/") + "/$newPageNumber.$extension" - } else { - url.substringBeforeLast("/") + "/$newPageNumber" - } - } -} \ No newline at end of file From bf546a9ce66200a0ee9d52242a56379a3d205779 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:02:01 +0200 Subject: [PATCH 072/434] Add files via upload --- .../kotlin/tsuki/site/all/nsfw/AsmHentai.kt | 46 + .../kotlin/tsuki/site/all/nsfw/BuonDua.kt | 19 + .../kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt | 90 ++ .../kotlin/tsuki/site/all/nsfw/HentaiFox.kt | 136 +++ .../tsuki/site/all/nsfw/HitomiLaParser.kt | 798 ++++++++++++ .../kotlin/tsuki/site/all/nsfw/IMHentai.kt | 248 ++++ .../kotlin/tsuki/site/all/nsfw/Kiutaku.kt | 35 + .../kotlin/tsuki/site/all/nsfw/MangaCrazy.kt | 15 + .../kotlin/tsuki/site/all/nsfw/Misskon.kt | 152 +++ .../kotlin/tsuki/site/all/nsfw/Multporn.kt | 235 ++++ .../tsuki/site/all/nsfw/NHentaiParser.kt | 1085 +++++++++++++++++ .../tsuki/site/all/nsfw/NHentaiXXXParser.kt | 187 +++ 12 files changed, 3046 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt diff --git a/src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt new file mode 100644 index 00000000..2be0d833 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt @@ -0,0 +1,46 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix + +import org.jsoup.nodes.Element +import java.util.Locale + +@MangaSourceParser("ASMHENTAI", "AsmHentai", type = ContentType.HENTAI) +internal class AsmHentai(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.ASMHENTAI, "asmhentai.com") { + + override val selectGallery = ".preview_item" + override val selectGalleryLink = ".image a" + override val selectGalleryImg = ".image img" + override val pathTagUrl = "/tags/?page=" + override val selectTags = ".tags_page ul.tags" + override val selectAuthor = "div.tags:contains(Artists:) .tag_list a span.tag" + override val idImg = "fimg" + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale.CHINESE, + Locale("tr"), + ), + ) + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".tag")?.html()?.substringBefore("<") ?: it.html().substringBefore("<") + MangaTag( + key = key, + title = name, + source = source, + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt b/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt new file mode 100644 index 00000000..3d7d8e6a --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt @@ -0,0 +1,19 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.GalleryParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) +internal class BuonDua(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( + "buondua.com", + "buondua.us", + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt new file mode 100644 index 00000000..c560534a --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt @@ -0,0 +1,90 @@ +package tsuki.site.all.nsfw + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.SortOrder + +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import java.util.EnumSet +import java.util.Locale + + +@MangaSourceParser("HENTAIENVY", "HentaiEnvy", type = ContentType.HENTAI) +internal class HentaiEnvy(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIENVY, "hentaienvy.com", pageSize = 24) { + override val selectGalleryLink = "a" + override val selectGalleryTitle = "div.title" + override val selectTags = ".tags_items" + override val selectTag = ".gt_right_tags ul:contains(Tags:)" + override val selectAuthor = ".gt_right_tags ul:contains(Artists:) a" + override val selectLanguageChapter = ".gt_right_tags ul:contains(Languages:) a" + override val idImg = "fimg" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.JAPANESE, + Locale.CHINESE, + Locale("es"), + Locale("ru"), + Locale("ko"), + Locale.GERMAN, + Locale("pt"), + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("/search/?s_key=") + append(query?.urlEncoded()) + append("&") + } + + else -> { + if (filter.tags.isNotEmpty()) { + if (filter.locale != null) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) + } + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + if (order == SortOrder.POPULARITY) { + append("/popular") + } + append("/?") + } + } else if (filter.locale != null) { + val locale = filter.locale + append("/language/") + append(locale?.toLanguagePath()) + append("/?") + } else { + append("/?") + } + } + } + append("page=") + append(page) + + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt new file mode 100644 index 00000000..4d309389 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt @@ -0,0 +1,136 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.SortOrder + +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("HENTAIFOX", "HentaiFox", type = ContentType.HENTAI) +internal class HentaiFox(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIFOX, "hentaifox.com") { + override val selectGallery = ".lc_galleries .thumb, .related_galleries .thumb" + override val pathTagUrl = "/tags/popular/pag/" + override val selectTags = ".list_tags" + override val selectTag = "ul.tags" + override val selectLanguageChapter = "ul.languages a.tag_btn" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("/search/?q=") + append(query?.urlEncoded()) + if (page > 1) { + append("&page=") + append(page.toString()) + } + } + + else -> { + if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { + append("/search/?q=") + append(buildQuery(filter.tags, filter.locale)) + if (page > 1) { + append("&page=") + append(page.toString()) + } + + if (order == SortOrder.POPULARITY) { + append("&sort=popular") + } + } else if (filter.tags.isNotEmpty()) { + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + } + append("/") + if (order == SortOrder.POPULARITY) { + append("popular/") + } + + if (page > 1) { + append("/pag/") + append(page.toString()) + append("/") + } + } else if (filter.locale != null) { + val locale = filter.locale + append("/language/") + append(locale?.toLanguagePath()) + append("/") + if (order == SortOrder.POPULARITY) { + append("popular/") + } + + if (page > 1) { + append("/pag/") + append(page.toString()) + append("/") + } + } else { + if (page > 2) { + append("/pag/") + append(page.toString()) + append("/") + } else if (page > 1) { + append("/page/") + append(page.toString()) + append("/") + } + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".list_tag")?.text() ?: it.html().substringBefore("<") + MangaTag( + key = key, + title = name, + source = source, + ) + } + + private fun buildQuery(tags: Collection<MangaTag>, language: Locale?): String { + val joiner = StringUtil.StringJoiner(" ") + tags.forEach { tag -> + joiner.add(tag.key) + } + language?.let { lc -> + joiner.add(lc.toLanguagePath()) + } + return joiner.complete() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt new file mode 100644 index 00000000..6bbe24e6 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt @@ -0,0 +1,798 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseBytes +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.runCatchingCancellable +import tsuki.util.selectFirstOrThrow +import tsuki.util.splitByWhitespace +import tsuki.util.toAbsoluteUrl +import tsuki.util.toCamelCase +import tsuki.util.toTitleCase +import tsuki.util.urlDecode + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.security.MessageDigest +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedList +import java.util.Locale +import kotlin.math.min + +@OptIn(ExperimentalUnsignedTypes::class) +@MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) +internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser(context, MangaParserSource.HITOMILA) { + override val configKeyDomain = ConfigKey.Domain("hitomi.la") + + private val cdnDomain = "gold-usergeneratedcontent.net" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private val ltnBaseUrl get() = "https://ltn.$cdnDomain" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.POPULARITY_YEAR, + ) + + private val localeMap: Map<Locale, String> = mapOf( + Locale.forLanguageTag("id") to "indonesian", + Locale.forLanguageTag("jv") to "javanese", + Locale.forLanguageTag("ca") to "catalan", + Locale.forLanguageTag("ceb") to "cebuano", + Locale.forLanguageTag("cs") to "czech", + Locale.forLanguageTag("da") to "danish", + Locale.forLanguageTag("de") to "german", + Locale.forLanguageTag("et") to "estonian", + Locale.ENGLISH to "english", + Locale.forLanguageTag("es") to "spanish", + Locale.forLanguageTag("eo") to "esperanto", + Locale.forLanguageTag("fr") to "french", + Locale.forLanguageTag("it") to "italian", + Locale.forLanguageTag("hi") to "hindi", + Locale.forLanguageTag("hu") to "hungarian", + Locale.forLanguageTag("pl") to "polish", + Locale.forLanguageTag("pt") to "portuguese", + Locale.forLanguageTag("vi") to "vietnamese", + Locale.forLanguageTag("tr") to "turkish", + Locale.forLanguageTag("ru") to "russian", + Locale.forLanguageTag("uk") to "ukrainian", + Locale.forLanguageTag("ar") to "arabic", + Locale.KOREAN to "korean", + Locale.CHINESE to "chinese", + Locale.JAPANESE to "japanese", + ) + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableLocales = localeMap.keys, + ) + + private fun Locale?.getSiteLang(): String = when (this) { + null -> "all" + else -> localeMap[this] ?: "all" + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { + ('a'..'z').map { alphabet -> + async { + val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() + + doc.select(".posts > li").mapNotNull { element -> + val num = + element.ownText().let { + Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 + } + + if (num > 100) { + val url = element.selectFirst("a") + val href = + url?.attrAsRelativeUrl("href") + ?: return@mapNotNull null + + MangaTag( + title = url.ownText().toTagTitle(), + key = href.tagUrlToTag(), + source = source, + ) + } else { + null + } + } + } + }.awaitAll().flatten().toSet() + } + + private var cachedSearchIds: List<Int> = emptyList() + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> = when { + filter.query.isNullOrEmpty() -> { + if (filter.tags.isEmpty()) { + when (order) { + SortOrder.POPULARITY_TODAY -> { + getGalleryIDsFromNozomi( + "popular", + "today", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_WEEK -> { + getGalleryIDsFromNozomi( + "popular", + "week", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_MONTH -> { + getGalleryIDsFromNozomi( + "popular", + "month", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + SortOrder.POPULARITY_YEAR -> { + getGalleryIDsFromNozomi( + "popular", + "year", + filter.locale.getSiteLang(), + offset.nextOffsetRange(), + ) + } + + else -> { + getGalleryIDsFromNozomi(null, "index", filter.locale.getSiteLang(), offset.nextOffsetRange()) + } + } + } else { + if (offset == 0) { + cachedSearchIds = + hitomiSearch( + filter.tags.joinToString(" ") { it.key }, + order, + filter.locale.getSiteLang(), + ).toList() + } + cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) + } + } + + else -> { + if (offset == 0) { + cachedSearchIds = hitomiSearch(filter.query ?: "", order).toList() + } + cachedSearchIds.subList(offset, min(offset + 25, cachedSearchIds.size)) + } + }.toMangaList() + + private fun Int.nextOffsetRange(): LongRange { + val bytes = this * 4L + return bytes.until(bytes + 100L) + } + + private suspend fun hitomiSearch( + query: String, + sortByPopularity: SortOrder = SortOrder.UPDATED, + language: String = "all", + ): Set<Int> = + coroutineScope { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .splitByWhitespace() + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList<String>() + val negativeTerms = LinkedList<String>() + + for (term in terms) { + if (term.startsWith("-")) { + negativeTerms.push(term.removePrefix("-")) + } else if (term.isNotBlank()) { + positiveTerms.push(term) + } + } + + val positiveResults = positiveTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val negativeResults = negativeTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val results = when { + sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) + sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) + sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) + sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) + sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) + else -> emptySet() + }.toMutableSet() + + fun filterPositive(newResults: Set<Int>) { + when { + results.isEmpty() -> results.addAll(newResults) + else -> results.retainAll(newResults) + } + } + + fun filterNegative(newResults: Set<Int>) { + results.removeAll(newResults) + } + + // positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + // negative results + negativeResults.forEach { + filterNegative(it.await()) + } + + results + } + + // search.js + private suspend fun getGalleryIDsForQuery( + query: String, + language: String = "all", + ): Set<Int> { + query.replace("_", " ").let { + if (it.indexOf(':') > -1) { + val sides = it.split(":") + val ns = sides[0] + var tag = sides[1] + + var area: String? = ns + var lang = language + when (ns) { + "female", "male" -> { + area = "tag" + tag = it + } + + "language" -> { + area = null + lang = tag + tag = "index" + } + } + + return getGalleryIDsFromNozomi(area, tag, lang) + } + + val key = hashTerm(it) + val node = getGalleryNodeAtAddress(0) + val data = bSearch(key, node) ?: return emptySet() + + return getGalleryIDsFromData(data) + } + } + + private suspend fun getGalleryIDsFromData(data: Pair<Long, Int>): Set<Int> { + val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.data" + val (offset, length) = data + require(length in 1..100000000) { + "Length $length is too long" + } + + val inbuf = getRangedResponse(url, offset.until(offset + length)) + + val galleryIDs = mutableSetOf<Int>() + + val buffer = + ByteBuffer + .wrap(inbuf) + .order(ByteOrder.BIG_ENDIAN) + + val numberOfGalleryIDs = buffer.int + + val expectedLength = numberOfGalleryIDs * 4 + 4 + + require(numberOfGalleryIDs in 1..10000000) { + "number_of_galleryids $numberOfGalleryIDs is too long" + } + require(inbuf.size == expectedLength) { + "inbuf.byteLength ${inbuf.size} != expected_length $expectedLength" + } + + for (i in 0.until(numberOfGalleryIDs)) + galleryIDs.add(buffer.int) + + return galleryIDs + } + + private suspend fun bSearch( + key: UByteArray, + node: Node, + ): Pair<Long, Int>? { + fun compareArrayBuffers( + dv1: UByteArray, + dv2: UByteArray, + ): Int { + val top = min(dv1.size, dv2.size) + + for (i in 0.until(top)) { + if (dv1[i] < dv2[i]) { + return -1 + } else if (dv1[i] > dv2[i]) { + return 1 + } + } + + return 0 + } + + fun locateKey( + key: UByteArray, + node: Node, + ): Pair<Boolean, Int> { + for (i in node.keys.indices) { + val cmpResult = compareArrayBuffers(key, node.keys[i]) + + if (cmpResult <= 0) { + return Pair(cmpResult == 0, i) + } + } + + return Pair(false, node.keys.size) + } + + fun isLeaf(node: Node): Boolean { + for (subnode in node.subNodeAddresses) + if (subnode != 0L) { + return false + } + + return true + } + + if (node.keys.isEmpty()) { + return null + } + + val (there, where) = locateKey(key, node) + if (there) { + return node.datas[where] + } else if (isLeaf(node)) { + return null + } + + val nextNode = getGalleryNodeAtAddress(node.subNodeAddresses[where]) + return bSearch(key, nextNode) + } + + private suspend fun getGalleryIDsFromNozomi( + area: String?, + tag: String, + language: String, + range: LongRange? = null, + ): Set<Int> { + val nozomiAddress = when (area) { + null -> "$ltnBaseUrl/$tag-$language.nozomi" + else -> "$ltnBaseUrl/$area/$tag-$language.nozomi" + } + + val bytes = getRangedResponse(nozomiAddress, range) + val nozomi = mutableSetOf<Int>() + + val arrayBuffer = ByteBuffer + .wrap(bytes) + .order(ByteOrder.BIG_ENDIAN) + + while (arrayBuffer.hasRemaining()) + nozomi.add(arrayBuffer.int) + + return nozomi + } + + private val galleriesIndexVersion = suspendLazy { + webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() + } + + private data class Node( + val keys: List<UByteArray>, + val datas: List<Pair<Long, Int>>, + val subNodeAddresses: List<Long>, + ) + + private fun decodeNode(data: ByteArray): Node { + val buffer = ByteBuffer + .wrap(data) + .order(ByteOrder.BIG_ENDIAN) + + val uData = data.toUByteArray() + + val numberOfKeys = buffer.int + val keys = ArrayList<UByteArray>() + + for (i in 0.until(numberOfKeys)) { + val keySize = buffer.int + + check(keySize in 1..32) { "Invalid key size $keySize" } + + keys.add(uData.sliceArray(buffer.position().until(buffer.position() + keySize))) + buffer.position(buffer.position() + keySize) + } + + val numberOfDatas = buffer.int + val datas = ArrayList<Pair<Long, Int>>() + + for (i in 0.until(numberOfDatas)) { + val offset = buffer.long + val length = buffer.int + + datas.add(Pair(offset, length)) + } + + val numberOfSubNodeAddresses = 16 + 1 + val subNodeAddresses = ArrayList<Long>() + + for (i in 0.until(numberOfSubNodeAddresses)) { + val subNodeAddress = buffer.long + subNodeAddresses.add(subNodeAddress) + } + + return Node(keys, datas, subNodeAddresses) + } + + private suspend fun getGalleryNodeAtAddress(address: Long): Node { + val url = "$ltnBaseUrl/galleriesindex/galleries.${galleriesIndexVersion.get()}.index" + + val nodedata = getRangedResponse(url, address.until(address + 464)) + + return decodeNode(nodedata) + } + + private suspend fun getRangedResponse( + url: String, + range: LongRange? = null, + ): ByteArray { + val rangeHeaders = when (range) { + null -> Headers.headersOf() + else -> Headers.headersOf("Range", "bytes=${range.first}-${range.last}") + } + + return webClient.httpGet(url, rangeHeaders).parseBytes() + } + + private fun hashTerm(term: String): UByteArray { + return sha256(term.toByteArray()).copyOfRange(0, 4).toUByteArray() + } + + private fun sha256(data: ByteArray): ByteArray { + return MessageDigest.getInstance("SHA-256").digest(data) + } + + private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope { + map { id -> + async { + runCatchingCancellable { + val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { + val baseUri = it.request.url.toString() + val html = it.parseRaw() + Jsoup.parse(rewriteTnPaths(html), baseUri) + } + + Manga( + id = generateUid(id.toString()), + title = doc.selectFirstOrThrow("h1").text(), + url = id.toString(), + coverUrl = + "https:" + + doc.selectFirstOrThrow("picture > img") + .attr("data-src"), + publicUrl = + doc.selectFirstOrThrow("h1 > a") + .attrAsRelativeUrl("href") + .toAbsoluteUrl(domain), + authors = emptySet(), + tags = emptySet(), + contentRating = ContentRating.ADULT, + rating = RATING_UNKNOWN, + altTitles = emptySet(), + state = null, + source = source, + ) + }.getOrNull() + } + }.awaitAll().filterNotNull() + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + val author = + json.optJSONArray("artists") + ?.mapJSON { it.getString("artist").toCamelCase() } + ?.joinToString() + + return manga.copy( + title = json.getString("title"), + largeCoverUrl = + json.getJSONArray("files").getJSONObject(0).let { + val hash = it.getString("hash") + val imageId = imageIdFromHash(hash) + val subDomain = 'a' + subdomainOffset(imageId) + + "https://${subDomain}tn.$cdnDomain/webpbigtn/${thumbPathFromHash(hash)}/$hash.webp" + }, + authors = setOfNotNull(author), + publicUrl = json.getString("galleryurl").toAbsoluteUrl(domain), + tags = + buildSet + { + json.optJSONArray("characters") + ?.mapToTags("character") + ?.let(::addAll) + json.optJSONArray("tags") + ?.mapToTags("tag") + ?.let(::addAll) + json.optJSONArray("artists") + ?.mapToTags("artist") + ?.let(::addAll) + json.optJSONArray("parodys") + ?.mapToTags("parody") + ?.let(::addAll) + json.optJSONArray("groups") + ?.mapToTags("group") + ?.let(::addAll) + }, + chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + url = manga.url, + title = json.getStringOrNull("title"), + scanlator = json.getString("type").toTitleCase(), + number = 1f, + volume = 0, + branch = json.getString("language_localname"), + source = source, + uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), + ), + ), + ) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) + + private fun JSONArray.mapToTags(key: String): Set<MangaTag> { + val tags = ArraySet<MangaTag>(length()) + mapJSON { + MangaTag( + title = + it.getString(key).toCamelCase().let { title -> + if (it.getStringOrNull("female")?.toIntOrNull() == 1) { + "$title ♀" + } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { + "$title ♂" + } else { + title + } + }, + key = it.getString("url").tagUrlToTag(), + source = source, + ).let(tags::add) + } + return tags + } + + private fun String.tagUrlToTag(): String { + val urlContent = this.split("/") + val ns = urlContent[1] + val tag = + urlContent[2] + .substringBeforeLast("-") + .urlDecode() + .replace(" ", "_") + + return if (tag.split(":")[0] in listOf("female", "male")) { + tag + } else { + "$ns:$tag" + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${seed.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + + // any better way to get List<Int> from this json? + return json.getJSONArray("related").let { + 0.until(it.length()).map { i -> it.getInt(i) } + }.toMangaList() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") + .parseRaw() + .substringAfter("var galleryinfo = ") + .let(::JSONObject) + + return json.getJSONArray("files").mapJSON { image -> + val hash = image.getString("hash") + val commonId = commonImageId() + val imageId = imageIdFromHash(hash) + val subDomain = subdomainOffset(imageId) + 1 + val thumbSubdomain = 'a' + subdomainOffset(imageId) + MangaPage( + id = generateUid(hash), + url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", + preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", + source = source, + ) + } + } + +// / ---> + + private var scriptLastRetrieval: Long = -1L + private val mutex = Mutex() + private var subdomainOffsetDefault = 0 + private val subdomainOffsetMap = mutableMapOf<Int, Int>() + private var commonImageId = "" + + private suspend fun refreshScript() = mutex.withLock { + if (scriptLastRetrieval == -1L || (scriptLastRetrieval + 60000) < System.currentTimeMillis()) { + val ggScript = webClient.httpGet("$ltnBaseUrl/gg.js?_=${System.currentTimeMillis()}").parseRaw() + + subdomainOffsetDefault = Regex("var o = (\\d)").find(ggScript)!!.groupValues[1].toInt() + val o = Regex("o = (\\d); break;").find(ggScript)!!.groupValues[1].toInt() + + subdomainOffsetMap.clear() + Regex("case (\\d+):").findAll(ggScript).forEach { + val case = it.groupValues[1].toInt() + subdomainOffsetMap[case] = o + } + + commonImageId = Regex("b: '(.+)'").find(ggScript)!!.groupValues[1] + + scriptLastRetrieval = System.currentTimeMillis() + } + } + + // m <-- gg.js + private suspend fun subdomainOffset(imageId: Int): Int { + refreshScript() + return subdomainOffsetMap[imageId] ?: subdomainOffsetDefault + } + + // b <-- gg.js + private suspend fun commonImageId(): String { + refreshScript() + return commonImageId + } + + // s <-- gg.js + private fun imageIdFromHash(hash: String): Int { + val match = Regex("(..)(.)$").find(hash) + return match!!.groupValues.let { it[2] + it[1] }.toInt(16) + } + + // real_full_path_from_hash <-- common.js + private fun thumbPathFromHash(hash: String): String { + return hash.replace(Regex("""^.*(..)(.)$"""), "$2/$1") + } + + // rewrite_tn_paths <-- common.js + private suspend fun rewriteTnPaths(html: String): String { + val thumbUrlRegex = Regex( + """(?<protocol>//)(?<host>[a-z0-9.-]+\.(?:hitomi\.la|${Regex.escape(cdnDomain)}))/(?<pathAfterHost>(?:avif|webp)?(?:small)?(?:big|small|medium)tn/[0-9a-f]/[0-9a-f]{2}/[0-9a-f]{64}\.(?:webp|avif|gif|png|jpe?g))""", + ) + + var resultHtml = html + thumbUrlRegex.findAll(html).forEach { matchResult -> + val originalUrl = matchResult.value + val groups = matchResult.groups + + val pathAfterHost = groups["pathAfterHost"]?.value ?: return@forEach + val newTnSubdomain = subdomainFromURL(originalUrl, "tn") + val correctedUrl = "${groups["protocol"]!!.value}$newTnSubdomain.$cdnDomain/$pathAfterHost" + + if (originalUrl != correctedUrl) { + resultHtml = resultHtml.replace(originalUrl, correctedUrl) + } + } + return resultHtml + } + + private suspend fun subdomainFromURL(url: String, base: String?): String { + val resultSubdomain = base ?: "b" + + // This regex extracts the last 3 hex characters from the hash in the URL + // The hash is 64 characters, so we look for the 61st character onward + val hashRegex = Regex("""/([0-9a-f]{61}[0-9a-f]{3})[./]""") + val fullHashMatch = hashRegex.find(url) + ?: // If no hash is found, default to "a" + base (typically "atn") + return "a$resultSubdomain" + + val fullHash = fullHashMatch.groupValues[1] + + val lastThreeChars = fullHash.takeLast(3) + val lastDigit = lastThreeChars.last() + val lastTwoDigits = lastThreeChars.take(2) + + val imageId = "$lastDigit$lastTwoDigits".toIntOrNull(16) + + return if (imageId != null) { + ('a' + subdomainOffset(imageId)).toString() + resultSubdomain + } else { + "a$resultSubdomain" + } + } + + private fun String.toTagTitle(): String { + return toCamelCase() + .replace("♂", "(male)") + .replace("♀", "(female)") + } +} diff --git a/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt new file mode 100644 index 00000000..74875748 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt @@ -0,0 +1,248 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import androidx.collection.ArraySet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Element +import java.util.ArrayList +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("IMHENTAI", "ImHentai", type = ContentType.HENTAI) +internal class ImHentai(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.IMHENTAI, 20) { + + override val availableSortOrders: Set<SortOrder> = + EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.RATING) + + override val configKeyDomain = ConfigKey.Domain("imhentai.xxx") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale("es"), + Locale.FRENCH, + Locale("kr"), + Locale.GERMAN, + Locale("ru"), + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.COMICS, + ContentType.IMAGE_SET, + ContentType.ARTIST_CG, + ContentType.GAME_CG, + ), + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + val url = buildString { + append("https://") + append(domain) + append("/search/?page=") + append(page.toString()) + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("&key=") + append(query?.urlEncoded()) + } + + else -> { + + if (filter.tags.isNotEmpty()) { + append("&key=") + filter.tags.joinTo(this, separator = ",") { it.key } + } + + var types = "&m=1&d=1&w=1&i=1&a=1&g=1" + if (filter.types.isNotEmpty()) { + types = "&m=0&d=0&w=0&i=0&a=0&g=0" + filter.types.forEach { + when (it) { + ContentType.MANGA -> types = types.replace("&m=0", "&m=1") + ContentType.DOUJINSHI -> types = types.replace("&d=0", "&d=1") + ContentType.COMICS -> types = types.replace("&w=0", "&w=1") + ContentType.IMAGE_SET -> types = types.replace("&i=0", "&i=1") + ContentType.ARTIST_CG -> types = types.replace("&a=0", "&a=1") + ContentType.GAME_CG -> types = types.replace("&g=0", "&g=1") + else -> {} + } + } + } + append(types) + + + var lang = "&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1" + filter.locale?.let { + lang = "&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0" + lang = lang.replace("${it.language}=0", "${it.language}=1") + } + append(lang) + + when (order) { + SortOrder.UPDATED -> append("<=1&pp=0") + SortOrder.POPULARITY -> append("<=0&pp=1") + SortOrder.RATING -> append("<=0&pp=0") + else -> append("<=1&pp=0") + } + } + } + + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.galleries div.thumb").map { div -> + val a = div.selectFirstOrThrow(".inner_thumb a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = a.selectFirst("img")?.src(), + title = div.selectFirst(".caption")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + //Tags are deliberately reduced because there are too many and this slows down the application. + //only the most popular ones are taken. + private suspend fun fetchAvailableTags(): Set<MangaTag> { + return coroutineScope { + (1..3).map { page -> + async { fetchTagsPage(page) } + } + }.awaitAll().flattenTo(ArraySet(360)) + } + + private suspend fun fetchTagsPage(page: Int): Set<MangaTag> { + val url = "https://$domain/tags/popular/?page=$page" + val root = webClient.httpGet(url).parseHtml() + return root.parseTags() + } + + private fun Element.parseTags() = select("div.stags a.tag_btn").mapToSet { + val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') + MangaTag( + key = href, + title = it.selectFirstOrThrow("h3.list_tag").text().toTitleCase(sourceLocale), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val author = doc.selectFirst("li:contains(Artists) a.tag")?.ownTextOrNull() + manga.copy( + tags = doc.body().select("li:contains(Tags) a.tag").mapNotNullToSet { + val href = it.attr("href").substringAfterLast("tag/").substringBeforeLast('/') + MangaTag( + key = href, + title = it.ownText().toTitleCase(sourceLocale), + source = source, + ) + }, + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = null, + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0, + branch = doc.selectFirst("li:contains(Language) a.tag")?.ownTextOrNull()?.toTitleCase(sourceLocale), + source = source, + ), + ), + ) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.related") + return root.select("div.thumb").mapNotNull { div -> + val a = div.selectFirstOrThrow(".inner_thumb a") + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = a.selectFirst("img")?.src(), + title = div.selectFirst(".caption")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val totalPages = doc.selectFirstOrThrow(".pages").text().replace("Pages: ", "").toInt() + val baseImg = doc.requireElementById("append_thumbs").selectFirstOrThrow("img") + val baseUrl = baseImg.selectFirstParentOrThrow("a").attrAsRelativeUrl("href").replace("/1/", + "/$/" + ) + val baseThumbUrl = baseImg.src()?.replace("/1t.", $$"/$t.") + val pages = ArrayList<MangaPage>(totalPages) + repeat(totalPages) { i -> + val url = baseUrl.replace("$", (i + 1).toString()) + pages.add( + MangaPage( + id = generateUid(url), + url = url, + preview = baseThumbUrl?.replace("$", (i + 1).toString()), + source = source, + ), + ) + } + return pages + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + val img = doc.body().requireElementById("gimg") + return img.requireSrc() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt b/src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt new file mode 100644 index 00000000..1b07b72a --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt @@ -0,0 +1,35 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.network.CommonHeaders +import tsuki.parsers.GalleryParser + +import okhttp3.Interceptor +import okhttp3.Response + +@MangaSourceParser("KIUTAKU", "Kiutaku", type = ContentType.OTHER) +internal class Kiutaku(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.KIUTAKU, "kiutaku.com") { + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url.toString() + + val headers = if (url.contains("wp-content")) { + request.headers.newBuilder() + .removeAll(CommonHeaders.REFERER) + .build() + } else { + request.headers + } + + val newRequest = request.newBuilder() + .headers(headers) + .build() + + return chain.proceed(newRequest) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt new file mode 100644 index 00000000..cf2b8e77 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt @@ -0,0 +1,15 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.parsers.MadaraParser + +import java.util.Locale + +@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) +internal class MangaCrazy(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { + override val sourceLocale: Locale = Locale.ENGLISH +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt new file mode 100644 index 00000000..31b15685 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt @@ -0,0 +1,152 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import java.util.EnumSet + +@MangaSourceParser("MISSKON", "MissKon", type = ContentType.HENTAI) +internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MISSKON, 24) { + + override val configKeyDomain = ConfigKey.Domain("misskon.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities(isSearchSupported = true) + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/page/$page/") + append("?s=") + append((filter.query?.urlEncoded() ?: "")) + } + order == SortOrder.POPULARITY -> { + append("/top3/") + } + else -> { + append("/page/$page") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("article.item-list").map { article -> + val titleEl = article.selectFirst(".post-box-title")!! + val href = titleEl.selectFirst("a")?.attrAsRelativeUrl("href") + ?: article.parseFailed("Cannot find manga link") + + Manga( + id = generateUid(href), + title = titleEl.text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = article.selectFirst(".post-thumbnail img")?.absUrl("data-src").orEmpty(), + tags = setOf(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val postInnerEl = doc.selectFirst("article > .post-inner")!! + + return manga.copy( + tags = postInnerEl.select(".post-tag > a").mapToSet { a -> + MangaTag( + key = a.text().lowercase(), + title = a.text(), + source = source + ) + }, + chapters = listOf( + MangaChapter( + id = manga.id, + title = "Oneshot", // 1 album, idk + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0L, + branch = null, + source = source + ) + ) + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val basePageUrl = doc.selectFirst("link[rel=canonical]")?.absUrl("href") + ?: chapter.url.toAbsoluteUrl(domain) + + val pages = mutableListOf<MangaPage>() + val pageLinks = doc.select("div.post-inner div.page-link:nth-child(1) .post-page-numbers") + + if (pageLinks.isEmpty()) { + // Single page gallery + return doc.select("div.post-inner > div.entry > p > img") + .mapNotNull { img -> img.absUrl("data-src") } + .mapIndexed { i, url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + // Multi-page gallery + pageLinks.forEachIndexed { index, pageEl -> + val pageDoc = when (index) { + 0 -> doc + else -> { + val url = "$basePageUrl${pageEl.text()}/" + webClient.httpGet(url).parseHtml() + } + } + + pages.addAll( + pageDoc.select("div.post-inner > div.entry > p > img") + .mapNotNull { img -> img.absUrl("data-src") } + .map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + ) + } + return pages + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt new file mode 100644 index 00000000..cade229a --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt @@ -0,0 +1,235 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MULTPORN", "Multporn", type = ContentType.HENTAI) +internal class Multporn(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MULTPORN, 42) { + + override val configKeyDomain = ConfigKey.Domain("multporn.net") + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add(CommonHeaders.USER_AGENT, "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/81.0.4044.122 Safari/537.36") + .add(CommonHeaders.CONTENT_TYPE, "application/x-www-form-urlencoded; charset=UTF-8") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + init { + setFirstPage(0) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableLocales = setOf( + Locale("en"), + Locale("de"), + Locale("ru"), + Locale("zh"), + Locale("es"), + ), + availableContentTypes = EnumSet.of( + ContentType.COMICS, + ContentType.HENTAI, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + when { + !query.isNullOrEmpty() -> { + append("/search?search_api_views_fulltext=") + val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> + part.urlEncoded() + } + append(encodedQuery) + append("&undefined=Search") + append("&page=$page") + } + + filter.tags.isNotEmpty() -> { + val tag = filter.tags.first() + append("/category/") + append(tag.key) + + append("?sort_by=") + append( + when (order) { + SortOrder.NEWEST -> "created" + else -> "title" // default + } + ) + + append("&page=0,") + append(page) + } + + else -> { + append("/new") + append("?type=") + if (filter.types.isNotEmpty()) { + filter.types.oneOrThrowIfMany()?.let { + append( + when (it) { + ContentType.COMICS -> "1" + ContentType.HENTAI -> "2" + else -> "All" // all + }, + ) + } + } else append("All") + + + filter.locale?.let { + append("&language=") + append( + when (it) { + Locale("en") -> "1" + Locale("de") -> "2" + Locale("ru") -> "3" + Locale("zh") -> "4" + Locale("es") -> "5" + else -> "All" + } + ) + } + + append("&field_user_discription_value=All") + + append("&sort_by=") + append( + when (order) { + SortOrder.NEWEST -> "created&sort_order=DESC" + SortOrder.NEWEST_ASC -> "created&sort_order=ASC" + SortOrder.UPDATED -> "changed&sort_order=DESC" + SortOrder.UPDATED_ASC -> "changed&sort_order=ASC" + else -> "created&sort_order=DESC" // default + } + ) + + append("&undefined=Apply") + append("&page=$page") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".masonry-item").map { div -> + val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") + val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() + Manga( + id = generateUid(href), + title = div.select(".views-field-title").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = coverUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val authors = (doc.select(".field:has(.field-label:contains(Author:)) .links a").map { it.text() } + + parseUnlabelledAuthorNames(doc)).distinct() + + val tags = listOf("Tags", "Section", "Characters") + .flatMap { type -> + doc.select(".field:has(.field-label:contains($type:)) .links a").map { it.text() } + } + .distinct() + .map { tag -> + MangaTag( + title = tag, + key = tag.lowercase().replace(" ", "_"), + source = source, + ) + }.toSet() + + val isOngoing = doc.select(".field .links a").any { it.text() == "Ongoings" } + + return manga.copy( + authors = authors.toSet(), + tags = tags, + description = buildString { + append("Pages: ") + append(doc.select(".jb-image img").size) + append("\n\n") + doc.select(".field:has(.field-label:contains(Section:)) .links a").joinTo(this, prefix = "Section: ") { it.text() } + doc.select(".field:has(.field-label:contains(Characters:)) .links a").joinTo(this, prefix = "\n\nCharacters: ") { it.text() } + }, + state = if (isOngoing) MangaState.ONGOING else MangaState.FINISHED, + chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + title = "Oneshot", + number = 1f, + volume = 0, + url = manga.url, + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ) + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".jb-image img").map { img -> + val url = img.attrAsAbsoluteUrl("src") + .replace("/styles/juicebox_2k/public", "") + .substringBefore("?") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseUnlabelledAuthorNames(document: Document): List<String> { + val authorClasses = listOf( + "field-name-field-author", + "field-name-field-authors-gr", + "field-name-field-img-group", + "field-name-field-hentai-img-group", + "field-name-field-rule-63-section" + ) + return authorClasses.flatMap { className -> + document.select(".$className a").map { it.text().trim() } + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt new file mode 100644 index 00000000..e39dc81f --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt @@ -0,0 +1,1085 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.MangaParserAuthProvider + +import tsuki.model.* +import tsuki.util.* + +import org.json.JSONObject +import java.util.EnumSet +import java.util.Locale + + +// removed first server, it has always problems +private const val TWO_IMG_SERVER = "2" +private const val THREE_IMG_SERVER = "3" +private const val FOUR_IMG_SERVER = "4" + +@MangaSourceParser("NHENTAI", "NHentai.net", type = ContentType.HENTAI) +internal class NHentaiParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.NHENTAI, 25), + MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain("nhentai.net") + + private var imageServer: String = "" + private var thumbServer: String = "" + + private var displayFullTitle: Boolean = true + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, // latest (default) + SortOrder.POPULARITY, // popular (all time) + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + TWO_IMG_SERVER to "First server", + THREE_IMG_SERVER to "Second server", + FOUR_IMG_SERVER to "Third server", + ), + defaultValue = TWO_IMG_SERVER, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + private val selectedServer: String + get() = config[preferredServerKey].toString() + + companion object { + val LANG_MAP = mapOf( + "en" to "english", + "ja" to "japanese", + "zh" to "chinese", + ) + } + + private val nhTagMap: Map<String, String> by lazy { + val raw = """ +2937=big breasts +35762=sole female +35763=sole male +8010=group +14283=anal +19440=lolicon +24201=stockings +10314=schoolgirl uniform +13720=nakadashi +29859=blowjob +8378=glasses +20905=full color +32341=shotacon +27553=rape +15658=bondage +23895=yaoi +27473=mosaic censorship +13989=ahegao +22942=incest +21712=males only +1207=milf +19018=dark skin +22945=double penetration +25614=paizuri +20035=x-ray +779=futanari +23237=tankoubon +21572=multi-work series +20525=defloration +14971=sex toys +8653=netorare +3735=swimsuit +19954=yuri +15348=ffm threesome +8368=full censorship +15408=femdom +29224=impregnation +29013=dilf +85295=twintails +31044=collar +85288=ponytail +24380=pantyhose +9260=cheating +28031=sister +16828=hairy +31880=bbm +30555=big penis +15782=crossdressing +31775=tentacles +27384=mind break +19175=bikini +8739=story arc +30473=muscle +24102=lactation +7752=schoolboy uniform +20617=mind control +9083=big ass +29023=tomgirl +81774=kemonomimi +1590=sweating +9162=masturbation +7256=mmf threesome +28550=teacher +190=maid +8693=uncensored +19899=exhibitionism +6343=pregnant +8050=females only +6817=unusual pupils +25871=lingerie +10988=anthology +20282=footjob +15853=mother +15785=harem +14072=huge breasts +30035=gender bender +1643=kissing +130025=anal intercourse +1033=handjob +12824=condom +31386=catgirl +10476=urination +3666=garter belt +26130=fingering +81707=beauty mark +22079=drugs +105833=gloves +4435=gag +25601=small breasts +5820=piercing +12695=prostitution +16228=demon girl +7155=cunnilingus +22950=tanlines +832=elf +31012=blindfold +17773=kimono +2820=scat +29182=blackmail +23132=bunny girl +32484=stomach deformation +2515=virginity +27063=filming +7142=bbw +21989=inflation +88846=horns +104227=tail +26953=bukkake +28800=bloomers +25050=gyaru +24676=rimjob +23632=big areolae +16533=sleeping +73750=bald +18567=monster +35972=sole dickgirl +18328=thigh high boots +5810=strap-on +29565=school swimsuit +32996=deepthroat +370=business suit +7550=monster girl +1067=inseki +50585=webtoon +12523=bestiality +27697=leotard +30645=dick growth +29631=inverted nipples +29366=tomboy +24412=bodysuit +15492=scanmark +9406=enema +35970=dickgirl on dickgirl +29399=daughter +18613=military +11941=replaced +6525=nurse +9661=cervix penetration +33129=slave +4573=corruption +5529=urethra insertion +10542=snuff +683=squirting +51399=crotch tattoo +122908=very long hair +7838=magical girl +24726=apron +23183=breast expansion +20074=latex +28426=hairy armpits +27217=guro +31285=fox girl +106119=no penetration +24764=drunk +9990=prostate massage +35968=dickgirl on male +2956=old man +32752=shibari +6900=miko +2153=wings +706=birth +10794=breast feeding +14069=ryona +25822=smell +5357=humiliation +5962=spanking +2531=transformation +21538=bike shorts +31101=incomplete +32745=chikan +16236=shemale +36957=bisexual +26952=tall girl +25663=oppai loli +7995=big nipples +32602=fisting +106733=hair buns +1088=bdsm +21283=masked face +15225=blowjob face +2633=leg lock +27378=artbook +35971=male on dickgirl +27112=tiara +107705=facial hair +24933=eyepatch +4549=torture +30206=tribadism +1037=oni +89056=hidden sex +13136=facesitting +3391=nun +25766=gokkun +5200=pegging +17531=cosplaying +28521=voyeurism +19479=nipple fuck +17349=tracksuit +22221=blood +50505=oyakodon +50486=tail plug +560=twins +23965=chloroform +15425=vore +25457=possession +129668=eye-covering bang +24984=orgasm denial +144644=extraneous ads +28589=hotpants +17752=foot licking +32282=piss drinking +19390=cousin +32589=feminization +11376=body modification +20362=gyaru-oh +28778=large insertions +27720=smegma +10811=double vaginal +3614=triple penetration +3455=chastity belt +2452=scar +31319=yandere +7354=amputee +28335=giantess +26848=waitress +28349=cbt +24967=sumata +104893=vtuber +8516=emotionless sex +26380=demon +17591=robot +17801=solo action +13640=frottage +25996=gaping +23035=aunt +23967=huge penis +31846=body writing +25744=cheerleader +24708=cowgirl +25085=swinging +18322=brother +101724=leash +10354=milking +97795=pixie cut +11089=body swap +32224=eggs +10606=pasties +3947=onahole +14573=tall man +10604=dog +14362=low lolicon +15242=lab coat +4935=farting +13468=shimapan +5620=double anal +14138=freckles +50390=josou seme +15119=dog girl +93324=fishnets +22025=prolapse +15471=asphyxiation +21774=human pet +31337=kunoichi +15712=eyemask +30126=big clit +92409=thick eyebrows +109360=cumflation +7208=catboy +31687=randoseru +24529=bride +19561=big balls +24450=chinese dress +121738=focus anal +22967=diaper +29347=miniguy +29001=parasite +25296=armpit licking +6220=orc +7546=witch +30895=sunglasses +7372=corset +28119=nose hook +8429=machine +7684=armpit sex +14516=wolf girl +15045=niece +13882=tutor +8391=public use +30811=christmas +104245=small penis +266=sundress +17501=phimosis +17800=tickling +25794=widow +7288=vomit +1215=unusual teeth +72471=dickgirls only +107503=soushuuhen +138044=exposed clothing +1352=slime +31986=age regression +23917=long tongue +24115=angel +114993=shimaidon +13722=moral degeneration +26898=age progression +27120=selfcest +7577=vampire +17676=ghost +88103=clothed female nude male +13515=coach +141098=nipple stimulation +9116=unbirth +5936=time stop +18420=all the way through +72139=clothed paizuri +27530=ball sucking +16518=coprophagia +28869=stuck in wall +2527=bandages +24621=insect +11399=metal armor +106006=large tattoo +3843=fundoshi +20120=multiple paizuri +8400=goblin +129321=mesuiki +124610=mouth mask +10693=dougi +31371=mecha girl +21450=minigirl +10685=double blowjob +118056=petplay +20789=policewoman +3031=underwater +31173=first person perspective +78262=shaved head +19064=pubic stubble +14280=bunny boy +25949=gothic lolita +23463=wrestling +16947=horse +11247=skinsuit +11073=living clothes +30786=watermarked +23073=assjob +52826=dark sclera +107478=drill hair +23225=non-h +109930=domination loss +20170=poor grammar +138200=gender change +16759=artistcg +80978=nudity only +15749=oil +30176=petrification +25848=human cattle +559=ttf threesome +14010=snake girl +11276=multiple penises +90671=original +18024=touhou project +1841=kantai collection +35605=fate grand order +20925=the idolmaster +972=granblue fantasy +78245=azur lane +17137=neon genesis evangelion +3185=love live +391=girls und panzer +11219=pokemon +15021=sailor moon +4505=mahou shoujo lyrical nanoha +128408=blue archive +10222=fate stay night +27431=to love-ru +13159=naruto +123503=genshin impact +3984=sword art online +3603=street fighter +22174=one piece +16285=puella magi madoka magica +91195=princess connect +12232=my hero academia +3163=king of fighters +26172=k-on +7259=touken ranbu +19080=code geass +37544=love live sunshine +17077=cardcaptor sakura +27547=the melancholy of haruhi suzumiya +13508=final fantasy vii +10954=shingeki no kyojin +25430=vocaloid +32687=free +4577=toheart2 +22146=dead or alive +20025=gochuumon wa usagi desu ka +8485=dragon ball z +5037=bleach +3218=bakemonogatari +12624=ore no imouto ga konna ni kawaii wake ga nai +37109=kono subarashii sekai ni syukufuku o +4369=monster hunter +127065=hololive +74788=girls frontline +24886=fate kaleid liner prisma illya +6999=toaru kagaku no railgun +22032=boku wa tomodachi ga sukunai +18350=ragnarok online +21674=dragon quest iii +14345=ojamajo doremi +7832=darkstalkers +24135=ah my goddess +32394=samurai spirits +1283=queens blade +16639=haikyuu +13924=yu-gi-oh +79467=kimetsu no yaiba +18238=danganronpa +26336=yu-gi-oh zexal +16984=persona 4 +18569=kuroko no basuke +1910=smile precure +30587=sakura taisen +16166=mahou sensei negima +12285=ranma 12 +8470=infinite stratos +32363=toaru majutsu no index +22708=saki +8708=to heart +108082=arknights +16707=detective conan +22210=guilty gear +947=gundam seed destiny +22677=tenchi muyo +23429=pretty cure +18512=strike witches +31027=lucky star +7408=league of legends +394=love hina +23201=kanon +27704=amagami +127052=nijisanji +70802=kemono friends +52098=persona 5 +22215=super robot wars +27567=hayate no gotoku +35251=osomatsu-san +7633=pripara +34823=ensemble stars +37914=re zero kara hajimeru isekai seikatsu +74918=bang dream +15041=martian successor nadesico +24783=dragon ball +120519=love live nijigasaki high school idol club +2803=love plus +5085=senki zesshou symphogear +28474=zero no tsukaima +15197=gundam build fighters +15427=dragon quest iv +1163=rozen maiden +23859=yu-gi-oh arc-v +75023=dragon quest xi +2112=dungeon ni deai o motomeru no wa machigatteiru darou ka +36418=voiceroid +28281=mitsudomoe +11624=the legend of zelda +14694=fullmetal alchemist +16847=dragon quest v +2497=urusei yatsura +5671=tengen toppa gurren lagann +22754=amagi brilliant park +20606=tsukihime +5165=gundam build fighters try +4114=macross frontier +20763=inazuma eleven +14550=sister princess +19083=jojos bizarre adventure +21052=fate hollow ataraxia +29922=teitoku +51810=gudao +16643=producer +13848=reimu hakurei +25125=asuka langley soryu +17279=sakuya izayoi +10496=patchouli knowledge +37739=shielder +3206=shinji ikari +38068=gran +4675=sanae kochiya +21779=rei ayanami +14040=fate testarossa +3870=flandre scarlet +23902=remilia scarlet +21688=atago +11373=marisa kirisame +35128=kashima +17154=sakura kinomoto +31462=satori komeiji +30080=kaga +10802=alice margatroid +17017=aya shameimaru +17862=yukari yakumo +5340=shimakaze +18935=nanoha takamachi +18896=shirou emiya +16555=rin tosaka +16130=rito yuuki +15890=reisen udongein inaba +7724=takao +27060=jeanne darc +78989=jeanne alter +7718=naruto uzumaki +5337=nami +22975=chun-li +17502=illyasviel von einzbern +20111=tifa lockhart +21131=youmu konpaku +18026=kazuto kirigaya +92923=shikikan +29856=saber +71442=minamoto no raikou +1843=asuna yuuki +51419=gudako +7488=mai shiranui +9835=koishi komeiji +16916=kasumi +30026=maki nishikino +143975=sensei +26906=izuku midoriya +37275=scathach +7696=momiji inubashiri +38039=astolfo +27794=mikoto misaka +20062=hamakaze +78285=artoria pendragon +34860=katsuki bakugou +3328=homura akemi +37687=djeeta +32200=suzuya +21108=rin shibuya +35964=nico yazawa +27494=levi ackerman +609=eren jaeger +11920=sakura haruno +20427=sailor mercury +24714=chino kafuu +31456=mikan yuuki +866=koyomi araragi +12149=kyousuke kousaka +277=haruka nanase +19926=haruna +3763=haruhi suzumiya +26427=mio akiyama +25439=hinata hyuga +17811=ran yakumo +14857=kongou +18548=kotori minami +32364=rider +15641=madoka kaname +2613=hong meiling +491=makoto tachibana +20702=koakuma +15315=tomoyo daidouji +10730=shigure +14265=touma kamijou +80311=bb +4203=mami tomoe +37706=kazuma satou +33070=umi sonoda +27172=yuyuko saigyouji +3353=yuuka kazami +2078=nagato +6311=arisu tachibana +647=belldandy +9274=maya +24889=sena kashiwazaki +15125=golden darkness +6555=sailor jupiter +25695=mika jougasaki +50929=shuten douji +33077=sailor mars +8293=minami nitta +7451=lelouch vi britannia +389=rika jougasaki +22469=prinz eugen +16108=azusa nakano +12812=tenryuu +7311=ami mizuno +6642=byakuren hijiri +7097=suwako moriya +19172=miki hoshii +9657=ayane +29433=c.c. +25220=sakura matou +14499=tsunade +10665=tenshi hinanai +16564=miku hatsune +29190=kallen stadtfeld +3312=kirino kousaka +1234=yuki nagato +26261=ranma saotome +19002=rin kaenbyou +12748=nico robin +32765=rin matsuoka +4241=fumika sagisawa +1729=tamaki kousaka +23997=ruri gokou +29684=sailor venus +19160=nitori kawashiro +27302=uzuki shimamura +23216=android 18 +8489=hibiki +7333=suguha kirigaya +1267=kodaka hasegawa +2345=morrigan aensland +9371=yamato +26087=inazuma +27532=archer +26587=miho nishizumi +12346=utsuho reiuji +37108=megumin +22407=takane shijou +15914=sasuke uchiha +2774=kyouko sakura +80930=abigail williams +81288=gudao | ritsuka fujimaru +73756=nightingale +6109=eri ayase +27492=akagi +17899=sakura kasugano +32137=cirno +11760=yui kotegawa +75029=eli ayase +11740=sailor moon +49158=narmaya +29693=ikazuchi +126586=aether +20918=iori minase +24832=misato katsuragi +2883=kasen ibara +6932=souji okita +28555=tamamo-no-mae +14428=kokoa hoto +26783=taihou +12763=rumia +401=nakoruru +72475=musashi miyamoto +23122=maho nishizumi +29188=eirin yagokoro +466=usagi tsukino +29638=kyon +15995=makoto kino +11744=amatsukaze +6175=cammy white +30331=ichika orimura +23473=mikuru asahina +28807=ruri hoshino +2572=hatate himekaidou +15291=chen +23386=fujiwara no mokou +9237=shoukaku +28763=tewi inaba +23851=gilgamesh +10672=aqua +9702=ro-500 +31074=keine kamishirasawa +32443=charlotte dunois +2196=sayaka miki +1645=zuikaku +5925=akatsuki +4196=hestia +33171=shiho nishizumi +19534=hayate yagami +79507=belfast +12433=kaede takagaki +12872=warrior +8170=len kagamine +50415=rem +14409=momoka sakurai +2211=mari illustrious makinami +99075=kokkoro +1907=rei hino +15651=miyu edelfelt +26169=musashi +8053=lum +50596=you watanabe +9883=kagami hiiragi +24509=darjeeling +11992=lala satalin deviluke +32683=hachiman hikigaya +31076=kuroko shirai +20836=red saber +12902=isuzu sento +10379=bianca whitaker +16181=nozomi toujou +27774=bismarck +28219=yui hirasawa +1271=momo velia deviluke +49852=subaru natsuki +5918=shinobu oshino +28056=link +25605=rangiku matsumoto +35313=cagliostro +18453=hero +75102=nozomi tojo +20722=mutsu +29170=yuma tsukumo +9486=nue houjuu +33049=ritsuko akizuki +23626=murakumo +20323=tsumugi kotobuki +16566=ritsu tainaka +14016=yuu narukami +11609=yoko ritona +107011=chloe von einzbern +52132=riko sakurauchi +32114=onpu segawa +11924=kagerou imaizumi + """.trimIndent() + raw.lineSequence() + .mapNotNull { line -> + val parts = line.split('=') + if (parts.size == 2 && parts[0].isNotBlank() && parts[1].isNotBlank()) { + parts[0] to parts[1] + } else null + }.toMap() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = nhTagMap.map { (id, name) -> + MangaTag( + title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, + key = id, // numeric tag ID + source = source, + ) + }.toSet(), + availableLocales = setOf( + Locale.ENGLISH, + Locale.JAPANESE, + Locale.CHINESE, + ), + ) + + private suspend fun ensureNhConfig() { + if (imageServer.isBlank() || thumbServer.isBlank()) { + try { + val json = webClient.httpGet("https://$domain/api/v2/config").parseJson() + val imgServers = json.getJSONArray("image_servers") + .let { (0 until it.length()).map { i -> it.getString(i) } } + val thbServers = json.getJSONArray("thumb_servers") + .let { (0 until it.length()).map { i -> it.getString(i) } } + imageServer = imgServers[selectedServer.toInt() - 1] + thumbServer = thbServers[selectedServer.toInt() - 1] + } catch (_: Exception) { + imageServer = "https://i$selectedServer.nhentai.net" + thumbServer = "https://t$selectedServer.nhentai.net" + } + } + } + + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + ensureNhConfig() + + val directId = extractGalleryId(filter.query) + if (directId != null) { + return try { + listOf(fetchMangaById(directId)) + } catch (_: Exception) { + emptyList() + } + } + + val selectedTag = filter.tags.firstOrNull() + if (selectedTag != null) { + val url = "https://$domain/api/v2/galleries/tagged?tag_id=${selectedTag.key}&page=$page" + val json = webClient.httpGet(url).parseJson() + val items = json.getJSONArray("result") + return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } + } + + val lang = filter.locale?.let { LANG_MAP[it.language] } + val searchQuery = buildString { + if (!filter.query.isNullOrBlank()) { + append("title:\"${filter.query}\"") + } + if (lang != null) { + if (isNotEmpty()) append(" ") + append("language:$lang") + } + } + + val url: String + when (order) { + SortOrder.UPDATED -> { + url = if (searchQuery.isEmpty()) { + "https://$domain/api/v2/galleries?page=$page" + } else { + "https://$domain/api/v2/search?query=${searchQuery.urlEncoded()}&sort=date&page=$page" + } + } + else -> { + val sort = when (order) { + SortOrder.POPULARITY -> "popular" + SortOrder.POPULARITY_TODAY -> "popular-today" + SortOrder.POPULARITY_WEEK -> "popular-week" + SortOrder.POPULARITY_MONTH -> "popular-month" + else -> "popular" + } + val finalQuery = searchQuery.ifBlank { + lang?.let { "language:$it" } ?: "language:english" + } + url = "https://$domain/api/v2/search?query=${finalQuery.urlEncoded()}&sort=$sort&page=$page" + } + } + + val json = webClient.httpGet(url).parseJson() + val items = json.getJSONArray("result") + return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } + } + + private fun extractGalleryId(query: String?): Int? { + if (query.isNullOrBlank()) return null + val trimmed = query.trim() + return when { + trimmed.startsWith("id:") -> trimmed.removePrefix("id:").toIntOrNull() + trimmed.all(Char::isDigit) -> trimmed.toIntOrNull() + else -> null + } + } + + private suspend fun fetchMangaById(id: Int): Manga { + ensureNhConfig() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + return json.toMangaDetails() + } + + override suspend fun getDetails(manga: Manga): Manga { + ensureNhConfig() + val id = manga.url.removeSurrounding("/g/", "/").toInt() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + return json.toMangaDetails().copy( + coverUrl = manga.coverUrl?.ifBlank { + "${thumbServer}/${json.getJSONObject("cover").optString("path")}" + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + ensureNhConfig() + val id = chapter.url.removeSurrounding("/g/", "/").toInt() + val json = webClient.httpGet("https://$domain/api/v2/galleries/$id").parseJson() + val pagesArray = json.getJSONArray("pages") + return (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + val path = pageObj.getString("path") + MangaPage( + id = generateUid(path), + url = "$imageServer/$path", + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val titleObj = optJSONObject("title") + val rawTitle = titleObj?.optString("english") + ?: titleObj?.optString("pretty") + ?: optString("english_title") + ?: optString("japanese_title") + ?: "Gallery $id" + + val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() + val thumbPath = optJSONObject("thumbnail")?.optString("path") + ?: optString("thumbnail").takeIf { !it.isNullOrBlank() } + ?: "galleries/$id/thumb.webp" + + return Manga( + id = generateUid("/g/$id/"), + title = title, + altTitles = emptySet(), + url = "/g/$id/", + publicUrl = "https://$domain/g/$id/", + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = "$thumbServer/$thumbPath", + tags = extractTags(), + state = null, + authors = emptySet(), + source = source, + ) + } + + private fun JSONObject.toMangaDetails(): Manga { + val id = getInt("id") + val titleObj = optJSONObject("title") + val rawTitle = titleObj?.optString("english") + ?: titleObj?.optString("japanese") + ?: titleObj?.optString("pretty") + ?: "Gallery $id" + val title = if (displayFullTitle) rawTitle else rawTitle.shortenTitle() + + val thumbPath = getJSONObject("thumbnail").getString("path") + val coverPath = getJSONObject("cover").optString("path", thumbPath) + val numPages = optInt("num_pages") + val uploadDate = optLong("upload_date") * 1000 + + val tags = extractTags() + val artists = getTagsOfType("artist") + val groups = getTagsOfType("group") + val author = artists.joinToString(", ").ifBlank { groups.joinToString(", ") } + + var langName: String? + var translated = false + val languageCandidates = mutableListOf<String>() + + for (tag in tags) { + val parts = tag.key.split(":", limit = 2) + if (parts.size == 2 && parts[0] == "language") { + val name = parts[1] + when { + name.equals("translated", ignoreCase = true) -> translated = true + else -> languageCandidates.add(name) + } + } + } + + langName = languageCandidates.firstOrNull { it.equals("chinese", ignoreCase = true) } + ?: languageCandidates.firstOrNull { it.equals("japanese", ignoreCase = true) } + ?: languageCandidates.firstOrNull { it.equals("english", ignoreCase = true) } + ?: languageCandidates.firstOrNull() + + val description = buildString { + langName?.let { + val displayLang = it.replaceFirstChar { c -> c.uppercase() } + append(if (translated) "$displayLang (translated)" else displayLang) + append("\n") + } + append("Pages: $numPages") + } + + return Manga( + id = generateUid("/g/$id/"), + title = title, + altTitles = listOfNotNull(titleObj?.optString("japanese")).toSet(), + url = "/g/$id/", + publicUrl = "https://$domain/g/$id/", + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = "$thumbServer/$coverPath", + largeCoverUrl = "$imageServer/$coverPath", + tags = tags, + state = null, + authors = setOfNotNull(author.ifBlank { null }), + description = description, + chapters = listOf( + MangaChapter( + id = generateUid("/g/$id/"), + title = title, + number = 1f, + volume = 0, + url = "/g/$id/", + scanlator = groups.joinToString(", ").ifBlank { null }, + uploadDate = uploadDate, + branch = null, + source = source, + ) + ), + source = source, + ) + } + + private fun JSONObject.extractTags(): Set<MangaTag> { + val tagsArray = optJSONArray("tags") ?: return emptySet() + return (0 until tagsArray.length()).map { i -> + val tag = tagsArray.getJSONObject(i) + val name = tag.getString("name") + MangaTag( + title = name.replace("-", " ").replaceFirstChar { it.uppercase() }, + key = "${tag.getString("type")}:$name", + source = source, + ) + }.toSet() + } + + private fun JSONObject.getTagsOfType(type: String): List<String> { + val tagsArray = optJSONArray("tags") ?: return emptyList() + return (0 until tagsArray.length()).mapNotNull { i -> + val tag = tagsArray.getJSONObject(i) + if (tag.getString("type") == type) tag.getString("name") else null + } + } + + private val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") + private fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() + + override val authUrl: String get() = "https://$domain" + override suspend fun isAuthorized(): Boolean = true + override suspend fun getUsername(): String = "" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt new file mode 100644 index 00000000..c63aca26 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -0,0 +1,187 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.ErrorMessages +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("NHENTAIXXX", "NHentai.xxx", type = ContentType.HENTAI) +internal class NHentaiXXXParser(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.NHENTAIXXX, "nhentai.xxx", 25) { + + val supportedImageFormats = setOf("jpg", "webp", "jpeg", "png", "bmp") + + override val selectGallery = "div.galleries_box .gallery_item, #related-container .gallery_item" + override val selectGalleryLink = "a" + override val selectGalleryTitle = ".caption" + override val pathTagUrl = "/tags/popular?page=" + override val selectTitle = "h1" + override val selectTags = "div.tags_items" + override val selectTag = ".tags:contains(Tags)" + override val selectAuthor = ".tags:contains(Artists) span.tag_name" + override val selectLanguageChapter = + ".tags:contains(Languages) a:not([href=\"/language/translated/\"]) span.tag_name" + override val idImg = "fimg" + + override val availableSortOrders: Set<SortOrder> = + EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf(Locale.ENGLISH, Locale.JAPANESE, Locale.CHINESE), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + + val tags = filter.tags + val lang = filter.locale + if (tags.isNotEmpty() && lang != null) { + throw IllegalArgumentException(ErrorMessages.FILTER_BOTH_LOCALE_GENRES_NOT_SUPPORTED) + } + + when { + lang != null -> { + append("/language/") + append(lang.toLanguagePath()) + append("/?") + } + + else -> { + append("/search/?key=") + + val joiner = StringUtil.StringJoiner("+") + tags.forEach { tag -> + joiner.add(tag.title) + } + + if (!query.isNullOrEmpty()) { + joiner.add(query.urlEncoded()) + } + append(joiner.complete()) + } + } + + when (order) { + SortOrder.POPULARITY, SortOrder.RELEVANCE -> append("&sort=popular") + SortOrder.UPDATED -> {} + else -> {} + } + + if (page > 1) { + append("&page=") + append(page.toString()) + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(doc: Document): List<Manga> { + return doc.select(selectGallery).map { div -> + val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(selectGalleryTitle).text().cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = div.selectFirstOrThrow(selectGalleryImg).src(), + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = null, + chapters = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val totalPages = doc.selectFirstOrThrow(selectTotalPage).text().toInt() + val firstPageUrl = doc.requireElementById(idImg).requireSrc() + return (1..totalPages).map { + val url = replacePageNumber(firstPageUrl, it) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + return if (hasError(page.url)) { + findAlternativeUrl(page.url) ?: page.url + } else { + page.url + } + } + + private suspend fun hasError(url: String): Boolean { + return try { + webClient.httpHead(url) + false + } catch (e: Exception) { + true + } + } + + private suspend fun findAlternativeUrl(originalUrl: String): String? { + val lastSegment = originalUrl.substringAfterLast("/") + val oldFormat = lastSegment.substringAfterLast(".", "") + + return supportedImageFormats.firstNotNullOfOrNull { format -> + val newUrl = originalUrl.replace(".$oldFormat", ".$format") + try { + webClient.httpHead(newUrl) + newUrl + } catch (e: Exception) { + null // Skip errors and continue checking other formats + } + } + } + + override fun Element.parseTags() = select("a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".tag_name")?.text() ?: it.text() + MangaTag( + key = key, + title = name.toTitleCase(sourceLocale), + source = source, + ) + } + + private fun replacePageNumber(url: String, newPageNumber: Int): String { + val lastSegment = url.substringAfterLast("/") + val extension = lastSegment.substringAfterLast(".", "") + + return if (extension.isNotEmpty()) { + url.substringBeforeLast("/") + "/$newPageNumber.$extension" + } else { + url.substringBeforeLast("/") + "/$newPageNumber" + } + } +} \ No newline at end of file From c65b604169dbcd3f8c0194f9aa0e207225420923 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:02:31 +0200 Subject: [PATCH 073/434] Delete src/main/kotlin/tsuki/site/en/hentais directory --- .../tsuki/site/en/hentais/AllPornComic.kt | 18 - .../kotlin/tsuki/site/en/hentais/ComicLand.kt | 193 --------- .../tsuki/site/en/hentais/HentaiNexus.kt | 379 ------------------ .../tsuki/site/en/hentais/HentaiXComic.kt | 34 -- .../kotlin/tsuki/site/en/hentais/HeyToon.kt | 273 ------------- .../kotlin/tsuki/site/en/hentais/Hiperdex.kt | 19 - .../kotlin/tsuki/site/en/hentais/MadaraDex.kt | 185 --------- .../kotlin/tsuki/site/en/hentais/Manga18.kt | 12 - .../tsuki/site/en/hentais/MangaDistrict.kt | 103 ----- .../tsuki/site/en/hentais/Manhwa18CC.kt | 144 ------- .../tsuki/site/en/hentais/Manhwa18Com.kt | 247 ------------ .../kotlin/tsuki/site/en/hentais/Manhwa210.kt | 242 ----------- .../tsuki/site/en/hentais/ManhwaRead.kt | 294 -------------- .../kotlin/tsuki/site/en/hentais/Milftoon.kt | 15 - .../kotlin/tsuki/site/en/hentais/Novelcrow.kt | 16 - .../kotlin/tsuki/site/en/hentais/OmegaScan.kt | 12 - 16 files changed, 2186 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manga18.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt delete mode 100644 src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt diff --git a/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt b/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt deleted file mode 100644 index d7ae523c..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/AllPornComic.kt +++ /dev/null @@ -1,18 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - - -@MangaSourceParser("ALLPORNCOMIC", "AllPornComic", "en", ContentType.HENTAI) -internal class AllPornComic(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.ALLPORNCOMIC, "allporncomic.com", pageSize = 24) { - override val withoutAjax = true - override val listUrl = "porncomic/" - override val tagPrefix = "porncomic-cat/" - override val datePattern = "MMMM dd, yyyy" -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt b/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt deleted file mode 100644 index 3923793f..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/ComicLand.kt +++ /dev/null @@ -1,193 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.util.* -import tsuki.model.* - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject - -@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) -internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( - context, MangaParserSource.COMICLAND, 20) { - - override val configKeyDomain = ConfigKey.Domain("comicland.org") - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = request.newBuilder() - .header("Referer", "https://$domain/") - .header("Origin", "https://$domain") - .build() - return chain.proceed(newRequest) - } - - override val availableSortOrders = setOf(SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = false, - isTagsExclusionSupported = false, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - isAuthorSearchSupported = false - ) - - private val api = "https://api.comicland.org/api" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val offset = (page - 1) * 20 - val endpoint = - if (!filter.query.isNullOrBlank()) { - "$api/comic/search?q=${filter.query.urlEncoded()}" - } else { - val category = filter.tags.firstOrNull()?.key - when (category) { - "official" -> "$api/comics/official" - "ongoing" -> "$api/comics" - "popular" -> "$api/comics/popular" - else -> "$api/comics" - } - } - - val separator = if (endpoint.contains("?")) "&" else "?" - val response = webClient.httpGet("$endpoint${separator}offset=$offset&limit=20") - val root = JSONObject(response.body.string()) - - val data = root.optJSONObject("data") - ?: return emptyList() - - val array = data.optJSONArray("list") - ?: data.optJSONArray("items") - ?: return emptyList() - - val result = ArrayList<Manga>() - for (i in 0 until array.length()) { - val comic = - array.getJSONObject(i) - val slug = - comic.optString("slug") - result.add( - Manga( - id = generateUid(slug), - title = comic.optString("title"), - url = slug, - publicUrl = "https://comicland.org/comic/$slug", - coverUrl = comic.optString("cover_url"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - contentRating = ContentRating.ADULT, - source = source - ) - ) - } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - val response = webClient.httpGet("$api/comic/detail?slug=${manga.url}") - val root = JSONObject(response.body.string()) - val comic = root.optJSONObject("data") - ?: return manga - val chapters = ArrayList<MangaChapter>() - val chapterArray = comic.optJSONArray("chapters") - - if (chapterArray != null) { - for (i in 0 until chapterArray.length()) { - val chapter = chapterArray.getJSONObject(i) - - val index = chapter.optDouble("chapter_index", 0.0) - chapters.add( - MangaChapter( - id = generateUid("${comic.optString("slug")}/$index"), - title = chapter.optString("title"), - number = index.toFloat(), - volume = 0, - url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", - scanlator = null, - uploadDate = 0, - branch = null, - source = source - ) - ) - } - } - return manga.copy( - title = comic.optString("title", manga.title), - description = comic.optString("description", ""), - coverUrl = comic.optString("cover_url", manga.coverUrl), - authors = parseNames(comic.optJSONArray("authors")), - tags = emptySet(), - chapters = chapters - ) - } - - private fun parseNames(array: org.json.JSONArray?): Set<String> { - if (array == null) - return emptySet() - return buildSet { - for (i in 0 until array.length()) { - add( - array.getJSONObject(i) - .optString("name") - ) - } - } - } - - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val slug = chapter.url - .substringAfter("/comic/") - .substringBefore("/chapter/") - val index = chapter.url - .substringAfter("/chapter/") - val response = webClient.httpGet("$api/chapter/pages_by_index?slug=$slug&index=$index") - val root = JSONObject(response.body.string()) - val data = root.optJSONObject("data") - ?: return emptyList() - val pages = data.optJSONArray("pages") - ?: return emptyList() - return buildList { - for (i in 0 until pages.length()) { - val image = pages.getString(i) - add( - MangaPage( - id = generateUid(image), - url = image, - preview = null, - source = source - ) - ) - } - } - } - - private val categories = mapOf( - "recommended" to "/comics", - "official" to "/comics/official", - "ongoing" to "/comics?status=ongoing", - "popular" to "/comics/popular" - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = categories.map { (key, _) -> - MangaTag( - title = key.replaceFirstChar { it.uppercase() }, - key = key, - source = source - ) - }.toSet(), - availableContentRating = emptySet() - ) - } -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt deleted file mode 100644 index b9996659..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/HentaiNexus.kt +++ /dev/null @@ -1,379 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.parsers.GalleryAdultsParser - -import tsuki.util.* -import tsuki.model.* - -import org.json.JSONArray -import org.jsoup.nodes.Document -import java.text.SimpleDateFormat -import java.util.Base64 - -private const val SERVER_PNG = "png" // old method uses now image_fallback -private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format - -@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) -internal class HentaiNexus(context: MangaLoaderContext) : - GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { - override val selectGallery = "div.container div.columns div.column" - override val selectGalleryLink = "a" - override val selectGalleryTitle = ".card-header" - override val selectTitle = ".title" - override val selectTag = "tr:contains(Tags) td:nth-child(2) span.tag a" - override val selectAuthor = "tr:contains(Artist) td:nth-child(2) a" - override val selectLanguageChapter = "" - override val selectUrlChapter = "" - override val selectTotalPage = "div.column.is-2-fullhd" - - val selectReadUrl = "a:contains(Read Online)" - val selectPublisher = "tr:contains(Publisher) td:nth-child(2)" - val selectPublishedDate = "tr:contains(Published) td:nth-child(2)" - val selectDescription = "tr:contains(Description) td:nth-child(2)" - - var mangaInternalId: String = "" /* use as a flag for reloading data */ - var mangaPages: List<MangaPage> = listOf() - - private val pageCache = object : LinkedHashMap<String, List<String>>(10, 0.75f, true) { - override fun removeEldestEntry( - eldest: MutableMap.MutableEntry<String, List<String>> - ): Boolean { - return size > 5 - } - } - - private val preferredImageModeKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG to "Old method (png)", - _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP to "New method (webp/avif)", - ), - defaultValue = _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG, - ) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(preferredImageModeKey) - } - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isMultipleTagsSupported = true, - isAuthorSearchSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val document = webClient.httpGet("https://$domain/explore/categories/tag").parseHtml() - val tags = document.select("div.container div.columns div.column").mapToSet {div -> - val tag = div.selectFirstOrThrow("a").attr("href") - .substring(8) // href="/?q=tag:value" - .urlDecode() - .trim(' ', '\"') - MangaTag( - title = tag.replace(Regex("\\b[a-z]")) { it.value.uppercase(sourceLocale) }, - key = tag, - source = source, - ) - } - - return MangaListFilterOptions( - availableTags = tags - ) - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - val url = buildString { - append("https://$domain/page/$page") - when { - !filter.query.isNullOrEmpty() -> { - val query = filter.query - append("?q=") - append(query?.urlEncoded()) - } - - else -> { - val queries = mutableListOf<String>() - - if (!filter.author.isNullOrEmpty()) { - queries.add("artist:${filter.author}") - } - - filter.tags.map { - val key = it.key - when { - key.split(" ").count() > 1 -> { - "\"${key.replace(Regex("\\s+")) { "+" }}\"" - } - else -> { - key - } - } - }.also { - if (it.count() > 0) { - queries.add("tag:${ it.joinToString("+") }") - } - } - - if (queries.count() > 0) { - append("?q=${queries.joinToString("+")}") - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - override fun parseMangaList(doc: Document): List<Manga> { - return doc.select(selectGallery).map { div -> - val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = div.select(selectGalleryTitle).text().cleanupTitle(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = div.selectFirst(selectGalleryImg)?.src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val tags = doc.select(selectTag).mapToSet { - val value = it.attr("href").substring(8).urlDecode().trim('\"') /* /?q=tag:"blow job" */ - MangaTag( - title = value.replace(Regex("\\b[a-z]")) { x -> x.value.uppercase(sourceLocale) }, - key = value, - source = source - ) - } - val authors = doc.select(selectAuthor).mapToSet { - it.attr("href").substring(11).urlDecode() /* /?q=artist:Danchino */ - } - - mangaPages = getPagesInternal(manga.url, doc) - mangaInternalId = manga.url.split("/").last() - - val format = SimpleDateFormat("dd MMMM yyyy") - - return manga.copy( - title = doc.select(selectTitle).text().cleanupTitle(), - tags = tags, - authors = authors, - chapters = listOf( - MangaChapter( - id = manga.id, - title = manga.title, - number = 0f, - volume = 0, - url = manga.url, - scanlator = doc.select(selectPublisher).text().replace(Regex(" \\([\\d,]+\\)")) { "" }, - uploadDate = format.parseSafe(doc.select(selectPublishedDate).text()), - branch = "English", - source = source, - ), - ), - description = doc.selectFirst(selectDescription)?.text() ?: "" - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { - mangaPages = getPagesInternal(chapter.url) - mangaInternalId = chapter.url.split("/").last() - - // preload image urls - val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") - if (reader != null && !pageCache.containsKey(reader)) { - pageCache[reader] = getPageUrlInternal(reader) - } - } - - return mangaPages - } - - override suspend fun getPageUrl(page: MangaPage): String { - val readerUrl = page.url.substringBefore("#") - - val pages = pageCache[readerUrl] - ?: getPageUrlInternal(readerUrl).also { - pageCache[readerUrl] = it - } - - val index = page.url.substringAfter("#").toInt() - 1 - - return pages[index] - } - - private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List<MangaPage> { - val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() - val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") - - return document.select(selectTotalPage).mapIndexed { index, element -> - val url = "$readUrl#${index + 1}" - MangaPage( - id = generateUid(url), - url = url, - preview = element.select("img").attr("src"), - source = source, - ) - } - } - - private suspend fun getPageUrlInternal(pageUrl: String): List<String> { - val doc = webClient - .httpGet(pageUrl.toAbsoluteUrl(domain)) - .parseHtml() - val script = doc.select("script") - .firstOrNull { it.data().contains("initReader") } - ?: throw Exception("Reader script not found") - val encryptedPagesData = script.data() - .substringAfter("initReader(\"") - .substringBefore("\",") - .replace("\\n", "") - .replace("\\r", "") - .trim() - val decryptedString = decrypt(encryptedPagesData) - .replace("\\/", "/") - val jsonArray = JSONArray(decryptedString) - val mode = config[preferredImageModeKey] ?: _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP - return buildList { - for (i in 0 until jsonArray.length()) { - val item = jsonArray.getJSONObject(i) - val url = when (mode) { - _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG -> { - val image = item.optString("image") - .takeIf { it.isNotBlank() } - val fallback = item.optString("image_fallback") - .takeIf { it.isNotBlank() } - fallback ?: image - } - _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP -> { - var found: String? = null - val keys = item.keys() - while (keys.hasNext()) { - val value = item.optString(keys.next()) - if ( - value.startsWith("http") && - ( - value.contains(".webp") || - value.contains(".avif") || - value.contains(".jpg") || - value.contains(".jpeg") || - value.contains(".png") - ) - ) { - found = value - break - } - } - found - } - else -> null - } - if (!url.isNullOrBlank()) { - add(normalizeImageUrl(url)) - } - } - } - } - - private fun normalizeImageUrl(url: String): String { - return when { - url.startsWith("http") -> url - - url.startsWith("//") -> - "https:$url" - - url.startsWith("/") -> - "https://$domain$url" - - else -> - "https://$domain/$url" - } - } - - private fun decrypt(encodedData: String): String { - val xorKey = listOf('h','e','n','t','a','i','n','e','x','u','s','.','c','o','m') - val keyLength = minOf(xorKey.size, 64) - - // Decode base64 string into characters (1 byte per char, unsigned) - val decodedBytes = Base64.getDecoder().decode(encodedData) - val decodedChars = decodedBytes.map { (it.toInt() and 0xFF).toChar() }.toMutableList() - - // XOR first 64 characters with the key - for (i in 0 until keyLength) { - decodedChars[i] = (decodedChars[i].code xor xorKey[i].code).toChar() - } - - val decodedString = decodedChars.joinToString("") - - // Prime sieve: first 16 primes - val sieve = BooleanArray(257) - val primeIndexes = mutableListOf<Int>() - var j = 2 - while (primeIndexes.size < 16) { - if (!sieve[j]) { - primeIndexes.add(j) - for (k in j * 2..256 step j) sieve[k] = true - } - j++ - } - - // Hash from first 64 chars - var hash = 0 - for (k in 0 until 64) { - hash = hash xor decodedString[k].code - repeat(8) { - hash = if ((hash and 1) == 1) { - (hash ushr 1) xor 12 - } else { - hash ushr 1 - } - } - } - hash = hash and 7 - - // RC4 key scheduling - val s = IntArray(256) { it } - j = 0 - for (i in 0 until 256) { - j = (j + s[i] + decodedString[i % 64].code) % 256 - s[i] = s[j].also { s[j] = s[i] } - } - - // Decrypt rest of the chars - val result = StringBuilder() - var i = 0 - j = 0 - var keyStream = 0 - var rnd = 0 - val step = primeIndexes[hash] - - for (n in 0 until decodedString.length - 64) { - i = (i + step) % 256 - j = (keyStream + s[(j + s[i]) % 256]) % 256 - keyStream = (keyStream + i + s[i]) % 256 - s[i] = s[j].also { s[j] = s[i] } - - rnd = s[(j + s[(i + s[(rnd + keyStream) % 256]) % 256]) % 256] - val decryptedChar = decodedString[n + 64].code xor rnd - result.append(decryptedChar.toChar()) - } - - return result.toString() - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt b/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt deleted file mode 100644 index d24318c4..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/HentaiXComic.kt +++ /dev/null @@ -1,34 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.model.MangaTag - -import tsuki.util.parseHtml -import tsuki.util.toTitleCase - - - - -@MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) -internal class HentaixComic(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.HENTAIXCOMIC, "hentaixcomic.com", 16) { - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/?s=&post_type=wp-manga").parseHtml() - val set = mutableSetOf<MangaTag>() - val titles = mutableSetOf<String>() - doc.select("div.checkbox-group input[type=checkbox]").forEach { input -> - val key = input.attr("value") - val title = input.nextElementSibling()?.text()?.toTitleCase() - if (key.isNotEmpty() && !title.isNullOrEmpty() && titles.add(title.lowercase())) { - set.add(MangaTag(key = key, title = title, source = source)) - } - } - return set - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt b/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt deleted file mode 100644 index 12f76972..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/HeyToon.kt +++ /dev/null @@ -1,273 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON - -import okhttp3.Headers -import org.jsoup.nodes.Document -import java.util.Calendar -import java.util.EnumSet - -@MangaSourceParser("HEYTOON", "HeyToon", "en", ContentType.HENTAI) -internal class HeyToon(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.HEYTOON, pageSize = 54, searchPageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("heytoon.net") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getAvailableGenres(), - ) - - private val headers = Headers.headersOf("X-Requested-With", "XMLHttpRequest") - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - return when { - !query.isNullOrEmpty() -> { - if (page > 1) return emptyList() - - val url = "https://$domain/api/complete-search?keyword=${query.urlEncoded()}" - val response = webClient.httpGet(url, headers).parseJsonArray() - - response.mapJSON { comic -> - val linkComic = comic.getString("linkComic") - val id = linkComic.substringAfterLast("/").substringBefore(".html") - Manga( - id = generateUid(id), - url = linkComic.toRelativeUrl(domain), - publicUrl = linkComic.toAbsoluteUrl(domain), - title = comic.getString("title"), - coverUrl = comic.getStringOrNull("raw_thumb"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - contentRating = null, - largeCoverUrl = null, - description = null, - chapters = null, - ) - } - } - - else -> { - val url = buildString { - append("https://") - append(domain) - append("/en/genres") - - filter.tags.firstOrNull()?.let { - append("/") - append(it.key) - } - - append("?orderBy=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - else -> append("latest") - } - - if (page > 1) { - append("&page=") - append(page) - } - } - - val doc = webClient.httpGet(url).parseHtml() - parseMangaList(doc) - } - } - } - - private fun parseMangaList(doc: Document): List<Manga> { - return doc.select("div.comicItemCon a").map { element -> - val href = element.attrAsRelativeUrl("href") - val id = href.substringAfterLast("/").substringBefore(".html").substringBefore("-") - val img = element.selectFirstOrThrow("img[alt!=badge]") - Manga( - id = generateUid(id), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = img.attr("title").ifEmpty { img.attr("alt") }, - coverUrl = img.attr("data-src").toAbsoluteUrl(domain), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - contentRating = null, - largeCoverUrl = null, - description = null, - chapters = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirstOrThrow("#titleSubWrapper h1.titCon").text() - val description = doc.selectFirst("#modal_detail .cont_area p")?.text() - val genres = parseGenres(doc) - val status = parseStatus(doc) - val chapters = parseChapters(doc) - - return manga.copy( - title = title, - description = description, - tags = genres, - state = status, - coverUrl = doc.selectFirst("meta[property=og:image]")?.attr("content") ?: manga.coverUrl, - chapters = chapters, - ) - } - - private fun parseGenres(doc: Document): Set<MangaTag> { - return doc.select("div.previewItemCon a[href*=/genres/]").mapNotNullToSet { a -> - val key = a.attr("href").substringAfterLast("/genres/").substringBefore("?") - if (key.isEmpty()) return@mapNotNullToSet null - MangaTag( - key = key, - title = key, - source = source, - ) - } - } - - private fun parseStatus(doc: Document): MangaState? { - return doc.select(".badgeArea span").eachText().let { badges -> - when { - badges.any { it.contains("Up") } -> MangaState.ONGOING - badges.any { it.contains("Completed") } -> MangaState.FINISHED - else -> null - } - } - } - - private fun parseChapters(doc: Document): List<MangaChapter> { - return doc.select(".episodeListConPC a#episodeItemCon").mapChapters(reversed = false) { i, element -> - val href = element.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - title = element.selectFirstOrThrow(".comicInfo p.episodeStitle").text(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - uploadDate = parseDateOrNull(element.selectFirst(".comicInfo .episodeDate")?.text()), - branch = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - - return doc.select("#comicContent img").map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun getAvailableGenres(): Set<MangaTag> { - val genres = listOf( - "Detective", - "Spin-Off", - "Mommy", - "Uncensored", - "New", - "In-Law", - "Cheating", - "MILF", - "Harem", - "College", - "Business", - "Supernatural", - "Thriller", - "Adventure", - "Romance", - "Drama", - ) - - return genres.mapToSet { genre -> - MangaTag( - key = genre, - title = genre, - source = source, - ) - } - } - - private fun parseDateOrNull(dateStr: String?): Long { - if (dateStr.isNullOrEmpty()) return 0 - return try { - val parts = dateStr.split(" ") - if (parts.size != 3) return 0 - - val month = when (parts[0].lowercase()) { - "jan" -> Calendar.JANUARY - "feb" -> Calendar.FEBRUARY - "mar" -> Calendar.MARCH - "apr" -> Calendar.APRIL - "may" -> Calendar.MAY - "jun" -> Calendar.JUNE - "jul" -> Calendar.JULY - "aug" -> Calendar.AUGUST - "sep" -> Calendar.SEPTEMBER - "oct" -> Calendar.OCTOBER - "nov" -> Calendar.NOVEMBER - "dec" -> Calendar.DECEMBER - else -> return 0 - } - - val day = parts[1].removeSuffix(",").toIntOrNull() ?: return 0 - val year = parts[2].toIntOrNull() ?: return 0 - - Calendar.getInstance().apply { - set(Calendar.YEAR, year) - set(Calendar.MONTH, month) - set(Calendar.DAY_OF_MONTH, day) - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } catch (e: Exception) { - 0 - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt b/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt deleted file mode 100644 index 3086eae2..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Hiperdex.kt +++ /dev/null @@ -1,19 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.HiperParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -import okhttp3.Headers - -@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) -internal class Hiperdex(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("x-cfg-auth", "yceqt7qgu004") - .build() -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt b/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt deleted file mode 100644 index 2c6a29a8..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/MadaraDex.kt +++ /dev/null @@ -1,185 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.network.CommonHeaders -import tsuki.network.UserAgents -import tsuki.parsers.MadaraParser -import tsuki.exception.ParseException - -import tsuki.model.ContentType -import tsuki.model.MangaChapter -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource - -import tsuki.util.generateUid -import tsuki.util.getCookies -import tsuki.util.insertCookies -import tsuki.util.nullIfEmpty -import tsuki.util.parseHtml -import tsuki.util.toAbsoluteUrl -import tsuki.util.toRelativeUrl - -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import org.jsoup.nodes.Document - -private const val F_URL = "fullUrl=" - -@MangaSourceParser("MADARADEX", "MadaraDex", "en", ContentType.HENTAI) -internal class MadaraDex(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MADARADEX, "madaradex.org") { - - init { - context.cookieJar.insertCookies(domain, "wpmanga-adault=1") - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.remove(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders() - .newBuilder() - .set(CommonHeaders.USER_AGENT, UserAgents.CHROME_DESKTOP) - .set(CommonHeaders.REFERER, "https://$domain/") - .build() - - override val authUrl: String - get() = "https://$domain" - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name.contains("cm_uaid") - } - } - - override val listUrl = "title/" - override val tagPrefix = "genre/" - override val postReq = true - override val stylePage = "" - - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - - val fullUrl = chapter.url.toAbsoluteUrl(domain) - - val doc = webClient - .httpGet(fullUrl) - .parseHtml() - - - val images = extractImages(doc) - - - if (images.isEmpty()) { - throw ParseException( - "No images found, try to log in", - fullUrl - ) - } - - - return images.map { image -> - - val imageUrl = image - .toRelativeUrl(domain) - .toHttpUrl() - .newBuilder() - .fragment(_root_ide_package_.tsuki.site.en.hentais.F_URL + fullUrl) - .build() - .toString() - - - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - - private fun extractImages(document: Document): List<String> { - - val pageBreakImages = - document - .select("div.page-break img") - .mapNotNull { - it.attr("data-src") - .ifBlank { - it.attr("src") - } - .nullIfEmpty() - } - - - if (pageBreakImages.isNotEmpty()) { - return pageBreakImages - } - - - return document - .body() - .select(selectBodyPage) - .flatMap { body -> - body.select(selectPage) - .flatMap { page -> - page.select("img") - .mapNotNull { - it.attr("src") - .nullIfEmpty() - } - } - } - } - - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val url = request.url - - val fullUrl = url.fragment - ?.substringAfter(_root_ide_package_.tsuki.site.en.hentais.F_URL, "") - - if (!fullUrl.isNullOrEmpty()) { - - copyCookies() - - val cleanUrl = url.newBuilder() - .fragment(null) - .build() - - val newRequest = request.newBuilder() - .header( - CommonHeaders.REFERER, - fullUrl - ) - .header( - CommonHeaders.USER_AGENT, - UserAgents.CHROME_DESKTOP - ) - .url(cleanUrl) - .build() - - return chain.proceed(newRequest) - } - - return super.intercept(chain) - } - - - private fun copyCookies() { - val cookies = context.cookieJar.getCookies(domain) - - cookies.forEach { - context.cookieJar.insertCookies( - "cdn.$domain", - "${it.name}=${it.value}" - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt b/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt deleted file mode 100644 index 03d27b50..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Manga18.kt +++ /dev/null @@ -1,12 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.Manga18Parser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) -internal class Manga18(context: MangaLoaderContext) : - Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt deleted file mode 100644 index 83ddbbcf..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/MangaDistrict.kt +++ /dev/null @@ -1,103 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.* -import tsuki.util.* - -import org.jsoup.nodes.Document -import java.text.SimpleDateFormat -import java.util.EnumSet - -@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) -internal class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { - - override val tagPrefix = "publication-genre/" - override val withoutAjax: Boolean = true - override val datePattern: String = "MMMM d, yyyy" - override val stylePage: String = "?style=list" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) - - override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - - val chapters = doc.body() - .select("li.wp-manga-chapter") - .mapNotNull { li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - - val name = a.selectFirst("p")?.text() - ?: a.ownText() - - val dateText = - li.selectFirst("a.c-new-tag")?.attr("title") - ?: li.selectFirst("span.chapter-release-date i")?.text() - - MangaChapter( - id = generateUid(href), - title = name, - number = extractChapterNumber(name) ?: 0f, - volume = 0, - url = link, - uploadDate = parseChapterDate(dateFormat, dateText), - scanlator = null, - branch = null, - source = source, - ) - } - .sortedWith( - compareBy<MangaChapter> { it.number } - .thenBy { it.title } - ) - - return chapters - } - - private fun extractChapterNumber(name: String): Float? { - val match = Regex( - """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", - RegexOption.IGNORE_CASE - ).find(name) - - return match - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/series/").parseHtml() - val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") - - return elements.mapNotNullToSet { a -> - val href = a.attr("href") - .removeSuffix("/") - .substringAfterLast(tagPrefix, "") - - if (href.isBlank()) return@mapNotNullToSet null - - MangaTag( - key = href, - title = a.text() - .replace(Regex("""\s*\(\d+\)"""), "") - .replace(Regex("""[^\x20-\x7E]"""), "") - .trim() - .toTitleCase(), - source = source, - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt deleted file mode 100644 index 60e71e88..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18CC.kt +++ /dev/null @@ -1,144 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.* -import tsuki.util.* - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import java.util.HashSet -import java.util.Locale - -@MangaSourceParser("MANHWA18CC", "Manhwa18.cc", "en", ContentType.HENTAI) -internal class Manhwa18CC(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANHWA18CC, "manhwa18.cc", 24) { - - override val datePattern = "dd MMM yyyy" - override val sourceLocale: Locale = Locale.ENGLISH - override val listUrl = "webtoons/" - override val tagPrefix = "webtoon-genre/" - override val withoutAjax = true - override val selectTestAsync = "ul.row-content-chapter" - override val selectDate = "span.chapter-time" - override val selectChapter = "li.a-h" - override val selectBodyPage = "div.read-content" - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - @Volatile - private var tagsCache: Set<MangaTag>? = null - private val tagsMutex = Mutex() - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTagsFromSite().also { tagsCache = it } - } - } - - private suspend fun fetchTagsFromSite(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() - val list = doc.body().selectFirstOrThrow("div.sub-menu").select("ul li") - val keySet = HashSet<String>(list.size) - return list.mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") - if (href.isEmpty() || !keySet.add(href)) { - return@mapNotNullToSet null - } - MangaTag( - key = href, - title = a.ownText().ifEmpty { - a.selectFirst(".menu-image-title")?.text() ?: return@mapNotNullToSet null - }.toTitleCase(), - source = source, - ) - } - } - - override suspend fun getFilterOptions() = super.getFilterOptions().copy( - availableStates = emptySet(), - availableContentRating = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - when { - !query.isNullOrEmpty() -> { - append("/search?q=") - append(query.urlEncoded()) - append("&page=") - append(page.toString()) - } - else -> { - val tag = filter.tags.oneOrThrowIfMany() - if (filter.tags.isNotEmpty()) { - append("/$tagPrefix") - append(tag?.key.orEmpty()) - } else { - append("/$listUrl") - } - if (page > 1) { - append(page.toString()) - } - append("?orderby=") - when (order) { - SortOrder.POPULARITY -> append("trending") - SortOrder.UPDATED -> append("latest") - SortOrder.ALPHABETICAL -> append("alphabet") - SortOrder.RATING -> append("rating") - else -> append("latest") - } - } - } - } - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.manga-lists div.manga-item").map { div -> - val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(div.host ?: domain), - coverUrl = div.selectFirst("img")?.src(), - title = div.selectFirstOrThrow("h3").text(), - altTitles = emptySet(), - rating = div.selectFirst(".item-rate span")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val root = doc.body().selectFirstOrThrow(selectBodyPage) - return root.select("img").map { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt deleted file mode 100644 index 380c07bb..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Manhwa18Com.kt +++ /dev/null @@ -1,247 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) -internal class Manhwa18Com(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { - - override val configKeyDomain = ConfigKey.Domain("manhwa18.com") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, - SortOrder.NEWEST, SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) - - private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - private val numericRegex = Regex("""\d+""") - - @Volatile - private var tagsCache: Set<MangaTag>? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - ) - - private suspend fun getOrFetchTags(): Set<MangaTag> { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTags().also { tagsCache = it } - } - } - - private suspend fun fetchTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() - val list = doc.select("[data-genre-id]") - if (list.isEmpty()) return emptySet() - return list.mapNotNull { item -> - val id = item.attr("data-genre-id") - val name = item.text().trim() - if (id.isNotEmpty() && name.isNotEmpty()) { - MangaTag(name.toTitleCase(Locale.ENGLISH), id, source) - } else null - }.toSet() - } - - private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { - doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } - doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } - val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") - ?: doc.selectFirst(".au-cover-art")?.attr("style") - if (styleUrl != null) { - val url = styleUrl.substringAfter("url('").substringBefore("')") - .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } - if (url.isNotEmpty()) { - return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) - } - } - return fallback - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - val url = buildString { - append("https://$domain/tim-kiem?page=$page") - if (query != null) { - append("&q=") - append(query.urlEncoded()) - } - append("&accept_genres=") - if (filter.tags.isNotEmpty()) { - append(filter.tags.joinToString(",") { it.key }) - } - append("&reject_genres=") - if (filter.tagsExclude.isNotEmpty()) { - append(filter.tagsExclude.joinToString(",") { it.key }) - } - append("&sort=") - append( - when (order) { - SortOrder.ALPHABETICAL -> "az" - SortOrder.POPULARITY -> "top" - SortOrder.UPDATED -> "update" - SortOrder.NEWEST -> "new" - SortOrder.RATING -> "like" - else -> "update" - } - ) - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - append( - when (it) { - MangaState.ONGOING -> "1" - MangaState.FINISHED -> "3" - MangaState.PAUSED -> "2" - else -> "" - } - ) - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".thumb-item-flow").map { element -> - val a = element.selectFirst("a") ?: return@map null - val absUrl = a.attrAsAbsoluteUrl("href") - val title = element.selectFirst(".series-title a")?.text() ?: a.text() - val cover = element.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg") ?: "" - Manga( - id = generateUid(absUrl.toRelativeUrl(domain)), - title = title, - altTitles = emptySet(), - url = absUrl.toRelativeUrl(domain), - publicUrl = absUrl, - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - }.filterNotNull() - } - - private val detailsCacheLock = Any() - - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 - } - - override suspend fun getDetails(manga: Manga): Manga { - synchronized(detailsCacheLock) { - detailsCache[manga.url]?.let { return it } - } - - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title - val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) - val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() - - val statusText = doc.select(".au-stat") - .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } - ?.selectFirst(".v")?.text()?.trim() - val state = when (statusText?.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "on hold" -> MangaState.PAUSED - else -> null - } - - val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> - val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } - ?: return@mapNotNull null - MangaTag(a.text().trim(), slug, source) - }.toSet() - - val chapters = parseChapters(doc) - - val result = manga.copy( - title = title, - coverUrl = coverUrl, - description = description, - state = state, - tags = tags, - authors = emptySet(), - chapters = chapters, - ) - synchronized(detailsCacheLock) { - detailsCache[manga.url] = result - } - return result - } - - private fun parseChapters(doc: org.jsoup.nodes.Document): List<MangaChapter> { - return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> - val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) - val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null - val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() - val uploadDate = dateStr?.let { - runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) - } ?: 0L - val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() - ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() - ?: 0f - MangaChapter( - id = generateUid(chUrl), - title = chTitle, - number = number, - volume = 0, - url = chUrl, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ) - }.sortedBy { it.number } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-content img.lazy").mapNotNull { img -> - val src = img.attrAsAbsoluteUrl("data-src") - .takeIf { it.isNotEmpty() } - ?: img.attrAsAbsoluteUrl("src").takeIf { it.isNotEmpty() } - ?: return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt deleted file mode 100644 index 2eaec65d..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Manhwa210.kt +++ /dev/null @@ -1,242 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) -internal class Manhwa210(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { - - override val configKeyDomain = ConfigKey.Domain("manhwa210.com") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - @Volatile - private var tagsCache: Set<MangaTag>? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - private suspend fun getOrFetchTags(): Set<MangaTag> { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTagsFromSite().also { tagsCache = it } - } - } - - private suspend fun fetchTagsFromSite(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain").parseHtml() - return doc.select("ul.grid.grid-cols-2 a").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text(), - source = source, - ) - } - } - - private val detailsCacheLock = Any() - - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 - } - - private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) - - override suspend fun getDetails(manga: Manga): Manga { - synchronized(detailsCacheLock) { - detailsCache[manga.url]?.let { return it } - } - - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() - - val result = manga.copy( - altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), - state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - else -> null - }, - tags = root.select("div.mt-2:contains(Genres) a.bg-gray-500").mapToSet { a -> - MangaTag( - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - title = a.text(), - source = source, - ) - }, - authors = setOfNotNull(author), - description = root.selectFirst("meta[name=description]")?.attrOrNull("content"), - chapters = root.select("div.justify-between ul.overflow-y-auto.overflow-x-hidden a") - .mapNotNull { a -> - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() - val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() - val number = extractChapterNumber(name) - MangaChapter( - id = generateUid(href), - title = name, - number = number, - volume = 0, - url = href, - scanlator = null, - uploadDate = runCatching { chapterDateFormat.parse(dateText)?.time ?: 0L }.getOrDefault(0L), - branch = null, - source = source, - ) - } - .sortedBy { it.number } - ) - synchronized(detailsCacheLock) { - detailsCache[manga.url] = result - } - return result - } - - private fun extractChapterNumber(title: String): Float { - val regex = Regex("""(?i)(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""") - regex.find(title)?.let { match -> - return match.groupValues[1].toFloatOrNull() ?: 0f - } - Regex("""(\d+(?:\.\d+)?)""").find(title)?.let { - return it.value.toFloatOrNull() ?: 0f - } - return 0f - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append("https://") - append(domain) - val query = filter.query - when { - !query.isNullOrEmpty() -> { - append("/search") - append("?filter[name]=") - append(query.urlEncoded()) - if (page > 1) { - append("&page=") - append(page) - } - append("&sort=") - append( - when (order) { - SortOrder.POPULARITY -> "-views" - SortOrder.UPDATED -> "-updated_at" - SortOrder.NEWEST -> "-created_at" - SortOrder.ALPHABETICAL -> "name" - SortOrder.ALPHABETICAL_DESC -> "-name" - else -> "-updated_at" - }, - ) - } - filter.tags.isNotEmpty() -> { - val tag = filter.tags.first() - append("/genre/") - append(tag.key) - append("?page=") - append(page) - } - else -> { - append("/list") - append("?sort=") - append( - when (order) { - SortOrder.POPULARITY -> "-views" - SortOrder.UPDATED -> "-updated_at" - SortOrder.NEWEST -> "-created_at" - SortOrder.ALPHABETICAL -> "name" - SortOrder.ALPHABETICAL_DESC -> "-name" - else -> "-updated_at" - }, - ) - append("&page=") - append(page) - } - } - if (filter.states.isNotEmpty()) { - append("&filter[status]=") - filter.states.forEach { - append( - when (it) { - MangaState.ONGOING -> "2," - MangaState.FINISHED -> "1," - else -> "1,2" - }, - ) - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.select("div.grid div.relative").map { div -> - val href = div.selectFirst("a[href^=/manga/]")?.attrOrNull("href") - ?: div.parseFailed("Cant find manga image!") - val coverUrl = div.selectFirst("div.cover")?.attr("style") - ?.substringAfter("url('")?.substringBefore("')") - Manga( - id = generateUid(href), - title = div.select("div.p-2 a.text-ellipsis").text(), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = coverUrl.orEmpty(), - tags = setOf(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.text-center img.lazy").mapNotNull { img -> - val url = img.requireSrc() - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt b/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt deleted file mode 100644 index faecb6fa..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/ManhwaRead.kt +++ /dev/null @@ -1,294 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import java.util.Base64 -import org.json.JSONArray -import org.json.JSONObject -import org.jsoup.nodes.Document -import java.nio.charset.StandardCharsets -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANHWAREAD", "ManhwaRead", "en", ContentType.HENTAI) -internal class ManhwaRead(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANHWAREAD, pageSize = 12) { - - override val configKeyDomain = ConfigKey.Domain("manhwaread.com") - private val baseUrl = "https://$domain" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "$baseUrl/") - .build() - - private val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.NEWEST, -// SortOrder.ALPHABETICAL, // A-Z -// SortOrder.RATING, // rating -// SortOrder.POPULARITY, // all time -// SortOrder.POPULARITY_TODAY, // daily -// SortOrder.POPULARITY_WEEK, // weekly -// SortOrder.POPULARITY_MONTH, // monthly -// SortOrder.POPULARITY_YEAR, // yearly - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = false, - isSearchWithFiltersSupported = true, - ) - - companion object { - val GENRE_ID_MAP = mapOf( - "Action" to "650", "Adventure" to "645", "Comedy" to "536", - "Drama" to "530", "Ecchi" to "537", "Fantasy" to "646", - "Hentai" to "531", "Horror" to "590", "Isekai" to "2735", - "Mahou Shoujo" to "2696", "Mystery" to "626", "Psychological" to "591", - "Romance" to "538", "Sci-Fi" to "688", "Slice of Life" to "532", - "Sports" to "677", "Supernatural" to "544", "Thriller" to "580" - ) - val allTags: Set<MangaTag> = run { - val genreTags = GENRE_ID_MAP.keys.map { name -> - MangaTag(name, "genre:${GENRE_ID_MAP[name]}", MangaParserSource.MANHWAREAD) - } - genreTags.toSet() - } - } - - private val tags by lazy { allTags } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val sortBy = when (order) { - SortOrder.UPDATED -> "release" - SortOrder.NEWEST -> "new" -// SortOrder.ALPHABETICAL -> "alphabet" -// SortOrder.RATING -> "rating" -// SortOrder.POPULARITY -> "all_top" -// SortOrder.POPULARITY_TODAY -> "daily_top" -// SortOrder.POPULARITY_WEEK -> "weekly_top" -// SortOrder.POPULARITY_MONTH -> "monthly_top" -// SortOrder.POPULARITY_YEAR -> "yearly_top" - // doenst work idk why? search resulsts are different too - else -> "release" - } - - val url = buildString { - append(baseUrl) - if (page > 1) { - append("/page/") - append(page) - append("/") - } - - val params = linkedMapOf<String, String?>() - params["s"] = filter.query ?: "" - params["sortby"] = sortBy - - if (!filter.query.isNullOrBlank()) { - params["keyword_mode"] = "contain" - params["s_mode"] = "AND" - } - - filter.states.oneOrThrowIfMany()?.let { state -> - params["status"] = when (state) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "on-hold" - MangaState.ABANDONED -> "canceled" - else -> null - } - } - - append(params.toQueryString()) - - val includedGenres = mutableListOf<String>() - val includedTags = mutableListOf<String>() - val excludedTags = mutableListOf<String>() - - filter.tags.forEach { tag -> - val key = tag.key - when { - key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) - } - } - filter.tagsExclude.forEach { tag -> - val key = tag.key - when { - key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) - } - } - - includedGenres.forEach { append("&genres[]=$it") } - includedTags.forEach { append("&including[]=$it") } - excludedTags.forEach { append("&excluding[]=$it") } - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List<Manga> { - return doc.select(".main-container .manga-item").mapNotNull { card -> - val link = card.selectFirst("a.manga-item__link") ?: return@mapNotNull null - val href = link.absUrl("href") - val title = link.text().trim() - val cover = card.selectFirst(".manga-item__img img")?.absUrl("src") - Manga( - id = generateUid(href), - url = href.removePrefix(baseUrl), - publicUrl = href, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst("#mangaSummary .manga-titles h1")?.text() ?: manga.title - val cover = doc.selectFirst("head meta[property=og:image]")?.absUrl("content") ?: manga.coverUrl - - val author = doc.select("#mangaSummary .text-primary:contains(Author:) + .flex a span:first-child").text().trim() - val artist = doc.select("#mangaSummary .text-primary:contains(Artist:) + .flex a span:first-child").text().trim() - val publisher = doc.select("#mangaSummary .text-primary:contains(Publisher:) + .flex a span:first-child").text().trim() - - val statusText = doc.selectFirst("#mangaSummary .manga-status")?.attr("data-status") - val state = when (statusText) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "canceled" -> MangaState.ABANDONED - "on-hold" -> MangaState.PAUSED - else -> null - } - - val genres = doc.select("#mangaSummary .manga-genres a").map { it.text() } - val tags = doc.select("#mangaSummary .text-primary:contains(Tags:) + .flex a span:first-child").map { it.text() } - val allTags = (genres + tags).map { name -> - val genreId = GENRE_ID_MAP[name] - if (genreId != null) MangaTag(name, "genre:$genreId", source) - else MangaTag(name, "tag:$name", source) - }.toSet() - - val descBuilder = StringBuilder() - if (publisher.isNotEmpty()) descBuilder.append("Publisher: $publisher\n") - doc.selectFirst("#mangaDesc > .manga-desc__content")?.text()?.let { descBuilder.append("\n$it") } - doc.selectFirst("#mangaSummary .manga-titles h2")?.text()?.split("|")?.joinToString("\n") { "- " + it.trim() }?.let { altTitles -> - if (altTitles.isNotEmpty()) descBuilder.append("\n\nAlternative titles:\n$altTitles") - } - - val chapters = parseChapters(doc) - - val result = manga.copy( - title = title, - coverUrl = cover, - authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), - state = state, - tags = allTags, - description = descBuilder.toString().trim(), - chapters = chapters, - ) - - detailsCache[manga.url] = result - return result - } - - private fun parseChapters(doc: Document): List<MangaChapter> { - return doc.select("#chaptersList > a.chapter-item").mapNotNull { el -> - val href = el.absUrl("href") - val name = el.selectFirst("span.chapter-item__name")?.text() ?: return@mapNotNull null - val dateStr = el.selectFirst("span.chapter-item__date")?.text() - val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L - MangaChapter( - id = generateUid(href), - title = name, - number = 0f, - volume = 0, - url = href.removePrefix(baseUrl), - uploadDate = uploadDate, - scanlator = null, - branch = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val script = doc.selectFirst("script:containsData(chapterData)") - ?: throw Exception("Chapter data not found") - val dataMatch = CHAPTER_DATA_REGEX.find(script.data()) ?: throw Exception("Chapter data regex failed") - val jsonStr = dataMatch.groupValues[1] - val chapterData = JSONObject(jsonStr) - val base = chapterData.getString("base") - val encodedData = chapterData.getString("data") - val decodedBytes = Base64.getDecoder().decode(encodedData) - val decodedStr = String(decodedBytes, StandardCharsets.UTF_8) - val imagesArray = JSONArray(decodedStr) - - return (0 until imagesArray.length()).map { i -> - val imgObj = imagesArray.getJSONObject(i) - val imgSrc = imgObj.getString("src") - val imgUrl = "$base/$imgSrc" - MangaPage( - id = generateUid(imgUrl), - url = imgUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - - private fun SimpleDateFormat.parseSafe(date: String): Long { - return try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } - } - - private fun Map<String, String?>.toQueryString(): String { - val sb = StringBuilder() - for ((key, value) in this) { - if (!value.isNullOrEmpty()) { - if (sb.isNotEmpty()) sb.append("&") - sb.append(key).append("=").append(value.urlEncoded()) - } - } - return if (sb.isNotEmpty()) "?$sb" else "" - } - private val CHAPTER_DATA_REGEX = Regex("""var\s+chapterData\s*=\s*(\{.*\})""") -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt b/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt deleted file mode 100644 index 20e9bf21..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Milftoon.kt +++ /dev/null @@ -1,15 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("MILFTOON", "MilfToon", "en", ContentType.HENTAI) -internal class MilfToon(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MILFTOON, "milftoon.xxx", 20) { - override val withoutAjax = true - override val listUrl = "comics/" -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt b/src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt deleted file mode 100644 index 1aa38146..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/Novelcrow.kt +++ /dev/null @@ -1,16 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) -internal class Novelcrow(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { - override val withoutAjax = true - override val tagPrefix = "genres-2/" - override val listUrl = "" -} diff --git a/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt b/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt deleted file mode 100644 index b26e0d9e..00000000 --- a/src/main/kotlin/tsuki/site/en/hentais/OmegaScan.kt +++ /dev/null @@ -1,12 +0,0 @@ -package tsuki.site.en.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.HeanCmsParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) -internal class OmegaScans(context: MangaLoaderContext) : - HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file From e1a9fa8875afd3ad0f00032ca479d75d4041d67f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:02:46 +0200 Subject: [PATCH 074/434] Add files via upload --- .../kotlin/tsuki/site/en/nsfw/AllPornComic.kt | 18 + .../kotlin/tsuki/site/en/nsfw/ComicLand.kt | 193 +++++++++ .../kotlin/tsuki/site/en/nsfw/HentaiNexus.kt | 379 ++++++++++++++++++ .../kotlin/tsuki/site/en/nsfw/HentaiXComic.kt | 34 ++ src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt | 273 +++++++++++++ .../kotlin/tsuki/site/en/nsfw/Hiperdex.kt | 19 + .../kotlin/tsuki/site/en/nsfw/MadaraDex.kt | 185 +++++++++ src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt | 12 + .../tsuki/site/en/nsfw/MangaDistrict.kt | 103 +++++ .../kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt | 144 +++++++ .../kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt | 247 ++++++++++++ .../kotlin/tsuki/site/en/nsfw/Manhwa210.kt | 242 +++++++++++ .../kotlin/tsuki/site/en/nsfw/ManhwaRead.kt | 294 ++++++++++++++ .../kotlin/tsuki/site/en/nsfw/Milftoon.kt | 15 + .../kotlin/tsuki/site/en/nsfw/NovelCrow.kt | 17 + .../kotlin/tsuki/site/en/nsfw/OmegaScan.kt | 12 + 16 files changed, 2187 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt diff --git a/src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt b/src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt new file mode 100644 index 00000000..ffb752e4 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt @@ -0,0 +1,18 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + + +@MangaSourceParser("ALLPORNCOMIC", "AllPornComic", "en", ContentType.HENTAI) +internal class AllPornComic(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.ALLPORNCOMIC, "allporncomic.com", pageSize = 24) { + override val withoutAjax = true + override val listUrl = "porncomic/" + override val tagPrefix = "porncomic-cat/" + override val datePattern = "MMMM dd, yyyy" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt new file mode 100644 index 00000000..e738c63e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt @@ -0,0 +1,193 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.util.* +import tsuki.model.* + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject + +@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) +internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( + context, MangaParserSource.COMICLAND, 20) { + + override val configKeyDomain = ConfigKey.Domain("comicland.org") + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = request.newBuilder() + .header("Referer", "https://$domain/") + .header("Origin", "https://$domain") + .build() + return chain.proceed(newRequest) + } + + override val availableSortOrders = setOf(SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + isAuthorSearchSupported = false + ) + + private val api = "https://api.comicland.org/api" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val offset = (page - 1) * 20 + val endpoint = + if (!filter.query.isNullOrBlank()) { + "$api/comic/search?q=${filter.query.urlEncoded()}" + } else { + val category = filter.tags.firstOrNull()?.key + when (category) { + "official" -> "$api/comics/official" + "ongoing" -> "$api/comics" + "popular" -> "$api/comics/popular" + else -> "$api/comics" + } + } + + val separator = if (endpoint.contains("?")) "&" else "?" + val response = webClient.httpGet("$endpoint${separator}offset=$offset&limit=20") + val root = JSONObject(response.body.string()) + + val data = root.optJSONObject("data") + ?: return emptyList() + + val array = data.optJSONArray("list") + ?: data.optJSONArray("items") + ?: return emptyList() + + val result = ArrayList<Manga>() + for (i in 0 until array.length()) { + val comic = + array.getJSONObject(i) + val slug = + comic.optString("slug") + result.add( + Manga( + id = generateUid(slug), + title = comic.optString("title"), + url = slug, + publicUrl = "https://comicland.org/comic/$slug", + coverUrl = comic.optString("cover_url"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + contentRating = ContentRating.ADULT, + source = source + ) + ) + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet("$api/comic/detail?slug=${manga.url}") + val root = JSONObject(response.body.string()) + val comic = root.optJSONObject("data") + ?: return manga + val chapters = ArrayList<MangaChapter>() + val chapterArray = comic.optJSONArray("chapters") + + if (chapterArray != null) { + for (i in 0 until chapterArray.length()) { + val chapter = chapterArray.getJSONObject(i) + + val index = chapter.optDouble("chapter_index", 0.0) + chapters.add( + MangaChapter( + id = generateUid("${comic.optString("slug")}/$index"), + title = chapter.optString("title"), + number = index.toFloat(), + volume = 0, + url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", + scanlator = null, + uploadDate = 0, + branch = null, + source = source + ) + ) + } + } + return manga.copy( + title = comic.optString("title", manga.title), + description = comic.optString("description", ""), + coverUrl = comic.optString("cover_url", manga.coverUrl), + authors = parseNames(comic.optJSONArray("authors")), + tags = emptySet(), + chapters = chapters + ) + } + + private fun parseNames(array: org.json.JSONArray?): Set<String> { + if (array == null) + return emptySet() + return buildSet { + for (i in 0 until array.length()) { + add( + array.getJSONObject(i) + .optString("name") + ) + } + } + } + + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val slug = chapter.url + .substringAfter("/comic/") + .substringBefore("/chapter/") + val index = chapter.url + .substringAfter("/chapter/") + val response = webClient.httpGet("$api/chapter/pages_by_index?slug=$slug&index=$index") + val root = JSONObject(response.body.string()) + val data = root.optJSONObject("data") + ?: return emptyList() + val pages = data.optJSONArray("pages") + ?: return emptyList() + return buildList { + for (i in 0 until pages.length()) { + val image = pages.getString(i) + add( + MangaPage( + id = generateUid(image), + url = image, + preview = null, + source = source + ) + ) + } + } + } + + private val categories = mapOf( + "recommended" to "/comics", + "official" to "/comics/official", + "ongoing" to "/comics?status=ongoing", + "popular" to "/comics/popular" + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = categories.map { (key, _) -> + MangaTag( + title = key.replaceFirstChar { it.uppercase() }, + key = key, + source = source + ) + }.toSet(), + availableContentRating = emptySet() + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt new file mode 100644 index 00000000..baf65329 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt @@ -0,0 +1,379 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.GalleryAdultsParser + +import tsuki.util.* +import tsuki.model.* + +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Base64 + +private const val SERVER_PNG = "png" // old method uses now image_fallback +private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format + +@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) +internal class HentaiNexus(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { + override val selectGallery = "div.container div.columns div.column" + override val selectGalleryLink = "a" + override val selectGalleryTitle = ".card-header" + override val selectTitle = ".title" + override val selectTag = "tr:contains(Tags) td:nth-child(2) span.tag a" + override val selectAuthor = "tr:contains(Artist) td:nth-child(2) a" + override val selectLanguageChapter = "" + override val selectUrlChapter = "" + override val selectTotalPage = "div.column.is-2-fullhd" + + val selectReadUrl = "a:contains(Read Online)" + val selectPublisher = "tr:contains(Publisher) td:nth-child(2)" + val selectPublishedDate = "tr:contains(Published) td:nth-child(2)" + val selectDescription = "tr:contains(Description) td:nth-child(2)" + + var mangaInternalId: String = "" /* use as a flag for reloading data */ + var mangaPages: List<MangaPage> = listOf() + + private val pageCache = object : LinkedHashMap<String, List<String>>(10, 0.75f, true) { + override fun removeEldestEntry( + eldest: MutableMap.MutableEntry<String, List<String>> + ): Boolean { + return size > 5 + } + } + + private val preferredImageModeKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG to "Old method (png)", + _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP to "New method (webp/avif)", + ), + defaultValue = _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(preferredImageModeKey) + } + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val document = webClient.httpGet("https://$domain/explore/categories/tag").parseHtml() + val tags = document.select("div.container div.columns div.column").mapToSet {div -> + val tag = div.selectFirstOrThrow("a").attr("href") + .substring(8) // href="/?q=tag:value" + .urlDecode() + .trim(' ', '\"') + MangaTag( + title = tag.replace(Regex("\\b[a-z]")) { it.value.uppercase(sourceLocale) }, + key = tag, + source = source, + ) + } + + return MangaListFilterOptions( + availableTags = tags + ) + } + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + val url = buildString { + append("https://$domain/page/$page") + when { + !filter.query.isNullOrEmpty() -> { + val query = filter.query + append("?q=") + append(query?.urlEncoded()) + } + + else -> { + val queries = mutableListOf<String>() + + if (!filter.author.isNullOrEmpty()) { + queries.add("artist:${filter.author}") + } + + filter.tags.map { + val key = it.key + when { + key.split(" ").count() > 1 -> { + "\"${key.replace(Regex("\\s+")) { "+" }}\"" + } + else -> { + key + } + } + }.also { + if (it.count() > 0) { + queries.add("tag:${ it.joinToString("+") }") + } + } + + if (queries.count() > 0) { + append("?q=${queries.joinToString("+")}") + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(doc: Document): List<Manga> { + return doc.select(selectGallery).map { div -> + val href = div.selectFirstOrThrow(selectGalleryLink).attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.select(selectGalleryTitle).text().cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = div.selectFirst(selectGalleryImg)?.src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val tags = doc.select(selectTag).mapToSet { + val value = it.attr("href").substring(8).urlDecode().trim('\"') /* /?q=tag:"blow job" */ + MangaTag( + title = value.replace(Regex("\\b[a-z]")) { x -> x.value.uppercase(sourceLocale) }, + key = value, + source = source + ) + } + val authors = doc.select(selectAuthor).mapToSet { + it.attr("href").substring(11).urlDecode() /* /?q=artist:Danchino */ + } + + mangaPages = getPagesInternal(manga.url, doc) + mangaInternalId = manga.url.split("/").last() + + val format = SimpleDateFormat("dd MMMM yyyy") + + return manga.copy( + title = doc.select(selectTitle).text().cleanupTitle(), + tags = tags, + authors = authors, + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 0f, + volume = 0, + url = manga.url, + scanlator = doc.select(selectPublisher).text().replace(Regex(" \\([\\d,]+\\)")) { "" }, + uploadDate = format.parseSafe(doc.select(selectPublishedDate).text()), + branch = "English", + source = source, + ), + ), + description = doc.selectFirst(selectDescription)?.text() ?: "" + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { + mangaPages = getPagesInternal(chapter.url) + mangaInternalId = chapter.url.split("/").last() + + // preload image urls + val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") + if (reader != null && !pageCache.containsKey(reader)) { + pageCache[reader] = getPageUrlInternal(reader) + } + } + + return mangaPages + } + + override suspend fun getPageUrl(page: MangaPage): String { + val readerUrl = page.url.substringBefore("#") + + val pages = pageCache[readerUrl] + ?: getPageUrlInternal(readerUrl).also { + pageCache[readerUrl] = it + } + + val index = page.url.substringAfter("#").toInt() - 1 + + return pages[index] + } + + private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List<MangaPage> { + val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() + val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") + + return document.select(selectTotalPage).mapIndexed { index, element -> + val url = "$readUrl#${index + 1}" + MangaPage( + id = generateUid(url), + url = url, + preview = element.select("img").attr("src"), + source = source, + ) + } + } + + private suspend fun getPageUrlInternal(pageUrl: String): List<String> { + val doc = webClient + .httpGet(pageUrl.toAbsoluteUrl(domain)) + .parseHtml() + val script = doc.select("script") + .firstOrNull { it.data().contains("initReader") } + ?: throw Exception("Reader script not found") + val encryptedPagesData = script.data() + .substringAfter("initReader(\"") + .substringBefore("\",") + .replace("\\n", "") + .replace("\\r", "") + .trim() + val decryptedString = decrypt(encryptedPagesData) + .replace("\\/", "/") + val jsonArray = JSONArray(decryptedString) + val mode = config[preferredImageModeKey] ?: _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP + return buildList { + for (i in 0 until jsonArray.length()) { + val item = jsonArray.getJSONObject(i) + val url = when (mode) { + _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG -> { + val image = item.optString("image") + .takeIf { it.isNotBlank() } + val fallback = item.optString("image_fallback") + .takeIf { it.isNotBlank() } + fallback ?: image + } + _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP -> { + var found: String? = null + val keys = item.keys() + while (keys.hasNext()) { + val value = item.optString(keys.next()) + if ( + value.startsWith("http") && + ( + value.contains(".webp") || + value.contains(".avif") || + value.contains(".jpg") || + value.contains(".jpeg") || + value.contains(".png") + ) + ) { + found = value + break + } + } + found + } + else -> null + } + if (!url.isNullOrBlank()) { + add(normalizeImageUrl(url)) + } + } + } + } + + private fun normalizeImageUrl(url: String): String { + return when { + url.startsWith("http") -> url + + url.startsWith("//") -> + "https:$url" + + url.startsWith("/") -> + "https://$domain$url" + + else -> + "https://$domain/$url" + } + } + + private fun decrypt(encodedData: String): String { + val xorKey = listOf('h','e','n','t','a','i','n','e','x','u','s','.','c','o','m') + val keyLength = minOf(xorKey.size, 64) + + // Decode base64 string into characters (1 byte per char, unsigned) + val decodedBytes = Base64.getDecoder().decode(encodedData) + val decodedChars = decodedBytes.map { (it.toInt() and 0xFF).toChar() }.toMutableList() + + // XOR first 64 characters with the key + for (i in 0 until keyLength) { + decodedChars[i] = (decodedChars[i].code xor xorKey[i].code).toChar() + } + + val decodedString = decodedChars.joinToString("") + + // Prime sieve: first 16 primes + val sieve = BooleanArray(257) + val primeIndexes = mutableListOf<Int>() + var j = 2 + while (primeIndexes.size < 16) { + if (!sieve[j]) { + primeIndexes.add(j) + for (k in j * 2..256 step j) sieve[k] = true + } + j++ + } + + // Hash from first 64 chars + var hash = 0 + for (k in 0 until 64) { + hash = hash xor decodedString[k].code + repeat(8) { + hash = if ((hash and 1) == 1) { + (hash ushr 1) xor 12 + } else { + hash ushr 1 + } + } + } + hash = hash and 7 + + // RC4 key scheduling + val s = IntArray(256) { it } + j = 0 + for (i in 0 until 256) { + j = (j + s[i] + decodedString[i % 64].code) % 256 + s[i] = s[j].also { s[j] = s[i] } + } + + // Decrypt rest of the chars + val result = StringBuilder() + var i = 0 + j = 0 + var keyStream = 0 + var rnd = 0 + val step = primeIndexes[hash] + + for (n in 0 until decodedString.length - 64) { + i = (i + step) % 256 + j = (keyStream + s[(j + s[i]) % 256]) % 256 + keyStream = (keyStream + i + s[i]) % 256 + s[i] = s[j].also { s[j] = s[i] } + + rnd = s[(j + s[(i + s[(rnd + keyStream) % 256]) % 256]) % 256] + val decryptedChar = decodedString[n + 64].code xor rnd + result.append(decryptedChar.toChar()) + } + + return result.toString() + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt new file mode 100644 index 00000000..b9df12d4 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt @@ -0,0 +1,34 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import tsuki.util.parseHtml +import tsuki.util.toTitleCase + + + + +@MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) +internal class HentaixComic(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.HENTAIXCOMIC, "hentaixcomic.com", 16) { + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/?s=&post_type=wp-manga").parseHtml() + val set = mutableSetOf<MangaTag>() + val titles = mutableSetOf<String>() + doc.select("div.checkbox-group input[type=checkbox]").forEach { input -> + val key = input.attr("value") + val title = input.nextElementSibling()?.text()?.toTitleCase() + if (key.isNotEmpty() && !title.isNullOrEmpty() && titles.add(title.lowercase())) { + set.add(MangaTag(key = key, title = title, source = source)) + } + } + return set + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt new file mode 100644 index 00000000..b5dd4e77 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt @@ -0,0 +1,273 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON + +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.util.Calendar +import java.util.EnumSet + +@MangaSourceParser("HEYTOON", "HeyToon", "en", ContentType.HENTAI) +internal class HeyToon(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.HEYTOON, pageSize = 54, searchPageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("heytoon.net") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getAvailableGenres(), + ) + + private val headers = Headers.headersOf("X-Requested-With", "XMLHttpRequest") + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query + return when { + !query.isNullOrEmpty() -> { + if (page > 1) return emptyList() + + val url = "https://$domain/api/complete-search?keyword=${query.urlEncoded()}" + val response = webClient.httpGet(url, headers).parseJsonArray() + + response.mapJSON { comic -> + val linkComic = comic.getString("linkComic") + val id = linkComic.substringAfterLast("/").substringBefore(".html") + Manga( + id = generateUid(id), + url = linkComic.toRelativeUrl(domain), + publicUrl = linkComic.toAbsoluteUrl(domain), + title = comic.getString("title"), + coverUrl = comic.getStringOrNull("raw_thumb"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + contentRating = null, + largeCoverUrl = null, + description = null, + chapters = null, + ) + } + } + + else -> { + val url = buildString { + append("https://") + append(domain) + append("/en/genres") + + filter.tags.firstOrNull()?.let { + append("/") + append(it.key) + } + + append("?orderBy=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + else -> append("latest") + } + + if (page > 1) { + append("&page=") + append(page) + } + } + + val doc = webClient.httpGet(url).parseHtml() + parseMangaList(doc) + } + } + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("div.comicItemCon a").map { element -> + val href = element.attrAsRelativeUrl("href") + val id = href.substringAfterLast("/").substringBefore(".html").substringBefore("-") + val img = element.selectFirstOrThrow("img[alt!=badge]") + Manga( + id = generateUid(id), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = img.attr("title").ifEmpty { img.attr("alt") }, + coverUrl = img.attr("data-src").toAbsoluteUrl(domain), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + contentRating = null, + largeCoverUrl = null, + description = null, + chapters = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirstOrThrow("#titleSubWrapper h1.titCon").text() + val description = doc.selectFirst("#modal_detail .cont_area p")?.text() + val genres = parseGenres(doc) + val status = parseStatus(doc) + val chapters = parseChapters(doc) + + return manga.copy( + title = title, + description = description, + tags = genres, + state = status, + coverUrl = doc.selectFirst("meta[property=og:image]")?.attr("content") ?: manga.coverUrl, + chapters = chapters, + ) + } + + private fun parseGenres(doc: Document): Set<MangaTag> { + return doc.select("div.previewItemCon a[href*=/genres/]").mapNotNullToSet { a -> + val key = a.attr("href").substringAfterLast("/genres/").substringBefore("?") + if (key.isEmpty()) return@mapNotNullToSet null + MangaTag( + key = key, + title = key, + source = source, + ) + } + } + + private fun parseStatus(doc: Document): MangaState? { + return doc.select(".badgeArea span").eachText().let { badges -> + when { + badges.any { it.contains("Up") } -> MangaState.ONGOING + badges.any { it.contains("Completed") } -> MangaState.FINISHED + else -> null + } + } + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + return doc.select(".episodeListConPC a#episodeItemCon").mapChapters(reversed = false) { i, element -> + val href = element.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = element.selectFirstOrThrow(".comicInfo p.episodeStitle").text(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + uploadDate = parseDateOrNull(element.selectFirst(".comicInfo .episodeDate")?.text()), + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + return doc.select("#comicContent img").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun getAvailableGenres(): Set<MangaTag> { + val genres = listOf( + "Detective", + "Spin-Off", + "Mommy", + "Uncensored", + "New", + "In-Law", + "Cheating", + "MILF", + "Harem", + "College", + "Business", + "Supernatural", + "Thriller", + "Adventure", + "Romance", + "Drama", + ) + + return genres.mapToSet { genre -> + MangaTag( + key = genre, + title = genre, + source = source, + ) + } + } + + private fun parseDateOrNull(dateStr: String?): Long { + if (dateStr.isNullOrEmpty()) return 0 + return try { + val parts = dateStr.split(" ") + if (parts.size != 3) return 0 + + val month = when (parts[0].lowercase()) { + "jan" -> Calendar.JANUARY + "feb" -> Calendar.FEBRUARY + "mar" -> Calendar.MARCH + "apr" -> Calendar.APRIL + "may" -> Calendar.MAY + "jun" -> Calendar.JUNE + "jul" -> Calendar.JULY + "aug" -> Calendar.AUGUST + "sep" -> Calendar.SEPTEMBER + "oct" -> Calendar.OCTOBER + "nov" -> Calendar.NOVEMBER + "dec" -> Calendar.DECEMBER + else -> return 0 + } + + val day = parts[1].removeSuffix(",").toIntOrNull() ?: return 0 + val year = parts[2].toIntOrNull() ?: return 0 + + Calendar.getInstance().apply { + set(Calendar.YEAR, year) + set(Calendar.MONTH, month) + set(Calendar.DAY_OF_MONTH, day) + set(Calendar.HOUR_OF_DAY, 0) + set(Calendar.MINUTE, 0) + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + }.timeInMillis + } catch (e: Exception) { + 0 + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt b/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt new file mode 100644 index 00000000..d3212cfd --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt @@ -0,0 +1,19 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import okhttp3.Headers + +@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) +internal class Hiperdex(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("x-cfg-auth", "yceqt7qgu004") + .build() +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt new file mode 100644 index 00000000..a038cb64 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt @@ -0,0 +1,185 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.CommonHeaders +import tsuki.network.UserAgents +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.insertCookies +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.jsoup.nodes.Document + +private const val F_URL = "fullUrl=" + +@MangaSourceParser("MADARADEX", "MadaraDex", "en", ContentType.HENTAI) +internal class MadaraDex(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MADARADEX, "madaradex.org") { + + init { + context.cookieJar.insertCookies(domain, "wpmanga-adault=1") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders() + .newBuilder() + .set(CommonHeaders.USER_AGENT, UserAgents.CHROME_DESKTOP) + .set(CommonHeaders.REFERER, "https://$domain/") + .build() + + override val authUrl: String + get() = "https://$domain" + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name.contains("cm_uaid") + } + } + + override val listUrl = "title/" + override val tagPrefix = "genre/" + override val postReq = true + override val stylePage = "" + + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + + val fullUrl = chapter.url.toAbsoluteUrl(domain) + + val doc = webClient + .httpGet(fullUrl) + .parseHtml() + + + val images = extractImages(doc) + + + if (images.isEmpty()) { + throw ParseException( + "No images found, try to log in", + fullUrl + ) + } + + + return images.map { image -> + + val imageUrl = image + .toRelativeUrl(domain) + .toHttpUrl() + .newBuilder() + .fragment(_root_ide_package_.tsuki.site.en.hentais.F_URL + fullUrl) + .build() + .toString() + + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + + private fun extractImages(document: Document): List<String> { + + val pageBreakImages = + document + .select("div.page-break img") + .mapNotNull { + it.attr("data-src") + .ifBlank { + it.attr("src") + } + .nullIfEmpty() + } + + + if (pageBreakImages.isNotEmpty()) { + return pageBreakImages + } + + + return document + .body() + .select(selectBodyPage) + .flatMap { body -> + body.select(selectPage) + .flatMap { page -> + page.select("img") + .mapNotNull { + it.attr("src") + .nullIfEmpty() + } + } + } + } + + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + + val fullUrl = url.fragment + ?.substringAfter(_root_ide_package_.tsuki.site.en.hentais.F_URL, "") + + if (!fullUrl.isNullOrEmpty()) { + + copyCookies() + + val cleanUrl = url.newBuilder() + .fragment(null) + .build() + + val newRequest = request.newBuilder() + .header( + CommonHeaders.REFERER, + fullUrl + ) + .header( + CommonHeaders.USER_AGENT, + UserAgents.CHROME_DESKTOP + ) + .url(cleanUrl) + .build() + + return chain.proceed(newRequest) + } + + return super.intercept(chain) + } + + + private fun copyCookies() { + val cookies = context.cookieJar.getCookies(domain) + + cookies.forEach { + context.cookieJar.insertCookies( + "cdn.$domain", + "${it.name}=${it.value}" + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt new file mode 100644 index 00000000..3a05bd69 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.Manga18Parser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) +internal class Manga18(context: MangaLoaderContext) : + Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt new file mode 100644 index 00000000..b2df4c8d --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt @@ -0,0 +1,103 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) +internal class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { + + override val tagPrefix = "publication-genre/" + override val withoutAjax: Boolean = true + override val datePattern: String = "MMMM d, yyyy" + override val stylePage: String = "?style=list" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + + val chapters = doc.body() + .select("li.wp-manga-chapter") + .mapNotNull { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + + val name = a.selectFirst("p")?.text() + ?: a.ownText() + + val dateText = + li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst("span.chapter-release-date i")?.text() + + MangaChapter( + id = generateUid(href), + title = name, + number = extractChapterNumber(name) ?: 0f, + volume = 0, + url = link, + uploadDate = parseChapterDate(dateFormat, dateText), + scanlator = null, + branch = null, + source = source, + ) + } + .sortedWith( + compareBy<MangaChapter> { it.number } + .thenBy { it.title } + ) + + return chapters + } + + private fun extractChapterNumber(name: String): Float? { + val match = Regex( + """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", + RegexOption.IGNORE_CASE + ).find(name) + + return match + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/series/").parseHtml() + val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") + + return elements.mapNotNullToSet { a -> + val href = a.attr("href") + .removeSuffix("/") + .substringAfterLast(tagPrefix, "") + + if (href.isBlank()) return@mapNotNullToSet null + + MangaTag( + key = href, + title = a.text() + .replace(Regex("""\s*\(\d+\)"""), "") + .replace(Regex("""[^\x20-\x7E]"""), "") + .trim() + .toTitleCase(), + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt new file mode 100644 index 00000000..da6ccd3e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt @@ -0,0 +1,144 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.util.HashSet +import java.util.Locale + +@MangaSourceParser("MANHWA18CC", "Manhwa18.cc", "en", ContentType.HENTAI) +internal class Manhwa18CC(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANHWA18CC, "manhwa18.cc", 24) { + + override val datePattern = "dd MMM yyyy" + override val sourceLocale: Locale = Locale.ENGLISH + override val listUrl = "webtoons/" + override val tagPrefix = "webtoon-genre/" + override val withoutAjax = true + override val selectTestAsync = "ul.row-content-chapter" + override val selectDate = "span.chapter-time" + override val selectChapter = "li.a-h" + override val selectBodyPage = "div.read-content" + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + @Volatile + private var tagsCache: Set<MangaTag>? = null + private val tagsMutex = Mutex() + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTagsFromSite().also { tagsCache = it } + } + } + + private suspend fun fetchTagsFromSite(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + val list = doc.body().selectFirstOrThrow("div.sub-menu").select("ul li") + val keySet = HashSet<String>(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().ifEmpty { + a.selectFirst(".menu-image-title")?.text() ?: return@mapNotNullToSet null + }.toTitleCase(), + source = source, + ) + } + } + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableStates = emptySet(), + availableContentRating = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query + val url = buildString { + append("https://") + append(domain) + + when { + !query.isNullOrEmpty() -> { + append("/search?q=") + append(query.urlEncoded()) + append("&page=") + append(page.toString()) + } + else -> { + val tag = filter.tags.oneOrThrowIfMany() + if (filter.tags.isNotEmpty()) { + append("/$tagPrefix") + append(tag?.key.orEmpty()) + } else { + append("/$listUrl") + } + if (page > 1) { + append(page.toString()) + } + append("?orderby=") + when (order) { + SortOrder.POPULARITY -> append("trending") + SortOrder.UPDATED -> append("latest") + SortOrder.ALPHABETICAL -> append("alphabet") + SortOrder.RATING -> append("rating") + else -> append("latest") + } + } + } + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.manga-lists div.manga-item").map { div -> + val href = div.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: div.parseFailed("Link not found") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(div.host ?: domain), + coverUrl = div.selectFirst("img")?.src(), + title = div.selectFirstOrThrow("h3").text(), + altTitles = emptySet(), + rating = div.selectFirst(".item-rate span")?.ownText()?.toFloatOrNull()?.div(5f) ?: -1f, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val root = doc.body().selectFirstOrThrow(selectBodyPage) + return root.select("img").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt new file mode 100644 index 00000000..4821e237 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt @@ -0,0 +1,247 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) +internal class Manhwa18Com(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { + + override val configKeyDomain = ConfigKey.Domain("manhwa18.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, + SortOrder.NEWEST, SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) + + private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) + private val numericRegex = Regex("""\d+""") + + @Volatile + private var tagsCache: Set<MangaTag>? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + ) + + private suspend fun getOrFetchTags(): Set<MangaTag> { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTags().also { tagsCache = it } + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/tim-kiem?q=").parseHtml() + val list = doc.select("[data-genre-id]") + if (list.isEmpty()) return emptySet() + return list.mapNotNull { item -> + val id = item.attr("data-genre-id") + val name = item.text().trim() + if (id.isNotEmpty() && name.isNotEmpty()) { + MangaTag(name.toTitleCase(Locale.ENGLISH), id, source) + } else null + }.toSet() + } + + private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { + doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } + doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } + val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") + ?: doc.selectFirst(".au-cover-art")?.attr("style") + if (styleUrl != null) { + val url = styleUrl.substringAfter("url('").substringBefore("')") + .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } + if (url.isNotEmpty()) { + return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) + } + } + return fallback + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + val url = buildString { + append("https://$domain/tim-kiem?page=$page") + if (query != null) { + append("&q=") + append(query.urlEncoded()) + } + append("&accept_genres=") + if (filter.tags.isNotEmpty()) { + append(filter.tags.joinToString(",") { it.key }) + } + append("&reject_genres=") + if (filter.tagsExclude.isNotEmpty()) { + append(filter.tagsExclude.joinToString(",") { it.key }) + } + append("&sort=") + append( + when (order) { + SortOrder.ALPHABETICAL -> "az" + SortOrder.POPULARITY -> "top" + SortOrder.UPDATED -> "update" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "like" + else -> "update" + } + ) + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.ONGOING -> "1" + MangaState.FINISHED -> "3" + MangaState.PAUSED -> "2" + else -> "" + } + ) + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".thumb-item-flow").map { element -> + val a = element.selectFirst("a") ?: return@map null + val absUrl = a.attrAsAbsoluteUrl("href") + val title = element.selectFirst(".series-title a")?.text() ?: a.text() + val cover = element.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg") ?: "" + Manga( + id = generateUid(absUrl.toRelativeUrl(domain)), + title = title, + altTitles = emptySet(), + url = absUrl.toRelativeUrl(domain), + publicUrl = absUrl, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title + val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) + val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() + + val statusText = doc.select(".au-stat") + .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } + ?.selectFirst(".v")?.text()?.trim() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "on hold" -> MangaState.PAUSED + else -> null + } + + val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> + val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } + ?: return@mapNotNull null + MangaTag(a.text().trim(), slug, source) + }.toSet() + + val chapters = parseChapters(doc) + + val result = manga.copy( + title = title, + coverUrl = coverUrl, + description = description, + state = state, + tags = tags, + authors = emptySet(), + chapters = chapters, + ) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private fun parseChapters(doc: org.jsoup.nodes.Document): List<MangaChapter> { + return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> + val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) + val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null + val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() + val uploadDate = dateStr?.let { + runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + } ?: 0L + val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() + ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() + ?: 0f + MangaChapter( + id = generateUid(chUrl), + title = chTitle, + number = number, + volume = 0, + url = chUrl, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#chapter-content img.lazy").mapNotNull { img -> + val src = img.attrAsAbsoluteUrl("data-src") + .takeIf { it.isNotEmpty() } + ?: img.attrAsAbsoluteUrl("src").takeIf { it.isNotEmpty() } + ?: return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt new file mode 100644 index 00000000..aa24fe8a --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt @@ -0,0 +1,242 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) +internal class Manhwa210(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { + + override val configKeyDomain = ConfigKey.Domain("manhwa210.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + private suspend fun getOrFetchTags(): Set<MangaTag> { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTagsFromSite().also { tagsCache = it } + } + } + + private suspend fun fetchTagsFromSite(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain").parseHtml() + return doc.select("ul.grid.grid-cols-2 a").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + } + } + + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() + + val result = manga.copy( + altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), + state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + else -> null + }, + tags = root.select("div.mt-2:contains(Genres) a.bg-gray-500").mapToSet { a -> + MangaTag( + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + title = a.text(), + source = source, + ) + }, + authors = setOfNotNull(author), + description = root.selectFirst("meta[name=description]")?.attrOrNull("content"), + chapters = root.select("div.justify-between ul.overflow-y-auto.overflow-x-hidden a") + .mapNotNull { a -> + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.text-ellipsis")?.text().orEmpty() + val dateText = a.parent()?.selectFirst("span.timeago")?.attr("datetime").orEmpty() + val number = extractChapterNumber(name) + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + scanlator = null, + uploadDate = runCatching { chapterDateFormat.parse(dateText)?.time ?: 0L }.getOrDefault(0L), + branch = null, + source = source, + ) + } + .sortedBy { it.number } + ) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private fun extractChapterNumber(title: String): Float { + val regex = Regex("""(?i)(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""") + regex.find(title)?.let { match -> + return match.groupValues[1].toFloatOrNull() ?: 0f + } + Regex("""(\d+(?:\.\d+)?)""").find(title)?.let { + return it.value.toFloatOrNull() ?: 0f + } + return 0f + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + val query = filter.query + when { + !query.isNullOrEmpty() -> { + append("/search") + append("?filter[name]=") + append(query.urlEncoded()) + if (page > 1) { + append("&page=") + append(page) + } + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + } + filter.tags.isNotEmpty() -> { + val tag = filter.tags.first() + append("/genre/") + append(tag.key) + append("?page=") + append(page) + } + else -> { + append("/list") + append("?sort=") + append( + when (order) { + SortOrder.POPULARITY -> "-views" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.ALPHABETICAL_DESC -> "-name" + else -> "-updated_at" + }, + ) + append("&page=") + append(page) + } + } + if (filter.states.isNotEmpty()) { + append("&filter[status]=") + filter.states.forEach { + append( + when (it) { + MangaState.ONGOING -> "2," + MangaState.FINISHED -> "1," + else -> "1,2" + }, + ) + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.grid div.relative").map { div -> + val href = div.selectFirst("a[href^=/manga/]")?.attrOrNull("href") + ?: div.parseFailed("Cant find manga image!") + val coverUrl = div.selectFirst("div.cover")?.attr("style") + ?.substringAfter("url('")?.substringBefore("')") + Manga( + id = generateUid(href), + title = div.select("div.p-2 a.text-ellipsis").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = coverUrl.orEmpty(), + tags = setOf(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.text-center img.lazy").mapNotNull { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt new file mode 100644 index 00000000..66f0bdcb --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt @@ -0,0 +1,294 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import java.util.Base64 +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.nio.charset.StandardCharsets +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANHWAREAD", "ManhwaRead", "en", ContentType.HENTAI) +internal class ManhwaRead(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWAREAD, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain("manhwaread.com") + private val baseUrl = "https://$domain" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + private val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, // A-Z + SortOrder.RATING, // rating + SortOrder.POPULARITY, // all time + SortOrder.POPULARITY_TODAY, // daily + SortOrder.POPULARITY_WEEK, // weekly + SortOrder.POPULARITY_MONTH, // monthly + SortOrder.POPULARITY_YEAR, // yearly + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + companion object { + val GENRE_ID_MAP = mapOf( + "Action" to "650", "Adventure" to "645", "Comedy" to "536", + "Drama" to "530", "Ecchi" to "537", "Fantasy" to "646", + "Hentai" to "531", "Horror" to "590", "Isekai" to "2735", + "Mahou Shoujo" to "2696", "Mystery" to "626", "Psychological" to "591", + "Romance" to "538", "Sci-Fi" to "688", "Slice of Life" to "532", + "Sports" to "677", "Supernatural" to "544", "Thriller" to "580" + ) + val allTags: Set<MangaTag> = run { + val genreTags = GENRE_ID_MAP.keys.map { name -> + MangaTag(name, "genre:${GENRE_ID_MAP[name]}", MangaParserSource.MANHWAREAD) + } + genreTags.toSet() + } + } + + private val tags by lazy { allTags } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortBy = when (order) { + SortOrder.UPDATED -> "release" + SortOrder.NEWEST -> "new" +// SortOrder.ALPHABETICAL -> "alphabet" +// SortOrder.RATING -> "rating" +// SortOrder.POPULARITY -> "all_top" +// SortOrder.POPULARITY_TODAY -> "daily_top" +// SortOrder.POPULARITY_WEEK -> "weekly_top" +// SortOrder.POPULARITY_MONTH -> "monthly_top" +// SortOrder.POPULARITY_YEAR -> "yearly_top" + // doenst work idk why? search resulsts are different too + else -> "release" + } + + val url = buildString { + append(baseUrl) + if (page > 1) { + append("/page/") + append(page) + append("/") + } + + val params = linkedMapOf<String, String?>() + params["s"] = filter.query ?: "" + params["sortby"] = sortBy + + if (!filter.query.isNullOrBlank()) { + params["keyword_mode"] = "contain" + params["s_mode"] = "AND" + } + + filter.states.oneOrThrowIfMany()?.let { state -> + params["status"] = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on-hold" + MangaState.ABANDONED -> "canceled" + else -> null + } + } + + append(params.toQueryString()) + + val includedGenres = mutableListOf<String>() + val includedTags = mutableListOf<String>() + val excludedTags = mutableListOf<String>() + + filter.tags.forEach { tag -> + val key = tag.key + when { + key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) + } + } + filter.tagsExclude.forEach { tag -> + val key = tag.key + when { + key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + } + } + + includedGenres.forEach { append("&genres[]=$it") } + includedTags.forEach { append("&including[]=$it") } + excludedTags.forEach { append("&excluding[]=$it") } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select(".main-container .manga-item").mapNotNull { card -> + val link = card.selectFirst("a.manga-item__link") ?: return@mapNotNull null + val href = link.absUrl("href") + val title = link.text().trim() + val cover = card.selectFirst(".manga-item__img img")?.absUrl("src") + Manga( + id = generateUid(href), + url = href.removePrefix(baseUrl), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("#mangaSummary .manga-titles h1")?.text() ?: manga.title + val cover = doc.selectFirst("head meta[property=og:image]")?.absUrl("content") ?: manga.coverUrl + + val author = doc.select("#mangaSummary .text-primary:contains(Author:) + .flex a span:first-child").text().trim() + val artist = doc.select("#mangaSummary .text-primary:contains(Artist:) + .flex a span:first-child").text().trim() + val publisher = doc.select("#mangaSummary .text-primary:contains(Publisher:) + .flex a span:first-child").text().trim() + + val statusText = doc.selectFirst("#mangaSummary .manga-status")?.attr("data-status") + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "canceled" -> MangaState.ABANDONED + "on-hold" -> MangaState.PAUSED + else -> null + } + + val genres = doc.select("#mangaSummary .manga-genres a").map { it.text() } + val tags = doc.select("#mangaSummary .text-primary:contains(Tags:) + .flex a span:first-child").map { it.text() } + val allTags = (genres + tags).map { name -> + val genreId = GENRE_ID_MAP[name] + if (genreId != null) MangaTag(name, "genre:$genreId", source) + else MangaTag(name, "tag:$name", source) + }.toSet() + + val descBuilder = StringBuilder() + if (publisher.isNotEmpty()) descBuilder.append("Publisher: $publisher\n") + doc.selectFirst("#mangaDesc > .manga-desc__content")?.text()?.let { descBuilder.append("\n$it") } + doc.selectFirst("#mangaSummary .manga-titles h2")?.text()?.split("|")?.joinToString("\n") { "- " + it.trim() }?.let { altTitles -> + if (altTitles.isNotEmpty()) descBuilder.append("\n\nAlternative titles:\n$altTitles") + } + + val chapters = parseChapters(doc) + + val result = manga.copy( + title = title, + coverUrl = cover, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + state = state, + tags = allTags, + description = descBuilder.toString().trim(), + chapters = chapters, + ) + + detailsCache[manga.url] = result + return result + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + return doc.select("#chaptersList > a.chapter-item").mapNotNull { el -> + val href = el.absUrl("href") + val name = el.selectFirst("span.chapter-item__name")?.text() ?: return@mapNotNull null + val dateStr = el.selectFirst("span.chapter-item__date")?.text() + val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L + MangaChapter( + id = generateUid(href), + title = name, + number = 0f, + volume = 0, + url = href.removePrefix(baseUrl), + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val script = doc.selectFirst("script:containsData(chapterData)") + ?: throw Exception("Chapter data not found") + val dataMatch = CHAPTER_DATA_REGEX.find(script.data()) ?: throw Exception("Chapter data regex failed") + val jsonStr = dataMatch.groupValues[1] + val chapterData = JSONObject(jsonStr) + val base = chapterData.getString("base") + val encodedData = chapterData.getString("data") + val decodedBytes = Base64.getDecoder().decode(encodedData) + val decodedStr = String(decodedBytes, StandardCharsets.UTF_8) + val imagesArray = JSONArray(decodedStr) + + return (0 until imagesArray.length()).map { i -> + val imgObj = imagesArray.getJSONObject(i) + val imgSrc = imgObj.getString("src") + val imgUrl = "$base/$imgSrc" + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private fun SimpleDateFormat.parseSafe(date: String): Long { + return try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } + } + + private fun Map<String, String?>.toQueryString(): String { + val sb = StringBuilder() + for ((key, value) in this) { + if (!value.isNullOrEmpty()) { + if (sb.isNotEmpty()) sb.append("&") + sb.append(key).append("=").append(value.urlEncoded()) + } + } + return if (sb.isNotEmpty()) "?$sb" else "" + } + private val CHAPTER_DATA_REGEX = Regex("""var\s+chapterData\s*=\s*(\{.*\})""") +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt b/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt new file mode 100644 index 00000000..0c0b284d --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt @@ -0,0 +1,15 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MILFTOON", "MilfToon", "en", ContentType.HENTAI) +internal class MilfToon(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MILFTOON, "milftoon.xxx", 20) { + override val withoutAjax = true + override val listUrl = "comics/" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt b/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt new file mode 100644 index 00000000..29029202 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt @@ -0,0 +1,17 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) +internal class Novelcrow(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { + + override val withoutAjax = true + override val tagPrefix = "genres-2/" + override val listUrl = "" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt new file mode 100644 index 00000000..20860b12 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HeanCmsParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) +internal class OmegaScans(context: MangaLoaderContext) : + HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file From 074fcb399f8ce3e651ace3d4b9a83b7b71668bf6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:03:09 +0200 Subject: [PATCH 075/434] Delete src/main/kotlin/tsuki/site/id/hentais directory --- .../tsuki/site/id/hentais/BacaKomikpics.kt | 186 ------------------ .../tsuki/site/id/hentais/DoujinDesu.kt | 23 --- .../kotlin/tsuki/site/id/hentais/Kanzenin.kt | 12 -- .../kotlin/tsuki/site/id/hentais/Komiktap.kt | 104 ---------- .../tsuki/site/id/hentais/MangaSusuku.kt | 18 -- .../tsuki/site/id/hentais/SekteDoujin.kt | 12 -- 6 files changed, 355 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt delete mode 100644 src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt diff --git a/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt b/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt deleted file mode 100644 index 4f46ebbe..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/BacaKomikpics.kt +++ /dev/null @@ -1,186 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import okhttp3.Headers -import org.json.JSONArray -import org.jsoup.nodes.Document -import java.util.EnumSet - -@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) -internal class BacaKomikpics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance - - override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = false, // weird handling of website - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = when (order) { - SortOrder.POPULARITY -> "https://$domain/komik-populer/" - SortOrder.UPDATED -> { - if (page == 0) "https://$domain/?s=&post_type=komik_series" - else "https://$domain/page/${page + 1}/?s&post_type=komik_series" - } - else -> "https://$domain/komik-populer/" - } - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List<Manga> { - return doc.select("a.card").mapNotNull { card -> - val href = card.attrAsAbsoluteUrl("href") - val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null - val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() - val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() - val status = card.selectFirst(".status-label")?.text()?.trim() - Manga( - id = generateUid(href), - url = href.toRelativeUrl(domain), - publicUrl = href, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = cover ?: "", - tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), - state = when (status) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst(".post-title h1")?.text()?.trim() - ?: doc.selectFirst("h1.entry-title")?.text()?.trim() - ?: manga.title - - val description = doc.selectFirst(".sinopsis")?.text()?.trim() - ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" - - val cover = doc.selectFirst(".series-thumb img")?.attr("src") - ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl - - val tags = doc.select("a.genre-link").mapNotNull { a -> - val genre = a.text().trim() - val key = a.attr("href").substringAfter("/genre/").substringBefore("/") - if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null - }.toSet() - - val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() - ?.substringAfter("Status :")?.trim() - val state = when { - statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING - statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED - statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED - else -> null - } - - val chapters = extractChapters(doc) - - return manga.copy( - title = title, - description = description, - coverUrl = cover, - tags = tags, - state = state, - authors = emptySet(), - chapters = chapters, - ) - } - - private fun extractChapters(doc: Document): List<MangaChapter> { - val script = doc.select("script").find { it.data().contains("chapterData") } - ?: return emptyList() - val data = script.data() - - val prefixIndex = data.indexOf("chapterData") - if (prefixIndex == -1) return emptyList() - - val windowStart = (prefixIndex - 500).coerceAtLeast(0) - val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) - val window = data.substring(windowStart, windowEnd) - - val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) - val match = arrayRegex.find(window) ?: return emptyList() - val jsonStr = match.groupValues[1].replace("\\/", "/") - - return try { - val jsonArray = JSONArray(jsonStr) - (0 until jsonArray.length()).mapNotNull { i -> - val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null - val chTitle = obj.optString("title", "") - val chUrl = obj.optString("url", "") - val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) - .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) - MangaChapter( - id = generateUid(chUrl), - title = chTitle, - number = number, - volume = 0, - url = chUrl.toRelativeUrl(domain), - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ) - } - } catch (e: Exception) { - e.printStackTrace() - emptyList() - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> - var src = img.attr("src") - if (src.isBlank()) src = img.attr("data-src") - if (src.isBlank()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun String.nullIfEmpty() = ifBlank { null } -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt b/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt deleted file mode 100644 index 36a8d1e6..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/DoujinDesu.kt +++ /dev/null @@ -1,23 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.DoujinDesuParser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -import java.util.EnumSet - -@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) -internal class DoujinDesu(context: MangaLoaderContext) : - DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { - - override val defaultTypes: String = "doujinshi,manga,manhwa" - - override val availableContentTypes: Set<ContentType> = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.MANHWA - ) -} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt b/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt deleted file mode 100644 index 08aa259a..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/Kanzenin.kt +++ /dev/null @@ -1,12 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MangaThemesia - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType - -@MangaSourceParser("KANZENIN", "Kanzenin", "id", ContentType.HENTAI) -internal class Kanzenin(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.KANZENIN, "kanzenin.info") diff --git a/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt b/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt deleted file mode 100644 index 6f4ec9a7..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/Komiktap.kt +++ /dev/null @@ -1,104 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.MangaParserAuthProvider -import tsuki.parsers.MangaThemesia -import tsuki.exception.ParseException - -import tsuki.model.ContentType -import tsuki.model.MangaChapter -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource - -import tsuki.util.generateUid -import tsuki.util.parseHtml -import tsuki.util.selectFirstOrThrow -import tsuki.util.toAbsoluteUrl - -import okhttp3.Interceptor -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.Response -import okhttp3.ResponseBody.Companion.asResponseBody -import org.json.JSONArray -import org.json.JSONObject -import java.util.Base64 - -@MangaSourceParser("KOMIKTAP", "Komiktap", "id", ContentType.HENTAI) -internal class Komiktap(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.KOMIKTAP, "komiktap.info", pageSize = 25), MangaParserAuthProvider, Interceptor { - - private val selectScript = "script[src^='data:text/javascript;base64,']" - private val selectTestScript = "script:containsData(ts_reader.run)" - private val selectPage = "div#readerarea img" - - override val authUrl: String get() = "https://$domain/" - - override suspend fun isAuthorized(): Boolean { - val response = webClient.httpGet("https://$domain/") - return response.headers["x-sucuri-cache"] != null || - response.headers["x-sucuri-id"] == null - } - - override suspend fun getUsername(): String = "" - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(chapterUrl).parseHtml() - - val test = doc.select(selectTestScript) - if (test.isEmpty()) { - return doc.select(selectPage).mapNotNull { img -> - val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - val images = try { - val base64Script = doc.select(selectScript) - var decode = "" - for (elem in base64Script) { - val src = elem.attr("src") - if (src.startsWith("data:text/javascript;base64,")) { - decode = Base64.getDecoder() - .decode(src.removePrefix("data:text/javascript;base64,")) - .toString(Charsets.UTF_8) - if (decode.startsWith("ts_reader.run")) break - } - } - - if (decode.startsWith("ts_reader.run")) { - extractImagesFromScript(decode) - } else { - val scriptElem = doc.selectFirstOrThrow(selectTestScript) - extractImagesFromScript(scriptElem.data()) - } - } catch (e: Exception) { - throw ParseException("Failed to extract images: ${e.message}", chapterUrl) - } - - return (0 until images.length()).map { i -> - val url = images.getString(i) - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - private fun extractImagesFromScript(scriptBody: String): JSONArray { - val json = scriptBody.substringAfter('(').substringBeforeLast(')') - return JSONObject(json) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - val mime = response.headers["Content-Type"] - if (response.isSuccessful && mime == "application/octet-stream") { - val newBody = response.body.source().asResponseBody("image/jpeg".toMediaTypeOrNull()!!) - return response.newBuilder().body(newBody).build() - } - return response - } -} diff --git a/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt b/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt deleted file mode 100644 index bb10e48f..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/MangaSusuku.kt +++ /dev/null @@ -1,18 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MangaThemesia - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -import java.text.SimpleDateFormat -import java.util.Locale - -@MangaSourceParser("MANGASUSUKU", "MangaSusuku", "id", ContentType.HENTAI) -internal class MangaSusuku(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com") { - override val mangaDirectory = "komik" - override val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) -} diff --git a/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt b/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt deleted file mode 100644 index 698f9fad..00000000 --- a/src/main/kotlin/tsuki/site/id/hentais/SekteDoujin.kt +++ /dev/null @@ -1,12 +0,0 @@ -package tsuki.site.id.hentais - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MangaThemesia - -import tsuki.model.MangaParserSource -import tsuki.model.ContentType - -@MangaSourceParser("SEKTEDOUJIN", "SekteDoujin", "id", ContentType.HENTAI) -internal class SekteDoujin(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.SEKTEDOUJIN, "sektedoujin.cc") From 90d4a5a0cbcaf227ed8c8edfa47b8153141c53d1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:03:25 +0200 Subject: [PATCH 076/434] Add files via upload --- .../tsuki/site/id/nsfw/BacaKomikpics.kt | 186 ++++++++++++++++++ .../kotlin/tsuki/site/id/nsfw/DoujinDesu.kt | 23 +++ .../kotlin/tsuki/site/id/nsfw/Kanzenin.kt | 12 ++ .../kotlin/tsuki/site/id/nsfw/Komiktap.kt | 104 ++++++++++ .../kotlin/tsuki/site/id/nsfw/MangaSusuku.kt | 18 ++ .../kotlin/tsuki/site/id/nsfw/SekteDoujin.kt | 12 ++ 6 files changed, 355 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt diff --git a/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt new file mode 100644 index 00000000..e6b9cbba --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt @@ -0,0 +1,186 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.Headers +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) +internal class BacaKomikpics(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance + + override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") + + override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = false, // weird handling of website + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = when (order) { + SortOrder.POPULARITY -> "https://$domain/komik-populer/" + SortOrder.UPDATED -> { + if (page == 0) "https://$domain/?s=&post_type=komik_series" + else "https://$domain/page/${page + 1}/?s&post_type=komik_series" + } + else -> "https://$domain/komik-populer/" + } + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a.card").mapNotNull { card -> + val href = card.attrAsAbsoluteUrl("href") + val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null + val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() + val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() + val status = card.selectFirst(".status-label")?.text()?.trim() + Manga( + id = generateUid(href), + url = href.toRelativeUrl(domain), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover ?: "", + tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), + state = when (status) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst(".post-title h1")?.text()?.trim() + ?: doc.selectFirst("h1.entry-title")?.text()?.trim() + ?: manga.title + + val description = doc.selectFirst(".sinopsis")?.text()?.trim() + ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" + + val cover = doc.selectFirst(".series-thumb img")?.attr("src") + ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl + + val tags = doc.select("a.genre-link").mapNotNull { a -> + val genre = a.text().trim() + val key = a.attr("href").substringAfter("/genre/").substringBefore("/") + if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null + }.toSet() + + val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() + ?.substringAfter("Status :")?.trim() + val state = when { + statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING + statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED + statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED + else -> null + } + + val chapters = extractChapters(doc) + + return manga.copy( + title = title, + description = description, + coverUrl = cover, + tags = tags, + state = state, + authors = emptySet(), + chapters = chapters, + ) + } + + private fun extractChapters(doc: Document): List<MangaChapter> { + val script = doc.select("script").find { it.data().contains("chapterData") } + ?: return emptyList() + val data = script.data() + + val prefixIndex = data.indexOf("chapterData") + if (prefixIndex == -1) return emptyList() + + val windowStart = (prefixIndex - 500).coerceAtLeast(0) + val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) + val window = data.substring(windowStart, windowEnd) + + val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) + val match = arrayRegex.find(window) ?: return emptyList() + val jsonStr = match.groupValues[1].replace("\\/", "/") + + return try { + val jsonArray = JSONArray(jsonStr) + (0 until jsonArray.length()).mapNotNull { i -> + val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null + val chTitle = obj.optString("title", "") + val chUrl = obj.optString("url", "") + val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) + .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) + MangaChapter( + id = generateUid(chUrl), + title = chTitle, + number = number, + volume = 0, + url = chUrl.toRelativeUrl(domain), + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ) + } + } catch (e: Exception) { + e.printStackTrace() + emptyList() + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> + var src = img.attr("src") + if (src.isBlank()) src = img.attr("data-src") + if (src.isBlank()) return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun String.nullIfEmpty() = ifBlank { null } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt new file mode 100644 index 00000000..c39d1b4c --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt @@ -0,0 +1,23 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.DoujinDesuParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.util.EnumSet + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) +internal class DoujinDesu(context: MangaLoaderContext) : + DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { + + override val defaultTypes: String = "doujinshi,manga,manhwa" + + override val availableContentTypes: Set<ContentType> = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.MANHWA + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt b/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt new file mode 100644 index 00000000..7b24b607 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("KANZENIN", "Kanzenin", "id", ContentType.HENTAI) +internal class Kanzenin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KANZENIN, "kanzenin.info") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt b/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt new file mode 100644 index 00000000..dc618062 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt @@ -0,0 +1,104 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.MangaParserAuthProvider +import tsuki.parsers.MangaThemesia +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl + +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.Response +import okhttp3.ResponseBody.Companion.asResponseBody +import org.json.JSONArray +import org.json.JSONObject +import java.util.Base64 + +@MangaSourceParser("KOMIKTAP", "Komiktap", "id", ContentType.HENTAI) +internal class Komiktap(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KOMIKTAP, "komiktap.info", pageSize = 25), MangaParserAuthProvider, Interceptor { + + private val selectScript = "script[src^='data:text/javascript;base64,']" + private val selectTestScript = "script:containsData(ts_reader.run)" + private val selectPage = "div#readerarea img" + + override val authUrl: String get() = "https://$domain/" + + override suspend fun isAuthorized(): Boolean { + val response = webClient.httpGet("https://$domain/") + return response.headers["x-sucuri-cache"] != null || + response.headers["x-sucuri-id"] == null + } + + override suspend fun getUsername(): String = "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + + val test = doc.select(selectTestScript) + if (test.isEmpty()) { + return doc.select(selectPage).mapNotNull { img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + val images = try { + val base64Script = doc.select(selectScript) + var decode = "" + for (elem in base64Script) { + val src = elem.attr("src") + if (src.startsWith("data:text/javascript;base64,")) { + decode = Base64.getDecoder() + .decode(src.removePrefix("data:text/javascript;base64,")) + .toString(Charsets.UTF_8) + if (decode.startsWith("ts_reader.run")) break + } + } + + if (decode.startsWith("ts_reader.run")) { + extractImagesFromScript(decode) + } else { + val scriptElem = doc.selectFirstOrThrow(selectTestScript) + extractImagesFromScript(scriptElem.data()) + } + } catch (e: Exception) { + throw ParseException("Failed to extract images: ${e.message}", chapterUrl) + } + + return (0 until images.length()).map { i -> + val url = images.getString(i) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + private fun extractImagesFromScript(scriptBody: String): JSONArray { + val json = scriptBody.substringAfter('(').substringBeforeLast(')') + return JSONObject(json) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val mime = response.headers["Content-Type"] + if (response.isSuccessful && mime == "application/octet-stream") { + val newBody = response.body.source().asResponseBody("image/jpeg".toMediaTypeOrNull()!!) + return response.newBuilder().body(newBody).build() + } + return response + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt b/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt new file mode 100644 index 00000000..2c35927a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt @@ -0,0 +1,18 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("MANGASUSUKU", "MangaSusuku", "id", ContentType.HENTAI) +internal class MangaSusuku(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com") { + override val mangaDirectory = "komik" + override val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt b/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt new file mode 100644 index 00000000..f567453e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("SEKTEDOUJIN", "SekteDoujin", "id", ContentType.HENTAI) +internal class SekteDoujin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SEKTEDOUJIN, "sektedoujin.cc") \ No newline at end of file From 71b62a467dc1ba970ff9d0dfe3458c12fcd3cebf Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:03:55 +0200 Subject: [PATCH 077/434] Update and rename LerHentais.kt to LerHentais.kt --- src/main/kotlin/tsuki/site/pt/{hentais => nsfw}/LerHentais.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/main/kotlin/tsuki/site/pt/{hentais => nsfw}/LerHentais.kt (91%) diff --git a/src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt b/src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt similarity index 91% rename from src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt rename to src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt index c822f422..58fd52ac 100644 --- a/src/main/kotlin/tsuki/site/pt/hentais/LerHentais.kt +++ b/src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt @@ -1,4 +1,4 @@ -package tsuki.site.pt.hentais +package tsuki.site.pt.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -16,4 +16,4 @@ internal class LerHentais(context: MangaLoaderContext) : override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() .set("x-flux-node", "G2ZsDdWhUwdU82Vw") .build() -} \ No newline at end of file +} From 212a3c444d0a2153fe1fa719155fe3b78062c76d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:04:32 +0200 Subject: [PATCH 078/434] Update and rename TruyenTranhDamMyy.kt to TruyenTranhDamMyy.kt --- .../tsuki/site/vi/{hentai => nsfw}/TruyenTranhDamMyy.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/main/kotlin/tsuki/site/vi/{hentai => nsfw}/TruyenTranhDamMyy.kt (95%) diff --git a/src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt b/src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt similarity index 95% rename from src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt rename to src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt index 1223b3e0..d0cf32c7 100644 --- a/src/main/kotlin/tsuki/site/vi/hentai/TruyenTranhDamMyy.kt +++ b/src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt @@ -1,4 +1,4 @@ -package tsuki.site.vi.hentai +package tsuki.site.vi.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -43,4 +43,4 @@ internal class TruyenTranhDamMyy(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From a42fccc8a2674a3706898331e34a609e55c23555 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:05:09 +0200 Subject: [PATCH 079/434] Update and rename Xiutaku.kt to Xiutaku.kt --- src/main/kotlin/tsuki/site/zh/{hentai => nsfw}/Xiutaku.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) rename src/main/kotlin/tsuki/site/zh/{hentai => nsfw}/Xiutaku.kt (92%) diff --git a/src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt b/src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt similarity index 92% rename from src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt rename to src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt index 62811403..ef6e1072 100644 --- a/src/main/kotlin/tsuki/site/zh/hentai/Xiutaku.kt +++ b/src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt @@ -1,4 +1,4 @@ -package tsuki.site.zh.hentai +package tsuki.site.zh.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -14,4 +14,4 @@ internal class Xiutaku(context: MangaLoaderContext) : override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions(availableTags = fetchTags()) -} \ No newline at end of file +} From c69e0cd1f599e3213e814ae18197d785e958fe5f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:10:06 +0200 Subject: [PATCH 080/434] Update EnThunderScans.kt --- src/main/kotlin/tsuki/site/en/EnThunderScans.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt index db4ba9b6..528827eb 100644 --- a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -5,7 +5,6 @@ import tsuki.MangaSourceParser import tsuki.parsers.MangaThemesia import tsuki.model.Manga -import tsuki.model.MangaListFilterCapabilities import tsuki.model.MangaParserSource import tsuki.util.parseHtml From 14e51cdf0ff17cc6ae39e067e265a8d8a14bbda7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:14:17 +0200 Subject: [PATCH 081/434] Update HentaiNexus.kt --- src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt index baf65329..5a089664 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt @@ -47,10 +47,10 @@ internal class HentaiNexus(context: MangaLoaderContext) : private val preferredImageModeKey = ConfigKey.PreferredImageServer( presetValues = mapOf( - _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG to "Old method (png)", - _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP to "New method (webp/avif)", + SERVER_PNG to "Old method (png)", + SERVER_WEBP to "New method (webp/avif)", ), - defaultValue = _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG, + defaultValue = SERVER_PNG, ) override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { @@ -249,19 +249,19 @@ internal class HentaiNexus(context: MangaLoaderContext) : val decryptedString = decrypt(encryptedPagesData) .replace("\\/", "/") val jsonArray = JSONArray(decryptedString) - val mode = config[preferredImageModeKey] ?: _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP + val mode = config[preferredImageModeKey] ?: SERVER_WEBP return buildList { for (i in 0 until jsonArray.length()) { val item = jsonArray.getJSONObject(i) val url = when (mode) { - _root_ide_package_.tsuki.site.en.hentais.SERVER_PNG -> { + SERVER_PNG -> { val image = item.optString("image") .takeIf { it.isNotBlank() } val fallback = item.optString("image_fallback") .takeIf { it.isNotBlank() } fallback ?: image } - _root_ide_package_.tsuki.site.en.hentais.SERVER_WEBP -> { + SERVER_WEBP -> { var found: String? = null val keys = item.keys() while (keys.hasNext()) { @@ -376,4 +376,4 @@ internal class HentaiNexus(context: MangaLoaderContext) : return result.toString() } -} \ No newline at end of file +} From 3f244c5b1aabe598282e1de0f2ba0fe08f21b8d1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:15:00 +0200 Subject: [PATCH 082/434] Update MadaraDex.kt --- src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt index a038cb64..a8e28e68 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt @@ -88,7 +88,7 @@ internal class MadaraDex(context: MangaLoaderContext) : .toRelativeUrl(domain) .toHttpUrl() .newBuilder() - .fragment(_root_ide_package_.tsuki.site.en.hentais.F_URL + fullUrl) + .fragment(F_URL + fullUrl) .build() .toString() @@ -143,7 +143,7 @@ internal class MadaraDex(context: MangaLoaderContext) : val url = request.url val fullUrl = url.fragment - ?.substringAfter(_root_ide_package_.tsuki.site.en.hentais.F_URL, "") + ?.substringAfter(F_URL, "") if (!fullUrl.isNullOrEmpty()) { @@ -182,4 +182,4 @@ internal class MadaraDex(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 9aa8d1b818d3fd1f310e37f1c3fd716d1d78dcdd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:22:59 +0200 Subject: [PATCH 083/434] add android 5+ support --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 89b74704..4bd7e26a 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -17,10 +17,11 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.supervisorScope -import java.util.Base64 import java.util.concurrent.TimeUnit import java.util.concurrent.ScheduledThreadPoolExecutor import java.util.concurrent.Semaphore +import okio.ByteString +import okio.ByteString.Companion.decodeBase64 internal class VrfSigner { fun interceptor() = Interceptor { chain -> @@ -65,7 +66,7 @@ internal class VrfSigner { for ((table, key, iv) in STAGES) { data = encryptStage(data, table, key, iv) } - return Base64.getUrlEncoder().withoutPadding().encodeToString(data) + return ByteString.of(*data).base64Url().trimEnd('=') } private fun encryptStage(data: ByteArray, table: ByteArray, key: ByteArray, iv: Int): ByteArray { @@ -80,7 +81,7 @@ internal class VrfSigner { } companion object { - private val decoder = Base64.getDecoder() + private fun decodeB64(str: String) = str.decodeBase64()!!.toByteArray() private const val TABLE_1 = "yINlmUNho8VYJT+ibTIP+9ESiULpVEtMOoD6U6lRE0R/xwXo/Xp9NrUgC4cw/Lmo33vUyjUE40kUoEWIr/fxfNNcq2s79ShQ5NhNrFnJ4hXPwOu/SuXzIbuTQKGFvfm08E9jvCfqAtoDqvQq3dVWPQFmJjgvkISBeXY3BgANR+yVnjGbcxZ47d6kLNfZPIayTq3/YGySb1KuVZodWp/WGNAO5pfMcpaK53Hhs0allBszaMaxuouOwdxbwgxIw6YunSsXjI05Yi0j9j4eHKfSXR8Ifo/Od+8iamRfCXTyvm7NGRGYdcQ0ywcK/u6RXhrbcCm4t2eCtrDgQVecJGkQ+A==" private const val KEY_1 = "0Ec58JOY3uBzJK9m3zqIOpdlF7UFiax9DmA=" @@ -90,9 +91,9 @@ internal class VrfSigner { private const val KEY_3 = "DELOJgPsVaCcblDtTGMdHzM=" private val STAGES: List<Triple<ByteArray, ByteArray, Int>> = listOf( - Triple(decoder.decode(TABLE_1), decoder.decode(KEY_1), 0x5A), - Triple(decoder.decode(TABLE_2), decoder.decode(KEY_2), 0x35), - Triple(decoder.decode(TABLE_3), decoder.decode(KEY_3), 0xBA), + Triple(decodeB64(TABLE_1), decodeB64(KEY_1), 0x5A), + Triple(decodeB64(TABLE_2), decodeB64(KEY_2), 0x35), + Triple(decodeB64(TABLE_3), decodeB64(KEY_3), 0xBA), ) } } @@ -604,6 +605,9 @@ internal abstract class MangaFireParser( @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire (Spanish)", "es") class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") + @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish 2)", "es") + class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es-419") + @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") @@ -612,4 +616,7 @@ internal abstract class MangaFireParser( @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire (Portuguese)", "pt") class PortugueseBR(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") + + @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese 2)", "pt") + class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") } From 68ea2338698520d27f235186a8d09f08b979d133 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:47:39 +0200 Subject: [PATCH 084/434] Create Softkomik.kt --- src/main/kotlin/tsuki/site/id/Softkomik.kt | 476 +++++++++++++++++++++ 1 file changed, 476 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Softkomik.kt diff --git a/src/main/kotlin/tsuki/site/id/Softkomik.kt b/src/main/kotlin/tsuki/site/id/Softkomik.kt new file mode 100644 index 00000000..218ed40f --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Softkomik.kt @@ -0,0 +1,476 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import java.net.URLDecoder +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +@MangaSourceParser("SOFTKOMIK", "Softkomik", "id") +internal class Softkomik(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SOFTKOMIK, pageSize = 24), MangaParserAuthProvider { + + override val authUrl: String get() = "https://$domain/akun/login" + override suspend fun isAuthorized(): Boolean { + val cookies = context.cookieJar.getCookies(domain) + return cookies.any { it.name == "tokkey" } + } + override suspend fun getUsername(): String = "" + + override val configKeyDomain = ConfigKey.Domain("softkomik.co") + + private val apiUrl = "https://v2.softdevices.my.id" + private val coverUrl = "https://cover.softdevices.my.id/softkomik-cover" + + private val cdnUrls = listOf( + "https://psy1.komik.im", + "https://image.komik.im/softkomik", + "https://cdn1.softkomik.online/softkomik", + "https://cd1.softkomik.online/softkomik", + "https://f1.softkomik.com/file/softkomik-image", + "https://img.softdevices.my.id/softkomik-image", + "https://image.softkomik.com/softkomik", + ) + + private val loginRequiredGenres = setOf("ecchi", "mature") + private val requiredLoginSuffix = "login-required" + private val requiredLoginFragment = "#$requiredLoginSuffix" + + override val availableSortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + ) + + private val customClient: OkHttpClient by lazy { + context.httpClient.newBuilder() + .addInterceptor(::apiAuthInterceptor) + .addInterceptor(::imageInterceptor) + .build() + } + + override val webClient by lazy { + tsuki.network.OkHttpWebClient(customClient, source) + } + + private fun pageHeaders() = getRequestHeaders().newBuilder() + .add("rsc", "1") + .build() + + private fun apiHeaders(session: SessionDto): Headers { + val builder = Headers.Builder() + .add("Accept", "application/json") + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + if (session.sign.isNotEmpty()) { + builder.add("X-Token", session.token.cleanB64()) + builder.add("X-Sign", session.sign.take(64)) + } else { + builder.add("Authorization", "Bearer ${session.token}") + } + return builder.build() + } + + private data class SessionDto(val ex: Long, val token: String, val sign: String) + private val sessionsByKey = ConcurrentHashMap<String, SessionDto>() + private val sessionKeyChapterList = "chapter-list" + private val sessionKeyChapterImage = "chapter-image" + + private suspend fun getSessionAsync(route: SessionRoute): SessionDto { + sessionsByKey[route.key]?.takeIf { it.ex > System.currentTimeMillis() + 30_000L }?.let { return it } + + try { + val apiHeaders = Headers.Builder() + .add("Accept", "application/json") + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("X-Requested-With", "XMLHttpRequest") + .build() + val res = webClient.httpGet(route.sessionApiUrl, apiHeaders).parseJson() + val rawSign = res.optString("sign", "") + val session = SessionDto( + ex = res.optLong("ex", System.currentTimeMillis() + 60_000L), + token = res.optString("token", ""), + sign = rawSign.substringBefore('|'), + ) + sessionsByKey[route.key] = session + return session + } catch (_: Exception) { + val token = getBearerTokenFromCookie() + if (token != null) { + val session = SessionDto( + ex = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(2), + token = token, + sign = "", + ) + sessionsByKey[route.key] = session + return session + } + throw Exception("Gagal mendapatkan session. Coba lagi.") + } + } + + private fun getSessionBlocking(route: SessionRoute): SessionDto = runBlocking { + getSessionAsync(route) + } + + private fun getBearerTokenFromCookie(): String? { + val cookies = context.cookieJar.getCookies(domain) + val tokkey = cookies.firstOrNull { it.name == "tokkey" }?.value ?: return null + val decoded = runCatching { URLDecoder.decode(tokkey, Charsets.UTF_8.name()) }.getOrDefault(tokkey) + return decoded.cleanB64() + } + + private fun String.cleanB64(): String = + substringBefore('=').let { it + "=".repeat((4 - (it.length % 4)) % 4) } + + private fun apiAuthInterceptor(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.url.toString().startsWith(apiUrl)) { + return chain.proceed(request) + } + val route = resolveSessionRoute(request.url) + val session = getSessionBlocking(route) + val newRequest = request.newBuilder() + .apply { + if (session.sign.isNotEmpty()) { + header("X-Token", session.token.cleanB64()) + header("X-Sign", session.sign.take(64)) + } else { + header("Authorization", "Bearer ${session.token}") + } + } + .build() + return chain.proceed(newRequest) + } + + private fun imageInterceptor(chain: Interceptor.Chain): Response { + val request = chain.request() + val userAgent = request.header("User-Agent") + val normalizedUserAgent = userAgent?.replace( + Regex("""\s*Mobile Safari/\d+(?:\.\d+)*""", RegexOption.IGNORE_CASE), + "", + )?.trim()?.ifEmpty { null } + + val modRequest = if (normalizedUserAgent != userAgent) { + request.newBuilder().header("User-Agent", normalizedUserAgent ?: "").build() + } else request + + val response: Response? = try { + chain.proceed(modRequest) + } catch (_: java.net.UnknownHostException) { + null + } + + if (response?.isSuccessful == true) return response + + val currentHost = cdnUrls.firstOrNull { request.url.toString().startsWith(it) } + if (currentHost == null) return response ?: throw java.net.UnknownHostException(request.url.host) + + response?.close() + val imagePath = request.url.toString().removePrefix(currentHost).removePrefix("/") + for (newHost in cdnUrls) { + if (newHost == currentHost) continue + val newUrl = "$newHost/$imagePath".toHttpUrl() + try { + val newResp = chain.proceed(request.newBuilder().url(newUrl).build()) + if (newResp.isSuccessful) return newResp + newResp.close() + } catch (_: Exception) {} + } + throw java.net.UnknownHostException("All CDN hosts failed for $imagePath") + } + + private data class SessionRoute( + val key: String, + val sessionApiUrl: String, + val slug: String?, + val isChapterListRequest: Boolean, + val isChapterImageRequest: Boolean, + ) + + private fun resolveSessionRoute(url: okhttp3.HttpUrl): SessionRoute { + val segments = url.pathSegments + val komikIndex = segments.indexOf("komik") + val slug = if (komikIndex != -1) segments.getOrNull(komikIndex + 1) else null + val isChapterListRequest = komikIndex != -1 && segments.getOrNull(komikIndex + 2) == "chapter" + val isChapterImageRequest = isChapterListRequest && (segments.contains("imgs") || segments.contains("img")) + + val key = if (isChapterImageRequest) sessionKeyChapterImage else sessionKeyChapterList + val sessionApiUrl = if (isChapterImageRequest) { + "https://$domain/api/session/chapter/oioa" + } else { + "https://$domain/api/session/iuiuiwqw" + } + return SessionRoute( + key = key, + sessionApiUrl = sessionApiUrl, + slug = slug, + isChapterListRequest = isChapterListRequest, + isChapterImageRequest = isChapterImageRequest, + ) + } + + @Volatile + private var genresCache: Set<MangaTag>? = null + private val genresMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + ) + + private suspend fun getOrFetchGenres(): Set<MangaTag> { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: fetchGenres().also { genresCache = it } + } + } + + private suspend fun fetchGenres(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/komik/library", pageHeaders()).parseHtml() + val pageProps = extractNextJs(doc) ?: return emptySet() + val arr = pageProps.optJSONArray("genre") ?: return emptySet() + return LinkedHashSet<MangaTag>(arr.length()).apply { + for (i in 0 until arr.length()) { + val jo = arr.optJSONObject(i) ?: continue + val value = jo.optString("value", "").ifEmpty { jo.optString("label", "") } + val label = jo.optString("label", "").ifEmpty { value } + if (value.isNotEmpty() && label.isNotEmpty()) { + add(MangaTag(title = label, key = value, source = source)) + } + } + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.isNotEmpty()) { + return searchByQuery(query, page) + } + val url = buildString { + append("https://$domain/komik/library") + append("?page=").append(page) + append("&sortBy=").append(if (order == SortOrder.POPULARITY) "popular" else "newKomik") + filter.tags.firstOrNull()?.key?.let { + append("&genre=").append(it.urlEncoded()) + } + } + val doc = webClient.httpGet(url, pageHeaders()).parseHtml() + val pageProps = extractNextJs(doc) ?: return emptyList() + val libData = pageProps.optJSONObject("libData") ?: pageProps + val dataArray = libData.optJSONArray("data") ?: return emptyList() + return parseListData(dataArray) + } + + private suspend fun searchByQuery(query: String, page: Int): List<Manga> { + val urlPath = "$apiUrl/komik?name=${query.urlEncoded()}&search=true&limit=24&page=$page" + val session = getSessionAsync(resolveSessionRoute(urlPath.toHttpUrl())) + val headers = apiHeaders(session) + val json = webClient.httpGet(urlPath, headers).parseJson() + val dataArray = json.optJSONArray("data") ?: return emptyList() + return parseListData(dataArray) + } + + private fun parseListData(dataArray: JSONArray): List<Manga> { + val result = ArrayList<Manga>(dataArray.length()) + for (i in 0 until dataArray.length()) { + val jo = dataArray.optJSONObject(i) ?: continue + val slug = jo.optString("title_slug", "").ifEmpty { jo.optString("id", "") } + if (slug.isEmpty()) continue + val gambar = jo.optString("gambar", "").removePrefix("/") + result.add( + Manga( + id = generateUid(slug), + title = jo.optString("title", "Untitled"), + altTitles = emptySet(), + url = "/$slug", + publicUrl = "https://$domain/$slug", + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + coverUrl = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else "", + tags = emptySet(), + state = parseStatus(jo.optString("status")), + authors = emptySet(), + source = source, + ), + ) + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl, pageHeaders()).parseHtml() + val pageProps = extractNextJs(doc) + val detail = pageProps?.optJSONObject("data") + + val title = detail?.optString("title")?.takeIf { it.isNotBlank() } ?: manga.title + val gambar = detail?.optString("gambar", "")?.removePrefix("/") ?: "" + val coverImg = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else manga.coverUrl + val description = detail?.optString("sinopsis")?.takeIf { it.isNotBlank() } + val author = detail?.optString("author")?.takeIf { it.isNotBlank() } + val state = parseStatus(detail?.optString("status")) + + val tags = LinkedHashSet<MangaTag>() + detail?.optJSONArray("Genre")?.let { arr -> + for (i in 0 until arr.length()) { + val name = arr.optString(i, "").trim() + if (name.isNotEmpty()) tags.add(MangaTag(name, name.lowercase(), source)) + } + } + + val isLoginRequired = tags.any { it.title.lowercase() in loginRequiredGenres } + val slug = manga.url.trim('/').substringBefore('/') + val chapters = fetchChapterList(slug, isLoginRequired) + + return manga.copy( + title = title, + coverUrl = coverImg, + description = description, + tags = tags, + authors = setOfNotNull(author), + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChapterList(slug: String, isLoginRequired: Boolean): List<MangaChapter> { + val apiPath = "$apiUrl/komik/$slug/chapter?limit=9999999" + val headers = if (isLoginRequired) { + val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan untuk mengakses chapter") + Headers.Builder() + .add("Authorization", "Bearer $token") + .add("Referer", "https://$domain/$slug") + .add("Origin", "https://$domain") + .build() + } else { + val session = getSessionAsync(resolveSessionRoute(apiPath.toHttpUrl())) + apiHeaders(session) + } + + val json = webClient.httpGet(apiPath, headers).parseJson() + val chapterArray = json.optJSONArray("chapter") ?: return emptyList() + val chapters = ArrayList<MangaChapter>(chapterArray.length()) + for (i in 0 until chapterArray.length()) { + val ch = chapterArray.optJSONObject(i) ?: continue + val chStr = ch.optString("chapter", "") + if (chStr.isEmpty()) continue + val number = chStr.substringBefore(".").toFloatOrNull() ?: continue + var chapterUrl = "/$slug/chapter/$chStr" + if (isLoginRequired) chapterUrl += requiredLoginFragment + chapters.add( + MangaChapter( + id = generateUid(chapterUrl), + title = "Chapter ${formatChapterDisplay(chStr)}", + url = chapterUrl, + number = number, + volume = 0, + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ), + ) + } + chapters.sortBy { it.number } + return chapters + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val isLoginRequired = chapter.url.contains(requiredLoginFragment) + val cleanUrl = chapter.url.substringBefore('#') + + val doc = webClient.httpGet(cleanUrl.toAbsoluteUrl(domain), pageHeaders()).parseHtml() + val pageProps = extractNextJs(doc) + val data = pageProps?.optJSONObject("data") + val komik = data?.optJSONObject("komik") + val chapterData = data?.optJSONObject("data") + + val slug = cleanUrl.trim('/').substringBefore("/chapter/") + val chNum = cleanUrl.substringAfterLast("/chapter/").trim('/') + + var imageSrc = chapterData?.optJSONArray("imageSrc") ?: JSONArray() + val storageInter2 = chapterData?.optBoolean("storageInter2", false) ?: false + + if (imageSrc.length() == 0) { + val id = chapterData?.optString("_id") ?: komik?.optString("_id") ?: return emptyList() + imageSrc = fetchChapterImages(slug, chNum, id, isLoginRequired) + } + + if (imageSrc.length() == 0) return emptyList() + + val imageBaseUrl = if (storageInter2) cdnUrls[2] else cdnUrls[0] + return (0 until imageSrc.length()).mapNotNull { i -> + val path = imageSrc.optString(i, "").removePrefix("/") + if (path.isEmpty()) return@mapNotNull null + MangaPage( + id = generateUid(path), + url = "$imageBaseUrl/$path", + preview = null, + source = source, + ) + } + } + + private suspend fun fetchChapterImages( + slug: String, + chapter: String, + id: String, + isLoginRequired: Boolean, + ): JSONArray { + val url = "$apiUrl/komik/$slug/chapter/$chapter/imgs/$id" + val headers = if (isLoginRequired) { + val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan") + Headers.Builder() + .add("Authorization", "Bearer $token") + .add("Referer", "https://$domain/$slug/chapter/$chapter") + .add("Origin", "https://$domain") + .build() + } else { + val session = getSessionAsync(resolveSessionRoute(url.toHttpUrl())) + apiHeaders(session) + } + val json = webClient.httpGet(url, headers).parseJson() + return json.optJSONArray("imageSrc") ?: JSONArray() + } + + private fun extractNextJs(doc: org.jsoup.nodes.Document): JSONObject? { + val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() ?: return null + return runCatching { + JSONObject(raw).optJSONObject("props")?.optJSONObject("pageProps") + }.getOrNull() + } + + private fun parseStatus(text: String?): MangaState? = when { + text == null -> null + text.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING + text.contains("tamat", ignoreCase = true) || text.contains("completed", ignoreCase = true) -> MangaState.FINISHED + else -> null + } + + private fun formatChapterDisplay(chStr: String): String { + val parts = chStr.split(".") + val numPart = parts[0].toFloatOrNull() ?: return chStr + val formatted = if (numPart == numPart.toLong().toFloat()) numPart.toLong().toString() + else numPart.toString().trimEnd('0').trimEnd('.') + val suffix = parts.drop(1).joinToString(".") + return if (suffix.isNotEmpty()) "$formatted.$suffix" else formatted + } +} From 3739b4c7fb0559740375878c57ab0db0f85ab75b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 13:56:33 +0200 Subject: [PATCH 085/434] Update Batcave.kt --- src/main/kotlin/tsuki/site/en/Batcave.kt | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 6dcff078..5ace266b 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -31,8 +31,6 @@ internal class Batcave(context: MangaLoaderContext) : @Volatile private var genreList: List<Pair<String, Int>>? = null - @Volatile - private var publisherList: List<Pair<String, Int>>? = null private var filterFetchFailed = false private val rawHttpClient: OkHttpClient by lazy { @@ -88,7 +86,7 @@ internal class Batcave(context: MangaLoaderContext) : ) private suspend fun fetchFilters() { - if (genreList != null && publisherList != null) return + if (genreList != null) return if (filterFetchFailed) return try { @@ -103,15 +101,14 @@ internal class Batcave(context: MangaLoaderContext) : val root = JSONObject(rawJson) val filterItems = root.getJSONObject("filter_items") - genreList = parseFilterValues(filterItems, "g") - publisherList = parseFilterValues(filterItems, "p") + genreList = parseFilterValues(filterItems) } catch (_: Exception) { filterFetchFailed = true } } - private fun parseFilterValues(filterItems: JSONObject, key: String): List<Pair<String, Int>> { - val obj = filterItems.optJSONObject(key) ?: return emptyList() + private fun parseFilterValues(filterItems: JSONObject): List<Pair<String, Int>> { + val obj = filterItems.optJSONObject("g") ?: return emptyList() val values = obj.optJSONArray("values") ?: return emptyList() val result = mutableListOf<Pair<String, Int>>() for (i in 0 until values.length()) { @@ -127,9 +124,6 @@ internal class Batcave(context: MangaLoaderContext) : genreList?.forEach { (name, id) -> tags += MangaTag(name, "g_$id", source) } - publisherList?.forEach { (name, id) -> - tags += MangaTag(name, "p_$id", source) - } return MangaListFilterOptions( availableTags = tags, ) @@ -141,9 +135,7 @@ internal class Batcave(context: MangaLoaderContext) : val urlBuilder = StringBuilder().apply { append("https://$domain/ComicList/") - val pIds = filter.tags.filter { it.key.startsWith("p_") }.map { it.key.removePrefix("p_") } val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } - if (pIds.isNotEmpty()) append("p=${pIds.joinToString(",")}/") if (gIds.isNotEmpty()) append("g=${gIds.joinToString(",")}/") append("sort") if (page > 1) append("/page/$page/") @@ -200,7 +192,6 @@ internal class Batcave(context: MangaLoaderContext) : } private val detailsCacheLock = Any() - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } From 9f7c84c85c9ea1eb69de992e95c26e90b1f69bcb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 21:56:16 +0200 Subject: [PATCH 086/434] Update libs.versions.toml --- gradle/libs.versions.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 9553a389..b0cba018 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -12,8 +12,7 @@ json = "20251224" jsoup = "1.17.2" quickjs = "1.1.0" korte = "4.0.10" -#noinspection NewerVersionAvailable -serialization = "1.7.3" +serialization = "1.11.0" simplexml = "2.7.1" tsuki = "1.0.3" From bf0bbed925d28a558056b635e476b52b90f77ff4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:17:34 +0200 Subject: [PATCH 087/434] Update RavenScans.kt --- src/main/kotlin/tsuki/site/en/RavenScans.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/RavenScans.kt b/src/main/kotlin/tsuki/site/en/RavenScans.kt index 568a6219..1671b55b 100644 --- a/src/main/kotlin/tsuki/site/en/RavenScans.kt +++ b/src/main/kotlin/tsuki/site/en/RavenScans.kt @@ -8,6 +8,6 @@ import tsuki.model.MangaParserSource @MangaSourceParser("RAVENSCANS", "RavenScans", "en") internal class RavensScan(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.org") { + MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.net") { override val mangaDirectory = "series" } From 5c042d49e701a7b828beff66c810ece72b3d137a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 22:26:13 +0200 Subject: [PATCH 088/434] Create Submanhwa.kt --- src/main/kotlin/tsuki/site/es/Submanhwa.kt | 179 +++++++++++++++++++++ 1 file changed, 179 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/es/Submanhwa.kt diff --git a/src/main/kotlin/tsuki/site/es/Submanhwa.kt b/src/main/kotlin/tsuki/site/es/Submanhwa.kt new file mode 100644 index 00000000..02000e48 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/Submanhwa.kt @@ -0,0 +1,179 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("SUBMANHWA", "Submanhwa", "es") +internal class Submanhwa(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SUBMANHWA, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("submanhwa.com") + + private val dateFormat = SimpleDateFormat("dd MMM. yyyy", Locale.ENGLISH) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Accept-Language", "es-PE,es;q=0.9,en-US;q=0.8,en;q=0.7") + .set("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + val url = if (query.isNotEmpty()) { + "https://$domain/filterList?page=$page&sortBy=views&asc=false&alpha=${query.urlEncoded()}" + } else when (order) { + SortOrder.POPULARITY -> "https://$domain/filterList?page=$page&sortBy=views&asc=false" + SortOrder.UPDATED -> "https://$domain" // latest updates + else -> "https://$domain/filterList?page=$page&sortBy=views&asc=false" + } + + val doc = webClient.httpGet(url).parseHtml() + + val elements = if (order == SortOrder.UPDATED && query.isEmpty()) { + doc.select("div[class^=manga-item]") + } else { + doc.select(".series-card") + } + + return elements.mapNotNull { element -> + val a = element.selectFirst("a") ?: return@mapNotNull null + val href = a.absUrl("href") + val title = element.selectFirst("h3[class^=manga-title] a, .series-title")?.text() ?: a.text() + val coverUrl = element.selectFirst("img")?.absUrl("src") + + Manga( + id = generateUid(href), + url = href, + publicUrl = href, + title = title, + coverUrl = coverUrl, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst(".manga-title-centered")?.text() ?: manga.title + val coverUrl = doc.selectFirst("img")?.absUrl("src") ?: manga.coverUrl + val description = doc.selectFirst("h5:contains(Resumen) + p")?.text() + + val box = doc.selectFirst(".main-content > .boxed-modern") + + val state = when (box?.selectFirst(".detail-label:contains(Estado) + .detail-value span")?.text()?.lowercase()) { + "completa" -> MangaState.FINISHED + "en curso" -> MangaState.ONGOING + else -> null + } + + val author = box?.selectFirst(".detail-label:contains(Autor) + .detail-value a")?.text() + val artist = box?.selectFirst(".detail-label:contains(Artist) + .detail-value a")?.text() + val genres = box?.select(".detail-label:contains(Categor) + .detail-value a") + ?.map { MangaTag(it.text().lowercase(), it.text(), source) } + .orEmpty() + .toSet() + + val chapters = parseChapters(doc) + + return manga.copy( + title = title, + coverUrl = coverUrl, + description = description, + authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet(), + tags = genres, + state = state, + chapters = chapters, + ) + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + return doc.select(".chapters-grid [class^=chapter-card]").mapNotNull { element -> + val a = element.selectFirst("a.chapter-link") ?: return@mapNotNull null + val name = a.text() + val href = a.absUrl("href") + + val dateText = element.selectFirst("span:has(i.glyphicon-time)")?.text() + ?: element.selectFirst(".chapter-preview-meta > span")?.text() + val uploadDate = dateText?.let { parseDate(it) } ?: 0L + + val number = Regex("""\d+(\.\d+)?""").find(name)?.value?.toFloatOrNull() ?: 0f + + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + private fun parseDate(text: String): Long { + return try { + dateFormat.parse(text.trim())?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#all img").mapIndexed { idx, img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: "" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun Element.imgAttr(): String { + if (hasAttr("data-src")) return absUrl("data-src") + if (hasAttr("data-lazy-src")) return absUrl("data-lazy-src") + return absUrl("src") + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} From 6c2ba2a4a4d1175128e075b64479f5774022b4b1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:04:12 +0200 Subject: [PATCH 089/434] Update ShadowManga.kt --- src/main/kotlin/tsuki/site/es/ShadowManga.kt | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/es/ShadowManga.kt b/src/main/kotlin/tsuki/site/es/ShadowManga.kt index 1f61a251..dbb41480 100644 --- a/src/main/kotlin/tsuki/site/es/ShadowManga.kt +++ b/src/main/kotlin/tsuki/site/es/ShadowManga.kt @@ -1,5 +1,13 @@ package tsuki.site.es +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex @@ -8,14 +16,8 @@ import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet import kotlin.time.Duration.Companion.milliseconds @MangaSourceParser("SHADOWMANGA", "Shadow Manga", "es") From 989ce1c9d4bd1514fc019efcdaedf4ad0b9201da Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 26 Jul 2026 23:04:33 +0200 Subject: [PATCH 090/434] Create OlympusScanlation.kt --- .../kotlin/tsuki/site/es/OlympusScanlation.kt | 254 ++++++++++++++++++ 1 file changed, 254 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/es/OlympusScanlation.kt diff --git a/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt new file mode 100644 index 00000000..3045c159 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt @@ -0,0 +1,254 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import tsuki.network.OkHttpWebClient +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +@MangaSourceParser("OLYMPUSSCANLATION", "Olympus Scanlation", "es") +internal class OlympusScanlation(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.OLYMPUSSCANLATION, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("olympusxyz.com") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://panel.olympusxyz.com" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .set("X-Requested-With", "XMLHttpRequest") + .build() + + @Volatile + private var seriesList: List<MangaDto> = emptyList() + @Volatile + private var lastFetchTime: Long = 0L + private val seriesMutex = Mutex() + private val slugMap = ConcurrentHashMap<String, String>() + + private suspend fun fetchSeriesListIfNeeded() { + val now = System.currentTimeMillis() + if (seriesList.isNotEmpty() && (now - lastFetchTime) < TimeUnit.HOURS.toMillis(1)) return + seriesMutex.withLock { + if (seriesList.isNotEmpty() && (now - lastFetchTime) < TimeUnit.HOURS.toMillis(1)) return + val response = webClient.httpGet("$baseUrl/api/series/list", getRequestHeaders()) + val json = response.parseJson() + val dataArray = json.optJSONArray("data") ?: return + val comics = mutableListOf<MangaDto>() + for (i in 0 until dataArray.length()) { + val obj = dataArray.optJSONObject(i) ?: continue + if (obj.optString("type") != "comic") continue + val id = obj.getInt("id") + val name = obj.optString("name", "") + val slug = obj.optString("slug", "") + val cover = obj.optString("cover", "") + val summary = obj.optString("summary", "") + val statusId = obj.optJSONObject("status")?.optInt("id") ?: -1 + val genres = mutableListOf<String>() + obj.optJSONArray("genres")?.let { arr -> + for (j in 0 until arr.length()) { + arr.optJSONObject(j)?.optString("name", "")?.takeIf { it.isNotEmpty() }?.let(genres::add) + } + } + comics.add(MangaDto(id, name, slug, cover, summary, statusId, genres)) + slugMap[id.toString()] = slug + } + seriesList = comics + lastFetchTime = now + } + } + + private data class MangaDto( + val id: Int, val name: String, val slug: String, val cover: String, + val summary: String, val statusId: Int, val genres: List<String> + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + fetchSeriesListIfNeeded() + + val query = filter.query?.trim().orEmpty() + + if (query.isNotEmpty()) { + val filtered = seriesList.filter { it.name.contains(query, ignoreCase = true) } + val start = (page - 1) * pageSize + if (start >= filtered.size) return emptyList() + val end = (start + pageSize).coerceAtMost(filtered.size) + return filtered.subList(start, end).map { it.toManga() } + } + + val apiUrl = when (order) { + SortOrder.POPULARITY -> "$baseUrl/api/rankings?page=$page&period=total_ranking" + SortOrder.UPDATED -> "$baseUrl/api/new-chapters?page=$page" + else -> "$baseUrl/api/rankings?page=$page&period=total_ranking" + } + + val json = webClient.httpGet(apiUrl, getRequestHeaders()).parseJson() + val dataArray = json.optJSONArray("data") ?: return emptyList() + val mangas = mutableListOf<Manga>() + for (i in 0 until dataArray.length()) { + val obj = dataArray.optJSONObject(i) ?: continue + if (obj.optString("type") != "comic") continue + val id = obj.getInt("id") + val name = obj.optString("name", "") + val slug = obj.optString("slug", "") + val cover = obj.optString("cover", "") + slugMap[id.toString()] = slug + mangas.add(MangaDto(id, name, slug, cover, "", -1, emptyList()).toManga()) + } + return mangas + } + + private fun MangaDto.toManga() = Manga( + id = generateUid(id.toString()), + url = id.toString(), + publicUrl = "$baseUrl/series/comic-$slug", + title = name, + coverUrl = cover.takeIf { it.isNotEmpty() }?.let { + if (it.startsWith("http")) it else "$baseUrl/$it" + } ?: "", + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + + override suspend fun getDetails(manga: Manga): Manga { + fetchSeriesListIfNeeded() + val mangaId = manga.url + val slug = slugMap[mangaId] ?: return manga + + val url = "$baseUrl/api/series/$slug?type=comic" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val data = json.optJSONObject("data") ?: return manga + + val title = data.optString("name", manga.title) + val cover = data.optString("cover", manga.coverUrl) + val summary = data.optString("summary", "") + val statusId = data.optJSONObject("status")?.optInt("id") ?: -1 + val state = when (statusId) { + 1 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.FINISHED + 5 -> MangaState.ABANDONED + else -> null + } + val genres = mutableSetOf<MangaTag>() + data.optJSONArray("genres")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.optString("name", "")?.takeIf { it.isNotEmpty() }?.let { + genres.add(MangaTag(it.lowercase(), it, source)) + } + } + } + val chapters = fetchChapters(mangaId, slug) + + return manga.copy( + title = title, + coverUrl = cover?.takeIf { it.isNotEmpty() }?.let { + if (it.startsWith("http")) it else "$baseUrl/$it" + } ?: manga.coverUrl, + description = summary, + tags = genres, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChapters(mangaId: String, slug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + var page = 1 + var total = Int.MAX_VALUE + while (chapters.size < total) { + val url = "$apiBaseUrl/api/series/$slug/chapters?page=$page&direction=desc&type=comic" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val dataArray = json.optJSONArray("data") ?: break + total = json.optJSONObject("meta")?.optInt("total", dataArray.length()) ?: dataArray.length() + for (i in 0 until dataArray.length()) { + val ch = dataArray.optJSONObject(i) ?: continue + val id = ch.getInt("id") + val name = ch.optString("name", "") + val publishedAt = ch.optString("published_at", "") + val uploadDate = try { dateFormat.parse(publishedAt)?.time ?: 0L } catch (_: Exception) { 0L } + chapters.add( + MangaChapter( + id = generateUid("$mangaId/$id"), + title = "Capitulo $name", + number = name.toFloatOrNull() ?: 0f, + volume = 0, + url = "$mangaId/$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + page++ + } + return chapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("/") + if (parts.size != 2) return emptyList() + val mangaId = parts[0] + val chapterId = parts[1] + val slug = slugMap[mangaId] ?: return emptyList() + + val url = "$baseUrl/api/capitulo/comic-$slug/$chapterId" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val pagesArray = json.optJSONObject("chapter")?.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).map { i -> + val imgUrl = pagesArray.getString(i) + MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} From a1000e7ec7d33886fa78f267d0f005fafdaa498d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:11:47 +0200 Subject: [PATCH 091/434] Create MistScans.kt --- src/main/kotlin/tsuki/site/en/MistScans.kt | 326 +++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/MistScans.kt diff --git a/src/main/kotlin/tsuki/site/en/MistScans.kt b/src/main/kotlin/tsuki/site/en/MistScans.kt new file mode 100644 index 00000000..473832fc --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MistScans.kt @@ -0,0 +1,326 @@ +package tsuki.site.en + +import tsuki.network.OkHttpWebClient +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale + +// Todo: preloading chapter image fix + +@MangaSourceParser("MISTSCANS", "Mist Scans", "en") +internal class MistScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MISTSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("mistscans.com") + private val baseUrl = "https://$domain" + + private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + private var cachedGenres: List<Genre>? = null + + private data class Genre( + val name: String, + val id: String + ) + + private suspend fun fetchGenres(): List<Genre> { + if (cachedGenres != null) return cachedGenres!! + val html = webClient.httpGet("$baseUrl/series/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + val genres = doc.select("#series_tags_page > button").map { btn -> + Genre(btn.text(), btn.attr("tag")) + } + cachedGenres = genres + return genres + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ADDED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + return MangaListFilterOptions( + availableTags = genres.map { MangaTag(it.name, it.id, source) }.toSet(), + availableStates = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val genreIds = filter.tags.map { it.key } + + if (query.isNotEmpty() || genreIds.isNotEmpty()) { + return searchManga(query, genreIds) + } + + return when (order) { + SortOrder.UPDATED -> fetchLatest() + SortOrder.ADDED -> fetchAdded() + else -> fetchPopular() + } + } + + private suspend fun fetchPopular(): List<Manga> { + val html = webClient.httpGet(baseUrl, getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select(".series-splide .splide__slide").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun fetchLatest(): List<Manga> { + val html = webClient.httpGet("$baseUrl/latest/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select("div.grid > div.group").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun fetchAdded(): List<Manga> { + val html = webClient.httpGet("$baseUrl/series/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select("#searched_series_page > button").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun searchManga(query: String, genreIds: List<String>): List<Manga> { + val urlBuilder = "$baseUrl/series/".toHttpUrlOrNull()?.newBuilder()?.apply { + if (query.isNotEmpty()) addQueryParameter("q", query) + genreIds.forEach { addQueryParameter("genre", it) } + }?.build()?.toString() ?: return emptyList() + + val html = webClient.httpGet(urlBuilder, getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + + return doc.select("#searched_series_page > button") + .filter { element -> + val title = element.attr("title") + val matchesQuery = query.isEmpty() || title.contains(query, true) + + val entryGenres = try { + element.attr("tags").parseAsList() + } catch (_: Exception) { emptyList() } + val matchesGenres = genreIds.isEmpty() || + genreIds.all { genre -> entryGenres.any { it.equals(genre, true) } } + + matchesQuery && matchesGenres + } + .map { element -> popularMangaFromElement(element) } + } + + private fun popularMangaFromElement(element: Element): Manga { + val a = element.selectFirst("a[href]") ?: throw Exception("Link not found") + val title = a.attr("title") + val url = a.attr("abs:href") + val cover = element.getImageUrl("*[style*=background-image]") ?: "" + return Manga( + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = fetchDocument(manga.url) + + val title = doc.selectFirst("div.grid > h1")?.text() ?: manga.title + val cover = doc.getImageUrl("div[class*=photoURL]") ?: manga.coverUrl ?: "" + val status = doc.selectFirst("div:has(span:containsOwn(Status)) ~ div").parseStatus() + val author = doc.selectFirst("div:has(span:containsOwn(Author)) ~ div")?.text() + val artist = doc.selectFirst("div:has(span:containsOwn(Artist)) ~ div")?.text() + val type = doc.selectFirst("div:has(span:containsOwn(Type)) ~ div")?.text() + val genres = doc.select("div.grid:has(>h1) > div > a:not([title='Status'])").map { it.text() } + val altNames = doc.select("div.font-medium:containsOwn(Alternative titles) ~ div span").mapToSet { it.text() } + val description = buildString { + val synopsis = doc.selectFirst("#expand_content p")?.text().orEmpty() + append(synopsis) + }.takeIf { it.isNotEmpty() } + + val chapters = fetchChapters(doc) + + return manga.copy( + title = title, + altTitles = altNames, + coverUrl = cover, + description = description, + authors = listOfNotNull(author, artist).toSet(), + tags = (genres + listOfNotNull(type)).map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = status, + chapters = chapters, + ) + } + + private suspend fun fetchDocument(url: String): Document { + val fullUrl = if (url.startsWith("http")) url else "$baseUrl$url" + val html = webClient.httpGet(fullUrl, getRequestHeaders()).body.string() + return Jsoup.parse(html, fullUrl) + } + + private fun Element?.parseStatus(): MangaState? = when (this?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "dropped" -> MangaState.ABANDONED + "paused" -> MangaState.PAUSED + "completed" -> MangaState.FINISHED + else -> null + } + + private fun fetchChapters(document: Document): List<MangaChapter> { + val selector = "#chapters > a:not(:has(.text-sm span:matches(Upcoming))):not(:has(img[alt~=Coin]))" + return document.select(selector).map { element -> + val link = element.selectFirst("a[href]") ?: return@map null + val url = link.attr("abs:href") + val name = element.selectFirst(".text-sm")?.text() ?: "" + val dateStr = element.selectFirst(".text-xs")?.text()?.trim() ?: "" + val uploadDate = dateStr.parseDate() + + MangaChapter( + id = generateUid(url), + title = name, + number = 0f, + volume = 0, + url = url, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.filterNotNull().sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = fetchDocument(chapter.url) + + val cdnUrl = getCdnUrl(doc) + val uidPages = doc.select("#pages > img") + .map { it.attr("uid") } + .filter { it.isNotEmpty() } + + if (uidPages.isNotEmpty() && cdnUrl != null) { + return uidPages.mapIndexed { _, uid -> + MangaPage( + id = generateUid(uid), + url = "$cdnUrl/$uid", + preview = null, + source = source, + ) + } + } + + return doc.select("#pages > img").map { it.imgAttr() } + .filter { it.contains(Regex("""^(https?:)?//cdn\d*\.keyoapp\.com""")) } + .mapIndexed { _, imgUrl -> + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + private fun getCdnUrl(document: Document): String? { + val script = document.select("script").firstOrNull { + CDN_HOST_REGEX.containsMatchIn(it.html()) + } ?: return null + val cdnHost = CDN_HOST_REGEX.find(script.html())?.groups?.get(1)?.value + ?.replace(CDN_CLEAN_REGEX, "") ?: return null + return "https://$cdnHost/uploads" + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun Element.getImageUrl(selector: String): String? { + val element = selectFirst(selector) ?: return null + val style = element.attr("style") + val match = IMG_REGEX.find(style) ?: return null + val rawUrl = match.groups[1]?.value ?: return null + val url = rawUrl.toHttpUrlOrNull() ?: return null + return url.newBuilder().setQueryParameter("w", "480").build().toString() + } + + private fun Element.imgAttr(): String = when { + hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") + hasAttr("data-src") -> attr("abs:data-src") + else -> attr("abs:src") + } + + private fun String.parseDate(): Long { + val trimmed = trim() + if (trimmed.contains("ago")) return trimmed.parseRelativeDate() + return try { dateFormat.parse(trimmed)?.time ?: 0L } catch (_: Exception) { 0L } + } + + private fun String.parseRelativeDate(): Long { + val now = Calendar.getInstance().apply { + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + } + val number = split(" ").firstOrNull() + ?.replace("one", "1") + ?.replace("a", "1") + ?.toIntOrNull() ?: return 0L + val text = this.lowercase() + when { + "second" in text -> now.add(Calendar.SECOND, -number) + "minute" in text -> now.add(Calendar.MINUTE, -number) + "hour" in text -> now.add(Calendar.HOUR, -number) + "day" in text -> now.add(Calendar.DAY_OF_YEAR, -number) + "week" in text -> now.add(Calendar.WEEK_OF_YEAR, -number) + "month" in text -> now.add(Calendar.MONTH, -number) + "year" in text -> now.add(Calendar.YEAR, -number) + } + return now.timeInMillis + } + + private fun String.parseAsList(): List<String> { + return removeSurrounding("[", "]") + .split(",") + .map { it.trim().removeSurrounding("\"") } + .filter { it.isNotEmpty() } + } + + companion object { + val CDN_HOST_REGEX = """realUrl\s*=\s*`[^`]+//([^/]+)""".toRegex() + val CDN_CLEAN_REGEX = """\$\{[^}]*\}""".toRegex() + val IMG_REGEX = """url\(['"]?([^(['")])]+)""".toRegex() + } +} From 9282fdc2e9790e5daf88ed9486765717bac5a822 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:26:52 +0200 Subject: [PATCH 092/434] Create VortexScans.kt --- src/main/kotlin/tsuki/site/en/VortexScans.kt | 327 +++++++++++++++++++ 1 file changed, 327 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/VortexScans.kt diff --git a/src/main/kotlin/tsuki/site/en/VortexScans.kt b/src/main/kotlin/tsuki/site/en/VortexScans.kt new file mode 100644 index 00000000..53b501c5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/VortexScans.kt @@ -0,0 +1,327 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.* +import tsuki.util.* + +import org.json.JSONArray +import org.json.JSONObject +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("VORTEXSCANS", "Vortex Scans", "en") +internal class VortexScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VORTEXSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("vortexscans.org") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://api.$domain" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .build() + + private var cachedGenres: List<Pair<String, String>>? = null + + private suspend fun fetchGenres(): List<Pair<String, String>> { + if (cachedGenres != null) return cachedGenres!! + val json = webClient.httpGet("$apiBaseUrl/api/genres", getRequestHeaders()).parseJson() + val arr = json.optJSONArray("data") ?: json.optJSONArray("genres") ?: return emptyList() + val list = mutableListOf<Pair<String, String>>() + for (i in 0 until arr.length()) { + val obj = arr.optJSONObject(i) ?: continue + list.add(obj.optString("name") to obj.optString("id")) + } + cachedGenres = list + return list + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + val tags = mutableSetOf<MangaTag>() + + tags.add(MangaTag("Ongoing", "status_ongoing", source)) + tags.add(MangaTag("Completed", "status_completed", source)) + tags.add(MangaTag("Canceled", "status_canceled", source)) + tags.add(MangaTag("Dropped", "status_dropped", source)) + tags.add(MangaTag("Coming Soon", "status_coming_soon", source)) + tags.add(MangaTag("Mass Released", "status_mass_released", source)) + + tags.add(MangaTag("Manga", "type_manga", source)) + tags.add(MangaTag("Manhua", "type_manhua", source)) + tags.add(MangaTag("Manhwa", "type_manhwa", source)) + tags.add(MangaTag("Russian", "type_russian", source)) + tags.add(MangaTag("Spanish", "type_spanish", source)) + + genres.forEach { (name, id) -> + tags.add(MangaTag(name, "genre_$id", source)) + } + + return MangaListFilterOptions( + availableTags = tags, + availableStates = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val status = filter.tags.firstOrNull { it.key.startsWith("status_") }?.key?.removePrefix("status_") + val type = filter.tags.firstOrNull { it.key.startsWith("type_") }?.key?.removePrefix("type_") + val genreIds = filter.tags.filter { it.key.startsWith("genre_") }.map { it.key.removePrefix("genre_") } + + val (orderBy, orderDir) = when (order) { + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.ADDED -> "createdAt" to "desc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + else -> "totalViews" to "desc" + } + + val baseHttpUrl = "$apiBaseUrl/api/query".toHttpUrlOrNull() ?: return emptyList() + val urlBuilder = baseHttpUrl.newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + if (query.isNotEmpty()) addQueryParameter("searchTerm", query) + if (!status.isNullOrEmpty()) addQueryParameter("seriesStatus", status.uppercase()) + if (!type.isNullOrEmpty()) addQueryParameter("seriesType", type.uppercase()) + if (genreIds.isNotEmpty()) addQueryParameter("genreIds", genreIds.joinToString(",")) + addQueryParameter("orderBy", orderBy) + addQueryParameter("orderDirection", orderDir) + } + + val (mangas, _) = fetchSearchPageWithFallback(urlBuilder, page) + return mangas + } + + private suspend fun fetchSearchPageWithFallback( + initialUrlBuilder: HttpUrl.Builder, + startPage: Int + ): Pair<List<Manga>, Boolean> { + var currentPage = startPage + while (true) { + val url = initialUrlBuilder + .setQueryParameter("page", currentPage.toString()) + .build() + .toString() + + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val posts = json.optJSONArray("posts") ?: JSONArray() + val totalCount = json.optInt("totalCount", 0) + + val mangas = mutableListOf<Manga>() + for (i in 0 until posts.length()) { + val obj = posts.optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + mangas.add(obj.toManga()) + } + + val hasNext = totalCount > (currentPage * pageSize) + + if (mangas.isNotEmpty() || !hasNext) { + return mangas to hasNext + } + currentPage++ + } + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val slug = getString("slug") + val title = getString("postTitle") + val cover = optString("featuredImage", "") + return Manga( + id = generateUid("$slug#$id"), + url = "$slug#$id", + publicUrl = "$baseUrl/series/$slug", + title = title, + coverUrl = cover.ifEmpty { null }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val (slug, id) = manga.url.split("#") + val detailsUrl = "$apiBaseUrl/api/post?postSlug=$slug" + val json = webClient.httpGet(detailsUrl, getRequestHeaders()).parseJson() + val data = json.optJSONObject("post") ?: return manga + if (data.optBoolean("isNovel", false)) throw Exception("Novels not supported") + + val title = data.optString("postTitle", manga.title) + val cover = data.optString("featuredImage", "") + val desc = data.optString("postContent", "") + val altTitles = data.optString("alternativeTitles", "") + val author = data.optString("author", "") + val artist = data.optString("artist", "") + val seriesType = data.optString("seriesType", "") + val seriesStatus = data.optString("seriesStatus", "") + val genresArr = data.optJSONArray("genres") ?: JSONArray() + val chaptersArr = data.optJSONArray("chapters") ?: JSONArray() + val totalChapters = json.optInt("totalChapterCount", chaptersArr.length()) + + val state = when (seriesStatus.uppercase()) { + "ONGOING", "COMING_SOON", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (seriesType.isNotEmpty()) { + tags.add(MangaTag(seriesType.lowercase(), seriesType, source)) + } + for (i in 0 until genresArr.length()) { + val g = genresArr.optJSONObject(i) ?: continue + val name = g.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name.lowercase(), name, source)) + } + + val plainDesc = if (desc.isNotEmpty()) { + org.jsoup.Jsoup.parse(desc.replace("\n", "<br>")).text() + } else "" + val fullDesc = buildString { + append(plainDesc) + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Alternative Names: $altTitles") + } + }.trim() + + val chapters = if (totalChapters > chaptersArr.length()) { + fetchChaptersFromApi(id.toInt(), slug) + } else { + parseChaptersFromArray(chaptersArr, slug) + } + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = fullDesc.takeIf { it.isNotEmpty() }, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChaptersFromApi(postId: Int, slug: String): List<MangaChapter> { + val url = "$apiBaseUrl/api/chapters?postId=$postId" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val chaptersArr = json.optJSONObject("post")?.optJSONArray("chapters") ?: JSONArray() + return parseChaptersFromArray(chaptersArr, slug) + } + + private fun parseChaptersFromArray(arr: JSONArray, mangaSlug: String): List<MangaChapter> { + val list = mutableListOf<MangaChapter>() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + if (!ch.optBoolean("isAccessible", false)) continue + val id = ch.getInt("id") + val number = ch.optString("number", "") + val title = ch.optString("title", "") + val slug = ch.optString("slug", "") + val dateStr = ch.optString("createdAt", "") + val uploadDate = try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + + val cleanTitle = if (title == "null" || title.isEmpty()) null else title + val fullTitle = buildString { + append("Chapter $number") + if (cleanTitle != null) append(" - $cleanTitle") + } + + list.add( + MangaChapter( + id = generateUid("$mangaSlug#$id"), + title = fullTitle, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug#$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("#") + if (parts.size != 2) return emptyList() + val chapterId = parts[1].toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/chapter?chapterId=$chapterId", getRequestHeaders()).parseJson() + val chData = json.optJSONObject("chapter") ?: return emptyList() + if (chData.optBoolean("isShortLinkLocked")) throw Exception("Chapter locked (short link)") + if (chData.optBoolean("isLockedByCoins")) throw Exception("Chapter locked (coins required)") + if (chData.optBoolean("isPermanentlyLocked")) throw Exception("Chapter permanently locked") + + val imagesArr = chData.optJSONArray("images") ?: return emptyList() + val pages = mutableListOf<JSONObject>() + for (i in 0 until imagesArr.length()) { + pages.add(imagesArr.optJSONObject(i) ?: continue) + } + pages.sortBy { it.optInt("order", Int.MAX_VALUE) } + + return pages.mapIndexed { _, imgObj -> + val imgUrl = imgObj.optString("url", "").replace(" ", "%20") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val (_, id) = seed.url.split("#") + val postId = id.toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/recommendations?postId=$postId&limit=25", getRequestHeaders()).parseJson() + val recs = json.optJSONArray("recommendations") ?: return emptyList() + val mangas = mutableListOf<Manga>() + for (i in 0 until recs.length()) { + val obj = recs.optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + mangas.add(obj.toManga()) + } + return mangas + } +} From ff53d50c6c9176c934ebdc903648398122ff69ea Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 27 Jul 2026 15:53:30 +0200 Subject: [PATCH 093/434] Create WebtoonXYZ.kt --- src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt | 27 +++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt diff --git a/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt b/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt new file mode 100644 index 00000000..90858dbf --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt @@ -0,0 +1,27 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaParserSource + +@MangaSourceParser("WEBTOONXYZ", "WebtoonXYZ", "en") +internal class WebtoonXYZ(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.WEBTOONXYZ, "webtoon.xyz", 20) { + + override val datePattern = "dd MMMM yyyy" + + override val listUrl = "webtoons/" + + private val thumbnailRegex = Regex("-\\d+x\\d+(\\.[a-zA-Z]+)$") + + override fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { + return super.parseMangaList(doc).map { manga -> + manga.copy( + coverUrl = manga.coverUrl?.replace(thumbnailRegex, "$1") + ) + } + } +} From 3f46918c1d71f24320b26abc720f928abbf5b05e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:53:11 +0200 Subject: [PATCH 094/434] sort by number --- src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt index af409a2c..6a9af0d1 100644 --- a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -289,7 +289,7 @@ internal abstract class DoujinDesuParser( rating = obj.optDouble("rating", 0.0).toFloat() / 10f, tags = tags, coverUrl = coverUrl ?: manga.coverUrl, - chapters = chapters.reversed() + chapters = chapters.sortedBy { it.number } ) synchronized(detailsCacheLock) { From 7915402e3af7c1f6cd556b628c87f42159bea93f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:56:27 +0200 Subject: [PATCH 095/434] Update GalleryAdultsParser.kt --- src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt index fab5860a..40984908 100644 --- a/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt @@ -42,6 +42,12 @@ import org.jsoup.nodes.Element import java.util.EnumSet import java.util.Locale +/* +Default: One chapter per manga + +Override getDetails chapters section if source contains more than one chapter. + */ + internal abstract class GalleryAdultsParser( context: MangaLoaderContext, source: MangaParserSource, @@ -249,4 +255,4 @@ internal abstract class GalleryAdultsParser( .trim() protected open fun Locale.toLanguagePath() = getDisplayLanguage(Locale.ENGLISH).lowercase() -} \ No newline at end of file +} From 52586590d4c4a64e2f4d4ba619cd0be7215b8736 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:56:55 +0200 Subject: [PATCH 096/434] Update DoujinDesuParser.kt --- src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt index 6a9af0d1..a38950f0 100644 --- a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -38,6 +38,7 @@ import kotlinx.coroutines.delay import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock +// Only for the source id/DoujinDesu? internal abstract class DoujinDesuParser( context: MangaLoaderContext, From d515bedd68dfdc5ee4a15304636c928033431e70 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 10:59:43 +0200 Subject: [PATCH 097/434] sort by number --- src/main/kotlin/tsuki/parsers/GalleryParser.kt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/GalleryParser.kt b/src/main/kotlin/tsuki/parsers/GalleryParser.kt index ce61f0a8..d6ed991a 100644 --- a/src/main/kotlin/tsuki/parsers/GalleryParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryParser.kt @@ -119,7 +119,9 @@ internal abstract class GalleryParser( branch = null, source = source, ) - }.orEmpty() + } + ?.sortedBy { it.number } + .orEmpty() return manga.copy(chapters = chapters, description = description) } @@ -146,4 +148,4 @@ internal abstract class GalleryParser( ) }.toSet() } -} \ No newline at end of file +} From 50b2712a7afa945375c8214882dc60f17540de8f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:01:15 +0200 Subject: [PATCH 098/434] Update HeanCmsParser.kt --- src/main/kotlin/tsuki/parsers/HeanCmsParser.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt index af40847d..0245eaf2 100644 --- a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt +++ b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt @@ -183,9 +183,8 @@ internal abstract class HeanCmsParser( val data = response.getJSONArray("data").asTypedList<JSONObject>() val dateFormat = SimpleDateFormat(datePattern, Locale.ENGLISH) return manga.copy( - chapters = data.mapChapters(reversed = true) { i, it -> - val chapterUrl = - "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" + chapters = data.mapChapters(reversed = false) { i, it -> + val chapterUrl = "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" MangaChapter( id = generateUid(it.getLong("id")), title = it.getString("chapter_name"), @@ -197,7 +196,7 @@ internal abstract class HeanCmsParser( branch = null, source = source, ) - }, + }.sortedBy { it.number } ) } @@ -238,4 +237,4 @@ internal abstract class HeanCmsParser( } } -} \ No newline at end of file +} From 29e4c14cd39dafcef17720a49ba144705632a9c1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:02:48 +0200 Subject: [PATCH 099/434] Update LikeMangaParser.kt --- src/main/kotlin/tsuki/parsers/LikeMangaParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt index 6dafdb8b..fea3a4fb 100644 --- a/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt @@ -177,7 +177,7 @@ internal abstract class LikeMangaParser( val maxPageChapterSelect = doc.getElementById("nav_list_chapter_id_detail")?.select("a:not(.next)") var maxPageChapter = 1 if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { + maxPageChapterSelect.forEach { val i = it.text().toInt() if (i > maxPageChapter) { maxPageChapter = i @@ -212,7 +212,7 @@ internal abstract class LikeMangaParser( result } } - }.reversed(), + }.sortedBy { it.number } ) } @@ -306,4 +306,4 @@ internal abstract class LikeMangaParser( else -> dateFormat.parseSafe(date) } } -} \ No newline at end of file +} From 751cefe7e9a9ad1b2093fe0d41a11ee5799e2d71 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:03:34 +0200 Subject: [PATCH 100/434] Update LilianaParser.kt --- src/main/kotlin/tsuki/parsers/LilianaParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/LilianaParser.kt b/src/main/kotlin/tsuki/parsers/LilianaParser.kt index 44f5411d..6f2eb03a 100644 --- a/src/main/kotlin/tsuki/parsers/LilianaParser.kt +++ b/src/main/kotlin/tsuki/parsers/LilianaParser.kt @@ -201,7 +201,7 @@ internal abstract class LilianaParser( in abandoned -> MangaState.ABANDONED else -> null }, - chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = true) { i, element -> + chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = false) { i, element -> val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") MangaChapter( id = generateUid(href), @@ -215,7 +215,7 @@ internal abstract class LilianaParser( branch = null, source = source, ) - }, + }.sortedBy { it.number }, ) } From 51133e8d140d29674c7e7ebee6ee82ac06d32c8b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:06:04 +0200 Subject: [PATCH 101/434] Update MadaraParser.kt --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index 0af51490..1124cd73 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -685,7 +685,7 @@ internal abstract class MadaraParser( protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.body().select(selectChapter).mapChapters(reversed = false) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -705,7 +705,7 @@ internal abstract class MadaraParser( scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } protected open val postDataReq = "action=manga_get_chapters&manga=" @@ -721,7 +721,7 @@ internal abstract class MadaraParser( webClient.httpPost(url, emptyMap()).parseHtml() } val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.select(selectChapter).mapChapters(reversed = false) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -741,7 +741,7 @@ internal abstract class MadaraParser( scanlator = null, source = source, ) - } + }.sortedBy { it.number } } override suspend fun getRelatedManga(seed: Manga): List<Manga> { @@ -933,4 +933,4 @@ internal abstract class MadaraParser( return chunked(2).map { it.toInt(16).toByte() }.toByteArray() } } -} \ No newline at end of file +} From 5d542ad6e25d6fd345913676ca38369854d2a9d7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:06:50 +0200 Subject: [PATCH 102/434] Update MadthemeParser.kt --- src/main/kotlin/tsuki/parsers/MadthemeParser.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadthemeParser.kt b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt index 3cbd1f01..0336e3cf 100644 --- a/src/main/kotlin/tsuki/parsers/MadthemeParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt @@ -277,7 +277,7 @@ internal abstract class MadthemeParser( val slug = doc.selectFirstOrThrow("script:containsData(bookSlug)").data().substringAfter("bookSlug = \"") .substringBefore("\";") val docChapter = webClient.httpGet("https://$domain/api/manga/$slug/chapters?source=detail").parseHtml() - return docChapter.select(selectChapter).mapChapters(reversed = true) { i, li -> + return docChapter.select(selectChapter).mapChapters(reversed = false) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val dateText = li.selectFirst(selectDate)?.text() @@ -295,7 +295,7 @@ internal abstract class MadthemeParser( scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } protected open val selectPage = "div#chapter-images img" @@ -446,4 +446,4 @@ internal abstract class MadthemeParser( private const val IMAGE_FALLBACK_HOST = "sb.mbcdn.xyz" } -} \ No newline at end of file +} From 6c3149c53c9d17f6da099ac5c1adbd5222c8cde7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:14:25 +0200 Subject: [PATCH 103/434] Create ChapterNumber.kt --- src/main/kotlin/tsuki/util/ChapterNumber.kt | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 src/main/kotlin/tsuki/util/ChapterNumber.kt diff --git a/src/main/kotlin/tsuki/util/ChapterNumber.kt b/src/main/kotlin/tsuki/util/ChapterNumber.kt new file mode 100644 index 00000000..0b315ff9 --- /dev/null +++ b/src/main/kotlin/tsuki/util/ChapterNumber.kt @@ -0,0 +1,10 @@ +package tsuki.util + +/* +Extracts the first integer or decimal number from a string. +Returns the parsed [Float], or 0f if no number is found. +Useful for chapter titles like "Chapter 10 / Vol.1 / Ch.5.5 / 12.3" + */ +fun String.extractChapterNumber(): Float { + return Regex("""(\d+(?:\.\d+)?)""").find(this)?.value?.toFloatOrNull() ?: 0f +} From 382585f7658a4e45567df5893025c072fcc57175 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:38:43 +0200 Subject: [PATCH 104/434] Update ChapterNumber.kt --- src/main/kotlin/tsuki/util/ChapterNumber.kt | 29 +++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/util/ChapterNumber.kt b/src/main/kotlin/tsuki/util/ChapterNumber.kt index 0b315ff9..5a157d7b 100644 --- a/src/main/kotlin/tsuki/util/ChapterNumber.kt +++ b/src/main/kotlin/tsuki/util/ChapterNumber.kt @@ -1,10 +1,29 @@ package tsuki.util -/* -Extracts the first integer or decimal number from a string. -Returns the parsed [Float], or 0f if no number is found. -Useful for chapter titles like "Chapter 10 / Vol.1 / Ch.5.5 / 12.3" +import tsuki.model.MangaChapter + +private val CHAPTER_REGEX = Regex("""(?i)\b(?:chapter|ch\.?|chapitre|capítulo|cap\.?|episode|ep\.?)\s*(\d+(?:\.\d+)?)""") +private val FALLBACK_REGEX = Regex("""(\d+(?:\.\d+)?)""") + +/** + * Extracts the most likely chapter number from a string. + * Fast – no object allocations beyond the regex result. + * Returns [Float] or 0f if no number found. */ fun String.extractChapterNumber(): Float { - return Regex("""(\d+(?:\.\d+)?)""").find(this)?.value?.toFloatOrNull() ?: 0f + CHAPTER_REGEX.find(this)?.groupValues?.get(1)?.toFloatOrNull()?.let { return it } + return FALLBACK_REGEX.find(this)?.value?.toFloatOrNull() ?: 0f +} + +/** + * Sorts a list of [MangaChapter] by their chapter number (ascending). + * The number is extracted from the chapter title using [extractChapterNumber]. + * Chapters without a recognisable number appear first (treated as 0). + * + * Usage: `.sortChapters()` + */ +fun List<MangaChapter>.sortChapters(): List<MangaChapter> { + return map { it to (it.title?.extractChapterNumber() ?: 0f) } + .sortedBy { it.second } + .map { it.first } } From 34577aeaf7393d69f72b5cde45829967118c98b0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:39:18 +0200 Subject: [PATCH 105/434] Update MadaraParser.kt --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 20 +++++++------------ 1 file changed, 7 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index 1124cd73..e4258eba 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -31,7 +31,6 @@ import tsuki.util.src import tsuki.util.textOrNull import tsuki.util.toTitleCase import tsuki.util.urlEncoded -import tsuki.util.mapChapters import tsuki.util.selectOrThrow import tsuki.util.CryptoAES import tsuki.util.getCookies @@ -45,6 +44,7 @@ import tsuki.util.requireSrc import tsuki.util.selectLast import tsuki.util.toMutableMap import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber import androidx.collection.scatterSetOf import kotlinx.coroutines.async @@ -685,7 +685,7 @@ internal abstract class MadaraParser( protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = false) { i, li -> + return doc.body().select(selectChapter).map { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -694,13 +694,10 @@ internal abstract class MadaraParser( MangaChapter( id = generateUid(href), title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, url = link, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), source = source, scanlator = null, branch = null, @@ -721,7 +718,7 @@ internal abstract class MadaraParser( webClient.httpPost(url, emptyMap()).parseHtml() } val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.select(selectChapter).mapChapters(reversed = false) { i, li -> + return doc.select(selectChapter).map { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -731,13 +728,10 @@ internal abstract class MadaraParser( id = generateUid(href), url = link, title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, branch = null, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), scanlator = null, source = source, ) From fb941f063c14812f50d8b307931a6217439243c3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:45:58 +0200 Subject: [PATCH 106/434] Update GalleryParser.kt --- src/main/kotlin/tsuki/parsers/GalleryParser.kt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/GalleryParser.kt b/src/main/kotlin/tsuki/parsers/GalleryParser.kt index d6ed991a..f873b1d9 100644 --- a/src/main/kotlin/tsuki/parsers/GalleryParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryParser.kt @@ -23,7 +23,7 @@ import tsuki.util.attrAsRelativeUrl import tsuki.util.attrAsRelativeUrlOrNull import tsuki.util.attrOrNull import tsuki.util.attrOrThrow -import tsuki.util.mapChapters +import tsuki.util.extractChapterNumber import tsuki.util.mapNotNullToSet import tsuki.util.parseHtml import tsuki.util.parseSafe @@ -100,18 +100,20 @@ internal abstract class GalleryParser( } } + override suspend fun getDetails(manga: Manga): Manga { val content = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val description = content.selectFirst("div.article-info")?.text()?.trim() ?: "" val df = SimpleDateFormat("HH:mm dd-MM-yyyy") val time = content.selectFirst("div.article-info > small")?.text()?.trim() val chapters = content.selectFirst("nav.pagination")?.select("a.pagination-link") - ?.mapChapters { index, element -> + ?.map { element -> val relUrl = element.attrAsRelativeUrl("href") + val chapterText = element.ownText() MangaChapter( id = generateUid(relUrl), - title = null, - number = index + 1f, + title = chapterText, + number = chapterText.extractChapterNumber(), volume = 0, url = relUrl, scanlator = null, From ac0925c5ab780003e300632234b8ef6f154281fb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:47:59 +0200 Subject: [PATCH 107/434] Update HiperParser.kt --- src/main/kotlin/tsuki/parsers/HiperParser.kt | 22 ++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HiperParser.kt b/src/main/kotlin/tsuki/parsers/HiperParser.kt index c499a2dd..7c1310fc 100644 --- a/src/main/kotlin/tsuki/parsers/HiperParser.kt +++ b/src/main/kotlin/tsuki/parsers/HiperParser.kt @@ -314,8 +314,12 @@ abstract class HiperParser( id = generateUid("/$mangaPath/$slug/$number"), url = "/$mangaPath/$slug/$number", title = displayTitle, - number = number, volume = 0, uploadDate = uploadDate, - scanlator = null, branch = null, source = source + number = number, + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source ) } }.sortedBy { it.number } @@ -348,7 +352,12 @@ abstract class HiperParser( val url = page.optString("webpUrl", "").ifEmpty { page.optString("avifUrl", "") }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) } } catch (_: Exception) { emptyList() } } @@ -371,7 +380,12 @@ abstract class HiperParser( val url = page.optString("webpUrl", "").ifEmpty { page.optString("avifUrl", "") }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) } } From 71d667209772092534ce155a2877c5b34cb9942d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:51:58 +0200 Subject: [PATCH 108/434] Update HeanCmsParser.kt --- .../kotlin/tsuki/parsers/HeanCmsParser.kt | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt index 0245eaf2..2a222c84 100644 --- a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt +++ b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt @@ -29,9 +29,9 @@ import tsuki.util.json.getFloatOrDefault import tsuki.util.json.mapJSON import tsuki.util.json.mapJSONToSet import tsuki.util.json.unescapeJson -import tsuki.util.mapChapters import tsuki.util.parseJson import tsuki.util.parseSafe +import tsuki.util.extractChapterNumber import org.json.JSONArray import org.json.JSONObject @@ -182,22 +182,23 @@ internal abstract class HeanCmsParser( val response = webClient.httpGet(url).parseJson() val data = response.getJSONArray("data").asTypedList<JSONObject>() val dateFormat = SimpleDateFormat(datePattern, Locale.ENGLISH) - return manga.copy( - chapters = data.mapChapters(reversed = false) { i, it -> - val chapterUrl = "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" - MangaChapter( - id = generateUid(it.getLong("id")), - title = it.getString("chapter_name"), - number = i + 1f, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = dateFormat.parseSafe(it.getString("created_at").substringBefore("T")), - branch = null, - source = source, - ) - }.sortedBy { it.number } - ) + + val chapters = data.map { ch -> + val chapterUrl = "/series/${ch.getJSONObject("series").getString("series_slug")}/${ch.getString("chapter_slug")}" + val name = ch.getString("chapter_name") + MangaChapter( + id = generateUid(ch.getLong("id")), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = dateFormat.parseSafe(ch.getString("created_at").substringBefore("T")), + branch = null, + source = source, + ) + }.sortedBy { it.number } + return manga.copy(chapters = chapters) } protected open val selectPages = ".flex > img:not([alt])" From 47eb42209a574f7f0a7628e2e2b7c0d13d08c12b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:55:29 +0200 Subject: [PATCH 109/434] Update Manga18Parser.kt --- src/main/kotlin/tsuki/parsers/Manga18Parser.kt | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt index ca55945e..5d1a40dd 100644 --- a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -27,11 +27,11 @@ import tsuki.util.src import tsuki.util.textOrNull import tsuki.util.toTitleCase import tsuki.util.urlEncoded -import tsuki.util.mapChapters import tsuki.util.mapNotNullToSet import tsuki.util.nullIfEmpty import tsuki.util.parseSafe import tsuki.util.removeSuffix +import tsuki.util.extractChapterNumber import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -234,14 +234,15 @@ internal abstract class Manga18Parser( protected open suspend fun getChapters(doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.body().select(selectChapter).map { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") + val name = a.textOrNull() ?: "" val dateText = li.selectFirst(selectDate)?.text() MangaChapter( id = generateUid(href), - title = a.textOrNull(), - number = i + 1f, + title = name, + number = name.extractChapterNumber(), volume = 0, url = href, uploadDate = dateFormat.parseSafe(dateText), @@ -249,7 +250,7 @@ internal abstract class Manga18Parser( scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { @@ -273,4 +274,4 @@ internal abstract class Manga18Parser( ) } } -} \ No newline at end of file +} From 9dbafd7ea9829f9ebdf8644f8efa851c6c5db5ac Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:56:51 +0200 Subject: [PATCH 110/434] Update MangaboxParser.kt --- .../kotlin/tsuki/parsers/MangaboxParser.kt | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaboxParser.kt b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt index b14786c6..fd42fb09 100644 --- a/src/main/kotlin/tsuki/parsers/MangaboxParser.kt +++ b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt @@ -30,13 +30,13 @@ import tsuki.util.parseHtml import tsuki.util.selectFirstOrThrow import tsuki.util.src import tsuki.util.toTitleCase -import tsuki.util.mapChapters import tsuki.util.nullIfEmpty import tsuki.util.parseSafe import tsuki.util.removeSuffix import tsuki.util.requireSrc import tsuki.util.attrAsRelativeUrlOrNull import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -280,26 +280,24 @@ internal abstract class MangaboxParser( protected open suspend fun getChapters(doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.body().select(selectChapter).map { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") + val name = a.text() val dateText = li.select(selectDate).last()?.text() MangaChapter( id = generateUid(href), - title = a.text(), - number = i + 1f, + title = name, + number = name.extractChapterNumber(), volume = 0, url = href, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), source = source, scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } protected open val selectPage = "div#vungdoc img, div.container-chapter-reader img" @@ -456,4 +454,4 @@ internal abstract class MangaboxParser( private val backupImageRegex = Regex("""backupImage\s*=\s*\[([^]]+)]""") private val chapterImagesRegex = Regex("""chapterImages\s*=\s*\[([^]]+)]""") } -} \ No newline at end of file +} From 2d228ad05829f7adcceace5a9e48640a92e6ce69 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 11:57:58 +0200 Subject: [PATCH 111/434] Update MangaReaderParser.kt --- .../kotlin/tsuki/parsers/MangaReaderParser.kt | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt index 1cb7c034..133aa5af 100644 --- a/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt +++ b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt @@ -27,7 +27,6 @@ import tsuki.util.src import tsuki.util.textOrNull import tsuki.util.toTitleCase import tsuki.util.urlEncoded -import tsuki.util.mapChapters import tsuki.util.mapNotNullToSet import tsuki.util.nullIfEmpty import tsuki.util.oneOrThrowIfMany @@ -44,6 +43,7 @@ import tsuki.util.mimeType import tsuki.util.parseRaw import tsuki.util.runCatchingCancellable import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber import androidx.collection.ArrayMap import kotlinx.coroutines.runBlocking @@ -221,20 +221,23 @@ internal abstract class MangaReaderParser( override suspend fun getDetails(manga: Manga): Manga { val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + val chapters = docs.select(selectChapter).mapNotNull { element -> + val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val chapterNumEl = element.selectFirst(".chapternum") + val name = chapterNumEl?.textOrNull() ?: "" MangaChapter( id = generateUid(url), - title = element.selectFirst(".chapternum")?.textOrNull(), + title = name, url = url, - number = index + 1f, + number = name.extractChapterNumber(), volume = 0, scanlator = null, uploadDate = dateFormat.parseSafe(element.selectFirst(".chapterdate")?.text()), branch = null, source = source, ) - } + }.sortedBy { it.number } + return parseInfo(docs, manga, chapters) } @@ -458,4 +461,4 @@ internal abstract class MangaReaderParser( this, url, generateUid(url), ) } -} \ No newline at end of file +} From 2a865db57d1953fccad834ae441373eec00662e3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:06:19 +0200 Subject: [PATCH 112/434] Update MangaThemesia.kt --- .../kotlin/tsuki/parsers/MangaThemesia.kt | 31 ++++++++----------- 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 3e2a18fd..12af006c 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -1,5 +1,7 @@ package tsuki.parsers +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser @@ -24,7 +26,7 @@ import tsuki.util.toAbsoluteUrl import tsuki.util.attrAsRelativeUrl import tsuki.util.parseHtml import tsuki.util.urlEncoded -import tsuki.util.mapChapters +import tsuki.util.extractChapterNumber import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -168,11 +170,13 @@ abstract class MangaThemesia( parseMangaElement(element) } } - - override suspend fun getDetails(manga: Manga): Manga { + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() + val chaptersDeferred = async { loadChapters(doc, manga.url) } + val title = doc.selectFirst("h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1") ?.text() ?: manga.title @@ -229,9 +233,9 @@ abstract class MangaThemesia( ?: doc.selectFirst(".num")?.text()?.toFloatOrNull() val normalizedRating = if (rating != null && rating > 0) rating / 10f else RATING_UNKNOWN - val chapters = loadChapters(doc, manga.url) + val chapters = chaptersDeferred.await() - return manga.copy( + manga.copy( title = title, description = description, coverUrl = coverUrl, @@ -270,32 +274,23 @@ abstract class MangaThemesia( } private fun parseChapters(elements: Elements): List<MangaChapter> { - return elements.mapChapters(reversed = true) { i, element -> - val a = element.selectFirst("a") ?: return@mapChapters null + return elements.mapNotNull { element -> + val a = element.selectFirst("a") ?: return@mapNotNull null val href = a.attrAsRelativeUrl("href") val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() val dateStr = element.selectFirst(".chapterdate")?.text() - val number = extractChapterNumber(name) ?: (i + 1).toFloat() MangaChapter( id = generateUid(href), url = href, title = name, - number = number, + number = name.extractChapterNumber(), volume = 0, uploadDate = parseChapterDate(dateStr), scanlator = null, branch = null, source = source, ) - } - } - - private fun extractChapterNumber(title: String): Float? { - return Regex("""(?i)(?:chapter|ch\.?)\s*([0-9]+(?:\.[0-9]+)?)""") - .find(title) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() + }.sortedBy { it.number } } protected open fun parseChapterDate(date: String?): Long { From 9fd6e0f9eb5238ea2974be5b37eefcde215b9271 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:12:08 +0200 Subject: [PATCH 113/434] Update NatsuParser.kt --- src/main/kotlin/tsuki/parsers/NatsuParser.kt | 119 +++++++++++-------- 1 file changed, 67 insertions(+), 52 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt index fcaac8e2..8c69696e 100644 --- a/src/main/kotlin/tsuki/parsers/NatsuParser.kt +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -40,6 +40,8 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale import org.jsoup.nodes.Document +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope internal abstract class NatsuParser( context: MangaLoaderContext, @@ -215,66 +217,79 @@ internal abstract class NatsuParser( return slugs.mapNotNull { mangaMap[it] } } - @get:Synchronized + private val detailsCacheLock = Any() private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val slug = manga.url.removePrefix("/manga/").removeSuffix("/") - val id = getMangaIdFromSlug(slug) ?: return manga - - val url = "https://$domain/wp-json/wp/v2/manga/$id?_embed" - val obj = webClient.httpGet(url).parseJson() - val title = obj.getJSONObject("title").getString("rendered") - val description = obj.getJSONObject("content").getString("rendered") - .let { org.jsoup.Jsoup.parseBodyFragment(it).wholeText().trim() } - val cover = obj.optJSONObject("_embedded") - ?.optJSONArray("wp:featuredmedia") - ?.optJSONObject(0) - ?.optString("source_url", null) - - val embedded = obj.optJSONObject("_embedded") - val terms = embedded?.optJSONArray("wp:term") - - fun getTermNames(taxonomy: String): List<String> { - if (terms == null) return emptyList() - for (i in 0 until terms.length()) { - val termArray = terms.optJSONArray(i) ?: continue - if (termArray.length() > 0 && termArray.getJSONObject(0).optString("taxonomy") == taxonomy) - return (0 until termArray.length()).map { j -> termArray.getJSONObject(j).getString("name") } - } - return emptyList() + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } } - val genres = getTermNames("genre") + getTermNames("type") - val statusList = getTermNames("status") - val state = when { - statusList.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING - statusList.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED - statusList.any { it.equals("Cancelled", ignoreCase = true) } -> MangaState.ABANDONED - statusList.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED - else -> null + val result = try { + coroutineScope { + val slug = manga.url.removePrefix("/manga/").removeSuffix("/") + val id = getMangaIdFromSlug(slug) ?: return@coroutineScope manga + + val chaptersDeferred = async { loadChapters(id.toString(), "https://$domain/manga/$slug/") } + + val url = "https://$domain/wp-json/wp/v2/manga/$id?_embed" + val obj = webClient.httpGet(url).parseJson() + val title = obj.getJSONObject("title").getString("rendered") + val description = obj.getJSONObject("content").getString("rendered") + .let { org.jsoup.Jsoup.parseBodyFragment(it).wholeText().trim() } + val cover = obj.optJSONObject("_embedded") + ?.optJSONArray("wp:featuredmedia") + ?.optJSONObject(0) + ?.optString("source_url", null) + + val embedded = obj.optJSONObject("_embedded") + val terms = embedded?.optJSONArray("wp:term") + + fun getTermNames(taxonomy: String): List<String> { + if (terms == null) return emptyList() + for (i in 0 until terms.length()) { + val termArray = terms.optJSONArray(i) ?: continue + if (termArray.length() > 0 && termArray.getJSONObject(0).optString("taxonomy") == taxonomy) + return (0 until termArray.length()).map { j -> termArray.getJSONObject(j).getString("name") } + } + return emptyList() + } + + val genres = getTermNames("genre") + getTermNames("type") + val statusList = getTermNames("status") + val state = when { + statusList.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING + statusList.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED + statusList.any { it.equals("Cancelled", ignoreCase = true) } -> MangaState.ABANDONED + statusList.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED + else -> null + } + val authors = getTermNames("series-author") + val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() + val altTitles = fetchAltTitlesFromPage("https://$domain/manga/$slug/") + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + description = description, + coverUrl = cover ?: manga.coverUrl, + state = state, + authors = authors.toSet(), + tags = tags, + chapters = chapters, + altTitles = altTitles, + ) + } + } catch (_: Exception) { + manga } - val authors = getTermNames("series-author") - val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() - val chapters = loadChapters(id.toString(), "https://$domain/manga/$slug/") - val altTitles = fetchAltTitlesFromPage("https://$domain/manga/$slug/") - - val result = manga.copy( - title = title, - description = description, - coverUrl = cover ?: manga.coverUrl, - state = state, - authors = authors.toSet(), - tags = tags, - chapters = chapters, - altTitles = altTitles, - ) - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } @@ -334,7 +349,7 @@ internal abstract class NatsuParser( ) } } - }.reversed() + }.sortedBy { it.number } } private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US) From 3a6c4ea6cbdf3546f83d039ef1d8dfb74cbae4ae Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:14:03 +0200 Subject: [PATCH 114/434] Update ZeistMangaParser.kt --- src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt index d3b2084f..9b8ce3dc 100644 --- a/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt @@ -36,6 +36,7 @@ import tsuki.util.selectFirstOrThrow import tsuki.util.toRelativeUrl import tsuki.util.toTitleCase import tsuki.util.urlDecode +import tsuki.util.extractChapterNumber import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -309,7 +310,7 @@ internal abstract class ZeistMangaParser( webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList<JSONObject>() .reversed() val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return json.mapIndexedNotNull { i, j -> + return json.mapNotNull { j -> val name = j.getJSONObject("title").getString("\$t") val href = j.getJSONArray("link").asTypedList<JSONObject>().first { it.getString("rel") == "alternate" } @@ -318,20 +319,20 @@ internal abstract class ZeistMangaParser( val slug = mangaUrl.substringAfterLast('/') val slugChapter = href.substringAfterLast('/') if (slug == slugChapter) { - return@mapIndexedNotNull null + return@mapNotNull null } MangaChapter( id = generateUid(href), url = href, title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, branch = null, uploadDate = dateFormat.parseSafe(dateText), scanlator = null, source = source, ) - } + }.sortedBy { it.number } } protected open val selectPage = @@ -381,4 +382,4 @@ internal abstract class ZeistMangaParser( } } -} \ No newline at end of file +} From 74c73a397c7a5d85e9e75e108153261ada141a41 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:24:56 +0200 Subject: [PATCH 115/434] Update Atsumaru.kt --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 84 ++++++++++++----------- 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index d8a150f2..f5d23be5 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -253,6 +253,12 @@ internal class Atsumaru(context: MangaLoaderContext) : } } + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + // limit tags to 20 override suspend fun getDetails(manga: Manga): Manga { synchronized(detailsCacheLock) { @@ -262,13 +268,7 @@ internal class Atsumaru(context: MangaLoaderContext) : val result = coroutineScope { val mangaId = manga.url.substringAfterLast("/") - val pageDeferred = async { - webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() - } - val chaptersDeferred = async { - webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() - } - + val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } val pageJson = pageDeferred.await() val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga @@ -313,9 +313,6 @@ internal class Atsumaru(context: MangaLoaderContext) : } } - val chaptersJson = chaptersDeferred.await() - val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() - val scanlators = mutableMapOf<String, String>() mangaPage.optJSONArray("scanlators")?.let { scanArr -> for (i in 0 until scanArr.length()) { @@ -326,38 +323,14 @@ internal class Atsumaru(context: MangaLoaderContext) : } } - val chapters = (0 until chaptersArray.length()).map { i -> - val ch = chaptersArray.getJSONObject(i) - val chId = ch.getString("id") - val number = ch.optDouble("number", 0.0).toFloat() - val chTitle = ch.optString("title") - val scanId = ch.optString("scanlationMangaId").nullIfEmpty() - val scanName = scanId?.let { scanlators[it] } - val createdAt = ch.optLong("createdAt", 0L) - - MangaChapter( - id = generateUid("$mangaId/$chId"), - title = chTitle.ifBlank { "Chapter $number" }, - number = number, - volume = 0, - url = "$mangaId/$chId", - uploadDate = createdAt, - source = source, - scanlator = scanName, - branch = scanName, - ) - }.reversed() - - val groupedChapters = if (chapters.map { it.branch }.distinct().size > 1) { - chapters.map { it.copy(branch = it.branch ?: "Unknown") } - } else { - chapters.map { it.copy(branch = null, scanlator = null) } - } + val chaptersDeferred = async { loadChapters(mangaId, scanlators) } val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() } ?: emptySet() + val chapters = chaptersDeferred.await() + manga.copy( title = title, description = description, @@ -366,7 +339,7 @@ internal class Atsumaru(context: MangaLoaderContext) : state = state, tags = tagSet, rating = rating, - chapters = groupedChapters, + chapters = chapters, altTitles = altTitles, ) } @@ -376,12 +349,41 @@ internal class Atsumaru(context: MangaLoaderContext) : return result } - private val detailsCacheLock = Any() + private suspend fun loadChapters(mangaId: String, scanlators: Map<String, String>): List<MangaChapter> { + val chaptersJson = webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() + val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() + + val chapters = (0 until chaptersArray.length()).map { i -> + val ch = chaptersArray.getJSONObject(i) + val chId = ch.getString("id") + val number = ch.optDouble("number", 0.0).toFloat() + val chTitle = ch.optString("title") + val scanId = ch.optString("scanlationMangaId").nullIfEmpty() + val scanName = scanId?.let { scanlators[it] } + val createdAt = ch.optLong("createdAt", 0L) + + MangaChapter( + id = generateUid("$mangaId/$chId"), + title = chTitle.ifBlank { "Chapter $number" }, + number = number, + volume = 0, + url = "$mangaId/$chId", + uploadDate = createdAt, + source = source, + scanlator = scanName, + branch = scanName, + ) + }.sortedBy { it.number } - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + val distinctBranches = chapters.map { it.branch }.distinct() + return if (distinctBranches.size > 1) { + chapters.map { it.copy(branch = it.branch ?: "Unknown") } + } else { + chapters.map { it.copy(branch = null, scanlator = null) } + } } + override suspend fun getRelatedManga(seed: Manga): List<Manga> { val mangaId = seed.url.substringAfterLast("/") val pageJson = webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() From e1b021e65f33dd9eb8723f7cfcabc514b59ca66f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 12:47:32 +0200 Subject: [PATCH 116/434] Create HiveToons.kt --- src/main/kotlin/tsuki/site/en/HiveToons.kt | 322 +++++++++++++++++++++ 1 file changed, 322 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/HiveToons.kt diff --git a/src/main/kotlin/tsuki/site/en/HiveToons.kt b/src/main/kotlin/tsuki/site/en/HiveToons.kt new file mode 100644 index 00000000..46cabefb --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/HiveToons.kt @@ -0,0 +1,322 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.model.* +import tsuki.util.* +import tsuki.network.OkHttpWebClient +import org.json.JSONArray +import org.json.JSONObject +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("HIVETOONS", "HiveToons", "en") +internal class HiveToons(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.HIVETOONS, 20) { + + override val configKeyDomain = ConfigKey.Domain("hivetoons.org") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://api.$domain" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .build() + + private var cachedGenres: List<Pair<String, String>>? = null + + private suspend fun fetchGenres(): List<Pair<String, String>> { + if (cachedGenres != null) return cachedGenres!! + val arr = webClient.httpGet("$apiBaseUrl/api/genres", getRequestHeaders()).parseJsonArray() + val list = mutableListOf<Pair<String, String>>() + for (i in 0 until arr.length()) { + val obj = arr.optJSONObject(i) ?: continue + list.add(obj.optString("name") to obj.optString("id")) + } + cachedGenres = list + return list + } + + private val statusTags = listOf( + "Ongoing" to "status_ongoing", + "Completed" to "status_completed", + "Canceled" to "status_canceled", + "Dropped" to "status_dropped", + "Coming Soon" to "status_coming_soon", + ) + + private val typeTags = listOf( + "Manga" to "type_manga", + "Manhua" to "type_manhua", + "Manhwa" to "type_manhwa", + "Russian" to "type_russian", + "Spanish" to "type_spanish", + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.ALPHABETICAL, // asc + SortOrder.ALPHABETICAL_DESC // desc + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + val tags = mutableSetOf<MangaTag>() + + statusTags.forEach { (name, key) -> tags.add(MangaTag(name, key, source)) } + typeTags.forEach { (name, key) -> tags.add(MangaTag(name, key, source)) } + genres.forEach { (name, id) -> tags.add(MangaTag(name, "genre_$id", source)) } + + return MangaListFilterOptions( + availableTags = tags, + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val status = filter.tags.firstOrNull { it.key.startsWith("status_") }?.key?.removePrefix("status_") + val type = filter.tags.firstOrNull { it.key.startsWith("type_") }?.key?.removePrefix("type_") + val genreIds = filter.tags.filter { it.key.startsWith("genre_") }.map { it.key.removePrefix("genre_") } + + val (orderBy, orderDir) = when (order) { + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.ADDED -> "createdAt" to "desc" + SortOrder.ADDED_ASC -> "createdAt" to "asc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + SortOrder.ALPHABETICAL_DESC -> "postTitle" to "desc" + else -> "totalViews" to "desc" + } + + val baseHttpUrl = "$apiBaseUrl/api/query".toHttpUrlOrNull() ?: return emptyList() + val urlBuilder = baseHttpUrl.newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + if (query.isNotEmpty()) addQueryParameter("searchTerm", query) + if (!status.isNullOrEmpty()) addQueryParameter("seriesStatus", status.uppercase()) + if (!type.isNullOrEmpty()) addQueryParameter("seriesType", type.uppercase()) + if (genreIds.isNotEmpty()) addQueryParameter("genreIds", genreIds.joinToString(",")) + addQueryParameter("orderBy", orderBy) + addQueryParameter("orderDirection", orderDir) + } + + val (mangas, _) = fetchSearchPageWithFallback(urlBuilder, page) + return mangas + } + + private suspend fun fetchSearchPageWithFallback( + initialUrlBuilder: HttpUrl.Builder, + startPage: Int + ): Pair<List<Manga>, Boolean> { + var currentPage = startPage + while (true) { + val url = initialUrlBuilder + .setQueryParameter("page", currentPage.toString()) + .build() + .toString() + + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val posts = json.optJSONArray("posts") ?: JSONArray() + val totalCount = json.optInt("totalCount", 0) + val mangas = posts.toMangaList() + + val hasNext = totalCount > (currentPage * pageSize) + + if (mangas.isNotEmpty() || !hasNext) { + return mangas to hasNext + } + currentPage++ + } + } + + private fun JSONArray.toMangaList(): List<Manga> { + val list = mutableListOf<Manga>() + for (i in 0 until length()) { + val obj = optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + list.add(obj.toManga()) + } + return list + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val slug = getString("slug") + val title = getString("postTitle") + val cover = optString("featuredImage", "") + return Manga( + id = generateUid("$slug#$id"), + url = "$slug#$id", + publicUrl = "$baseUrl/series/$slug", + title = title, + coverUrl = cover.ifEmpty { null }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val (slug, id) = manga.url.split("#") + val detailsUrl = "$apiBaseUrl/api/post?postSlug=$slug" + val json = webClient.httpGet(detailsUrl, getRequestHeaders()).parseJson() + val data = json.optJSONObject("post") ?: return manga + if (data.optBoolean("isNovel", false)) throw Exception("Novels not supported") + + val title = data.optString("postTitle", manga.title) + val cover = data.optString("featuredImage", "") + val desc = data.optString("postContent", "") + val altTitles = data.optString("alternativeTitles", "") + val author = data.optString("author", "") + val artist = data.optString("artist", "") + val seriesType = data.optString("seriesType", "") + val seriesStatus = data.optString("seriesStatus", "") + val genresArr = data.optJSONArray("genres") ?: JSONArray() + val chaptersArr = data.optJSONArray("chapters") ?: JSONArray() + val totalChapters = json.optInt("totalChapterCount", chaptersArr.length()) + + val state = when (seriesStatus.uppercase()) { + "ONGOING", "COMING_SOON", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (seriesType.isNotEmpty()) { + tags.add(MangaTag(seriesType.lowercase(), seriesType, source)) + } + for (i in 0 until genresArr.length()) { + val g = genresArr.optJSONObject(i) ?: continue + val name = g.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name.lowercase(), name, source)) + } + + val plainDesc = if (desc.isNotEmpty()) { + org.jsoup.Jsoup.parse(desc.replace("\n", "<br>")).text() + } else "" + val fullDesc = buildString { + append(plainDesc) + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Alternative Names: $altTitles") + } + }.trim() + + val finalChaptersArr = if (totalChapters > chaptersArr.length()) { + val chaptersJson = webClient.httpGet("$apiBaseUrl/api/chapters?postId=${id.toInt()}", getRequestHeaders()).parseJson() + chaptersJson.optJSONObject("post")?.optJSONArray("chapters") ?: JSONArray() + } else { + chaptersArr + } + + val chapters = parseChaptersFromArray(finalChaptersArr, slug) + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = fullDesc.takeIf { it.isNotEmpty() }, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private fun parseChaptersFromArray(arr: JSONArray, mangaSlug: String): List<MangaChapter> { + val list = mutableListOf<MangaChapter>() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + if (!ch.optBoolean("isAccessible", false)) continue + val id = ch.getInt("id") + val number = ch.optString("number", "") + val title = ch.optString("title", "") + val slug = ch.optString("slug", "") + val dateStr = ch.optString("createdAt", "") + val uploadDate = try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + + val cleanTitle = if (title == "null" || title.isEmpty()) null else title + val fullTitle = buildString { + append("Chapter $number") + if (cleanTitle != null) append(" - $cleanTitle") + } + + list.add( + MangaChapter( + id = generateUid("$mangaSlug#$id"), + title = fullTitle, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug#$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("#") + if (parts.size != 2) return emptyList() + val chapterId = parts[1].toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/chapter?chapterId=$chapterId", getRequestHeaders()).parseJson() + val chData = json.optJSONObject("chapter") ?: return emptyList() + if (chData.optBoolean("isShortLinkLocked")) throw Exception("Chapter locked (short link)") + if (chData.optBoolean("isLockedByCoins")) throw Exception("Chapter locked (coins required)") + if (chData.optBoolean("isPermanentlyLocked")) throw Exception("Chapter permanently locked") + + val imagesArr = chData.optJSONArray("images") ?: return emptyList() + val pages = mutableListOf<JSONObject>() + for (i in 0 until imagesArr.length()) { + pages.add(imagesArr.optJSONObject(i) ?: continue) + } + pages.sortBy { it.optInt("order", Int.MAX_VALUE) } + + return pages.mapIndexed { _, imgObj -> + val imgUrl = imgObj.optString("url", "").replace(" ", "%20") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val (_, id) = seed.url.split("#") + val postId = id.toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/recommendations?postId=$postId&limit=25", getRequestHeaders()).parseJson() + val recs = json.optJSONArray("recommendations") ?: return emptyList() + return recs.toMangaList() + } +} From 8e479b555b941cf93d8ede4ffb4e1ca389f34f64 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:22:33 +0200 Subject: [PATCH 117/434] Create AnimeSama.kt --- src/main/kotlin/tsuki/site/fr/AnimeSama.kt | 297 +++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/AnimeSama.kt diff --git a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt new file mode 100644 index 00000000..ae8a972c --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt @@ -0,0 +1,297 @@ +package tsuki.site.fr + +import kotlinx.coroutines.Dispatchers +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import tsuki.network.OkHttpWebClient +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl +import java.util.EnumSet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import java.net.URLEncoder + +@MangaSourceParser("ANIMESAMA", "AnimeSama", "fr") +internal class AnimeSama(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ANIMESAMA, pageSize = 48) { + + override val configKeyDomain = ConfigKey.Domain("anime-sama.to") + private val baseUrl = "https://$domain" + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept-Language", "fr-FR") + .build() + + private var cachedGenres: Set<MangaTag>? = null + + private suspend fun fetchGenres(): Set<MangaTag> { + if (cachedGenres != null) return cachedGenres!! + val doc = webClient.httpGet("$baseUrl/catalogue", getRequestHeaders()).parseHtml() + val tags = doc.select("#list_genres #genreList label").mapNotNull { label -> + val input = label.selectFirst("input[name=genre[]]") ?: return@mapNotNull null + val value = input.attr("value") + val name = label.selectFirst("span")?.text() ?: return@mapNotNull null + MangaTag(name, value, source) + }.toSet() + cachedGenres = tags + return tags + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, // catalogue listing + SortOrder.UPDATED // latest additions on homepage + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = fetchGenres(), + availableStates = emptySet() + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + if (query.startsWith("http")) { + val url = query.toHttpUrlOrNull() + if (url != null && url.host == domain && url.pathSegments.contains("catalogue")) { + return fetchMangaFromUrl(url) + } + } + + if (query.isNotEmpty()) { + return fetchSearch(query, filter.tags.map { it.key }, page) + } + return when (order) { + SortOrder.UPDATED -> fetchLatest() + else -> fetchSearch("", filter.tags.map { it.key }, page) // POPULARITY or any other + } + } + + private suspend fun fetchMangaFromUrl(url: HttpUrl): List<Manga> { + val path = url.encodedPath.removeSuffix("/").substringBefore("/scan") + val mangaUrl = url.newBuilder().encodedPath("$path/").build() + val doc = webClient.httpGet(mangaUrl.toString(), getRequestHeaders()).parseHtml() + val manga = parseMangaDetails(doc, mangaUrl.toString()) + return listOfNotNull(manga) + } + + private suspend fun fetchLatest(): List<Manga> { + val doc = webClient.httpGet(baseUrl, getRequestHeaders()).parseHtml() + return doc.select("div#containerAjoutsScans > div").mapNotNull { el -> + parseCardElement(el, removeSuffix = "scan/vf/") + } + } + + private suspend fun fetchSearch(query: String, genreIds: List<String>, page: Int): List<Manga> { + val urlBuilder = "$baseUrl/catalogue".toHttpUrl().newBuilder() + .addQueryParameter("type[]", "Scans") + .addQueryParameter("page", page.toString()) + if (query.isNotEmpty()) urlBuilder.addQueryParameter("search", query) + genreIds.forEach { urlBuilder.addQueryParameter("genre[]", it) } + val doc = webClient.httpGet(urlBuilder.build().toString(), getRequestHeaders()).parseHtml() + return doc.select("div#list_catalog > div").mapNotNull { el -> parseCardElement(el) } + } + + private fun proxyImage(originalUrl: String?, width: Int = 480): String? { + if (originalUrl.isNullOrBlank()) return null + val encoded = URLEncoder.encode(originalUrl, "UTF-8") + return "https://wsrv.nl/?url=$encoded&w=$width" + } + + private fun parseCardElement(el: Element, removeSuffix: String = ""): Manga? { + val a = el.selectFirst("a") ?: return null + val rawUrl = a.absUrl("href") + val url = if (removeSuffix.isNotEmpty()) rawUrl.removeSuffix(removeSuffix) else rawUrl + val title = el.selectFirst("h2.card-title")?.text() ?: return null + val cover = proxyImage(el.selectFirst("img")?.absUrl("src")) + return Manga( + id = generateUid(url), url = url, publicUrl = url, + title = title, coverUrl = cover, altTitles = emptySet(), + rating = RATING_UNKNOWN, contentRating = null, tags = emptySet(), + state = null, authors = emptySet(), source = source + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url, getRequestHeaders()).parseHtml() + val detailedManga = parseMangaDetails(doc, manga.url) ?: manga + val chapters = fetchChapters(doc, manga.url) + return detailedManga.copy(chapters = chapters) + } + + private fun parseMangaDetails(doc: Document, url: String): Manga? { + val title = doc.selectFirst("div.my-2 h1")?.text() ?: return null + val cover = proxyImage(doc.selectFirst("img#coverOeuvre")?.absUrl("src")) + val author = doc.selectFirst("span.info-lbl:contains(Créateur) + span.info-val")?.text() + val statusText = doc.selectFirst("span.info-lbl:contains(État) + span.info-val")?.text() + val state = when (statusText?.lowercase()) { + "en cours" -> MangaState.ONGOING + "terminé" -> MangaState.FINISHED + else -> null + } + val description = doc.selectFirst("p#synopsisText")?.text() + val genres = doc.select("span.genre-pill").map { it.text() }.toSet() + val tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet() + + return Manga( + id = generateUid(url), url = url, publicUrl = url, + title = title, coverUrl = cover, altTitles = emptySet(), + rating = RATING_UNKNOWN, contentRating = null, tags = tags, + state = state, authors = setOfNotNull(author), description = description, + source = source + ) + } + + private suspend fun fetchChapters(mangaDoc: Document, mangaUrl: String): List<MangaChapter> { + val scriptContent = mangaDoc.select("script:containsData(panneauScan(\"nom\", \"url\"))").toString() + val lines = scriptContent.split(";").drop(1) + val scanPanels = lines.mapNotNull { line -> + SCAN_PANEL_REGEX.find(line)?.destructured?.let { (name, url) -> + if (!url.contains("va")) name to url else null + } + } + if (scanPanels.isEmpty()) return emptyList() + + val chapterLists = coroutineScope { + scanPanels.map { (scanTitle, scanUrl) -> + async { fetchScanGroupChapters(mangaUrl, scanTitle, scanUrl) } + }.awaitAll() + } + + val seen = mutableSetOf<String>() + val allChapters = mutableListOf<MangaChapter>() + for (list in chapterLists) { + for (ch in list) { + if (seen.add(ch.title.toString())) allChapters.add(ch) + } + } + + return allChapters + .sortedBy { ch -> ch.url.toHttpUrlOrNull()?.queryParameter("id")?.toIntOrNull() ?: 0 } + .sortedBy { it.number } + } + + private suspend fun fetchScanGroupChapters(mangaUrl: String, scanTitle: String, scanUrl: String): List<MangaChapter> { + val subUrl = mangaUrl.trimEnd('/') + "/" + scanUrl.trim('/') + val subDoc = webClient.httpGet(subUrl, getRequestHeaders()).parseHtml() + + var title: String? = null + val mainPageLink = subDoc.selectFirst("a:has(#imgOeuvre.grayscale)")?.absUrl("href") + if (!mainPageLink.isNullOrEmpty()) { + val mainDoc = webClient.httpGet(mainPageLink, getRequestHeaders()).parseHtml() + title = mainDoc.getWorkTitle() + } + if (title.isNullOrBlank()) title = subDoc.getWorkTitle() + if (title.isBlank()) return emptyList() + + val scanlator = scanTitle.replace(SCANS_REGEX, "").trim() + val chapterApiUrl = "$baseUrl/s2/scans/get_nb_chap_et_img.php".toHttpUrl().newBuilder() + .addQueryParameter("oeuvre", title).build() + val apiResponse = webClient.httpGet(chapterApiUrl.toString(), getRequestHeaders()).parseJson() + val imageCounts = mutableMapOf<String, Int>() + for (key in apiResponse.keys()) { + imageCounts[key] = apiResponse.optInt(key, 0) + } + + val html = subDoc.html() + val chapterList = mutableListOf<MangaChapter>() + var chapterDelay = 0 + + if (html.contains("resetListe()")) { + val commands = html.split(";") + for (command in commands) { + when { + CREATE_LIST_REGEX.find(command) != null -> { + val data = CREATE_LIST_REGEX.find(command)!!.groupValues[1].split(",") + val start = data[0].trim().toInt() + val end = data[1].trim().toInt() + for (i in start..end) { + chapterList.add(buildChapter(i.toString(), chapterList.size + 1, title, scanlator, chapterApiUrl)) + } + } + NEW_SP_REGEX.find(command) != null -> { + val chapterName = NEW_SP_REGEX.find(command)!!.groupValues[1] + chapterList.add(buildChapter(chapterName, chapterList.size + 1, title, scanlator, chapterApiUrl)) + chapterDelay++ + } + } + } + } + + val totalChapters = imageCounts.size + for (i in chapterList.size until totalChapters) { + val id = (chapterList.size + 1 - chapterDelay).toString() + chapterList.add(buildChapter(id, chapterList.size + 1, title, scanlator, chapterApiUrl)) + } + return chapterList + } + + private fun buildChapter(chapterName: String, id: Int, title: String, scanlator: String, baseChapterUrl: HttpUrl): MangaChapter { + val chapterUrl = baseChapterUrl.newBuilder() + .addQueryParameter("id", id.toString()) + .addQueryParameter("title", title) + .build() + return MangaChapter( + id = generateUid(chapterUrl.toString()), + title = "Chapitre $chapterName", number = id.toFloat(), volume = 0, + url = chapterUrl.toString(), uploadDate = 0, scanlator = scanlator, + branch = null, source = source + ) + } + + private fun Document.getWorkTitle(): String = selectFirst("#titreOeuvre")?.textNodes()?.firstOrNull()?.wholeText ?: "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = chapter.url.toHttpUrlOrNull() ?: return emptyList() + val chapterId = url.queryParameter("id") ?: return emptyList() + val rawTitle = url.queryParameter("title") ?: return emptyList() + val encodedTitle = withContext(Dispatchers.IO) { + URLEncoder.encode(rawTitle, "UTF-8") + } + val apiUrl = "$baseUrl/s2/scans/get_nb_chap_et_img.php".toHttpUrl().newBuilder() + .addQueryParameter("oeuvre", rawTitle) + .build() + val json = webClient.httpGet(apiUrl.toString(), getRequestHeaders()).parseJson() + val imageCount = json.optInt(chapterId, 0) + + return (1..imageCount).map { index -> + MangaPage( + id = generateUid("$rawTitle/$chapterId/$index"), + url = "$baseUrl/s2/scans/$encodedTitle/$chapterId/$index.jpg", + preview = null, + source = source + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + companion object { + private val SCAN_PANEL_REGEX = """panneauScan\("(.+?)", "(.+?)"\)""".toRegex() + private val CREATE_LIST_REGEX = """creerListe\((\d+,\s*\d+)\)""".toRegex() + private val NEW_SP_REGEX = """newSP\((\d+(\.\d+)?|"(.*?)")\)""".toRegex() + private val SCANS_REGEX = """(Scans|\(|\))""".toRegex() + } +} From 7899ee4cf6c49f2e1307362f750ff6ee65bac56e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:23:33 +0200 Subject: [PATCH 118/434] Update HiveToons.kt --- src/main/kotlin/tsuki/site/en/HiveToons.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/HiveToons.kt b/src/main/kotlin/tsuki/site/en/HiveToons.kt index 46cabefb..9a236ef1 100644 --- a/src/main/kotlin/tsuki/site/en/HiveToons.kt +++ b/src/main/kotlin/tsuki/site/en/HiveToons.kt @@ -124,10 +124,7 @@ internal class HiveToons(context: MangaLoaderContext): return mangas } - private suspend fun fetchSearchPageWithFallback( - initialUrlBuilder: HttpUrl.Builder, - startPage: Int - ): Pair<List<Manga>, Boolean> { + private suspend fun fetchSearchPageWithFallback(initialUrlBuilder: HttpUrl.Builder, startPage: Int): Pair<List<Manga>, Boolean> { var currentPage = startPage while (true) { val url = initialUrlBuilder From 8c0ef7819bd8cfd0967f96b51a199147a14c2fd5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 13:39:49 +0200 Subject: [PATCH 119/434] Create MangaPill.kt --- src/main/kotlin/tsuki/site/en/MangaPill.kt | 233 +++++++++++++++++++++ 1 file changed, 233 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/MangaPill.kt diff --git a/src/main/kotlin/tsuki/site/en/MangaPill.kt b/src/main/kotlin/tsuki/site/en/MangaPill.kt new file mode 100644 index 00000000..83c0db82 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaPill.kt @@ -0,0 +1,233 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.util.EnumSet + +@MangaSourceParser("MANGAPILL", "MangaPill", "en") +internal class MangaPill(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.MANGAPILL, 50) { + + override val configKeyDomain = ConfigKey.Domain("mangapill.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.NOVEL, + ContentType.DOUJINSHI, + ContentType.ONE_SHOT, + ContentType.OTHER, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrEmpty() || filter.tags.isNotEmpty() || filter.types.isNotEmpty() || filter.states.isNotEmpty()) { + return fetchSearch(page, filter) + } + return when (order) { + SortOrder.POPULARITY -> fetchPopular() + SortOrder.UPDATED -> fetchLatest() + else -> fetchSearch(page, filter) + } + } + + private suspend fun fetchPopular(): List<Manga> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + return doc.select("div:has(h4:contains(Trending)) > .grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private suspend fun fetchLatest(): List<Manga> { + val doc = webClient.httpGet("https://$domain/chapters").parseHtml() + return doc.select(".grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private suspend fun fetchSearch(page: Int, filter: MangaListFilter): List<Manga> { + val url = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .apply { + if (!filter.query.isNullOrEmpty()) addQueryParameter("q", filter.query) + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.NOVEL -> "novel" + ContentType.DOUJINSHI -> "doujinshi" + ContentType.ONE_SHOT -> "one-shot" + ContentType.OTHER -> "oel" + else -> "" + }) + } + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "publishing" + MangaState.FINISHED -> "finished" + MangaState.ABANDONED -> "discontinued" + MangaState.PAUSED -> "on hiatus" + MangaState.UPCOMING -> "not yet published" + else -> "" + }) + } + filter.tags.forEach { tag -> + addQueryParameter("genre", tag.key) + } + } + .build() + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private fun latestUpdatesFromElement(element: org.jsoup.nodes.Element): Manga { + val a = element.selectFirst("a[href^='/manga/']") ?: return Manga( + id = generateUid(""), + url = "", + publicUrl = "", + title = "", + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + val href = a.attr("href").removeSuffix("/") + val title = element.selectFirst("div.line-clamp-2")?.text() ?: "" + val img = element.selectFirst("img") ?: return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + val cover = img.attr("data-src") + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cover = doc.selectFirst("div.container > div:first-child > div:first-child > img") + ?.attr("data-src") ?: manga.coverUrl + val altTitle = doc.selectFirst("div.text-sm.text-secondary")?.text()?.nullIfEmpty() + val description = doc.select("div.container > div:first-child > div:last-child > div:nth-child(2) > p").text() + val statusText = doc.select("label.text-secondary").firstOrNull { it.text() == "Status" } + ?.nextElementSibling()?.text() + val state = when (statusText) { + "publishing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + "discontinued" -> MangaState.ABANDONED + "on hiatus" -> MangaState.PAUSED + "not yet published" -> MangaState.UPCOMING + else -> null + } + val tags = doc.select("div").firstOrNull { + it.selectFirst("label.text-secondary")?.text() == "Genres" + }?.select("a.text-sm.mr-1.text-brand")?.mapNotNullToSet { element -> + val key = element.attr("href").substringAfter("/search?genre=") + if (key.isBlank()) null + else MangaTag(key = key, title = element.text(), source = source) + } ?: emptySet() + + val chapters = doc.select("div#chapters a").mapNotNull { element -> + val href = element.attrAsRelativeUrl("href") + val rawTitle = element.text() + val chapterNumber = rawTitle.substringAfter("Chapter ").toFloatOrNull() ?: 0f + + MangaChapter( + id = generateUid(href), + title = rawTitle, + url = href, + number = chapterNumber, + volume = 0, + scanlator = null, + uploadDate = 0, + branch = null, + source = source, + ) + }.reversed() + + return manga.copy( + coverUrl = cover ?: manga.coverUrl, + altTitles = altTitle?.let { setOf(it) } ?: emptySet(), + description = description.nullIfEmpty(), + state = state, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("picture img").map { img -> + val url = img.attr("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + return doc.select("div.m-1 label input").mapNotNull { input -> + val title = input.attr("value") + val key = title.replace(" ", "+") + MangaTag(key = key, title = title, source = source) + }.toSet() + } +} From d5e7a736c776d80b1724d3b1891d0c88142c450b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:46:38 +0200 Subject: [PATCH 120/434] Update Mangaball.kt --- src/main/kotlin/tsuki/site/all/Mangaball.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt index 93f71060..3abbce21 100644 --- a/src/main/kotlin/tsuki/site/all/Mangaball.kt +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -605,7 +605,7 @@ internal abstract class MangaBallParser( companion object { - private const val BASE_URL = "https://${_root_ide_package_.tsuki.site.all.DOMAIN}" + private const val BASE_URL = "https://${DOMAIN}" private val GROUP_ID_REGEX = Regex("[a-z0-9]{24}") private val CHAPTER_IMAGES_REGEX = Regex("""const\s+chapterImages\s*=\s*JSON\.parse\(`([^`]+)`\)""") private val ADULT_TAG_TITLES = setOf( From c0ef60c1ff8ec07e16bd04e6cda00621407592eb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 28 Jul 2026 23:49:29 +0200 Subject: [PATCH 121/434] Create SwatManga.kt --- src/main/kotlin/tsuki/site/ar/SwatManga.kt | 244 +++++++++++++++++++++ 1 file changed, 244 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/SwatManga.kt diff --git a/src/main/kotlin/tsuki/site/ar/SwatManga.kt b/src/main/kotlin/tsuki/site/ar/SwatManga.kt new file mode 100644 index 00000000..0ed2ab0f --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/SwatManga.kt @@ -0,0 +1,244 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SWATMANGA", "Swat Manga", "ar", ContentType.MANGA) +internal class SwatManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SWATMANGA, 20) { + + override val configKeyDomain = ConfigKey.Domain("meshmanga.com") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override val availableSortOrders: Set<SortOrder> = LinkedHashSet( + listOf( + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val response = webClient.httpGet("https://appswat.com/v2/api/v2/genres/").parseJsonArray() + val tags = mutableSetOf<MangaTag>() + for (i in 0 until response.length()) { + val genreObj = response.getJSONObject(i) + tags.add(MangaTag( + key = genreObj.getInt("id").toString(), + title = genreObj.getString("name"), + source = source, + )) + } + return tags + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://appswat.com/v2/api/v2/series/?page_size=20") + + // Add page offset + if (page > 1) { + val offset = (page - 1) * 20 + append("&offset=$offset") + } + + // Add search query + if (!filter.query.isNullOrEmpty()) { + append("&search=${filter.query.urlEncoded()}") + } + + // Add sort order + when (order) { + SortOrder.POPULARITY -> append("&order_by=followers_count") + SortOrder.RATING -> append("&order_by=-rating") + else -> {} // Default relevance + } + + // Add genre filters + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&genres=${tag.key}") + } + } + } + + val response = webClient.httpGet(url).parseJson() + val results = response.getJSONArray("results") + + return (0 until results.length()).map { i -> + val item = results.getJSONObject(i) + parseMangaFromJson(item) + } + } + + private fun parseMangaFromJson(json: JSONObject): Manga { + val id = json.getInt("id") + val title = json.getString("title") + val slug = json.getString("slug") + + val status = json.getJSONObject("status") + val state = when (status.getString("name")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val poster = json.getJSONObject("poster") + val coverUrl = poster.optString("medium").nullIfEmpty() + ?: poster.optString("thumbnail").nullIfEmpty() + + val rating = json.optString("rating", "0.0").toFloatOrNull() ?: 0f + val normalizedRating = if (rating > 0) rating / 10f else RATING_UNKNOWN + + val genres = json.getJSONArray("genres") + val tags = mutableSetOf<MangaTag>() + for (i in 0 until genres.length()) { + val genre = genres.getJSONObject(i) + tags.add(MangaTag( + key = genre.getInt("id").toString(), + title = genre.getString("name"), + source = source, + )) + } + + // Extract translator/editor info for authors + val authors = mutableSetOf<String>() + json.optJSONObject("translator")?.let { translator -> + authors.add(translator.getString("name")) + } + json.optJSONObject("editor")?.let { editor -> + authors.add(editor.getString("name")) + } + + return Manga( + id = generateUid(id.toString()), + url = "/series/$id", + publicUrl = "https://meshmanga.com/series/$slug/", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = normalizedRating, + tags = tags, + authors = authors, + state = state, + source = source, + contentRating = ContentRating.SAFE, + ) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val seriesId = manga.url.substringAfter("/series/") + val chaptersDeferred = async { getChapters(seriesId) } + + // Get detailed manga info from the same API endpoint + val detailUrl = "https://appswat.com/v2/api/v2/series/?page_size=100" + val response = webClient.httpGet(detailUrl).parseJson() + val results = response.getJSONArray("results") + + // Find the specific manga by ID + var updatedManga = manga + for (i in 0 until results.length()) { + val item = results.getJSONObject(i) + if (item.getInt("id").toString() == seriesId) { + updatedManga = parseMangaFromJson(item) + break + } + } + + return@coroutineScope updatedManga.copy( + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfter("/chapters/") + val chapterUrl = "https://appswat.com/v2/api/v2/chapters/$chapterId/" + + val response = webClient.httpGet(chapterUrl).parseJson() + val images = response.getJSONArray("images") + + return (0 until images.length()).map { i -> + val imageObj = images.getJSONObject(i) + val imageUrl = imageObj.getString("image") + val order = imageObj.getInt("order") + + MangaPage( + id = generateUid("$chapterId-$order"), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun getChapters(seriesId: String): List<MangaChapter> { + val allChapters = mutableListOf<JSONObject>() + var page = 1 + + // Fetch all chapters with pagination + while (true) { + val chaptersUrl = "https://appswat.com/v2/api/v2/chapters/?page=$page&serie=$seriesId&order_by=-order&page_size=20" + val response = webClient.httpGet(chaptersUrl).parseJson() + val results = response.getJSONArray("results") + + if (results.length() == 0) break + + for (i in 0 until results.length()) { + allChapters.add(results.getJSONObject(i)) + } + + // Check if there's a next page - properly handle null case + val hasNext = !response.isNull("next") + if (!hasNext) break + page++ + } + + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + + return allChapters.mapIndexed { index, item -> + val chapterId = item.getInt("id") + val chapterNumber = item.optString("chapter", "").toFloatOrNull() ?: (index + 1f) + val title = item.getString("title") + val createdAt = item.getString("created_at") + + val uploadDate = try { + dateFormat.parse(createdAt)?.time ?: 0L + } catch (_: Exception) { + 0L + } + + MangaChapter( + id = generateUid(chapterId.toString()), + title = title, + number = chapterNumber, + volume = 0, + url = "/chapters/$chapterId", + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } +} From 0d884d2b7e807f69e074228f6b11d87c06fb559e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:48:32 +0200 Subject: [PATCH 122/434] Update Batcave.kt --- src/main/kotlin/tsuki/site/en/Batcave.kt | 101 +++++++++++++---------- 1 file changed, 59 insertions(+), 42 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 5ace266b..4d4b524c 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -4,44 +4,66 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.exception.ParseException import tsuki.network.OkHttpWebClient +import tsuki.network.CloudFlareHelper import tsuki.model.* import tsuki.util.* import okhttp3.FormBody import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import org.json.JSONObject +import tsuki.MangaParserAuthProvider +import tsuki.exception.AuthRequiredException +import tsuki.exception.ParseException import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) internal class Batcave(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { + PagedMangaParser(context, MangaParserSource.BATCAVE, 20), + MangaParserAuthProvider { override val configKeyDomain = ConfigKey.Domain("batcave.biz") + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("User-Agent", "Mozilla/5.0 (Linux; Android 14; SM-S928B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36") + .build() + + override val authUrl: String get() = "https://$domain" + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { it.name == "__guard_trust" } + } + override suspend fun getUsername(): String = "" + private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.ENGLISH) @Volatile private var genreList: List<Pair<String, Int>>? = null private var filterFetchFailed = false - private val rawHttpClient: OkHttpClient by lazy { - context.httpClient.newBuilder() + override val webClient: OkHttpWebClient by lazy { + val client = context.httpClient.newBuilder() .addInterceptor(::refererInterceptor) - .addInterceptor(::dleGuardInterceptor) + .addInterceptor { chain -> + val response = chain.proceed(chain.request()) + if (!response.isSuccessful && (response.code == 403 || response.code == 503)) { + try { + val protection = CloudFlareHelper.checkResponseForProtection(response) + if (protection == CloudFlareHelper.PROTECTION_CAPTCHA) { + response.close() + throw AuthRequiredException(source) + } + } catch (_: Exception) {} + } + response + } .build() - } - - private val apiClient: OkHttpWebClient by lazy { - OkHttpWebClient(rawHttpClient, source) + OkHttpWebClient(client, source) } private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { @@ -54,19 +76,18 @@ internal class Batcave(context: MangaLoaderContext) : return chain.proceed(request.newBuilder().header("Referer", referer).build()) } - private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { - val originalRequest = chain.request() - val response = chain.proceed(originalRequest) - if (response.request.url.pathSegments.firstOrNull() != "_c") { - return response - } - response.close() - val url = if (originalRequest.method == "GET") { - originalRequest.url.toString() - } else { - "https://$domain/" + // gets cookie when opening source + private suspend fun ensureGuard() { + if (isAuthorized()) return + repeat(2) { + try { + val request = Request.Builder().url("https://$domain/").build() + val response = context.httpClient.newCall(request).execute() + response.close() + if (isAuthorized()) return + } catch (_: Exception) {} } - context.requestBrowserAction(this, url) + throw AuthRequiredException(source) } override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { @@ -88,19 +109,17 @@ internal class Batcave(context: MangaLoaderContext) : private suspend fun fetchFilters() { if (genreList != null) return if (filterFetchFailed) return - try { - val doc = apiClient.httpGet("https://$domain/comix").parseHtml() + ensureGuard() + val doc = webClient.httpGet("https://$domain/comix").parseHtml() val script = doc.selectFirst("script:containsData(window.__XFILTER__)")?.data() ?: throw ParseException("Filter data not found", "https://$domain/comix") - val rawJson = script .substringAfter("window.__XFILTER__ = ") .substringBeforeLast(";") .trim() val root = JSONObject(rawJson) val filterItems = root.getJSONObject("filter_items") - genreList = parseFilterValues(filterItems) } catch (_: Exception) { filterFetchFailed = true @@ -124,12 +143,11 @@ internal class Batcave(context: MangaLoaderContext) : genreList?.forEach { (name, id) -> tags += MangaTag(name, "g_$id", source) } - return MangaListFilterOptions( - availableTags = tags, - ) + return MangaListFilterOptions(availableTags = tags) } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + ensureGuard() val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } if (query != null) return searchManga(query, page) @@ -149,7 +167,7 @@ internal class Batcave(context: MangaLoaderContext) : } return if (sortPair.first.isEmpty()) { - parseMangaList(apiClient.httpGet(url).parseHtml()) + parseMangaList(webClient.httpGet(url).parseHtml()) } else { val formBody = FormBody.Builder() .add("dlenewssortby", sortPair.first) @@ -158,21 +176,22 @@ internal class Batcave(context: MangaLoaderContext) : .add("set_direction_sort", "dle_direction_xfilter") .build() val request = Request.Builder().url(url).post(formBody).build() - val response = rawHttpClient.newCall(request).execute() + val response = context.httpClient.newCall(request).execute() parseMangaList(response.parseHtml()) } } private suspend fun searchManga(query: String, page: Int): List<Manga> { + ensureGuard() val encoded = query.urlEncoded() val url = if (page == 1) "https://$domain/search/$encoded/" else "https://$domain/search/$encoded/page/$page/" - return parseMangaList(apiClient.httpGet(url).parseHtml()) + return parseMangaList(webClient.httpGet(url).parseHtml()) } private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { - return doc.select("#dle-content > .readed").map { element -> - val a = element.selectFirst(".readed__title > a") ?: return@map null + return doc.select("#dle-content > .readed").mapNotNull { element -> + val a = element.selectFirst(".readed__title > a") ?: return@mapNotNull null val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") Manga( id = generateUid(a.attrAsRelativeUrl("href")), @@ -188,7 +207,7 @@ internal class Batcave(context: MangaLoaderContext) : authors = emptySet(), source = source, ) - }.filterNotNull() + } } private val detailsCacheLock = Any() @@ -197,16 +216,14 @@ internal class Batcave(context: MangaLoaderContext) : } override suspend fun getDetails(manga: Manga): Manga { + ensureGuard() synchronized(detailsCacheLock) { detailsCache[manga.url]?.let { return it } } - - val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") val description = doc.selectFirst("div.page__text")?.text() - val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer)) a")?.text() ?: doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { @@ -216,7 +233,6 @@ internal class Batcave(context: MangaLoaderContext) : val tags = doc.select("div.page__tags a").map { a -> MangaTag(a.text(), a.text().lowercase().replace(" ", "-"), source) }.toSet() + MangaTag("Comic", "comic", source) - val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() ?: throw ParseException("Chapter data script not found", manga.url) val json = JSONObject( @@ -237,7 +253,7 @@ internal class Batcave(context: MangaLoaderContext) : branch = null, source = source, ) - }.reversed() + }.sortedBy { it.number } val result = manga.copy( title = title, @@ -255,6 +271,7 @@ internal class Batcave(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + ensureGuard() val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) val id = Regex("^\\d+").find(rawId)?.value ?: rawId val jsonBody = JSONObject().apply { @@ -269,7 +286,7 @@ internal class Batcave(context: MangaLoaderContext) : .post(jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())) .build() - val response = rawHttpClient.newCall(request).execute() + val response = context.httpClient.newCall(request).execute() val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) val data = json.getJSONObject("data") val images = data.optJSONArray("images") ?: throw ParseException("No images found", chapter.url) From 48056a85255f6fba85a7bd98f26289823b559b8d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 29 Jul 2026 13:59:03 +0200 Subject: [PATCH 123/434] revert --- src/main/kotlin/tsuki/site/en/Batcave.kt | 101 ++++++++++------------- 1 file changed, 42 insertions(+), 59 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 4d4b524c..5ace266b 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -4,66 +4,44 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException import tsuki.network.OkHttpWebClient -import tsuki.network.CloudFlareHelper import tsuki.model.* import tsuki.util.* import okhttp3.FormBody import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import org.json.JSONObject -import tsuki.MangaParserAuthProvider -import tsuki.exception.AuthRequiredException -import tsuki.exception.ParseException import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) internal class Batcave(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BATCAVE, 20), - MangaParserAuthProvider { + PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { override val configKeyDomain = ConfigKey.Domain("batcave.biz") - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("User-Agent", "Mozilla/5.0 (Linux; Android 14; SM-S928B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Mobile Safari/537.36") - .build() - - override val authUrl: String get() = "https://$domain" - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { it.name == "__guard_trust" } - } - override suspend fun getUsername(): String = "" - private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.ENGLISH) @Volatile private var genreList: List<Pair<String, Int>>? = null private var filterFetchFailed = false - override val webClient: OkHttpWebClient by lazy { - val client = context.httpClient.newBuilder() + private val rawHttpClient: OkHttpClient by lazy { + context.httpClient.newBuilder() .addInterceptor(::refererInterceptor) - .addInterceptor { chain -> - val response = chain.proceed(chain.request()) - if (!response.isSuccessful && (response.code == 403 || response.code == 503)) { - try { - val protection = CloudFlareHelper.checkResponseForProtection(response) - if (protection == CloudFlareHelper.PROTECTION_CAPTCHA) { - response.close() - throw AuthRequiredException(source) - } - } catch (_: Exception) {} - } - response - } + .addInterceptor(::dleGuardInterceptor) .build() - OkHttpWebClient(client, source) + } + + private val apiClient: OkHttpWebClient by lazy { + OkHttpWebClient(rawHttpClient, source) } private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { @@ -76,18 +54,19 @@ internal class Batcave(context: MangaLoaderContext) : return chain.proceed(request.newBuilder().header("Referer", referer).build()) } - // gets cookie when opening source - private suspend fun ensureGuard() { - if (isAuthorized()) return - repeat(2) { - try { - val request = Request.Builder().url("https://$domain/").build() - val response = context.httpClient.newCall(request).execute() - response.close() - if (isAuthorized()) return - } catch (_: Exception) {} + private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { + val originalRequest = chain.request() + val response = chain.proceed(originalRequest) + if (response.request.url.pathSegments.firstOrNull() != "_c") { + return response + } + response.close() + val url = if (originalRequest.method == "GET") { + originalRequest.url.toString() + } else { + "https://$domain/" } - throw AuthRequiredException(source) + context.requestBrowserAction(this, url) } override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { @@ -109,17 +88,19 @@ internal class Batcave(context: MangaLoaderContext) : private suspend fun fetchFilters() { if (genreList != null) return if (filterFetchFailed) return + try { - ensureGuard() - val doc = webClient.httpGet("https://$domain/comix").parseHtml() + val doc = apiClient.httpGet("https://$domain/comix").parseHtml() val script = doc.selectFirst("script:containsData(window.__XFILTER__)")?.data() ?: throw ParseException("Filter data not found", "https://$domain/comix") + val rawJson = script .substringAfter("window.__XFILTER__ = ") .substringBeforeLast(";") .trim() val root = JSONObject(rawJson) val filterItems = root.getJSONObject("filter_items") + genreList = parseFilterValues(filterItems) } catch (_: Exception) { filterFetchFailed = true @@ -143,11 +124,12 @@ internal class Batcave(context: MangaLoaderContext) : genreList?.forEach { (name, id) -> tags += MangaTag(name, "g_$id", source) } - return MangaListFilterOptions(availableTags = tags) + return MangaListFilterOptions( + availableTags = tags, + ) } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - ensureGuard() val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } if (query != null) return searchManga(query, page) @@ -167,7 +149,7 @@ internal class Batcave(context: MangaLoaderContext) : } return if (sortPair.first.isEmpty()) { - parseMangaList(webClient.httpGet(url).parseHtml()) + parseMangaList(apiClient.httpGet(url).parseHtml()) } else { val formBody = FormBody.Builder() .add("dlenewssortby", sortPair.first) @@ -176,22 +158,21 @@ internal class Batcave(context: MangaLoaderContext) : .add("set_direction_sort", "dle_direction_xfilter") .build() val request = Request.Builder().url(url).post(formBody).build() - val response = context.httpClient.newCall(request).execute() + val response = rawHttpClient.newCall(request).execute() parseMangaList(response.parseHtml()) } } private suspend fun searchManga(query: String, page: Int): List<Manga> { - ensureGuard() val encoded = query.urlEncoded() val url = if (page == 1) "https://$domain/search/$encoded/" else "https://$domain/search/$encoded/page/$page/" - return parseMangaList(webClient.httpGet(url).parseHtml()) + return parseMangaList(apiClient.httpGet(url).parseHtml()) } private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { - return doc.select("#dle-content > .readed").mapNotNull { element -> - val a = element.selectFirst(".readed__title > a") ?: return@mapNotNull null + return doc.select("#dle-content > .readed").map { element -> + val a = element.selectFirst(".readed__title > a") ?: return@map null val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") Manga( id = generateUid(a.attrAsRelativeUrl("href")), @@ -207,7 +188,7 @@ internal class Batcave(context: MangaLoaderContext) : authors = emptySet(), source = source, ) - } + }.filterNotNull() } private val detailsCacheLock = Any() @@ -216,14 +197,16 @@ internal class Batcave(context: MangaLoaderContext) : } override suspend fun getDetails(manga: Manga): Manga { - ensureGuard() synchronized(detailsCacheLock) { detailsCache[manga.url]?.let { return it } } - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") val description = doc.selectFirst("div.page__text")?.text() + val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer)) a")?.text() ?: doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { @@ -233,6 +216,7 @@ internal class Batcave(context: MangaLoaderContext) : val tags = doc.select("div.page__tags a").map { a -> MangaTag(a.text(), a.text().lowercase().replace(" ", "-"), source) }.toSet() + MangaTag("Comic", "comic", source) + val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() ?: throw ParseException("Chapter data script not found", manga.url) val json = JSONObject( @@ -253,7 +237,7 @@ internal class Batcave(context: MangaLoaderContext) : branch = null, source = source, ) - }.sortedBy { it.number } + }.reversed() val result = manga.copy( title = title, @@ -271,7 +255,6 @@ internal class Batcave(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - ensureGuard() val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) val id = Regex("^\\d+").find(rawId)?.value ?: rawId val jsonBody = JSONObject().apply { @@ -286,7 +269,7 @@ internal class Batcave(context: MangaLoaderContext) : .post(jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())) .build() - val response = context.httpClient.newCall(request).execute() + val response = rawHttpClient.newCall(request).execute() val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) val data = json.getJSONObject("data") val images = data.optJSONArray("images") ?: throw ParseException("No images found", chapter.url) From a4dcdbcaa436b6aefa856a343f2c8a32e9ef46cc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 11:47:22 +0200 Subject: [PATCH 124/434] fix for missing mangas MangaPlus.kt --- src/main/kotlin/tsuki/site/all/MangaPlus.kt | 30 ++++++++++++++++++--- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaPlus.kt b/src/main/kotlin/tsuki/site/all/MangaPlus.kt index 512770f2..3bd6f19b 100644 --- a/src/main/kotlin/tsuki/site/all/MangaPlus.kt +++ b/src/main/kotlin/tsuki/site/all/MangaPlus.kt @@ -602,9 +602,26 @@ internal abstract class MangaPlusParser( .orEmpty().filterByLang().map { it.toManga() } } SortOrder.UPDATED -> { - val resp = apiCall("/web/web_homeV4?lang=$internalLangName&clang=$internalLangName") - resp.success?.webHomeView?.groups?.flatMap { it.titles }?.mapNotNull { it.title } - .orEmpty().filterByLang().map { it.toManga() } + val latestResult = apiCall("/web/web_homeV4?lang=$internalLangName&clang=$internalLangName") + val allTitlesResult = apiCall("/title_list/allV2") + + val latestTitles = latestResult.success?.webHomeView?.groups + ?.flatMap { it.titles } + ?.mapNotNull { it.title } + ?.filterByLang() + .orEmpty() + + val allTitlesGroups = allTitlesResult.success?.allTitlesView?.allTitlesGroup + .orEmpty() + + val titles = latestTitles.mapNotNull { latestTitle -> + allTitlesGroups + .firstOrNull { group -> group.titles.any { it.titleId == latestTitle.titleId } } + ?.titles + ?.firstOrNull { it.language == internalLangCode } + }.distinctBy { it.titleId } + + titles.map { it.toManga() } } SortOrder.ALPHABETICAL -> { val resp = apiCall("/title_list/allV2") @@ -639,7 +656,7 @@ internal abstract class MangaPlusParser( val resp = apiCall("/title_detailV3?title_id=${manga.url}&clang=$internalLangName") val detail = resp.success?.titleDetailView ?: throw IOException("No detail") - val completed = detail.nonAppearanceInfo.contains(Regex("completado|complete|completo", RegexOption.IGNORE_CASE)) + val completed = detail.nonAppearanceInfo.contains(CompletedRegex) || detail.viewingPeriodDescription.contains("latest 0 chapters") val hiatus = detail.nonAppearanceInfo.contains(Regex("on a hiatus", RegexOption.IGNORE_CASE)) val result = manga.copy( title = detail.title.name, @@ -752,6 +769,11 @@ internal abstract class MangaPlusParser( .build() } + companion object { + private val CompletedRegex = Regex("completado|completed?|completo", RegexOption.IGNORE_CASE) + private val HIATUS_REGEX = Regex("on a hiatus", RegexOption.IGNORE_CASE) + } + @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH", "eng", 0) From 147623cab6fd153ef79e890ba1d5545f34e0afdd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:11:38 +0200 Subject: [PATCH 125/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index caa9c109..d04b0401 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -49,8 +49,10 @@ internal class MangaK(context: MangaLoaderContext) : override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.UPDATED, SortOrder.NEWEST, - SortOrder.POPULARITY, SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, ) override val filterCapabilities: MangaListFilterCapabilities @@ -66,8 +68,6 @@ internal class MangaK(context: MangaLoaderContext) : availableStates = EnumSet.of( MangaState.ONGOING, MangaState.FINISHED, -// MangaState.PAUSED, -// MangaState.ABANDONED, ), availableContentTypes = EnumSet.of( ContentType.MANGA, @@ -118,6 +118,8 @@ internal class MangaK(context: MangaLoaderContext) : SortOrder.NEWEST -> "newest" SortOrder.POPULARITY -> "popular" SortOrder.RATING -> "rating" + SortOrder.POPULARITY_WEEK -> "views_7days" + SortOrder.POPULARITY_MONTH -> "views_30days" else -> "latest" }, ) @@ -180,8 +182,6 @@ internal class MangaK(context: MangaLoaderContext) : when (it) { MangaState.ONGOING -> "ongoing" MangaState.FINISHED -> "completed" -// MangaState.PAUSED -> "hiatus" -// MangaState.ABANDONED -> "cancelled" else -> null } ?: return@let ) @@ -319,4 +319,4 @@ internal class MangaK(context: MangaLoaderContext) : companion object { private val IMAGE_FALLBACK_REGEX = Regex("rx\\.qvzr[a-z]\\.org") } -} \ No newline at end of file +} From daeb58a549c30a33fcc917bcaf66d6c42108c0ed Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:16:17 +0200 Subject: [PATCH 126/434] fix NHentaiXXXParser --- .../kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index c63aca26..3540e3ef 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -68,15 +68,9 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : else -> { append("/search/?key=") - val joiner = StringUtil.StringJoiner("+") - tags.forEach { tag -> - joiner.add(tag.title) - } - - if (!query.isNullOrEmpty()) { - joiner.add(query.urlEncoded()) - } - append(joiner.complete()) + val parts = tags.map { it.title }.toMutableList() + if (!query.isNullOrEmpty()) parts.add(query.urlEncoded()) + append(parts.joinToString("+")) } } @@ -184,4 +178,4 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : url.substringBeforeLast("/") + "/$newPageNumber" } } -} \ No newline at end of file +} From cbb8f3c24a389f9e464dfcca3eca189c3b75c435 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:16:55 +0200 Subject: [PATCH 127/434] Update NHentaiXXXParser.kt --- src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index 3540e3ef..a7c36b5f 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -8,7 +8,6 @@ import tsuki.parsers.GalleryAdultsParser import tsuki.model.* import tsuki.util.* -import org.jsoup.internal.StringUtil import org.jsoup.nodes.Document import org.jsoup.nodes.Element import java.util.EnumSet From ac2e244f0c383577511da37e74f9fe119f2f47bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:17:14 +0200 Subject: [PATCH 128/434] Update NHentaiXXXParser.kt --- src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index a7c36b5f..d9372483 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -137,7 +137,7 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : return try { webClient.httpHead(url) false - } catch (e: Exception) { + } catch (_: Exception) { true } } @@ -151,7 +151,7 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : try { webClient.httpHead(newUrl) newUrl - } catch (e: Exception) { + } catch (_: Exception) { null // Skip errors and continue checking other formats } } From f5e8fba91519f4b5ae0c0a80b7f3dbdfc37c4d3d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:33:47 +0200 Subject: [PATCH 129/434] fix rare 502 error as well moved adult key Atsumaru.kt --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 273 +++++++++++++--------- 1 file changed, 165 insertions(+), 108 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index f5d23be5..dca36f0d 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -15,6 +15,53 @@ import org.json.JSONObject import java.util.EnumSet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.delay +import kotlin.math.pow +import kotlin.math.roundToLong +import kotlin.random.Random +import kotlin.time.Duration.Companion.milliseconds + +suspend fun <T> retryWithBackoff( + maxRetries: Int = 3, + baseDelay: Long = 800L, + maxDelay: Long = 10_000L, + factor: Double = 2.0, + jitter: Double = 0.3, // ± 30% + isTransient: (Exception) -> Boolean = { isTransientHttpException(it) }, + block: suspend () -> T +): T { + var lastException: Exception? = null + repeat(maxRetries) { attempt -> + try { + return block() + } catch (e: Exception) { + if (!isTransient(e)) throw e + lastException = e + if (attempt < maxRetries - 1) { + val delayMs = (baseDelay * factor.pow(attempt)) + .toLong() + .coerceAtMost(maxDelay) + .let { it + (it * jitter * (Random.nextDouble() * 2 - 1)).roundToLong() } + .coerceAtLeast(0) + delay(delayMs.milliseconds) + } + } + } + throw lastException!! +} + +fun isTransientHttpException(e: Exception): Boolean { + val msg = e.message.orEmpty() + return when { + msg.contains("502", ignoreCase = true) -> true + msg.contains("503", ignoreCase = true) -> true + msg.contains("504", ignoreCase = true) -> true + msg.contains("timeout", ignoreCase = true) -> true + msg.contains("connection reset", ignoreCase = true) -> true + msg.contains("unreachable", ignoreCase = true) -> true + else -> false + } +} @MangaSourceParser("ATSUMARU", "Atsumaru", "en") internal class Atsumaru(context: MangaLoaderContext) : @@ -22,6 +69,7 @@ internal class Atsumaru(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("atsu.moe") private val baseUrl = "https://atsu.moe" + private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) private val apiHeaders: Headers by lazy { Headers.Builder() @@ -31,6 +79,12 @@ internal class Atsumaru(context: MangaLoaderContext) : .build() } + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showAdultKey) + } + override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.POPULARITY, // trending (views) SortOrder.UPDATED, // recently added (dateAdded) @@ -138,14 +192,9 @@ internal class Atsumaru(context: MangaLoaderContext) : allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> = getSearchPage(page, order, filter) + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter, ): List<Manga> = getSearchPage(page, order, filter) private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query ?: "*" @@ -161,18 +210,20 @@ internal class Atsumaru(context: MangaLoaderContext) : val filterParts = mutableListOf("hidden:!=true", "views:>0") - val ratings = filter.contentRating - when { - ratings.contains(ContentRating.ADULT) && !ratings.contains(ContentRating.SAFE) -> { - filterParts.add("isAdult:=true") - } - ratings.contains(ContentRating.SAFE) && !ratings.contains(ContentRating.ADULT) -> { + val showAdult = config[showAdultKey] + + val explicitAdultFilter = filter.contentRating.contains(ContentRating.ADULT) + val explicitSafeFilter = filter.contentRating.contains(ContentRating.SAFE) + + if (explicitAdultFilter && !explicitSafeFilter) { + filterParts.add("isAdult:=true") + } else if (explicitSafeFilter && !explicitAdultFilter) { + filterParts.add("isAdult:=false") + } else { + if (!showAdult) { filterParts.add("isAdult:=false") } - // both or none: no filter shows all - else -> { /* no isAdult filter */ } } - filterParts.add("(mbContentRating:=[`Safe`,`Suggestive`,`Erotica`] || mbContentRating:!=*)") val types = filter.types.mapNotNull { ct -> when (ct) { @@ -205,21 +256,19 @@ internal class Atsumaru(context: MangaLoaderContext) : val includedTags = mutableListOf<String>() val excludedTags = mutableListOf<String>() - for (tag in filter.tags) { - val key = tag.key - when { - key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) - } - } - for (tag in filter.tagsExclude) { - val key = tag.key - when { - key.startsWith("genre:") -> excludedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + fun processTags(tags: Collection<MangaTag>, genreAcc: MutableList<String>, tagAcc: MutableList<String>) { + for (tag in tags) { + val key = tag.key + when { + key.startsWith("genre:") -> genreAcc.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> tagAcc.add(key.removePrefix("tag:")) + } } } + processTags(filter.tags, includedGenres, includedTags) + processTags(filter.tagsExclude, excludedGenres, excludedTags) + if (includedGenres.isNotEmpty()) { filterParts.add(includedGenres.joinToString(" && ") { "genreIds:=`$it`" }) } @@ -243,10 +292,11 @@ internal class Atsumaru(context: MangaLoaderContext) : else -> "released:desc" } addQueryParameter("sort_by", sort) - addQueryParameter("sort_by", sort) }.build() - val json = webClient.httpGet(url, apiHeaders).parseJson() + val json = retryWithBackoff { + webClient.httpGet(url, apiHeaders).parseJson() + } val hits = json.optJSONArray("hits") ?: return emptyList() return (0 until hits.length()).map { i -> hits.getJSONObject(i).getJSONObject("document").toManga() @@ -259,90 +309,94 @@ internal class Atsumaru(context: MangaLoaderContext) : override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } - // limit tags to 20 override suspend fun getDetails(manga: Manga): Manga { synchronized(detailsCacheLock) { detailsCache[manga.url]?.let { return it } } - val result = coroutineScope { - val mangaId = manga.url.substringAfterLast("/") - - val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } - val pageJson = pageDeferred.await() - val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga - - val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } - val description = mangaPage.optString("synopsis") ?: manga.description - - val rawRating = mangaPage.optDouble("avgRating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val posterObj = mangaPage.optJSONObject("poster") - val posterImage = posterObj?.optString("mediumImage") - val coverUrl = if (!posterImage.isNullOrEmpty()) { - "https://$domain/static/$posterImage" - } else manga.coverUrl - - val authors = mangaPage.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() - } ?: emptySet() - - val statusText = mangaPage.optString("status").orEmpty() - val state = when (statusText.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "canceled" -> MangaState.ABANDONED - else -> manga.state - } - - val tagSet = mutableSetOf<MangaTag>() - mangaPage.optJSONArray("genres")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) - } - } - mangaPage.optJSONArray("tags")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) - } - } - - val scanlators = mutableMapOf<String, String>() - mangaPage.optJSONArray("scanlators")?.let { scanArr -> - for (i in 0 until scanArr.length()) { - val sc = scanArr.optJSONObject(i) ?: continue - val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue - val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue - scanlators[id] = name + val result = try { + retryWithBackoff { + coroutineScope { + val mangaId = manga.url.substringAfterLast("/") + + val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } + val pageJson = pageDeferred.await() + val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga + + val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } + val description = mangaPage.optString("synopsis") ?: manga.description + + val rawRating = mangaPage.optDouble("avgRating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val posterObj = mangaPage.optJSONObject("poster") + val posterImage = posterObj?.optString("mediumImage") + val coverUrl = if (!posterImage.isNullOrEmpty()) { + "https://$domain/static/$posterImage" + } else manga.coverUrl + + val authors = mangaPage.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() + } ?: emptySet() + + val statusText = mangaPage.optString("status").orEmpty() + val state = when (statusText.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> manga.state + } + + val tagSet = mutableSetOf<MangaTag>() + mangaPage.optJSONArray("genres")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) + } + } + mangaPage.optJSONArray("tags")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) + } + } + + val scanlators = mutableMapOf<String, String>() + mangaPage.optJSONArray("scanlators")?.let { scanArr -> + for (i in 0 until scanArr.length()) { + val sc = scanArr.optJSONObject(i) ?: continue + val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue + val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue + scanlators[id] = name + } + } + + val chapters = loadChapters(mangaId, scanlators) + + val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() + } ?: emptySet() + + manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + state = state, + tags = tagSet, + rating = rating, + chapters = chapters, + altTitles = altTitles, + ) } } - - val chaptersDeferred = async { loadChapters(mangaId, scanlators) } - - val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() - } ?: emptySet() - - val chapters = chaptersDeferred.await() - - manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - state = state, - tags = tagSet, - rating = rating, - chapters = chapters, - altTitles = altTitles, - ) + } catch (_: Exception) { + manga } + synchronized(detailsCacheLock) { detailsCache[manga.url] = result } @@ -350,7 +404,9 @@ internal class Atsumaru(context: MangaLoaderContext) : } private suspend fun loadChapters(mangaId: String, scanlators: Map<String, String>): List<MangaChapter> { - val chaptersJson = webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() + val chaptersJson = retryWithBackoff { + webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() + } val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() val chapters = (0 until chaptersArray.length()).map { i -> @@ -383,7 +439,6 @@ internal class Atsumaru(context: MangaLoaderContext) : } } - override suspend fun getRelatedManga(seed: Manga): List<Manga> { val mangaId = seed.url.substringAfterLast("/") val pageJson = webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() @@ -400,7 +455,9 @@ internal class Atsumaru(context: MangaLoaderContext) : .addQueryParameter("mangaId", mangaId) .addQueryParameter("chapterId", chapterId) .build() - val json = webClient.httpGet(url, apiHeaders).parseJson() + val json = retryWithBackoff { + webClient.httpGet(url, apiHeaders).parseJson() + } val pagesArray = json.getJSONObject("readChapter").getJSONArray("pages") return (0 until pagesArray.length()).map { i -> val page = pagesArray.getJSONObject(i) From 5a45d36332d671c40f24ba50a2c888b3fe33b1d1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 31 Jul 2026 13:29:46 +0200 Subject: [PATCH 130/434] Create Manga3Asq.kt --- src/main/kotlin/tsuki/site/ar/Manga3Asq.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/Manga3Asq.kt diff --git a/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt b/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt new file mode 100644 index 00000000..16abf0d9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt @@ -0,0 +1,14 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA3ASQ", "3Asq", "ar") +internal class Manga3Asq(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGA3ASQ, "3asq.online") { + override val datePattern = "d MMM\u060c yyy" + override val withoutAjax = true +} From b1e7fea2a114c9efa070e3d62ad35fd0531664dd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:19:39 +0200 Subject: [PATCH 131/434] Update publish.yml --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2349a736..ff63bacb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,7 +6,7 @@ on: branches: - master paths: - - 'src/main/kotlin/org/koitharu/kotatsu/parsers/**' + - 'src/main/kotlin/tsuki/**' jobs: publish: @@ -46,4 +46,4 @@ jobs: tag_name: ${{ steps.meta.outputs.sha_short }} name: "${{ steps.meta.outputs.sha_short }}" files: build/libs/uma.jar - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} From 5c7bf08ee18d1fed861a5ad8752ecd75fdc19338 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:22:30 +0200 Subject: [PATCH 132/434] Update libs.versions.toml --- gradle/libs.versions.toml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index b0cba018..e1d8f1c6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -10,11 +10,9 @@ json = "20251224" # Keep Android 5, 6 compatible (with SDK 21 - 23) #noinspection NewerVersionAvailable jsoup = "1.17.2" -quickjs = "1.1.0" korte = "4.0.10" -serialization = "1.11.0" simplexml = "2.7.1" -tsuki = "1.0.3" +tsuki = "1.0.5" [plugins] kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } @@ -26,7 +24,6 @@ tsuki = { module = "com.github.UsagiApp:Tsuki", version.ref = "tsuki"} ksp-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp" } kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines" } -kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" } junit-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } @@ -36,6 +33,5 @@ okio = { module = "com.squareup.okio:okio", version.ref = "okio" } json = { module = "org.json:json", version.ref = "json" } androidx-collection = { module = "androidx.collection:collection", version.ref = "collection" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } -quickjs = { module = "io.webfolder:quickjs", version.ref = "quickjs" } korte = { module = "com.soywiz.korlibs.korte:korte-jvm", version.ref = "korte" } simplexml = { module = "org.simpleframework:simple-xml", version.ref = "simplexml" } From 519cbb3b93a8f416cec57cb8e34dca1a64f8bdc3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:26:45 +0200 Subject: [PATCH 133/434] Update build.gradle.kts --- build.gradle.kts | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index 4cf02d16..cfd7984a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } group = "org.usagi" -version = "1.0.3" +version = "1.0.0" tasks.test { useJUnitPlatform() @@ -54,7 +54,6 @@ publishing { dependencies { implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) implementation(libs.okhttp) implementation(libs.okio) implementation(libs.json) @@ -70,7 +69,6 @@ dependencies { testImplementation(libs.junit.params) testRuntimeOnly(libs.junit.launcher) testImplementation(libs.kotlinx.coroutines.test) - testImplementation(libs.quickjs) } tasks.register("buildJar") { From 00a8ea6ca6d6cf3d23ea4bbea53cb294b07cbfe8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 1 Aug 2026 00:30:58 +0200 Subject: [PATCH 134/434] Update HentaiNexus.kt --- .../kotlin/tsuki/site/en/nsfw/HentaiNexus.kt | 50 +++++++++++-------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt index 5a089664..aa0c4d90 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt @@ -2,16 +2,37 @@ package tsuki.site.en.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.config.ConfigKey import tsuki.parsers.GalleryAdultsParser +import tsuki.config.ConfigKey -import tsuki.util.* -import tsuki.model.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseSafe +import tsuki.util.urlDecode +import tsuki.util.urlEncoded -import org.json.JSONArray -import org.jsoup.nodes.Document import java.text.SimpleDateFormat import java.util.Base64 +import org.json.JSONArray +import org.jsoup.nodes.Document private const val SERVER_PNG = "png" // old method uses now image_fallback private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format @@ -194,34 +215,27 @@ internal class HentaiNexus(context: MangaLoaderContext) : if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { mangaPages = getPagesInternal(chapter.url) mangaInternalId = chapter.url.split("/").last() - - // preload image urls val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") if (reader != null && !pageCache.containsKey(reader)) { pageCache[reader] = getPageUrlInternal(reader) } } - return mangaPages } override suspend fun getPageUrl(page: MangaPage): String { val readerUrl = page.url.substringBefore("#") - val pages = pageCache[readerUrl] ?: getPageUrlInternal(readerUrl).also { pageCache[readerUrl] = it } - val index = page.url.substringAfter("#").toInt() - 1 - return pages[index] } private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List<MangaPage> { val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") - return document.select(selectTotalPage).mapIndexed { index, element -> val url = "$readUrl#${index + 1}" MangaPage( @@ -294,15 +308,9 @@ internal class HentaiNexus(context: MangaLoaderContext) : private fun normalizeImageUrl(url: String): String { return when { url.startsWith("http") -> url - - url.startsWith("//") -> - "https:$url" - - url.startsWith("/") -> - "https://$domain$url" - - else -> - "https://$domain/$url" + url.startsWith("//") -> "https:$url" + url.startsWith("/") -> "https://$domain$url" + else -> "https://$domain/$url" } } From 651bd867a56f18f4755818222d24e53cfbf041ad Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 01:49:06 +0200 Subject: [PATCH 135/434] Update ChapterNumber.kt --- src/main/kotlin/tsuki/util/ChapterNumber.kt | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/util/ChapterNumber.kt b/src/main/kotlin/tsuki/util/ChapterNumber.kt index 5a157d7b..eb1499b3 100644 --- a/src/main/kotlin/tsuki/util/ChapterNumber.kt +++ b/src/main/kotlin/tsuki/util/ChapterNumber.kt @@ -5,9 +5,8 @@ import tsuki.model.MangaChapter private val CHAPTER_REGEX = Regex("""(?i)\b(?:chapter|ch\.?|chapitre|capítulo|cap\.?|episode|ep\.?)\s*(\d+(?:\.\d+)?)""") private val FALLBACK_REGEX = Regex("""(\d+(?:\.\d+)?)""") -/** - * Extracts the most likely chapter number from a string. - * Fast – no object allocations beyond the regex result. +/* + * Extract from string * Returns [Float] or 0f if no number found. */ fun String.extractChapterNumber(): Float { @@ -16,10 +15,9 @@ fun String.extractChapterNumber(): Float { } /** - * Sorts a list of [MangaChapter] by their chapter number (ascending). + * Sorts a list of [MangaChapter] by their chapter number. * The number is extracted from the chapter title using [extractChapterNumber]. - * Chapters without a recognisable number appear first (treated as 0). - * + * draft code * Usage: `.sortChapters()` */ fun List<MangaChapter>.sortChapters(): List<MangaChapter> { From 9cd1158794f30e8333a69b1ef4e658ef0ba64f24 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:23:09 +0200 Subject: [PATCH 136/434] Add utility imports in Mangalek.kt Added utility imports for HTML parsing and URL handling. --- src/main/kotlin/tsuki/site/ar/Mangalek.kt | 113 ++++++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/Mangalek.kt diff --git a/src/main/kotlin/tsuki/site/ar/Mangalek.kt b/src/main/kotlin/tsuki/site/ar/Mangalek.kt new file mode 100644 index 00000000..db8adf8f --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Mangalek.kt @@ -0,0 +1,113 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource +import tsuki.model.MangaChapter +import tsuki.model.Manga +import tsuki.model.MangaTag + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.removeSuffix + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("MANGALEK", "Mangalek", "ar") +internal class Mangalek(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGALEK, "mangalik.net") { + + override val datePattern = "MMMM dd, yyyy" + override val stylePage = "" + + private val altDateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + + private fun parseDateTwoFormats(primary: java.text.DateFormat, date: String?): Long { + date ?: return 0L + return try { + primary.parse(date)?.time ?: 0L + } catch (_: Exception) { + try { + altDateFormat.parse(date)?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage // stylePage is empty, so just href + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = link, + uploadDate = parseDateTwoFormats(dateFormat, dateText), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { + val doc = if (postReq) { + val mangaId = document.select("div#manga-chapters-holder").attr("data-id") + val url = "https://$domain/wp-admin/admin-ajax.php" + val postData = postDataReq + mangaId + webClient.httpPost(url, postData).parseHtml() + } else { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + webClient.httpPost(url, emptyMap()).parseHtml() + } + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.select(selectChapter).map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = name.extractChapterNumber(), + volume = 0, + branch = null, + uploadDate = parseDateTwoFormats(dateFormat, dateText), + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + return doc.selectFirst("div.genres") + ?.select("a") + .orEmpty() + .mapNotNull { a -> + val genre = a.ownText() + if (genre.isBlank()) null + else MangaTag(key = genre.lowercase(), title = genre, source = source) + } + .toSet() + } +} From 4bd39d24d7fdba37742d486c050dc8deaee6fe8e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:24:14 +0200 Subject: [PATCH 137/434] Update ChapterNumber.kt --- src/main/kotlin/tsuki/util/ChapterNumber.kt | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/src/main/kotlin/tsuki/util/ChapterNumber.kt b/src/main/kotlin/tsuki/util/ChapterNumber.kt index eb1499b3..ff731e10 100644 --- a/src/main/kotlin/tsuki/util/ChapterNumber.kt +++ b/src/main/kotlin/tsuki/util/ChapterNumber.kt @@ -1,11 +1,9 @@ package tsuki.util -import tsuki.model.MangaChapter - private val CHAPTER_REGEX = Regex("""(?i)\b(?:chapter|ch\.?|chapitre|capítulo|cap\.?|episode|ep\.?)\s*(\d+(?:\.\d+)?)""") private val FALLBACK_REGEX = Regex("""(\d+(?:\.\d+)?)""") -/* +/** * Extract from string * Returns [Float] or 0f if no number found. */ @@ -13,15 +11,3 @@ fun String.extractChapterNumber(): Float { CHAPTER_REGEX.find(this)?.groupValues?.get(1)?.toFloatOrNull()?.let { return it } return FALLBACK_REGEX.find(this)?.value?.toFloatOrNull() ?: 0f } - -/** - * Sorts a list of [MangaChapter] by their chapter number. - * The number is extracted from the chapter title using [extractChapterNumber]. - * draft code - * Usage: `.sortChapters()` - */ -fun List<MangaChapter>.sortChapters(): List<MangaChapter> { - return map { it to (it.title?.extractChapterNumber() ?: 0f) } - .sortedBy { it.second } - .map { it.first } -} From 13fa2630dac9a7676896663e987f1775634879a0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:33:27 +0200 Subject: [PATCH 138/434] Create Azora.kt --- src/main/kotlin/tsuki/site/ar/Azora.kt | 438 +++++++++++++++++++++++++ 1 file changed, 438 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/Azora.kt diff --git a/src/main/kotlin/tsuki/site/ar/Azora.kt b/src/main/kotlin/tsuki/site/ar/Azora.kt new file mode 100644 index 00000000..f47fc5f0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Azora.kt @@ -0,0 +1,438 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.MangaParserSource +import tsuki.model.MangaChapter +import tsuki.model.Manga +import tsuki.model.MangaTag +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.json.toJSONObjectOrNull +import tsuki.util.mapNotNullToSet +import tsuki.util.parseSafe +import tsuki.util.src +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Document +import org.jsoup.parser.Parser +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("AZORAMANGA", "Azora Manga", "ar", ContentType.MANGA) +internal class Azora(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.AZORAMANGA, 24) { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + + override val configKeyDomain = ConfigKey.Domain("azorafly.com") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://api.") + append(domain) + append("/api/query") + + val params = mutableListOf<String>() + + params.add("page=$page") + params.add("perPage=24") + + if (!filter.query.isNullOrEmpty()) { + params.add("searchTerm=${filter.query!!.urlEncoded()}") + } + + if (filter.tags.isNotEmpty()) { + val genreIds = filter.tags.joinToString(",") { it.key } + params.add("genreIds=$genreIds") + } + + val (orderBy, orderDirection) = when (order) { + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.NEWEST -> "createdAt" to "desc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + else -> "lastChapterAddedAt" to "desc" + } + params.add("orderBy=$orderBy") + params.add("orderDirection=$orderDirection") + + if (params.isNotEmpty()) { + append("?") + append(params.joinToString("&")) + } + } + + val response = webClient.httpGet(url) + val body = response.body.string() + + val jsonArray = try { + JSONArray(body) + } catch (_: Exception) { + try { + val jsonObject = JSONObject(body) + when { + jsonObject.has("posts") -> jsonObject.getJSONArray("posts") + jsonObject.has("data") -> jsonObject.getJSONArray("data") + jsonObject.has("results") -> jsonObject.getJSONArray("results") + else -> JSONArray() + } + } catch (_: Exception) { + JSONArray() + } + } + return parseMangaList(jsonArray) + } + + private fun parseMangaList(json: JSONArray): List<Manga> { + val result = mutableListOf<Manga>() + + for (i in 0 until json.length()) { + val obj = json.getJSONObject(i) + val slug = obj.getString("slug") + val url = "/series/$slug" + val title = obj.getString("postTitle") + val coverUrl = obj.optString("featuredImage") + + val seriesStatus = obj.optString("seriesStatus", "") + val state = when (seriesStatus.uppercase()) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + else -> null + } + + val genresArray = obj.optJSONArray("genres") + var isNovel = false + val tags = if (genresArray != null) { + buildSet { + for (idx in 0 until genresArray.length()) { + val genre = genresArray.getJSONObject(idx) + val genreName = genre.getString("name") + if (genreName == "رواية") isNovel = true + add(MangaTag( + key = genre.getInt("id").toString(), + title = genreName, + source = source, + )) + } + } + } else { + emptySet() + } + + if (obj.optString("postType") == "رواية" || obj.optString("type") == "رواية") isNovel = true + + if (obj.optString("seriesType").equals("NOVEL", ignoreCase = true)) isNovel = true + + if (slug.contains("novel", ignoreCase = true)) isNovel = true + + if (isNovel) continue + + result.add( + Manga( + id = generateUid(url), + title = title, + altTitles = emptySet(), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = tags, + state = state, + authors = emptySet(), + source = source, + ) + ) + } + + return result + } + + private fun fetchAvailableTags(): Set<MangaTag> { + return setOf( + MangaTag("أكشن", "1", source), + MangaTag("حريم", "2", source), + MangaTag("زمكاني", "3", source), + MangaTag("سحر", "4", source), + MangaTag("شونين", "5", source), + MangaTag("مغامرات", "6", source), + MangaTag("خيال", "7", source), + MangaTag("رومانسي", "8", source), + MangaTag("كوميدي", "9", source), + MangaTag("مانهوا", "10", source), + MangaTag("إثارة", "11", source), + MangaTag("دراما", "12", source), + MangaTag("تاريخي", "13", source), + MangaTag("راشد", "14", source), + MangaTag("سينين", "15", source), + MangaTag("خارق للطبيعة", "16", source), + MangaTag("شياطين", "17", source), + MangaTag("حياة مدرسية", "18", source), + MangaTag("جوسي", "19", source), + MangaTag("مانها", "20", source), + MangaTag("ويبتون", "21", source), + MangaTag("شينين", "22", source), + MangaTag("قوة خارقة", "23", source), + MangaTag("خيال علمي", "24", source), + MangaTag("غموض", "25", source), + MangaTag("مأساة", "26", source), + MangaTag("شريحة من الحياة", "27", source), + MangaTag("فنون قتالية", "28", source), + MangaTag("شوجو", "29", source), + MangaTag("ايسكاي", "30", source), + MangaTag("مصاصي الدماء", "31", source), + MangaTag("اسبوعي", "32", source), + MangaTag("لعبة", "33", source), + MangaTag("نفسي", "34", source), + MangaTag("وحوش", "35", source), + MangaTag("الحياة اليومية", "36", source), + MangaTag("الحياة المدرسية", "37", source), + MangaTag("رعب", "38", source), + MangaTag("عسكري", "39", source), + MangaTag("رياضي", "40", source), + MangaTag("اتشي", "41", source), + MangaTag("ايشي", "42", source), + MangaTag("دموي", "43", source), + MangaTag("زومبي", "44", source), + MangaTag("مميز", "45", source), + MangaTag("ايسيكاي", "46", source), + MangaTag("فنتازيا", "47", source), + MangaTag("اشباح", "48", source), + MangaTag("إعادة إحياء", "49", source), + MangaTag("بطل غير اعتيادي", "50", source), + MangaTag("ثأر", "51", source), + MangaTag("اثارة", "52", source), + MangaTag("تراجيدي", "53", source), + MangaTag("طبخ", "54", source), + MangaTag("تناسخ", "55", source), + MangaTag("عودة بالزمن", "56", source), + MangaTag("انتقام", "57", source), + MangaTag("تجسيد", "58", source), + MangaTag("فانتازيا", "59", source), + MangaTag("عائلي", "60", source), + MangaTag("تجسد", "61", source), + MangaTag("العاب", "62", source), + MangaTag("عالم اخر", "63", source), + MangaTag("السفر عبر الزمن", "64", source), + MangaTag("خيالي", "65", source), + MangaTag("زمنكاني", "66", source), + MangaTag("مغامرة", "67", source), + MangaTag("طبي", "68", source), + MangaTag("عصور وسطى", "69", source), + MangaTag("ساموراي", "70", source), + MangaTag("مافيا", "71", source), + MangaTag("نظام", "72", source), + MangaTag("هوس", "73", source), + MangaTag("عصري", "74", source), + MangaTag("بطل مجنون", "75", source), + MangaTag("رعاية اطفال", "76", source), + MangaTag("زواج مدبر", "77", source), + MangaTag("تشويق", "78", source), + MangaTag("مكتبي", "79", source), + MangaTag("قوى خارقه", "80", source), + MangaTag("تحقيق", "81", source), + MangaTag("أيتام", "82", source), + MangaTag("جوسين", "83", source), + MangaTag("موسيقي", "84", source), + MangaTag("قصة حقيقة", "85", source), + MangaTag("موريم", "86", source), + MangaTag("موظفين", "87", source), + MangaTag("فيكتوري", "88", source), + MangaTag("مأساوي", "89", source), + MangaTag("عصر حديث", "90", source), + MangaTag("ندم", "91", source), + MangaTag("حياة جامعية", "92", source), + MangaTag("حاصد", "93", source), + MangaTag("الأرواح", "94", source), + MangaTag("جريمة", "95", source), + MangaTag("عاطفي", "96", source), + MangaTag("أكاديمي", "97", source), + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + if (doc.selectFirst("span.inline-block.border.text-foreground")?.text() == "رواية") { + return manga.copy( + state = MangaState.FINISHED, + chapters = emptyList(), + ) + } + + val chapters = loadChapters(manga, doc) + + val coverUrl = doc.selectFirst("section img")?.src() ?: manga.coverUrl + + val ratingText = doc.selectFirst("div:contains(التقييم) + div, span:contains(Rating)")?.text() + val rating = ratingText?.substringBefore("/")?.trim()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN + + val statusText = doc.selectFirst("div:contains(الحالة) + div, span:contains(Status)")?.text() + val state = when { + statusText?.contains("مستمر", ignoreCase = true) == true || + statusText?.contains("ongoing", ignoreCase = true) == true -> MangaState.ONGOING + statusText?.contains("مكتمل", ignoreCase = true) == true || + statusText?.contains("completed", ignoreCase = true) == true -> MangaState.FINISHED + else -> null + } + + val description = doc.selectFirst("meta[name=description]")?.attr("content") + + val tags = doc.select("a[href*='/series/?genres='], span.genre").mapNotNullToSet { element -> + val genreName = element.text().trim() + val genreId = element.attr("href").substringAfter("genres=").substringBefore("&") + .ifEmpty { genreName } + + if (genreName.isNotEmpty()) { + MangaTag( + key = genreId, + title = genreName, + source = source, + ) + } else { + null + } + } + + return manga.copy( + coverUrl = coverUrl, + rating = rating, + state = state, + tags = tags, + description = description, + chapters = chapters, + ) + } + + private fun loadChapters(manga: Manga, doc: Document): List<MangaChapter> { + val seriesSlug = manga.url.substringAfter("/series/", "").substringBefore('/') + if (seriesSlug.isEmpty()) return emptyList() + + val chaptersJson = extractChaptersFromAstroProps(doc) ?: return emptyList() + + val chapters = ArrayList<MangaChapter>(chaptersJson.length()) + for (i in 0 until chaptersJson.length()) { + val chapter = unboxSerializedValue(chaptersJson.opt(i)) as? JSONObject ?: continue + val slug = (unboxSerializedValue(chapter.opt("slug")) as? String) + ?.takeUnless { it.isBlank() || it == "null" } + ?: continue + val url = "/series/$seriesSlug/$slug" + val number = when (val rawNumber = unboxSerializedValue(chapter.opt("number"))) { + is Number -> rawNumber.toFloat() + is String -> rawNumber.toFloatOrNull() ?: 0f + else -> 0f + } + val title = (unboxSerializedValue(chapter.opt("title")) as? String) + ?.takeUnless { it.isBlank() || it == "null" } + val createdAt = unboxSerializedValue(chapter.opt("createdAt")) as? String + + chapters += MangaChapter( + id = generateUid(url), + title = when { + !title.isNullOrBlank() && number > 0f -> "الفصل ${formatChapterNumber(number)} - $title" + !title.isNullOrBlank() -> title + number > 0f -> "الفصل ${formatChapterNumber(number)}" + else -> slug + }, + number = number, + volume = 0, + url = url, + scanlator = null, + uploadDate = parseChapterDate(createdAt), + branch = null, + source = source, + ) + } + return chapters.sortedBy { it.number } + } + + private fun extractChaptersFromAstroProps(doc: Document): JSONArray? { + val propsEscaped = doc + .selectFirst("section[data-series-tab-panel=chapters] astro-island[props]") + ?.attr("props") + ?.takeUnless { it.isBlank() } + ?: return null + + val props = Parser.unescapeEntities(propsEscaped, false) + val root = props.toJSONObjectOrNull() ?: return null + (unboxSerializedValue(root.opt("initialChap")) as? JSONArray)?.let { return it } + val post = unboxSerializedValue(root.opt("post")) as? JSONObject ?: return null + return unboxSerializedValue(post.opt("chapters")) as? JSONArray + } + + private fun unboxSerializedValue(value: Any?): Any? { + if (value !is JSONArray || value.length() != 2) { + return value + } + return when (value.opt(0)) { + 0, 1, "0", "1" -> unboxSerializedValue(value.opt(1)) + else -> value + } + } + + private fun formatChapterNumber(number: Float): String { + return if (number % 1f == 0f) { + number.toInt().toString() + } else { + number.toString() + } + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + return synchronized(chapterDateFormat) { chapterDateFormat.parseSafe(date) } + } + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select("section[itemprop=articleBody] figure img").mapNotNull { img -> + val imageUrl = img.src()?.takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } +} From 81945af94fe62ecfb280154cb82f9db46acb1a51 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 10:42:58 +0200 Subject: [PATCH 139/434] Create DespairManga.kt --- src/main/kotlin/tsuki/site/ar/DespairManga.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/DespairManga.kt diff --git a/src/main/kotlin/tsuki/site/ar/DespairManga.kt b/src/main/kotlin/tsuki/site/ar/DespairManga.kt new file mode 100644 index 00000000..af695454 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/DespairManga.kt @@ -0,0 +1,11 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("DESPAIRMANGA", "Despair Manga", "ar") +internal class DespairManga(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.DESPAIRMANGA, "despair-manga.net") From 09e2681c819aab55f944ea58a3abf4bb4617b407 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 11:41:42 +0200 Subject: [PATCH 140/434] Create DetectiveConan.kt --- src/main/kotlin/tsuki/site/ar/DetectiveConan.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/DetectiveConan.kt diff --git a/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt new file mode 100644 index 00000000..247a4167 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt @@ -0,0 +1,14 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("DETECTIVECONAN", "DetectiveConan", "ar", ContentType.MANGA) +internal class DetectiveConan(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.DETECTIVECONAN, "manga.detectiveconanar.com") { + override val datePattern = "MMMM dd, yyyy" +} From b07b123759e4683458f6c8281dbfb20841ee36c3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:03:50 +0200 Subject: [PATCH 141/434] Create LavaScans.kt --- src/main/kotlin/tsuki/site/ar/LavaScans.kt | 239 +++++++++++++++++++++ 1 file changed, 239 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/LavaScans.kt diff --git a/src/main/kotlin/tsuki/site/ar/LavaScans.kt b/src/main/kotlin/tsuki/site/ar/LavaScans.kt new file mode 100644 index 00000000..80bcbde5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/LavaScans.kt @@ -0,0 +1,239 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Locale +import kotlin.time.Duration.Companion.hours + +@MangaSourceParser("LAVASCANS", "LavaScans", "ar") +internal class LavaScans(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.LAVASCANS, "lavascans.com") { + + override val dateFormat = SimpleDateFormat("yyyy/MM/dd", Locale("ar")) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + private val slugRegex = Regex("""^\d+-""") + private val listUrl = "https://$domain/$mangaDirectory/list-mode/" + private val listSelector = "div#content div.soralist ul li a.series" + + private var urlMapCache: Map<String, String> = emptyMap() + private var mapFetchedAt: Long = 0L + private val mapMutex = Mutex() + + private suspend fun getUrlMap(): Map<String, String> { + if (System.currentTimeMillis() - mapFetchedAt < 1.hours.inWholeMilliseconds) return urlMapCache + return mapMutex.withLock { + if (System.currentTimeMillis() - mapFetchedAt < 1.hours.inWholeMilliseconds) return urlMapCache + urlMapCache = fetchUrlMap() + mapFetchedAt = System.currentTimeMillis() + urlMapCache + } + } + + private suspend fun fetchUrlMap(): Map<String, String> { + return try { + val doc = webClient.httpGet(listUrl).parseHtml() + doc.select(listSelector).associate { a -> + val fullUrl = a.attr("abs:href") + val slug = fullUrl.removeSuffix("/").substringAfterLast("/") + val permaSlug = slug.replaceFirst(slugRegex, "") + permaSlug to slug + } + } catch (_: Exception) { + emptyMap() + } + } + + private val browseSelector = ".listupd .manga-card-v" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val page = page + 1 + + val url = if (query.isNotEmpty()) { + val pagePath = if (page > 1) "page/$page/" else "" + "https://$domain/$pagePath?s=${query.urlEncoded()}" + } else { + buildString { + append("https://$domain/$mangaDirectory/") + append("?page=$page") + append("&order=") + append(when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ADDED -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "update" + }) + filter.states.firstOrNull()?.let { + append("&status=") + append(when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "dropped" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + append("&type=") + append(when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + }) + } + filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } + } + } + + val doc = webClient.httpGet(url).parseHtml() + val selector = if (query.isNotEmpty()) ".legend-card" else browseSelector + val elements = doc.select(selector) + if (elements.isEmpty()) return emptyList() + + val list = elements.mapNotNull { element -> + parseMangaElement(element, isSearch = query.isNotEmpty()) + } + + return list.map { manga -> + if (manga.url.startsWith("/$mangaDirectory/")) { + val slug = manga.url.removeSuffix("/").substringAfterLast("/") + val permaSlug = slug.replaceFirst(slugRegex, "") + manga.copy(url = "/$mangaDirectory/$permaSlug/") + } else { + manga + } + } + } + + private fun parseMangaElement(element: Element, isSearch: Boolean): Manga? { + return if (isSearch) { + val link = element.selectFirst(".legend-title a") + ?: element.selectFirst(".legend-poster a") ?: return null + val title = link.text().trim().ifBlank { link.attr("title") } + val cover = element.selectFirst(".legend-poster img")?.imgAttr() + val href = link.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } else { + val titleEl = element.selectFirst("h3.card-v-title a") ?: return null + val title = titleEl.text().trim() + val cover = element.selectFirst(".card-v-cover img")?.imgAttr() + val href = titleEl.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val permaSlug = manga.url.removeSuffix("/").substringAfterLast("/") + val randomSlug = getUrlMap()[permaSlug] ?: permaSlug + val fullUrl = "https://$domain/$mangaDirectory/$randomSlug/" + val doc = webClient.httpGet(fullUrl).parseHtml() + + val detailsContainer = doc.selectFirst("div.lh-container") ?: return manga + val title = detailsContainer.select(".lh-title").text() + val cover = detailsContainer.select(".lh-poster img").first()?.imgAttr() ?: manga.coverUrl + val genres = detailsContainer.select(".lh-genres a").map { it.text() }.toSet() + val desc = detailsContainer.select("#manga-story").textOrNull()?.trim() + val statusText = detailsContainer.select(".status-badge-lux").text() + val state = parseStatus(statusText) + val chapters = loadChapters(doc, fullUrl) + + return manga.copy( + title = title, + coverUrl = cover, + description = desc, + tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = state, + chapters = chapters, + altTitles = emptySet(), + authors = emptySet() + ) + } + + override val chapterListSelector = "#chapters-list-container .ch-item:not(.locked)" + + override suspend fun loadChapters(doc: Document, mangaUrl: String): List<MangaChapter> { + val elements = doc.select(chapterListSelector) + return elements.map { el -> + val a = el.selectFirst("a") ?: throw Exception("Missing chapter link") + val href = a.attrAsRelativeUrl("href") + val name = el.selectFirst(".ch-num")?.text().orEmpty().ifBlank { + a.selectFirst(".chapternum")?.text() ?: a.ownText() + } + val dateStr = el.selectFirst(".ch-date")?.text() + val uploadDate = dateStr?.let { dateFormat.parse(it)?.time ?: 0L } ?: 0L + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } +} From d4c3f18d59139b3d354a57845ed4c9498f948a48 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:17:27 +0200 Subject: [PATCH 142/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 12af006c..ebc0e5c3 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -58,7 +58,7 @@ abstract class MangaThemesia( .build() protected open val mangaDirectory = "manga" - protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US) + protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("en")) protected open val withoutAjax = false protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" @@ -170,7 +170,7 @@ abstract class MangaThemesia( parseMangaElement(element) } } - + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() From 91ddd9a25a5969a4423787c170e8ca8ac10e4972 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:24:15 +0200 Subject: [PATCH 143/434] Create Hijala.kt --- src/main/kotlin/tsuki/site/ar/Hijala.kt | 145 ++++++++++++++++++++++++ 1 file changed, 145 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/Hijala.kt diff --git a/src/main/kotlin/tsuki/site/ar/Hijala.kt b/src/main/kotlin/tsuki/site/ar/Hijala.kt new file mode 100644 index 00000000..aa15f851 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Hijala.kt @@ -0,0 +1,145 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia +import tsuki.network.OkHttpWebClient + +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.text.SimpleDateFormat +import java.util.Locale +import javax.imageio.ImageIO + +@MangaSourceParser("HIJALA", "Hijala", "ar") +internal class Hijala(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.HIJALA, "hijala.com") { // adjust domain if needed + + override val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("ar")) + + override val webClient = OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(::unscrambleInterceptor) + .build(), + source + ) + + private fun unscrambleInterceptor(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + + if (url.host != SCRAMBLED_HOST) { + return chain.proceed(request) + } + + val leftUrl = url.queryParameter("leftImage") ?: error("Missing leftImage") + val rightUrl = url.queryParameter("rightImage") ?: error("Missing rightImage") + + // Fetch both pieces + val leftResponse = chain.proceed(request.newBuilder().url(leftUrl).build()) + val rightResponse = chain.proceed(request.newBuilder().url(rightUrl).build()) + + return try { + val leftBytes = leftResponse.body?.bytes() ?: error("Left image empty") + val rightBytes = rightResponse.body?.bytes() ?: error("Right image empty") + + val leftImage = ImageIO.read(ByteArrayInputStream(leftBytes)) + val rightImage = ImageIO.read(ByteArrayInputStream(rightBytes)) + + val width = leftImage.width + rightImage.width + val height = maxOf(leftImage.height, rightImage.height) + + val combined = BufferedImage(width, height, BufferedImage.TYPE_INT_RGB) + val g = combined.createGraphics() + g.drawImage(leftImage, 0, 0, null) + g.drawImage(rightImage, leftImage.width, 0, null) + g.dispose() + + val outputStream = ByteArrayOutputStream() + ImageIO.write(combined, "jpeg", outputStream) + val combinedBytes = outputStream.toByteArray() + + Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(combinedBytes.toResponseBody("image/jpeg".toMediaType())) + .build() + } finally { + leftResponse.close() + rightResponse.close() + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + val pages = mutableListOf<MangaPage>() + val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + val jsonMatch = jsonRegex.find(doc.html()) + if (jsonMatch != null) { + val jsonArray = org.json.JSONArray(jsonMatch.groupValues[1]) + for (i in 0 until jsonArray.length()) { + val url = jsonArray.getString(i) + pages.add(MangaPage(id = generateUid(url), url = url, preview = null, source = source)) + } + } else { + doc.select(pageSelector).forEach { img -> + val url = img.imgAttr() + if (url.isNotBlank()) { + pages.add(MangaPage(id = generateUid(url), url = url, preview = null, source = source)) + } + } + } + + // if chapter is not scrambled return + if (doc.selectFirst("#chapter-pages-js-before") == null) return pages + + // group into left/right pairs and create synthetic URLs + val pairs = mutableListOf<List<String>>() + var i = 0 + while (i < pages.size) { + val pair = pages.subList(i, minOf(i + 2, pages.size)) + if (pair.size == 2) { + pairs.add(pair.map { it.url }) + } + i += 2 + } + + return pairs.mapIndexed { _, pair -> + val imageUrl = HttpUrl.Builder() + .scheme("http") + .host(SCRAMBLED_HOST) + .addQueryParameter("leftImage", pair[0]) + .addQueryParameter("rightImage", pair[1]) + .build() + .toString() + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source + ) + } + } + + companion object { + private const val SCRAMBLED_HOST = "127.0.0.1" + } +} From 00404178db104f89a5989c40a99e0f228d136b46 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 13:26:48 +0200 Subject: [PATCH 144/434] Update DetectiveConan.kt --- src/main/kotlin/tsuki/site/ar/DetectiveConan.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt index 247a4167..5a7c52b1 100644 --- a/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt +++ b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt @@ -5,10 +5,19 @@ import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser import tsuki.model.ContentType +import tsuki.model.MangaListFilterCapabilities import tsuki.model.MangaParserSource +import tsuki.model.MangaTag @MangaSourceParser("DETECTIVECONAN", "DetectiveConan", "ar", ContentType.MANGA) internal class DetectiveConan(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.DETECTIVECONAN, "manga.detectiveconanar.com") { override val datePattern = "MMMM dd, yyyy" + + override suspend fun fetchAvailableTags(): Set<MangaTag> = emptySet() + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = false, + isYearRangeSupported = false, + ) } From c75de35ac29d4703b52d7f3a566e69c30aa41885 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:04:24 +0200 Subject: [PATCH 145/434] Update MadaraParser.kt --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 280 +++++++++--------- 1 file changed, 134 insertions(+), 146 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index e4258eba..e9bb7c75 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -74,10 +74,60 @@ internal abstract class MadaraParser( keys.add(userAgentKey) } - // Change these values only if the site does not support manga listings via ajax + /** + * AJAX listing mode. + * false (default) – uses POST requests to wp-admin/admin-ajax.php + * true – builds normal GET URLs (?s=, ?page=, /manga-genre/…) and parses the HTML directly. + */ protected open val withoutAjax = false protected open val authorSearchSupported = false + /** Date format pattern used to parse chapter release dates (SimpleDateFormat). */ + protected open val datePattern = "MMMM d, yyyy" + + /** + * Genre fetching: + * listUrl – the page path (relative to domain) that contains the genre list. + * The parser fetches https://domain/<listUrl> to scrape available genres. + * + * tagPrefix – the URL segment that immediately precedes the genre identifier in filter links. + * Used to extract the genre key from a link like /manga-genre/action/ -> "action". + */ + protected open val listUrl = "manga/" + protected open val tagPrefix = "manga-genre/" + + /** + * Chapter list handling: + * stylePage – suffix appended to chapter URLs to force all images to be shown in one long list. + * postReq – whether to use the old AJAX endpoint for chapter loading. + * true -> POST admin-ajax.php with action=manga_get_chapters&manga=<id> + * false -> POST /manga/<slug>/ajax/chapters/ with empty body + * postDataReq – form data used when postReq = true. + */ + protected open val stylePage = "?style=list" + protected open val postReq = false + protected open val postDataReq = "action=manga_get_chapters&manga=" + + /** Page extraction selectors */ + protected open val selectBodyPage = "div.main-col-inner div.reading-content" + protected open val selectPage = "div.page-break, div.page-box" + protected open val selectRequiredLogin = ".content-blocked, .login-required" + + /** Details page selectors */ + protected open val selectDesc = "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" + protected open val selectGenre = "div.genres-content a" + protected open val selectTestAsync = "div.listing-chapters_wrap" + protected open val selectState = + "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + + "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + + "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(Статус)," + + "div.post-content_item:contains(状态), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" + protected open val selectAlt = ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" + + /** Chapter list selectors */ + protected open val selectDate = "span.chapter-release-date i" + protected open val selectChapter = "li.wp-manga-chapter" + override val availableSortOrders: Set<SortOrder> = setupAvailableSortOrders() private fun setupAvailableSortOrders(): Set<SortOrder> { @@ -150,11 +200,6 @@ internal abstract class MadaraParser( } } - protected open val tagPrefix = "manga-genre/" - protected open val datePattern = "MMMM d, yyyy" - protected open val stylePage = "?style=list" - protected open val postReq = false - init { paginator.firstPage = 0 searchPaginator.firstPage = 0 @@ -286,100 +331,95 @@ internal abstract class MadaraParser( "in arrivo", ) - // can be changed to retrieve tags see getTags - protected open val listUrl = "manga/" - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { if (withoutAjax) { val pages = page + 1 - val url = buildString { - append("https://") - append(domain) - - if (pages > 1) { - append("/page/") - append(pages.toString()) - } - append("/?s=") - append(filter.query?.urlEncoded() ?: "") - - append("&post_type=wp-manga") + fun sortParam(): String = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "new-manga" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "" + else -> "" + } - // Known bug: in some cases, if there are no manga with the associated tags, the source returns the full list of manga - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { - append("&genre[]=") - append(it.key) + val url = if (filter.tags.isNotEmpty()) { + val genreSlug = filter.tags.first().key + buildString { + append("https://") + append(domain) + append("/manga-genre/$genreSlug/") + if (pages > 1) { + append("page/") + append(pages) + append("/") } + append("?m_orderby=") + append(sortParam()) } + } else { + buildString { + append("https://") + append(domain) - filter.states.forEach { - append("&status[]=") - when (it) { - MangaState.ONGOING -> append("on-going") - MangaState.FINISHED -> append("end") - MangaState.ABANDONED -> append("canceled") - MangaState.PAUSED -> append("on-hold") - MangaState.UPCOMING -> append("upcoming") - else -> throw IllegalArgumentException("$it not supported") + if (pages > 1) { + append("/page/") + append(pages.toString()) } - } + append("/?s=") + append(filter.query?.urlEncoded() ?: "") + + append("&post_type=wp-manga") - filter.contentRating.oneOrThrowIfMany()?.let { - append("&adult=") - append( + filter.states.forEach { + append("&status[]=") when (it) { - ContentRating.SAFE -> "0" - ContentRating.ADULT -> "1" - else -> "" - }, - ) - } + MangaState.ONGOING -> append("on-going") + MangaState.FINISHED -> append("end") + MangaState.ABANDONED -> append("canceled") + MangaState.PAUSED -> append("on-hold") + MangaState.UPCOMING -> append("upcoming") + else -> throw IllegalArgumentException("$it not supported") + } + } - if (filter.year != 0) { - append("&release=") - append(filter.year.toString()) - } + filter.contentRating.oneOrThrowIfMany()?.let { + append("&adult=") + append( + when (it) { + ContentRating.SAFE -> "0" + ContentRating.ADULT -> "1" + else -> "" + }, + ) + } - filter.author?.takeIf { it.isNotEmpty() }?.let { - append("&author=") - // should be like "minamida-usuke" - append(it.lowercase().replace(" ", "-")) - } + if (filter.year != 0) { + append("&release=") + append(filter.year.toString()) + } - // Support artist - //filter.artist?.let { - // append("&artist=") - // append(filter.artist) - //} - - - append("&m_orderby=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.NEWEST -> append("new-manga") - SortOrder.ALPHABETICAL -> append("alphabet") - SortOrder.RATING -> append("rating") - SortOrder.RELEVANCE -> {} - else -> {} + filter.author?.takeIf { it.isNotEmpty() }?.let { + append("&author=") + append(it.lowercase().replace(" ", "-")) + } + + append("&m_orderby=") + append(sortParam()) } } + val html = try { webClient.httpGet(url).parseHtml() } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", url) - } + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() + else throw ParseException("Can't fetch data from source!", url) } return parseMangaList(html) } else { - val payload = createRequestTemplate() - payload["page"] = page.toString() filter.query?.takeIf { it.isNotEmpty() }?.let { @@ -417,14 +457,6 @@ internal abstract class MadaraParser( payload["vars[tax_query][3][operator]"] = "IN" } - // Support artist - // filter.artist.let { - // payload["vars[tax_query][4][taxonomy]"] = "wp-manga-artist" - // payload["vars[tax_query][4][field]"] = "name" - // payload["vars[tax_query][4][terms][0]"] = filter.artist - // payload["vars[tax_query][4][operator]"] = "IN" - //} - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.year != 0) { payload["vars[tax_query][relation]"] = "AND" } @@ -435,103 +467,80 @@ internal abstract class MadaraParser( payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "desc" } - SortOrder.POPULARITY_ASC -> { payload["vars[meta_key]"] = "_wp_manga_views" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "asc" } - SortOrder.UPDATED -> { payload["vars[meta_key]"] = "_latest_update" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "desc" } - SortOrder.UPDATED_ASC -> { payload["vars[meta_key]"] = "_latest_update" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "asc" } - SortOrder.NEWEST -> { payload["vars[orderby]"] = "date" payload["vars[order]"] = "desc" } - SortOrder.NEWEST_ASC -> { payload["vars[orderby]"] = "date" payload["vars[order]"] = "asc" } - SortOrder.ALPHABETICAL -> { payload["vars[orderby]"] = "post_title" payload["vars[order]"] = "asc" } - SortOrder.ALPHABETICAL_DESC -> { payload["vars[orderby]"] = "post_title" payload["vars[order]"] = "desc" } - SortOrder.RATING -> { payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - payload["vars[orderby][query_avarage_reviews]"] = "DESC" payload["vars[orderby][query_total_reviews]"] = "DESC" } - SortOrder.RATING_ASC -> { payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - payload["vars[orderby][query_avarage_reviews]"] = "ASC" payload["vars[orderby][query_total_reviews]"] = "ASC" } - - SortOrder.RELEVANCE -> { - payload["vars[orderby]"] = "" - } - + SortOrder.RELEVANCE -> payload["vars[orderby]"] = "" else -> payload["vars[orderby]"] = "" } filter.states.forEach { payload["vars[meta_query][0][0][key]"] = "_wp_manga_status" payload["vars[meta_query][0][0][compare]"] = "IN" - payload["vars[meta_query][0][0][value][]"] = - when (it) { - MangaState.ONGOING -> "on-going" - MangaState.FINISHED -> "end" - MangaState.ABANDONED -> "canceled" - MangaState.PAUSED -> "on-hold" - MangaState.UPCOMING -> "upcoming" - else -> throw IllegalArgumentException("$it not supported") - } + payload["vars[meta_query][0][0][value][]"] = when (it) { + MangaState.ONGOING -> "on-going" + MangaState.FINISHED -> "end" + MangaState.ABANDONED -> "canceled" + MangaState.PAUSED -> "on-hold" + MangaState.UPCOMING -> "upcoming" + else -> throw IllegalArgumentException("$it not supported") + } } filter.contentRating.oneOrThrowIfMany()?.let { payload["vars[meta_query][0][1][key]"] = "manga_adult_content" - payload["vars[meta_query][0][1][value]"] = - when (it) { - ContentRating.SAFE -> "" - ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" - else -> "" - } + payload["vars[meta_query][0][1][value]"] = when (it) { + ContentRating.SAFE -> "" + ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" + else -> "" + } } val html = try { - webClient.httpPost( - "https://$domain/wp-admin/admin-ajax.php", - payload, - ).parseHtml() + webClient.httpPost("https://$domain/wp-admin/admin-ajax.php", payload).parseHtml() } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", domain) - } + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() + else throw ParseException("Can't fetch data from source!", domain) } return parseMangaList(html) } @@ -614,18 +623,6 @@ internal abstract class MadaraParser( } } - protected open val selectDesc = - "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" - protected open val selectGenre = "div.genres-content a" - protected open val selectTestAsync = "div.listing-chapters_wrap" - protected open val selectState = - "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + - "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + - "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(Статус)," + - "div.post-content_item:contains(状态), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" - protected open val selectAlt = - ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" - protected open suspend fun createMangaTag(a: Element): MangaTag? { return MangaTag( key = a.attr("href").removeSuffix("/").substringAfterLast('/'), @@ -679,10 +676,6 @@ internal abstract class MadaraParser( ) } - - protected open val selectDate = "span.chapter-release-date i" - protected open val selectChapter = "li.wp-manga-chapter" - protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) return doc.body().select(selectChapter).map { li -> @@ -705,7 +698,6 @@ internal abstract class MadaraParser( }.sortedBy { it.number } } - protected open val postDataReq = "action=manga_get_chapters&manga=" protected open suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { val doc = if (postReq) { @@ -761,10 +753,6 @@ internal abstract class MadaraParser( } } - protected open val selectBodyPage = "div.main-col-inner div.reading-content" - protected open val selectPage = "div.page-break, div.page-box" - protected open val selectRequiredLogin = ".content-blocked, .login-required" - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() From 8843450d57f2a8bc7d8e12bcc311ccbef06761d6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:06:00 +0200 Subject: [PATCH 146/434] Update LavaScans.kt --- src/main/kotlin/tsuki/site/ar/LavaScans.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/ar/LavaScans.kt b/src/main/kotlin/tsuki/site/ar/LavaScans.kt index 80bcbde5..9c84d9de 100644 --- a/src/main/kotlin/tsuki/site/ar/LavaScans.kt +++ b/src/main/kotlin/tsuki/site/ar/LavaScans.kt @@ -31,7 +31,7 @@ import java.text.SimpleDateFormat import java.util.Locale import kotlin.time.Duration.Companion.hours -@MangaSourceParser("LAVASCANS", "LavaScans", "ar") +@MangaSourceParser("LAVASCANS", "Lava Scans", "ar") internal class LavaScans(context: MangaLoaderContext) : MangaThemesia(context, MangaParserSource.LAVASCANS, "lavascans.com") { From c12b4ca1a4845b53e85baf0b4a4e9053d26ba8db Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 14:19:37 +0200 Subject: [PATCH 147/434] Create RocksManga.kt --- src/main/kotlin/tsuki/site/ar/RocksManga.kt | 285 ++++++++++++++++++++ 1 file changed, 285 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/RocksManga.kt diff --git a/src/main/kotlin/tsuki/site/ar/RocksManga.kt b/src/main/kotlin/tsuki/site/ar/RocksManga.kt new file mode 100644 index 00000000..803d8c92 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/RocksManga.kt @@ -0,0 +1,285 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded + +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("ROCKSMANGA", "Rocks Manga", "ar") +internal class RocksManga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.ROCKSMANGA, "rocksmanga.com") { + + override val datePattern = "MMMM d, yyyy" + override val withoutAjax = true + override val stylePage = "" + override val selectBodyPage = "div.reading-content" + override val selectPage = "img" + override val selectDesc = ".description" + override val selectGenre = "div.genres-content a" + override val selectChapter = "ul.scroll-sm li.item" + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val pages = page + 1 + val url = buildString { + append("https://") + append(domain) + + if (!filter.query.isNullOrEmpty()) { + // Text search + if (pages > 1) { + append("/page/") + append(pages) + } + append("/?s=") + append(filter.query!!.urlEncoded()) + append("&post_type=wp-manga") + } else { + // Browse with optional genre / type filter + val genreTag = filter.tags.firstOrNull() + val type = filter.types.firstOrNull() + + if (genreTag != null) { + append("/manga-genre/") + append(genreTag.key) + append("/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + } else if (type != null) { + val typeKey = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.ONE_SHOT -> "one-shot" + else -> null + } + if (typeKey != null) { + append("/manga-type/") + append(typeKey) + append("/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + } + } else { + // Default browse – only one sort order is available, so hardcode it + append("/manga") + if (pages > 1) { + append("/page/") + append(pages) + } + append("/?m_orderby=latest") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + if (doc.location().removeSuffix("/").equals("https://$domain", ignoreCase = true)) { + return emptyList() + } + return parseMangaList(doc) + } + + override fun parseMangaList(doc: Document): List<Manga> { + val items = doc.select("div.original.card-lg div.unit") + return items.map { unit -> + val posterLink = unit.selectFirstOrThrow("a.poster") + val href = posterLink.attr("href").toRelativeUrl(domain) + val img = posterLink.selectFirst("img") + val info = unit.selectFirst("div.info") + val titleLink = info?.selectFirst("a") + val title = titleLink?.text()?.trim().orEmpty() + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = img?.src(), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("div.info h1")?.text() ?: manga.title + val coverUrl = doc.selectFirst("div.poster img")?.src() ?: manga.coverUrl + val description = doc.selectFirst("div.description")?.html()?.takeIf { it.isNotBlank() } + + val metaElements = doc.select("div.meta > div") + var author: String? = null + var status: MangaState? = null + val altTitles = mutableSetOf<String>() + + metaElements.forEach { element -> + val label = element.selectFirst("span")?.text()?.trim() ?: return@forEach + val value = element.selectFirst("a")?.text()?.trim() ?: element.ownText().trim() + + when { + label.contains("المؤلف") || label.contains("الكاتب") -> author = value + label.contains("الحالة") || label.contains("الوضع") -> { + status = when (value.lowercase()) { + "مستمر", "مستمرة", "ongoing" -> MangaState.ONGOING + "مكتمل", "مكتملة", "completed", "complete" -> MangaState.FINISHED + "متوقف", "متوقفة", "hiatus" -> MangaState.PAUSED + "ملغي", "ملغية", "cancelled", "dropped" -> MangaState.ABANDONED + else -> null + } + } + label.contains("الأسماء البديلة") || label.contains("أسماء أخرى") -> { + value.split(",", "،", ";").forEach { name -> + val trimmed = name.trim() + if (trimmed.isNotEmpty()) altTitles.add(trimmed) + } + } + } + } + + val tags = doc.select("div.genres a, div.tags a").mapNotNullToSet { a -> + val href = a.attr("href").removeSuffix("/").substringAfterLast("/") + val name = a.text().trim() + if (href.isNotEmpty() && name.isNotEmpty()) { + MangaTag(key = href, title = name, source = source) + } else null + } + + val ratingText = doc.selectFirst("div.rating span.score")?.text() + val rating = ratingText?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN + + val chapters = getChapters(manga, doc) + + val isAdult = doc.selectFirst("div.adult-content") != null || + tags.any { it.key in setOf("adult", "mature", "18+", "ecchi", "smut") } + + manga.copy( + title = title, + altTitles = manga.altTitles + altTitles, + coverUrl = coverUrl, + largeCoverUrl = coverUrl, + description = description, + tags = tags, + state = status, + authors = setOfNotNull(author), + rating = rating, + chapters = chapters, + contentRating = if (isAdult) ContentRating.ADULT else ContentRating.SAFE, + publicUrl = fullUrl, + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("ar")) + fun SimpleDateFormat.parseSafeOrZero(date: String): Long = + try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } + + return doc.body().select("ul.scroll-sm li.item").map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attr("href").toRelativeUrl(domain) + val link = href + stylePage + + val chapterNum = a.attr("title").toFloatOrNull() ?: 0f + val name = "Chapter ${chapterNum.toInt().toString().removeSuffix(".0")}" + + val dateText = li.selectFirst("span.time")?.text().orEmpty() + val scanlator = li.selectFirst("span.user span")?.text() + + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = chapterNum, + volume = 0, + branch = null, + uploadDate = dateFormat.parseSafeOrZero(dateText), + scanlator = scanlator, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val container = doc.selectFirst("#ch-images") + ?: doc.parseFailed("Container #ch-images not found") + val images = container.select("img.preload-image") + + return images.mapNotNull { img -> + val url = img.attr("data-src").ifEmpty { img.attr("src") } + .takeIf { it.isNotEmpty() } ?: return@mapNotNull null + val relativeUrl = url.toRelativeUrl(domain) + MangaPage( + id = generateUid(relativeUrl), + url = relativeUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val menu = doc.selectFirst("#nav-menu li:contains(التصنيفات) ul") + ?: return emptySet() + return menu.select("li a").mapNotNull { a -> + val href = a.attr("href") + val slug = href.substringAfter("/manga-genre/").substringBefore("/") + if (slug.isEmpty()) null + else MangaTag(key = slug, title = a.text().trim(), source = source) + }.toSet() + } +} From 0594baeee4800d62ebec8ff1db647bd5f49ea336 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 2 Aug 2026 22:08:59 +0200 Subject: [PATCH 148/434] Update MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 4bd7e26a..c9612cce 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -252,6 +252,7 @@ internal abstract class MangaFireParser( .addPathSegments("api/titles") .addQueryParameter("page", page.toString()) .addQueryParameter("limit", "50") + .addQueryParameter("languages[]", siteLang) if (!filter.query.isNullOrBlank()) { urlBuilder.addQueryParameter("keyword", filter.query) From 33c6016fdfe82fc988e94e25e5c8de5cd9ca0aef Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 10:43:28 +0200 Subject: [PATCH 149/434] Delete src/main/resources/assets/i18n directory --- .../assets/i18n/messages_en.properties | 27 ------------------- .../assets/i18n/messages_es.properties | 26 ------------------ .../assets/i18n/messages_pt_br.properties | 26 ------------------ 3 files changed, 79 deletions(-) delete mode 100644 src/main/resources/assets/i18n/messages_en.properties delete mode 100644 src/main/resources/assets/i18n/messages_es.properties delete mode 100644 src/main/resources/assets/i18n/messages_pt_br.properties diff --git a/src/main/resources/assets/i18n/messages_en.properties b/src/main/resources/assets/i18n/messages_en.properties deleted file mode 100644 index fd7fb06e..00000000 --- a/src/main/resources/assets/i18n/messages_en.properties +++ /dev/null @@ -1,27 +0,0 @@ -author_filter_title=Author -artist_filter_title=Artist -year_filter_title=Year of Released -status_filter_title=Status -status_filter_completed=Completed -status_filter_ongoing=Ongoing -status_filter_canceled=Canceled -status_filter_on_hold=On Hold -order_by_filter_title=Order By -order_by_filter_relevance=Relevance -order_by_filter_latest=Latest -order_by_filter_az=A-Z -order_by_filter_rating=Rating -order_by_filter_trending=Trending -order_by_filter_views=Most Views -order_by_filter_new=New -genre_condition_filter_title=Genre condition -genre_condition_filter_or=OR -genre_condition_filter_and=AND -adult_content_filter_title=Adult Content -adult_content_filter_all=All -adult_content_filter_none=None -adult_content_filter_only=Only -genre_filter_header=Genres filter may not work for all sources -genre_filter_title=Genres -genre_missing_warning=Press 'Reset' to attempt to show the genres -alt_names_heading=Alternative Names: diff --git a/src/main/resources/assets/i18n/messages_es.properties b/src/main/resources/assets/i18n/messages_es.properties deleted file mode 100644 index ba7132da..00000000 --- a/src/main/resources/assets/i18n/messages_es.properties +++ /dev/null @@ -1,26 +0,0 @@ -author_filter_title=Autor -artist_filter_title=Artista -year_filter_title=Ao de lanzamiento -status_filter_title=Estado -status_filter_completed=Completado -status_filter_ongoing=En curso -status_filter_canceled=Cancelado -status_filter_on_hold=En pausa -order_by_filter_title=Ordenar por -order_by_filter_relevance=Relevancia -order_by_filter_latest=Reciente -order_by_filter_rating=Valoracin -order_by_filter_trending=Tendencia -order_by_filter_views=Vistas -order_by_filter_new=Nuevos -genre_condition_filter_title=Condicin de gnero -genre_condition_filter_or=O -genre_condition_filter_and=Y -adult_content_filter_title=Contenido adulto -adult_content_filter_all=Todos -adult_content_filter_none=Ninguno -adult_content_filter_only=Solo -genre_filter_header=Es posible que el filtro de gneros no funcione correctamente -genre_filter_title=Gneros -genre_missing_warning=Presione 'Restablecer' para intentar mostrar los gneros -alt_names_heading=Nombres Alternativos: \ No newline at end of file diff --git a/src/main/resources/assets/i18n/messages_pt_br.properties b/src/main/resources/assets/i18n/messages_pt_br.properties deleted file mode 100644 index 0c0a88b6..00000000 --- a/src/main/resources/assets/i18n/messages_pt_br.properties +++ /dev/null @@ -1,26 +0,0 @@ -author_filter_title=Autor -artist_filter_title=Artista -year_filter_title=Ano de lanamento -status_filter_title=Estado -status_filter_completed=Completo -status_filter_ongoing=Em andamento -status_filter_canceled=Cancelado -status_filter_on_hold=Pausado -order_by_filter_title=Ordenar por -order_by_filter_relevance=Relevncia -order_by_filter_latest=Recentes -order_by_filter_rating=Avaliao -order_by_filter_trending=Tendncia -order_by_filter_views=Visualizaes -order_by_filter_new=Novos -genre_condition_filter_title=Operador dos gneros -genre_condition_filter_or=OU -genre_condition_filter_and=E -adult_content_filter_title=Contedo adulto -adult_content_filter_all=Indiferente -adult_content_filter_none=Nenhum -adult_content_filter_only=Somente -genre_filter_header=O filtro de gneros pode no funcionar -genre_filter_title=Gneros -genre_missing_warning=Aperte 'Redefinir' para tentar mostrar os gneros -alt_names_heading=Nomes alternativos: \ No newline at end of file From 29a6649c47a916ffad9f394228101407e8de93de Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:16:08 +0200 Subject: [PATCH 150/434] Update DoujinDesuParser.kt --- .../kotlin/tsuki/parsers/DoujinDesuParser.kt | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt index a38950f0..151065aa 100644 --- a/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -45,9 +45,6 @@ internal abstract class DoujinDesuParser( source: MangaParserSource ): PagedMangaParser(context, source, pageSize = 18) { - protected abstract val defaultTypes: String - protected abstract val availableContentTypes: Set<ContentType> - override val configKeyDomain = ConfigKey.Domain("doujin.desu.xxx") override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { @@ -65,10 +62,12 @@ internal abstract class DoujinDesuParser( override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } - override val defaultSortOrder: SortOrder = SortOrder.UPDATED - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.NEWEST_ASC ) override val filterCapabilities = MangaListFilterCapabilities( @@ -80,8 +79,15 @@ internal abstract class DoujinDesuParser( override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = getOrFetchGenres(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = availableContentTypes, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.MANHWA + ), ) private suspend fun getOrFetchGenres(): Set<MangaTag> { @@ -149,6 +155,7 @@ internal abstract class DoujinDesuParser( SortOrder.POPULARITY -> "rating" SortOrder.ALPHABETICAL -> "title_asc" SortOrder.NEWEST -> "newest" + SortOrder.NEWEST_ASC -> "oldest" SortOrder.UPDATED -> "latest_chapter" else -> "latest_chapter" } From 7ec1077a3bf88476f84d63f95576f1bae1c1862b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:16:31 +0200 Subject: [PATCH 151/434] Refactor DoujinDesu class by removing properties Removed defaultTypes and availableContentTypes properties. --- src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt index c39d1b4c..55aab75b 100644 --- a/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt +++ b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt @@ -7,17 +7,6 @@ import tsuki.parsers.DoujinDesuParser import tsuki.model.ContentType import tsuki.model.MangaParserSource -import java.util.EnumSet - @MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) internal class DoujinDesu(context: MangaLoaderContext) : - DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { - - override val defaultTypes: String = "doujinshi,manga,manhwa" - - override val availableContentTypes: Set<ContentType> = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.MANHWA - ) -} \ No newline at end of file + DoujinDesuParser(context, MangaParserSource.DOUJINDESU) From ca386c381476053137408e592e36770b344cd8d0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:33:00 +0200 Subject: [PATCH 152/434] Update print statement from 'Hello' to 'Goodbye' --- src/main/kotlin/tsuki/site/all/MangaPlus.kt | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaPlus.kt b/src/main/kotlin/tsuki/site/all/MangaPlus.kt index 3bd6f19b..4440ecc7 100644 --- a/src/main/kotlin/tsuki/site/all/MangaPlus.kt +++ b/src/main/kotlin/tsuki/site/all/MangaPlus.kt @@ -4,8 +4,22 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.SinglePageMangaParser -import tsuki.model.* -import tsuki.util.* + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Interceptor @@ -771,7 +785,6 @@ internal abstract class MangaPlusParser( companion object { private val CompletedRegex = Regex("completado|completed?|completo", RegexOption.IGNORE_CASE) - private val HIATUS_REGEX = Regex("on a hiatus", RegexOption.IGNORE_CASE) } @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") From 42636c2507cf50111b45ebbfe3192f17428ef6e7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:35:04 +0200 Subject: [PATCH 153/434] Update MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 33 ++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index c9612cce..56c84dc2 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -7,8 +7,23 @@ import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient import tsuki.MangaParserAuthProvider -import tsuki.util.* -import tsuki.model.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson import okhttp3.Interceptor import org.jsoup.Jsoup @@ -236,8 +251,18 @@ internal abstract class MangaFireParser( availableStates = EnumSet.of( MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, ), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableDemographics = EnumSet.of(Demographic.SHOUNEN, Demographic.SHOUJO, Demographic.SEINEN, Demographic.JOSEI), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), + availableDemographics = EnumSet.of( + Demographic.SHOUNEN, + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.JOSEI + ), availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) ) From fde539cdf0d9f56c58dc4b8242a3860b662f73b9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 13:38:19 +0200 Subject: [PATCH 154/434] Update Mangaball.kt --- src/main/kotlin/tsuki/site/all/Mangaball.kt | 28 +++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt index 3abbce21..a6ffb97a 100644 --- a/src/main/kotlin/tsuki/site/all/Mangaball.kt +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -7,8 +7,32 @@ import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient import tsuki.network.WebClient -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.generateUid +import tsuki.util.insertCookies +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.ownTextOrNull +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.Headers import okhttp3.HttpUrl From 6817a381e437caa400ab82f5e545a67e50f62442 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:14:46 +0200 Subject: [PATCH 155/434] Update MangaThemesia.kt --- .../kotlin/tsuki/parsers/MangaThemesia.kt | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index ebc0e5c3..f4d90686 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -1,7 +1,5 @@ package tsuki.parsers -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser @@ -28,6 +26,8 @@ import tsuki.util.parseHtml import tsuki.util.urlEncoded import tsuki.util.extractChapterNumber +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.json.JSONArray @@ -220,13 +220,26 @@ abstract class MangaThemesia( doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() } } - val altnames = if (hasTable) { + val altnamesFromTable = if (hasTable) { table.selectFirst("tr:has(td:contains(Alternative)) td:last-child") ?.text() ?.trim() ?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } ?.let { setOf(it) } } else emptySet() + + val altnamesFromAlternative = doc.select(".alternative") + .firstOrNull() + ?.text() + ?.trim() + ?.takeIf { it.isNotBlank() } + ?.split(",") + ?.map { it.trim() } + ?.filter { it.isNotBlank() } + ?.toSet() ?: emptySet() + + val altnames = altnamesFromTable?.plus(altnamesFromAlternative) + val state = parseStatus(statusText) val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() From a37b4a272b91642fd6f5f056989007b057a1cbf5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:15:52 +0200 Subject: [PATCH 156/434] Update EnThunderScans.kt --- .../kotlin/tsuki/site/en/EnThunderScans.kt | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt index 528827eb..c8097aee 100644 --- a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -4,30 +4,11 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MangaThemesia -import tsuki.model.Manga import tsuki.model.MangaParserSource -import tsuki.util.parseHtml -import tsuki.util.toAbsoluteUrl - @MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") internal class EnThunderScans(context: MangaLoaderContext) : MangaThemesia(context, MangaParserSource.ENTHUNDERSCANS, "en-thunderscans.com", pageSize = 30) { - override val mangaDirectory = "comics" - override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" - - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val base = super.getDetails(manga) - val altTitle = doc.selectFirst(".alternative .desktop-titles, .alternative .mobile-titles.first") - ?.text() - ?.trim() - ?.takeIf { it.isNotBlank() } - return base.copy( - altTitles = if (altTitle != null) setOf(altTitle) else base.altTitles - ) - } } From 3d698e05408039fe9992a9b7b22e750ca63f1213 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 14:23:46 +0200 Subject: [PATCH 157/434] Create MangaTube.kt --- src/main/kotlin/tsuki/site/de/MangaTube.kt | 331 +++++++++++++++++++++ 1 file changed, 331 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/de/MangaTube.kt diff --git a/src/main/kotlin/tsuki/site/de/MangaTube.kt b/src/main/kotlin/tsuki/site/de/MangaTube.kt new file mode 100644 index 00000000..3ec18ec9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/de/MangaTube.kt @@ -0,0 +1,331 @@ +package tsuki.site.de + +import okhttp3.FormBody +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.network.OkHttpWebClient +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.urlEncoded +import java.io.IOException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.concurrent.TimeUnit + +@MangaSourceParser("MANGATUBE", "Manga Tube", "de") +internal class MangaTube(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGATUBE, 20) { + + override val configKeyDomain = ConfigKey.Domain("manga-tube.me") + internal val baseUrl = "https://$domain" + + internal val baseClient: OkHttpClient = context.httpClient.newBuilder() + .connectTimeout(1, TimeUnit.MINUTES) + .readTimeout(1, TimeUnit.MINUTES) + .writeTimeout(1, TimeUnit.MINUTES) + .build() + + override val webClient: OkHttpWebClient by lazy { + val client = baseClient.newBuilder() + .addInterceptor(ChallengeInterceptor(this)) + .addInterceptor { chain -> + val req = chain.request().newBuilder() + .header("User-Agent", getUserAgent()) + .build() + chain.proceed(req) + } + .build() + OkHttpWebClient(client, source) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + internal fun getUserAgent(): String = config[userAgentKey] + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY + ) + override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) + override suspend fun getFilterOptions() = MangaListFilterOptions() + + private val apiHeaders: Headers + get() = Headers.Builder() + .add("Accept", "application/json") + .add("Referer", baseUrl) + .build() + + private fun mangaApiHeaders(slug: String) = apiHeaders.newBuilder() + .add("Referer", "$baseUrl/series/$slug") + .add("Use-Parameter", "manga_slug") + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf(String::isNotEmpty) + if (query != null) return searchManga(query) + + return when (order) { + SortOrder.POPULARITY -> if (page == 1) popularManga() else emptyList() + SortOrder.UPDATED -> latestUpdates(page) + else -> emptyList() + } + } + + private suspend fun popularManga(): List<Manga> { + val json = webClient.httpGet("$baseUrl/api/home/top-manga", apiHeaders).parseJson() + return json?.getJSONObject("data")?.getJSONArray("manga")?.toMangaList().orEmpty() + } + + private suspend fun latestUpdates(page: Int): List<Manga> { + val offset = (page - 1) * LATEST_PAGE_SIZE + val json = webClient.httpGet("$baseUrl/api/home/updates?offset=$offset", apiHeaders).parseJson() + val published = json?.getJSONObject("data")?.getJSONArray("published") ?: return emptyList() + val mangas = (0 until published.length()).map { i -> + published.getJSONObject(i).getJSONObject("manga").toManga() + }.distinctBy { it.url } + return if (mangas.size < LATEST_PAGE_SIZE) mangas else mangas + } + + private suspend fun searchManga(query: String): List<Manga> { + val json = webClient.httpGet("$baseUrl/api/manga/quick-search?query=${query.urlEncoded()}", apiHeaders).parseJson() + return json?.getJSONArray("data")?.toMangaList().orEmpty() + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.substringAfterLast("/") + val json = webClient.httpGet("$baseUrl/api/manga/$slug", mangaApiHeaders(slug)).parseJson() + val data = json?.getJSONObject("data")?.getJSONObject("manga") ?: return manga + + val title = data.getString("title") + val cover = data.optString("cover", manga.coverUrl ?: "") + val description = data.optString("description", "").ifBlank { null } + val state = when (data.optInt("status", 0)) { 1 -> MangaState.ONGOING; 2 -> MangaState.FINISHED; else -> null } + val authors = data.optJSONArray("author")?.toNameList().orEmpty() + val artists = data.optJSONArray("artist")?.toNameList().orEmpty() + + val chaptersJson = webClient.httpGet("$baseUrl/api/manga/$slug/chapters", mangaApiHeaders(slug)).parseJson() + val chapters = chaptersJson?.getJSONObject("data")?.getJSONArray("chapters")?.toChapterList(slug).orEmpty() + .sortedWith(compareBy({ it.volume }, { it.number })) + + return manga.copy( + title = title, + coverUrl = cover, + largeCoverUrl = cover, + description = description, + state = state, + authors = authors + artists, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val slug = chapter.url.substringAfter("/api/manga/").substringBefore("/chapter/") + val json = webClient.httpGet("$baseUrl${chapter.url}", mangaApiHeaders(slug)).parseJson() + val pagesArray = json?.getJSONObject("data")?.getJSONObject("chapter")?.getJSONArray("pages") ?: return emptyList() + val pageList = (0 until pagesArray.length()).map { i -> + val obj = pagesArray.getJSONObject(i) + val url = obj.optString("url", "").ifEmpty { obj.optString("alt_source", "") } + obj.optInt("page", 0) to url + }.sortedBy { it.first } + return pageList.mapIndexed { _, (_, url) -> + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun JSONObject.toManga(): Manga = Manga( + id = generateUid(getString("url")), + url = getString("url"), + publicUrl = baseUrl + getString("url"), + title = getString("title"), + coverUrl = optString("cover", ""), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + + private fun JSONArray.toMangaList() = (0 until length()).map { getJSONObject(it).toManga() } + private fun JSONArray.toNameList() = (0 until length()).map { getJSONObject(it).getString("name") }.toSet() + + private fun JSONArray.toChapterList(slug: String) = (0 until length()).map { i -> + getJSONObject(i).toChapter(slug) + } + + private fun JSONObject.toChapter(slug: String): MangaChapter { + val id = getLong("id") + val number = optDouble("number", 0.0).toFloat() + val subNumber = optDouble("subNumber", 0.0).toFloat() + val volume = optDouble("volume", 0.0).toFloat() + val name = optString("name", "") + val publishedAt = optString("publishedAt", "") + + val title = buildString { + if (volume > 0) append("Vol. ${volume.toString().removeSuffix(".0")} ") + append("Ch. ${number.toString().removeSuffix(".0")}") + if (subNumber > 0) append(".${subNumber.toString().removeSuffix(".0")}") + if (name.isNotEmpty()) append(" - $name") + } + + return MangaChapter( + id = generateUid("/api/manga/$slug/chapter/$id"), + title = title, + number = number + subNumber / 10f, + volume = volume.toInt(), + url = "/api/manga/$slug/chapter/$id", + uploadDate = dateFormat.parseSafe(publishedAt), + source = source, + scanlator = null, + branch = null, + ) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) + + companion object { + private const val LATEST_PAGE_SIZE = 40 + } +} + +private class ChallengeInterceptor( + private val parser: MangaTube, +) : Interceptor { + + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest = chain.request() + if (originalRequest.header(CHALLENGE_BYPASSED_HEADER) == "1") { + return chain.proceed(originalRequest) + } + + if (originalRequest.url.encodedPath.startsWith("/api/") && needsBootstrap(originalRequest.url)) { + bootstrapSession() + } + + val response = chain.proceed(originalRequest) + val bodyString = response.peekBody(Long.MAX_VALUE).string() + if (!isChallengePage(bodyString)) { + return response + } + + response.close() + solveChallenge(bodyString) + + return chain.proceed( + originalRequest.newBuilder() + .header(CHALLENGE_BYPASSED_HEADER, "1") + .build() + ) + } + + private fun isChallengePage(body: String): Boolean = + body.contains("window.__challange") || body.contains("_challange =") + + private fun needsBootstrap(url: HttpUrl): Boolean { + val cookies = parser.context.cookieJar.loadForRequest(url) + return REQUIRED_API_COOKIES.any { required -> cookies.none { it.name == required } } + } + + private fun bootstrapSession() { + val request = Request.Builder() + .url(parser.baseUrl) + .headers(sourceHeaders()) + .build() + val response = parser.baseClient.newCall(request).execute() + val bodyString = response.peekBody(Long.MAX_VALUE).string() + if (isChallengePage(bodyString)) { + response.close() + solveChallenge(bodyString) + parser.baseClient.newCall(request).execute().close() + return + } + response.close() + } + + private fun solveChallenge(body: String) { + val regex = Regex("""_challange = (.+?);""") + val match = regex.find(body) + ?: throw IOException("Challenge payload not found") + + val json = JSONObject(match.groupValues[1]) + val tk = json.getString("tk") + val arg1 = json.getString("arg1") + val arg2 = json.getString("arg2") + val arg3 = json.getString("arg3") + val solution = solve(arg1, arg2, arg3) + + val challengeHeaders = sourceHeaders().newBuilder() + .add("Content-Type", "application/x-www-form-urlencoded") + .add("x-challange-token", tk) + .add("x-challange-arg1", arg1) + .add("x-challange-arg2", arg2) + .add("x-challange-arg3", arg3) + .add("x-challange-arg4", solution) + .build() + + Thread.sleep(1000) + + val postBody = FormBody.Builder().build() + val postRequest = Request.Builder() + .url("${parser.baseUrl}/") + .post(postBody) + .headers(challengeHeaders) + .build() + + parser.baseClient.newCall(postRequest).execute().use { resp -> + if (!resp.isSuccessful) throw IOException("Challenge validation failed: ${resp.code}") + } + } + + private fun solve(arg1: String, arg2: String, op: String): String { + val left = arg1.toLong(16).toDouble() + val right = arg2.toLong(16).toDouble() + val result = when (op) { + "a" -> left / right + "b" -> left * right + "c" -> left - right + "d" -> left + right + else -> throw IOException("Unknown challenge op: $op") + } + return if (result == result.toLong().toDouble()) result.toLong().toString() else result.toString() + } + + private fun sourceHeaders(): Headers = Headers.Builder() + .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + .add("User-Agent", parser.getUserAgent()) + .build() + + companion object { + private val REQUIRED_API_COOKIES = setOf("XSRF-TOKEN", "manga_tube_beta_session", "__mtbpass") + private const val CHALLENGE_BYPASSED_HEADER = "X-MangaTube-Challenge-Bypassed" + } +} From 7528598fba1be468b3d6e88be366e2aa32bf1637 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:30:39 +0200 Subject: [PATCH 158/434] Update HiperParser.kt --- src/main/kotlin/tsuki/parsers/HiperParser.kt | 38 ++++++++++++++------ 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HiperParser.kt b/src/main/kotlin/tsuki/parsers/HiperParser.kt index 7c1310fc..06f9e7cd 100644 --- a/src/main/kotlin/tsuki/parsers/HiperParser.kt +++ b/src/main/kotlin/tsuki/parsers/HiperParser.kt @@ -4,8 +4,24 @@ import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient -import tsuki.model.* -import tsuki.util.* + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.urlEncoded import okhttp3.Headers import okhttp3.Request @@ -314,11 +330,11 @@ abstract class HiperParser( id = generateUid("/$mangaPath/$slug/$number"), url = "/$mangaPath/$slug/$number", title = displayTitle, - number = number, - volume = 0, + number = number, + volume = 0, uploadDate = uploadDate, scanlator = null, - branch = null, + branch = null, source = source ) } @@ -353,9 +369,9 @@ abstract class HiperParser( page.optString("avifUrl", "") }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null MangaPage( - id = generateUid(url), - url = url, - preview = null, + id = generateUid(url), + url = url, + preview = null, source = source ) } @@ -381,9 +397,9 @@ abstract class HiperParser( page.optString("avifUrl", "") }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null MangaPage( - id = generateUid(url), - url = url, - preview = null, + id = generateUid(url), + url = url, + preview = null, source = source ) } From 04b1b878af603b08dd3b4c7155394298187a9a02 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:35:45 +0200 Subject: [PATCH 159/434] Update IMHentai.kt --- .../kotlin/tsuki/site/all/nsfw/IMHentai.kt | 37 ++++++++++++++++--- 1 file changed, 32 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt index 74875748..f53ab83b 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt @@ -5,8 +5,34 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.ownTextOrNull +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectFirstParentOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import androidx.collection.ArraySet import kotlinx.coroutines.async @@ -138,8 +164,9 @@ internal class ImHentai(context: MangaLoaderContext) : } } - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. + /** + * Reduced tags way too many + */ private suspend fun fetchAvailableTags(): Set<MangaTag> { return coroutineScope { (1..3).map { page -> @@ -245,4 +272,4 @@ internal class ImHentai(context: MangaLoaderContext) : val img = doc.body().requireElementById("gimg") return img.requireSrc() } -} \ No newline at end of file +} From c1ca0df67b51a93888e060796df5d2b30ae57088 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:37:22 +0200 Subject: [PATCH 160/434] Refactor Misskon class for improved functionality --- .../kotlin/tsuki/site/all/nsfw/Misskon.kt | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt index 31b15685..ed9c8f59 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt @@ -3,12 +3,28 @@ package tsuki.site.all.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey -import tsuki.model.ContentType -import tsuki.model.MangaParserSource import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import java.util.EnumSet @@ -114,7 +130,7 @@ internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, // Single page gallery return doc.select("div.post-inner > div.entry > p > img") .mapNotNull { img -> img.absUrl("data-src") } - .mapIndexed { i, url -> + .mapIndexed { _, url -> MangaPage( id = generateUid(url), url = url, @@ -149,4 +165,4 @@ internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, } return pages } -} \ No newline at end of file +} From ebadb56392e98de0ad9365d636b4278fa5c4c5ec Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:38:35 +0200 Subject: [PATCH 161/434] Update Multporn.kt --- .../kotlin/tsuki/site/all/nsfw/Multporn.kt | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt index cade229a..395e25dc 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt @@ -6,8 +6,30 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.CommonHeaders -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.splitByWhitespace +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.Headers import org.jsoup.nodes.Document @@ -232,4 +254,4 @@ internal class Multporn(context: MangaLoaderContext) : document.select(".$className a").map { it.text().trim() } } } -} \ No newline at end of file +} From 91458ffc7ff1d554bb399425b615d9937d6f4ab5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:39:30 +0200 Subject: [PATCH 162/434] Update NHentaiParser.kt --- .../tsuki/site/all/nsfw/NHentaiParser.kt | 23 +++++++++++++++---- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt index e39dc81f..1684790d 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt @@ -1,19 +1,32 @@ package tsuki.site.all.nsfw import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.MangaParserAuthProvider -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.urlEncoded import org.json.JSONObject import java.util.EnumSet import java.util.Locale - // removed first server, it has always problems private const val TWO_IMG_SERVER = "2" private const val THREE_IMG_SERVER = "3" @@ -1082,4 +1095,4 @@ internal class NHentaiParser(context: MangaLoaderContext) : override val authUrl: String get() = "https://$domain" override suspend fun isAuthorized(): Boolean = true override suspend fun getUsername(): String = "" -} \ No newline at end of file +} From 5f195b7150ee06c1075cebeb0f94fe2e38b7ebe3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:40:53 +0200 Subject: [PATCH 163/434] Update NHentaiXXXParser.kt --- .../tsuki/site/all/nsfw/NHentaiXXXParser.kt | 28 +++++++++++++++++-- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index d9372483..a33673ee 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -1,12 +1,34 @@ package tsuki.site.all.nsfw +import tsuki.ErrorMessages import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.ErrorMessages import tsuki.parsers.GalleryAdultsParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import org.jsoup.nodes.Document import org.jsoup.nodes.Element From b3cef666432e02f0c3d6d23dd646009334ab37b4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:46:13 +0200 Subject: [PATCH 164/434] Update MangaDexParser.kt --- .../kotlin/tsuki/site/all/MangaDexParser.kt | 53 ++++++++++++++++--- 1 file changed, 46 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaDexParser.kt b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt index e49e3983..d24ceffe 100644 --- a/src/main/kotlin/tsuki/site/all/MangaDexParser.kt +++ b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt @@ -7,16 +7,55 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.FlexibleMangaParser import tsuki.exception.ParseException -import tsuki.model.* +import tsuki.network.CommonHeaders + +import tsuki.model.ContentRating +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder import tsuki.model.search.MangaSearchQuery import tsuki.model.search.MangaSearchQueryCapabilities -import tsuki.model.search.QueryCriteria.* +import tsuki.model.search.QueryCriteria.Exclude +import tsuki.model.search.QueryCriteria.Include +import tsuki.model.search.QueryCriteria.Match import tsuki.model.search.SearchCapability import tsuki.model.search.SearchableField -import tsuki.model.search.SearchableField.* -import tsuki.network.CommonHeaders -import tsuki.util.* -import tsuki.util.json.* +import tsuki.model.search.SearchableField.AUTHOR +import tsuki.model.search.SearchableField.CONTENT_RATING +import tsuki.model.search.SearchableField.CONTENT_TYPE +import tsuki.model.search.SearchableField.DEMOGRAPHIC +import tsuki.model.search.SearchableField.LANGUAGE +import tsuki.model.search.SearchableField.ORIGINAL_LANGUAGE +import tsuki.model.search.SearchableField.PUBLICATION_YEAR +import tsuki.model.search.SearchableField.STATE +import tsuki.model.search.SearchableField.TAG +import tsuki.model.search.SearchableField.TITLE_NAME + +import tsuki.util.ChaptersListBuilder +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toIntUp +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.entries +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async @@ -537,4 +576,4 @@ internal class MangaDexParser(context: MangaLoaderContext) : FlexibleMangaParser val size: Int get() = data.size } -} \ No newline at end of file +} From d003b9a6f27f2846f08a7f8310ac7cf7fac36ef0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:50:39 +0200 Subject: [PATCH 165/434] Update ProjectSuki.kt --- src/main/kotlin/tsuki/site/all/ProjectSuki.kt | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/ProjectSuki.kt b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt index b3ac9881..c4d4f8a5 100644 --- a/src/main/kotlin/tsuki/site/all/ProjectSuki.kt +++ b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt @@ -5,30 +5,41 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl -import tsuki.util.* -import tsuki.model.* - -import androidx.collection.LruCache import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import java.util.EnumSet -import java.util.LinkedHashMap -import java.util.LinkedHashSet -import java.util.Locale import java.text.SimpleDateFormat import java.util.Calendar +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("PROJECTSUKI", "ProjectSuki") internal class ProjectSuki(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.PROJECTSUKI, pageSize = 30) { override val configKeyDomain = ConfigKey.Domain("projectsuki.com") - private val pagesCache = LruCache<String, List<MangaPage>>(50) override val defaultSortOrder: SortOrder = SortOrder.UPDATED override val availableSortOrders: Set<SortOrder> = EnumSet.of( @@ -385,9 +396,6 @@ internal class ProjectSuki(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val cached = pagesCache.get(chapter.url) - if (cached != null) return cached - val parts = chapter.url.toChapterParts() ?: return emptyList() val payload = JSONObject() @@ -428,11 +436,7 @@ internal class ProjectSuki(context: MangaLoaderContext) : ) ) } - - val result = pages.toList() - pagesCache.put(chapter.url, result) - - return result + return pages } @@ -487,4 +491,4 @@ internal class ProjectSuki(context: MangaLoaderContext) : private val IMAGE_EXTENSIONS = setOf(".jpg", ".png", ".jpeg", ".webp", ".avif", ".tiff") private val CHAPTER_NUMBER_REGEX = Regex("""(?:chapter|ch\.?)\s*(\d+)(?:\s*[.,-]\s*(\d+))?""", RegexOption.IGNORE_CASE) } -} \ No newline at end of file +} From 8b8395e74dd34f91b6460b25f9b0479dc68a5494 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:51:42 +0200 Subject: [PATCH 166/434] Update Webtoons.kt --- src/main/kotlin/tsuki/site/all/Webtoons.kt | 24 ++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt index 55a627ab..820e9221 100644 --- a/src/main/kotlin/tsuki/site/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -1,17 +1,32 @@ package tsuki.site.all +import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException -import tsuki.MangaLoaderContext -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -import kotlin.math.max import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.jsoup.nodes.Element @@ -20,6 +35,7 @@ import java.net.SocketException import java.text.DecimalFormat import java.util.Calendar import java.util.EnumSet +import kotlin.math.max private const val ID_SEARCH_PREFIX = "id:" From 8e2bfbba37e5929c1998e412fad82cd0ccc4d425 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:56:33 +0200 Subject: [PATCH 167/434] Update print statement from 'Hello' to 'Goodbye' --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 56c84dc2..55014a8c 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -632,7 +632,7 @@ internal abstract class MangaFireParser( class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish 2)", "es") - class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es-419") + class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") From 9a9a24541fb0620e773399d405f7a9f8a3c626b5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:19:04 +0200 Subject: [PATCH 168/434] Create Alawale.kt --- src/main/kotlin/tsuki/site/id/Alawale.kt | 319 +++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Alawale.kt diff --git a/src/main/kotlin/tsuki/site/id/Alawale.kt b/src/main/kotlin/tsuki/site/id/Alawale.kt new file mode 100644 index 00000000..b826a739 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Alawale.kt @@ -0,0 +1,319 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.urlEncoded + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("ALAWALE", "Alawale", "id") +internal class Alawale(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ALAWALE, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("alawale.net") + private val baseUrl = "https://$domain" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val doc = webClient.httpGet("$baseUrl/daftar-komik").parseHtml() + + val genreTags = doc.select("select[aria-label=Genre] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "genre:$value", title = opt.text().trim(), source = source) + }.toSet() + + val typeTags = doc.select("select[aria-label=Tipe] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "type:$value", title = opt.text().trim(), source = source) + }.toSet() + + return MangaListFilterOptions( + availableTags = genreTags + typeTags, + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "chapter" + SortOrder.ALPHABETICAL -> "title" + else -> "update" + } + + val url = "$baseUrl/api/komik".toHttpUrl().newBuilder() + .addQueryParameter("sort", sort) + .addQueryParameter("limit", pageSize.toString()) + .addQueryParameter("offset", ((page - 1) * pageSize).toString()) + + if (filter.query != null) { + url.addQueryParameter("q", filter.query.urlEncoded()) + } + + filter.tags.forEach { tag -> + when { + tag.key.startsWith("genre:") -> url.addQueryParameter("genre", tag.key.removePrefix("genre:")) + tag.key.startsWith("type:") -> url.addQueryParameter("type", tag.key.removePrefix("type:")) + } + } + filter.types.forEach { ct -> + val typeParam = when (ct) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Comic" + else -> null + } + if (typeParam != null) url.addQueryParameter("type", typeParam) + } + + val json = webClient.httpGet(url.build()).parseJson() + val items = json.optJSONArray("items") ?: return emptyList() + + return (0 until items.length()).map { i -> + val obj = items.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getString("title") + val type = obj.optString("type", "") + val coverPath = obj.optString("cover_remote", "") + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + Manga( + id = generateUid("/$slug"), + url = "/$slug", + publicUrl = "$baseUrl/$slug", + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl.ifBlank { null }, + tags = setOfNotNull(type.takeIf { it.isNotBlank() }?.let { MangaTag(it, it, source) }), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + + val apiJson = webClient.httpGet("$baseUrl/api/komik?slugs=$slug").parseJson() + val apiItem = apiJson?.optJSONArray("items")?.optJSONObject(0) + val title = apiItem?.optString("title") ?: manga.title + val coverPath = apiItem?.optString("cover_remote") ?: "" + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + val bookScript = doc.selectFirst("script[type=\"application/ld+json\"]:containsData(Book)") + var description: String? = null + var author: String? = null + val genres = mutableListOf<String>() + + if (bookScript != null) { + val json = JSONObject(bookScript.data()) + description = json.optString("description", null) + author = json.optJSONObject("author")?.optString("name") + val genreArray = json.optJSONArray("genre") + if (genreArray != null) { + for (i in 0 until genreArray.length()) { + genres.add(genreArray.getString(i)) + } + } + } + + if (author.isNullOrBlank()) { + author = doc.select(".detail-info .kv span:contains(Author) b")?.text()?.trim() + } + + val synopsisBody = doc.selectFirst(".syn-body")?.text()?.trim() + if (!synopsisBody.isNullOrBlank() && description.isNullOrBlank()) { + description = synopsisBody + } + + if (genres.isEmpty()) { + genres.addAll(doc.select(".detail-info .genres a.chip").map { it.text() }) + } + + val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() + val type = apiItem?.optString("type", "") + if (!type.isNullOrBlank()) { + tagSet += MangaTag(type, type, source) + } + + val chapters = parseChapters(doc, slug) + + return manga.copy( + title = title, + coverUrl = coverUrl.ifBlank { manga.coverUrl }, + description = description, + authors = setOfNotNull(author), + tags = tagSet, + chapters = chapters, + contentRating = ContentRating.SAFE, + ) + } + + private fun parseChapters(doc: Document, slug: String): List<MangaChapter> { + val chapterLinks = doc.select("div.chap-list a.chap-item") + .distinctBy { it.attr("abs:href") } + + return chapterLinks.map { link -> + val href = link.attr("href").removePrefix("/") + val chapterSlug = href.substringAfter("$slug/ch/") + val name = link.selectFirst("span")?.text()?.trim() ?: chapterSlug + val number = parseChapterNumber(chapterSlug) + + MangaChapter( + id = generateUid("/$href"), + url = "/$href", + title = name, + number = number, + volume = 0, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + private fun parseChapterNumber(chapterSlug: String): Float { + val clean = chapterSlug.substringBefore("-bahasa-indonesia") + val parts = clean.split("-") + return if (parts.size == 2) { + val main = parts[0].toFloatOrNull() ?: 0f + val sub = parts[1].toFloatOrNull() ?: 0f + main + sub / 10f + } else { + clean.toFloatOrNull() ?: 0f + } + } + + // RSC payloard + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = "$baseUrl/${chapter.url.removePrefix("/")}" + val rawHtml = webClient.httpGet(fullUrl).parseRaw() + val unescapedHtml = rawHtml.replace("\\/", "/") + + val imageRegex = Regex("""https://bmcdn\.my\.id/[^\s"\\]+\.jpg""") + val allUrls = imageRegex.findAll(unescapedHtml) + .map { it.value } + .distinct() + .toList() + + if (allUrls.isNotEmpty()) { + val pages = allUrls.mapNotNull { url -> + val filename = url.substringAfterLast("/") + val dashIndex = filename.indexOf('-') + if (dashIndex < 0) return@mapNotNull null + val pageNumStr = filename.substring(0, dashIndex) + val pageNum = pageNumStr.toIntOrNull() ?: return@mapNotNull null + pageNum to url + }.sortedBy { it.first } + .map { (_, url) -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + if (pages.isNotEmpty()) return pages + } + + val doc = Jsoup.parse(unescapedHtml) + val pageDivs = doc.select("div.reader-page") + if (pageDivs.isEmpty()) return emptyList() + + val totalPages = pageDivs.size + val firstImageUrl = doc.head() + .select("link[rel=preload][as=image]") + .map { it.attr("abs:href") } + .firstOrNull { it.isNotBlank() } + ?: return emptyList() + + val lastSegment = firstImageUrl.substringAfterLast("/") + val dashIndex = lastSegment.indexOf('-') + if (dashIndex < 0) return emptyList() + + val hashPart = lastSegment.substring(dashIndex + 1) + val basePath = firstImageUrl.substringBeforeLast("/") + "/" + + return (0 until totalPages).map { index -> + val url = "${basePath}${index}-${hashPart}" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val slug = seed.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + return doc.select("section.related .card").mapNotNull { card -> + val link = card.selectFirst("a") ?: return@mapNotNull null + val href = link.attr("href").removePrefix("/") + val title = link.selectFirst(".t")?.text() ?: return@mapNotNull null + val coverImg = link.selectFirst("img") + val coverUrl = coverImg?.attr("abs:src") + Manga( + id = generateUid("/$href"), + url = "/$href", + publicUrl = "$baseUrl/$href", + title = title, + coverUrl = coverUrl, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } +} From 5088d71bb5ea1ecbc1bd8a5c26375666e1c58329 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:19:15 +0200 Subject: [PATCH 169/434] Delete src/main/kotlin/tsuki/site/id/Alawale.kt --- src/main/kotlin/tsuki/site/id/Alawale.kt | 319 ----------------------- 1 file changed, 319 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/id/Alawale.kt diff --git a/src/main/kotlin/tsuki/site/id/Alawale.kt b/src/main/kotlin/tsuki/site/id/Alawale.kt deleted file mode 100644 index b826a739..00000000 --- a/src/main/kotlin/tsuki/site/id/Alawale.kt +++ /dev/null @@ -1,319 +0,0 @@ -package tsuki.site.id - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.ContentRating -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder - -import tsuki.util.generateUid -import tsuki.util.parseHtml -import tsuki.util.parseJson -import tsuki.util.parseRaw -import tsuki.util.urlEncoded - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.jsoup.Jsoup -import org.jsoup.nodes.Document -import java.util.EnumSet - -@MangaSourceParser("ALAWALE", "Alawale", "id") -internal class Alawale(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ALAWALE, pageSize = 30) { - - override val configKeyDomain = ConfigKey.Domain("alawale.net") - private val baseUrl = "https://$domain" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val doc = webClient.httpGet("$baseUrl/daftar-komik").parseHtml() - - val genreTags = doc.select("select[aria-label=Genre] option") - .mapNotNull { opt -> - val value = opt.`val`().trim() - if (value.isEmpty()) null - else MangaTag(key = "genre:$value", title = opt.text().trim(), source = source) - }.toSet() - - val typeTags = doc.select("select[aria-label=Tipe] option") - .mapNotNull { opt -> - val value = opt.`val`().trim() - if (value.isEmpty()) null - else MangaTag(key = "type:$value", title = opt.text().trim(), source = source) - }.toSet() - - return MangaListFilterOptions( - availableTags = genreTags + typeTags, - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS - ), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val sort = when (order) { - SortOrder.UPDATED -> "update" - SortOrder.POPULARITY -> "chapter" - SortOrder.ALPHABETICAL -> "title" - else -> "update" - } - - val url = "$baseUrl/api/komik".toHttpUrl().newBuilder() - .addQueryParameter("sort", sort) - .addQueryParameter("limit", pageSize.toString()) - .addQueryParameter("offset", ((page - 1) * pageSize).toString()) - - if (filter.query != null) { - url.addQueryParameter("q", filter.query.urlEncoded()) - } - - filter.tags.forEach { tag -> - when { - tag.key.startsWith("genre:") -> url.addQueryParameter("genre", tag.key.removePrefix("genre:")) - tag.key.startsWith("type:") -> url.addQueryParameter("type", tag.key.removePrefix("type:")) - } - } - filter.types.forEach { ct -> - val typeParam = when (ct) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Comic" - else -> null - } - if (typeParam != null) url.addQueryParameter("type", typeParam) - } - - val json = webClient.httpGet(url.build()).parseJson() - val items = json.optJSONArray("items") ?: return emptyList() - - return (0 until items.length()).map { i -> - val obj = items.getJSONObject(i) - val slug = obj.getString("slug") - val title = obj.getString("title") - val type = obj.optString("type", "") - val coverPath = obj.optString("cover_remote", "") - val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath - - Manga( - id = generateUid("/$slug"), - url = "/$slug", - publicUrl = "$baseUrl/$slug", - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = coverUrl.ifBlank { null }, - tags = setOfNotNull(type.takeIf { it.isNotBlank() }?.let { MangaTag(it, it, source) }), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.removePrefix("/") - val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() - - val apiJson = webClient.httpGet("$baseUrl/api/komik?slugs=$slug").parseJson() - val apiItem = apiJson?.optJSONArray("items")?.optJSONObject(0) - val title = apiItem?.optString("title") ?: manga.title - val coverPath = apiItem?.optString("cover_remote") ?: "" - val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath - - val bookScript = doc.selectFirst("script[type=\"application/ld+json\"]:containsData(Book)") - var description: String? = null - var author: String? = null - val genres = mutableListOf<String>() - - if (bookScript != null) { - val json = JSONObject(bookScript.data()) - description = json.optString("description", null) - author = json.optJSONObject("author")?.optString("name") - val genreArray = json.optJSONArray("genre") - if (genreArray != null) { - for (i in 0 until genreArray.length()) { - genres.add(genreArray.getString(i)) - } - } - } - - if (author.isNullOrBlank()) { - author = doc.select(".detail-info .kv span:contains(Author) b")?.text()?.trim() - } - - val synopsisBody = doc.selectFirst(".syn-body")?.text()?.trim() - if (!synopsisBody.isNullOrBlank() && description.isNullOrBlank()) { - description = synopsisBody - } - - if (genres.isEmpty()) { - genres.addAll(doc.select(".detail-info .genres a.chip").map { it.text() }) - } - - val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() - val type = apiItem?.optString("type", "") - if (!type.isNullOrBlank()) { - tagSet += MangaTag(type, type, source) - } - - val chapters = parseChapters(doc, slug) - - return manga.copy( - title = title, - coverUrl = coverUrl.ifBlank { manga.coverUrl }, - description = description, - authors = setOfNotNull(author), - tags = tagSet, - chapters = chapters, - contentRating = ContentRating.SAFE, - ) - } - - private fun parseChapters(doc: Document, slug: String): List<MangaChapter> { - val chapterLinks = doc.select("div.chap-list a.chap-item") - .distinctBy { it.attr("abs:href") } - - return chapterLinks.map { link -> - val href = link.attr("href").removePrefix("/") - val chapterSlug = href.substringAfter("$slug/ch/") - val name = link.selectFirst("span")?.text()?.trim() ?: chapterSlug - val number = parseChapterNumber(chapterSlug) - - MangaChapter( - id = generateUid("/$href"), - url = "/$href", - title = name, - number = number, - volume = 0, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }.sortedBy { it.number } - } - - private fun parseChapterNumber(chapterSlug: String): Float { - val clean = chapterSlug.substringBefore("-bahasa-indonesia") - val parts = clean.split("-") - return if (parts.size == 2) { - val main = parts[0].toFloatOrNull() ?: 0f - val sub = parts[1].toFloatOrNull() ?: 0f - main + sub / 10f - } else { - clean.toFloatOrNull() ?: 0f - } - } - - // RSC payloard - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = "$baseUrl/${chapter.url.removePrefix("/")}" - val rawHtml = webClient.httpGet(fullUrl).parseRaw() - val unescapedHtml = rawHtml.replace("\\/", "/") - - val imageRegex = Regex("""https://bmcdn\.my\.id/[^\s"\\]+\.jpg""") - val allUrls = imageRegex.findAll(unescapedHtml) - .map { it.value } - .distinct() - .toList() - - if (allUrls.isNotEmpty()) { - val pages = allUrls.mapNotNull { url -> - val filename = url.substringAfterLast("/") - val dashIndex = filename.indexOf('-') - if (dashIndex < 0) return@mapNotNull null - val pageNumStr = filename.substring(0, dashIndex) - val pageNum = pageNumStr.toIntOrNull() ?: return@mapNotNull null - pageNum to url - }.sortedBy { it.first } - .map { (_, url) -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - if (pages.isNotEmpty()) return pages - } - - val doc = Jsoup.parse(unescapedHtml) - val pageDivs = doc.select("div.reader-page") - if (pageDivs.isEmpty()) return emptyList() - - val totalPages = pageDivs.size - val firstImageUrl = doc.head() - .select("link[rel=preload][as=image]") - .map { it.attr("abs:href") } - .firstOrNull { it.isNotBlank() } - ?: return emptyList() - - val lastSegment = firstImageUrl.substringAfterLast("/") - val dashIndex = lastSegment.indexOf('-') - if (dashIndex < 0) return emptyList() - - val hashPart = lastSegment.substring(dashIndex + 1) - val basePath = firstImageUrl.substringBeforeLast("/") + "/" - - return (0 until totalPages).map { index -> - val url = "${basePath}${index}-${hashPart}" - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val slug = seed.url.removePrefix("/") - val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() - return doc.select("section.related .card").mapNotNull { card -> - val link = card.selectFirst("a") ?: return@mapNotNull null - val href = link.attr("href").removePrefix("/") - val title = link.selectFirst(".t")?.text() ?: return@mapNotNull null - val coverImg = link.selectFirst("img") - val coverUrl = coverImg?.attr("abs:src") - Manga( - id = generateUid("/$href"), - url = "/$href", - publicUrl = "$baseUrl/$href", - title = title, - coverUrl = coverUrl, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } -} From c21c715cc8ff69ad63779636953850eadfe55238 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:19:45 +0200 Subject: [PATCH 170/434] Create Alawale.kt --- src/main/kotlin/tsuki/site/id/Alawale.kt | 319 +++++++++++++++++++++++ 1 file changed, 319 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Alawale.kt diff --git a/src/main/kotlin/tsuki/site/id/Alawale.kt b/src/main/kotlin/tsuki/site/id/Alawale.kt new file mode 100644 index 00000000..b826a739 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Alawale.kt @@ -0,0 +1,319 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.urlEncoded + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("ALAWALE", "Alawale", "id") +internal class Alawale(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ALAWALE, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("alawale.net") + private val baseUrl = "https://$domain" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val doc = webClient.httpGet("$baseUrl/daftar-komik").parseHtml() + + val genreTags = doc.select("select[aria-label=Genre] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "genre:$value", title = opt.text().trim(), source = source) + }.toSet() + + val typeTags = doc.select("select[aria-label=Tipe] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "type:$value", title = opt.text().trim(), source = source) + }.toSet() + + return MangaListFilterOptions( + availableTags = genreTags + typeTags, + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "chapter" + SortOrder.ALPHABETICAL -> "title" + else -> "update" + } + + val url = "$baseUrl/api/komik".toHttpUrl().newBuilder() + .addQueryParameter("sort", sort) + .addQueryParameter("limit", pageSize.toString()) + .addQueryParameter("offset", ((page - 1) * pageSize).toString()) + + if (filter.query != null) { + url.addQueryParameter("q", filter.query.urlEncoded()) + } + + filter.tags.forEach { tag -> + when { + tag.key.startsWith("genre:") -> url.addQueryParameter("genre", tag.key.removePrefix("genre:")) + tag.key.startsWith("type:") -> url.addQueryParameter("type", tag.key.removePrefix("type:")) + } + } + filter.types.forEach { ct -> + val typeParam = when (ct) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Comic" + else -> null + } + if (typeParam != null) url.addQueryParameter("type", typeParam) + } + + val json = webClient.httpGet(url.build()).parseJson() + val items = json.optJSONArray("items") ?: return emptyList() + + return (0 until items.length()).map { i -> + val obj = items.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getString("title") + val type = obj.optString("type", "") + val coverPath = obj.optString("cover_remote", "") + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + Manga( + id = generateUid("/$slug"), + url = "/$slug", + publicUrl = "$baseUrl/$slug", + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl.ifBlank { null }, + tags = setOfNotNull(type.takeIf { it.isNotBlank() }?.let { MangaTag(it, it, source) }), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + + val apiJson = webClient.httpGet("$baseUrl/api/komik?slugs=$slug").parseJson() + val apiItem = apiJson?.optJSONArray("items")?.optJSONObject(0) + val title = apiItem?.optString("title") ?: manga.title + val coverPath = apiItem?.optString("cover_remote") ?: "" + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + val bookScript = doc.selectFirst("script[type=\"application/ld+json\"]:containsData(Book)") + var description: String? = null + var author: String? = null + val genres = mutableListOf<String>() + + if (bookScript != null) { + val json = JSONObject(bookScript.data()) + description = json.optString("description", null) + author = json.optJSONObject("author")?.optString("name") + val genreArray = json.optJSONArray("genre") + if (genreArray != null) { + for (i in 0 until genreArray.length()) { + genres.add(genreArray.getString(i)) + } + } + } + + if (author.isNullOrBlank()) { + author = doc.select(".detail-info .kv span:contains(Author) b")?.text()?.trim() + } + + val synopsisBody = doc.selectFirst(".syn-body")?.text()?.trim() + if (!synopsisBody.isNullOrBlank() && description.isNullOrBlank()) { + description = synopsisBody + } + + if (genres.isEmpty()) { + genres.addAll(doc.select(".detail-info .genres a.chip").map { it.text() }) + } + + val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() + val type = apiItem?.optString("type", "") + if (!type.isNullOrBlank()) { + tagSet += MangaTag(type, type, source) + } + + val chapters = parseChapters(doc, slug) + + return manga.copy( + title = title, + coverUrl = coverUrl.ifBlank { manga.coverUrl }, + description = description, + authors = setOfNotNull(author), + tags = tagSet, + chapters = chapters, + contentRating = ContentRating.SAFE, + ) + } + + private fun parseChapters(doc: Document, slug: String): List<MangaChapter> { + val chapterLinks = doc.select("div.chap-list a.chap-item") + .distinctBy { it.attr("abs:href") } + + return chapterLinks.map { link -> + val href = link.attr("href").removePrefix("/") + val chapterSlug = href.substringAfter("$slug/ch/") + val name = link.selectFirst("span")?.text()?.trim() ?: chapterSlug + val number = parseChapterNumber(chapterSlug) + + MangaChapter( + id = generateUid("/$href"), + url = "/$href", + title = name, + number = number, + volume = 0, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + private fun parseChapterNumber(chapterSlug: String): Float { + val clean = chapterSlug.substringBefore("-bahasa-indonesia") + val parts = clean.split("-") + return if (parts.size == 2) { + val main = parts[0].toFloatOrNull() ?: 0f + val sub = parts[1].toFloatOrNull() ?: 0f + main + sub / 10f + } else { + clean.toFloatOrNull() ?: 0f + } + } + + // RSC payloard + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = "$baseUrl/${chapter.url.removePrefix("/")}" + val rawHtml = webClient.httpGet(fullUrl).parseRaw() + val unescapedHtml = rawHtml.replace("\\/", "/") + + val imageRegex = Regex("""https://bmcdn\.my\.id/[^\s"\\]+\.jpg""") + val allUrls = imageRegex.findAll(unescapedHtml) + .map { it.value } + .distinct() + .toList() + + if (allUrls.isNotEmpty()) { + val pages = allUrls.mapNotNull { url -> + val filename = url.substringAfterLast("/") + val dashIndex = filename.indexOf('-') + if (dashIndex < 0) return@mapNotNull null + val pageNumStr = filename.substring(0, dashIndex) + val pageNum = pageNumStr.toIntOrNull() ?: return@mapNotNull null + pageNum to url + }.sortedBy { it.first } + .map { (_, url) -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + if (pages.isNotEmpty()) return pages + } + + val doc = Jsoup.parse(unescapedHtml) + val pageDivs = doc.select("div.reader-page") + if (pageDivs.isEmpty()) return emptyList() + + val totalPages = pageDivs.size + val firstImageUrl = doc.head() + .select("link[rel=preload][as=image]") + .map { it.attr("abs:href") } + .firstOrNull { it.isNotBlank() } + ?: return emptyList() + + val lastSegment = firstImageUrl.substringAfterLast("/") + val dashIndex = lastSegment.indexOf('-') + if (dashIndex < 0) return emptyList() + + val hashPart = lastSegment.substring(dashIndex + 1) + val basePath = firstImageUrl.substringBeforeLast("/") + "/" + + return (0 until totalPages).map { index -> + val url = "${basePath}${index}-${hashPart}" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val slug = seed.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + return doc.select("section.related .card").mapNotNull { card -> + val link = card.selectFirst("a") ?: return@mapNotNull null + val href = link.attr("href").removePrefix("/") + val title = link.selectFirst(".t")?.text() ?: return@mapNotNull null + val coverImg = link.selectFirst("img") + val coverUrl = coverImg?.attr("abs:src") + Manga( + id = generateUid("/$href"), + url = "/$href", + publicUrl = "$baseUrl/$href", + title = title, + coverUrl = coverUrl, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } +} From a68e7f5ea6c72d027934e6b853c2b2c7c45323b1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 14:21:28 +0200 Subject: [PATCH 171/434] Update MangaTube.kt --- src/main/kotlin/tsuki/site/de/MangaTube.kt | 23 ++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/tsuki/site/de/MangaTube.kt b/src/main/kotlin/tsuki/site/de/MangaTube.kt index 3ec18ec9..23dd282c 100644 --- a/src/main/kotlin/tsuki/site/de/MangaTube.kt +++ b/src/main/kotlin/tsuki/site/de/MangaTube.kt @@ -1,18 +1,11 @@ package tsuki.site.de -import okhttp3.FormBody -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + import tsuki.model.Manga import tsuki.model.MangaChapter import tsuki.model.MangaListFilter @@ -23,10 +16,20 @@ import tsuki.model.MangaParserSource import tsuki.model.MangaState import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder -import tsuki.network.OkHttpWebClient + import tsuki.util.generateUid import tsuki.util.parseJson import tsuki.util.urlEncoded + +import okhttp3.FormBody +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject import java.io.IOException import java.text.SimpleDateFormat import java.util.EnumSet From c1f65f4f52e74cae32821f889a4aa8ccfbcef358 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 15:39:39 +0200 Subject: [PATCH 172/434] Create LectorManga.kt --- src/main/kotlin/tsuki/site/es/LectorManga.kt | 229 +++++++++++++++++++ 1 file changed, 229 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/es/LectorManga.kt diff --git a/src/main/kotlin/tsuki/site/es/LectorManga.kt b/src/main/kotlin/tsuki/site/es/LectorManga.kt new file mode 100644 index 00000000..8b66c6d1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/LectorManga.kt @@ -0,0 +1,229 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import java.util.EnumSet + +// Mirror of zerocomics.net? + +@MangaSourceParser("LECTORMANGA", "Lector Manga", "es") +internal class LectorManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.LECTORMANGA, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("lectormangass.net") + private val baseUrl = "https://$domain" + private val apiBase = "https://api.zerocomics.net/api" + private val siteHeader = "lectormanga" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + private val apiHeaders: Headers + get() = Headers.Builder() + .add("Accept", "application/json") + .add("X-Site", siteHeader) + .add("User-Agent", config[userAgentKey]) + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val (sortParam, orderParam) = when (order) { + SortOrder.UPDATED -> "" to "" + SortOrder.RATING -> "rating" to "desc" + SortOrder.ALPHABETICAL -> "name" to "asc" + else -> "" to "" + } + + val url = "$apiBase/series".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("sort", sortParam) + .addQueryParameter("order", orderParam) + + if (filter.query != null) { + url.addQueryParameter("q", filter.query) + } + + val json = webClient.httpGet(url.build(), apiHeaders).parseJson() + val data = json.optJSONArray("data") ?: return emptyList() + + return (0 until data.length()).map { i -> + val obj = data.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getString("titulo") + val cover = obj.optString("portada", "") + Manga( + id = generateUid(slug), + url = slug, + publicUrl = "$baseUrl/comics/$slug", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url + val doc = webClient.httpGet("$baseUrl/comics/$slug").parseHtml() + + val title = doc.selectFirst("h1.info-title")?.text()?.trim() ?: manga.title + val cover = doc.selectFirst(".cover-image img")?.attr("abs:src") + ?: doc.selectFirst("meta[property='og:image']")?.attr("content") + ?: manga.coverUrl + + val description = doc.selectFirst(".info-desc-text")?.text()?.trim() + + val genres = doc.select(".genre-chips-wrap a.v-chip") + .map { it.text().trim() } + .filter { it.isNotBlank() } + .toSet() + + val author = doc.selectFirst("meta[name=author]")?.attr("content") + val artist = doc.selectFirst("meta[name=artist]")?.attr("content") + + val statusText = doc.selectFirst(".status-chip span")?.text()?.trim() + val state = when (statusText?.lowercase()) { + "en emisión" -> MangaState.ONGOING + "finalizado" -> MangaState.FINISHED + "pausado" -> MangaState.PAUSED + "abandonado" -> MangaState.ABANDONED + else -> null + } + + val chapters = parseChapters(doc) + + val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() + val type = doc.selectFirst("meta[name=type]")?.attr("content") + if (!type.isNullOrBlank()) tagSet += MangaTag(type, type, source) + + return manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + description = description, + authors = setOfNotNull(author, artist).ifEmpty { emptySet() }, + tags = tagSet, + state = state, + chapters = chapters, + contentRating = ContentRating.SAFE, + ) + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + val items = doc.select("div[data-chapter-num]") + return items.mapNotNull { item -> + val link = item.selectFirst("a") ?: return@mapNotNull null + val href = link.attr("href").removePrefix("/") // e.g., "comics/.../capitulo-20" + + val chapterSlug = href.substringAfterLast("/") // "capitulo-20" + val number = chapterSlug.substringAfter("capitulo-", "").toFloatOrNull() ?: 0f + + val titleDiv = link.selectFirst("div.d-flex.justify-space-between > div:first-child") + val title = titleDiv?.text()?.trim() ?: "Capítulo ${number.toInt()}" + + val dateDiv = link.selectFirst("div.text--disabled div.d-flex > div:first-child") + val dateText = dateDiv?.text()?.trim() + val uploadDate = parseRelativeDate(dateText) + + MangaChapter( + id = generateUid("/$href"), + url = "/$href", + title = title, + number = number, + volume = 0, + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val readerUrl = "$baseUrl/${chapter.url.removePrefix("/")}" + val doc = webClient.httpGet(readerUrl).parseHtml() + + val images = doc.select("div.reader-images img, img.reader-page, img[src*='bmcdn.my.id']") + if (images.isNotEmpty()) { + return images.map { img -> + val url = img.attr("abs:src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return doc.select("img").map { img -> + val url = img.attr("abs:src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseRelativeDate(text: String?): Long { + if (text.isNullOrBlank()) return 0L + val clean = text.trim().removePrefix("hace ").trim() + val parts = clean.split(" ") + if (parts.size != 2) return 0L + val amount = parts[0].toIntOrNull() ?: return 0L + val unit = parts[1].lowercase() + val now = System.currentTimeMillis() + val millis = when (unit) { + "h" -> amount * 60 * 60 * 1000L + "min", "minuto", "minutos" -> amount * 60 * 1000L + "día", "días", "dia", "dias" -> amount * 24 * 60 * 60 * 1000L + "sem", "semanas", "semana" -> amount * 7 * 24 * 60 * 60 * 1000L + "mes", "meses" -> amount * 30 * 24 * 60 * 60 * 1000L + "año", "años", "ano", "anos" -> amount * 365 * 24 * 60 * 60 * 1000L + else -> 0L + } + return if (millis > 0) now - millis else 0L + } +} From b4951fecba63bc49941bdabe0e4500105cc58831 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 4 Aug 2026 23:33:36 +0200 Subject: [PATCH 173/434] Create Toonily.kt --- src/main/kotlin/tsuki/site/en/Toonily.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/Toonily.kt diff --git a/src/main/kotlin/tsuki/site/en/Toonily.kt b/src/main/kotlin/tsuki/site/en/Toonily.kt new file mode 100644 index 00000000..145f349a --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Toonily.kt @@ -0,0 +1,14 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("TOONILY", "Toonily", "en") +internal class Toonily(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.TOONILY, "toonily.com") { + override val listUrl = "webtoon/" + override val datePattern = "MMMM dd, yyyy" +} From abcdf67df65bb06b1aa24bfa990f6e718d216b58 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:31:25 +0200 Subject: [PATCH 174/434] Create PhiliaScans.kt --- src/main/kotlin/tsuki/site/en/PhiliaScans.kt | 639 +++++++++++++++++++ 1 file changed, 639 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/PhiliaScans.kt diff --git a/src/main/kotlin/tsuki/site/en/PhiliaScans.kt b/src/main/kotlin/tsuki/site/en/PhiliaScans.kt new file mode 100644 index 00000000..05a7c28f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/PhiliaScans.kt @@ -0,0 +1,639 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSONNotNull +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe +import tsuki.util.runCatchingCancellable +import tsuki.util.toAbsoluteUrl + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONArray +import org.json.JSONObject +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import java.awt.RenderingHints +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.text.SimpleDateFormat +import javax.crypto.Cipher +import javax.crypto.Mac +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import javax.imageio.ImageIO + +@MangaSourceParser("PHILIASCANS", "Philia Scans", "en") +internal class PhiliaScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.PHILIASCANS, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("philiascans.org") + private val apiUrl get() = "https://$domain/api" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient: WebClient by lazy { + OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(this) + .build(), + source + ) + } + + override fun getRequestHeaders(): Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + val ua = config[userAgentKey].takeIf { it.isNotBlank() } + ?: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0" + builder["User-Agent"] = ua + return builder.build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.POPULARITY_WEEK, + SortOrder.RATING, SortOrder.NEWEST, SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS, + ), + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + + private suspend fun fetchTags(): Set<MangaTag> { + tagsCache?.let { return it } + val array = webClient.httpGet("$apiUrl/genres").parseJsonArray() + val tags = array.mapJSONNotNull { item -> + val key = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val title = item.getStringOrNull("name") ?: return@mapJSONNotNull null + MangaTag(key = key, title = title, source = source) + }.toSet() + tagsCache = tags + return tags + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = "$apiUrl/manga".toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + filter.query?.nullIfEmpty()?.let { addQueryParameter("q", it) } + + val (orderBy, direction) = when (order) { + SortOrder.UPDATED -> null to "desc" + SortOrder.POPULARITY -> "views" to "desc" + SortOrder.POPULARITY_WEEK -> "trending" to "desc" + SortOrder.RATING -> "rating" to "desc" + SortOrder.NEWEST -> "added" to "desc" + SortOrder.ALPHABETICAL -> "title" to "asc" + SortOrder.ALPHABETICAL_DESC -> "title" to "desc" + else -> null to "desc" + } + orderBy?.let { addQueryParameter("orderby", it) } + addQueryParameter("order", direction) + + filter.tags.forEach { addQueryParameter("genres", it.key) } + filter.states.forEach { state -> + stateToApi(state)?.let { addQueryParameter("statuses", it) } + } + filter.types.forEach { type -> + typeToApi(type)?.let { addQueryParameter("types", it) } + } + }.build() + + val items = webClient.httpGet(url).parseJson().optJSONArray("items") ?: return emptyList() + return items.mapJSONNotNull { item -> + val slug = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + Manga( + id = generateUid(slug), + url = "/series/$slug", + publicUrl = "https://$domain/series/$slug", + title = item.getStringOrNull("title") ?: return@mapJSONNotNull null, + altTitles = emptySet(), + coverUrl = item.getStringOrNull("coverImageUrl")?.toAbsoluteUrl(domain), + largeCoverUrl = null, + authors = emptySet(), + description = null, + tags = item.optJSONArray("genres")?.toTags().orEmpty(), + state = parseState(item.getStringOrNull("status")), + rating = item.getStringOrNull("ratingAvg")?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + contentRating = parseContentRating(item.getStringOrNull("contentRating")), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.slug + val details = webClient.httpGet("$apiUrl/manga/$slug").parseJson() + val chapters = webClient.httpGet("$apiUrl/manga/$slug/chapters").parseJson() + .optJSONArray("items") + .parseChapters(slug) + return manga.copy( + title = details.getStringOrNull("title") ?: manga.title, + altTitles = details.optJSONArray("alternativeTitles")?.let { array -> + (0 until array.length()).mapNotNullTo(LinkedHashSet()) { array.optString(it).nullIfEmpty() } + }.orEmpty(), + coverUrl = details.getStringOrNull("coverImageUrl")?.toAbsoluteUrl(domain) + ?: manga.coverUrl, + description = details.getStringOrNull("synopsis"), + tags = details.optJSONArray("genres")?.toTags().orEmpty(), + authors = buildSet { + addAll(details.optJSONArray("authors").toNames()) + addAll(details.optJSONArray("artists").toNames()) + }, + state = parseState(details.getStringOrNull("status")), + contentRating = parseContentRating(details.getStringOrNull("contentRating")), + chapters = chapters, + ) + } + + private fun JSONArray?.parseChapters(mangaSlug: String): List<MangaChapter> { + if (this == null) return emptyList() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + return mapJSONNotNull { item -> + val slug = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val number = item.optString("number") + val isLocked = !item.optBoolean("purchased") && item.optInt("coinPrice") != 0 + val rawTitle = item.getStringOrNull("title") + ?.takeIf { it != "null" && it != number } + val name = if (rawTitle != null) "Chapter $number - $rawTitle" else "Chapter $number" + MangaChapter( + id = generateUid("$mangaSlug/$slug"), + title = if (isLocked) "🔒 $name" else name, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug", + scanlator = item.optJSONObject("team")?.optString("name")?.nullIfEmpty(), + uploadDate = dateFormat.parseSafe(item.optString("publishedAt")), + branch = null, + source = source, + ) + }.reversed() + } + + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val segments = chapter.url.trim('/').split('/') + if (segments.size < 3) throw ParseException("Unexpected chapter url: ${chapter.url}", chapter.url) + val mangaSlug = segments[1] + val chapterSlug = segments[2] + + val viewerResponse = webClient.httpGet("$apiUrl/manga/$mangaSlug/chapters/$chapterSlug") + val viewer = viewerResponse.parseJson() + if (!viewer.optBoolean("hasAccess", true)) + throw ParseException("This chapter has to be unlocked with coins. Sign in through the browser and buy it first.", chapter.url) + val chapterJson = viewer.optJSONObject("chapter") + ?: throw ParseException("No chapter data in response", chapter.url) + val chapterId = chapterJson.optLong("id") + val isScrambled = chapterJson.optBoolean("scrambled") + System.err.println("PhiliaScans: chapterId=$chapterId, scrambled=$isScrambled") + + val token = try { + val content = ByteArray(0) + val request = okhttp3.Request.Builder() + .url("$apiUrl/reader/access-token") + .post(content.toRequestBody(null, 0, content.size)) + .headers(readerHeaders()) + .build() + val response = context.httpClient.newCall(request).execute() + val bodyString = response.body?.string() ?: throw ParseException("Empty response", chapter.url) + val json = JSONObject(bodyString) + json.optString("token").nullIfEmpty() + ?: throw ParseException("Token missing", chapter.url) + } catch (e: Exception) { + System.err.println("PhiliaScans: failed to get reader access token") + e.printStackTrace() + throw ParseException("Could not obtain reader token: ${e.message}", chapter.url) + } + + val headers = readerHeaders().newBuilder() + .add(HEADER_READER_TOKEN, token) + .build() + + val keysResponse = webClient.httpGet("$apiUrl/chapters/$chapterId/page-keys", headers) + val keys = keysResponse.parseJson() + val chapterKeyB64 = keys.optString("chapterKeyB64") + val gridSize = keys.optInt("gridSize", 1) + System.err.println("PhiliaScans: chapterKeyB64=$chapterKeyB64, gridSize=$gridSize") + + var payloadA: String? = null + var payloadB: String? = null + if (keys.optBoolean("sessionDefault")) { + try { + val open = webClient.httpPost("$apiUrl/chapters/$chapterId/open".toHttpUrl(), JSONObject(), headers).parseJson() + payloadA = open.getStringOrNull("payloadA") + val sessionId = open.getStringOrNull("sessionId") + if (sessionId != null) { + payloadB = runCatchingCancellable { + webClient.httpGet("$apiUrl/chapters/$chapterId/get-drm?session=$sessionId", headers) + .parseJson().optString("payloadB").nullIfEmpty() + }.getOrNull() + } + } catch (e: Exception) { + System.err.println("PhiliaScans: DRM key retrieval failed") + e.printStackTrace() + } + } + + val pages = chapterJson.optJSONArray("pages") ?: return emptyList() + return (0 until pages.length()) + .mapNotNull { pages.optJSONObject(it) } + .sortedBy { it.optInt("position") } + .mapIndexed { index, page -> + val url = page.optString("url").toAbsoluteUrl(domain) + val fragment = listOf( + if (isScrambled) "1" else "0", + page.getStringOrNull("mime") ?: "image/webp", + chapterKeyB64, + gridSize.toString(), + payloadA.orEmpty(), + payloadB.orEmpty(), + index.toString(), + ).joinToString(";") + MangaPage( + id = generateUid(url), + url = "$url#$fragment", + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + return try { + interceptInternal(chain) + } catch (e: Exception) { + System.err.println("PhiliaScans: unhandled exception in interceptor") + e.printStackTrace() + chain.proceed(chain.request()) + } + } + + private fun interceptInternal(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val fragment = request.url.fragment + if (fragment.isNullOrEmpty() || !response.isSuccessful || + !PROTECTED_IMAGE_REGEX.matches(request.url.pathSegments.last())) { + return response + } + + System.err.println("PhiliaScans: decrypting ${request.url}") + val parts = fragment.split(';') + if (parts.size < 7) { + System.err.println("PhiliaScans: fragment too short: $fragment") + return response + } + val isScrambled = parts[0] == "1" + val mimeType = parts[1] + val gridSize = parts[3].toIntOrNull() ?: return response + val pageIndex = parts[6].toIntOrNull() ?: return response + val chapterKey = try { + resolveChapterKey(parts[2], parts[4], parts[5]) + } catch (e: Exception) { + System.err.println("PhiliaScans: failed to resolve chapter key") + e.printStackTrace() + return response + } ?: return response + + val body = response.body + val raw = body.bytes() + if (raw.size < 6) { + System.err.println("PhiliaScans: response too small (${raw.size} bytes)") + return response + } + + val scheme = when { + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_AES -> "aesctr:" + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_CHACHA -> "chacha" + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_AES4 -> "aesctr4:" + else -> null + } + System.err.println("PhiliaScans: detected scheme=$scheme") + val offset = if (scheme != null) 2 else 0 + val header = ByteBuffer.wrap(raw, offset, 4).order(ByteOrder.BIG_ENDIAN) + val originalWidth = header.short.toInt() and 0xFFFF + val originalHeight = header.short.toInt() and 0xFFFF + val payload = raw.copyOfRange(offset + 4, raw.size) + + val plain = try { + when (scheme) { + "aesctr:", "aesctr4:" -> aesCtrDecrypt(payload, chapterKey, pageIndex, scheme) + "chacha" -> chaCha20(payload, hmacSha256(chapterKey, "cc:$pageIndex")) + else -> xorKeystream(payload, chapterKey, pageIndex) + } + } catch (e: Exception) { + System.err.println("PhiliaScans: decryption failed for page $pageIndex") + e.printStackTrace() + return response + } + + val contentType = mimeType.toMediaTypeOrNull() ?: body.contentType() + + if (isScrambled && scheme == null && gridSize >= 2) { + return try { + val unscrambledBytes = unscrambleWithAwt( + plain, chapterKey, pageIndex, gridSize, + originalWidth, originalHeight, formatNameFromMime(mimeType) + ) + response.newBuilder() + .body(unscrambledBytes.toResponseBody(contentType)) + .build() + } catch (e: Exception) { + System.err.println("PhiliaScans: unscramble failed for page $pageIndex") + e.printStackTrace() + response.newBuilder() + .body(plain.toResponseBody(contentType)) + .build() + } + } + + return response.newBuilder() + .body(plain.toResponseBody(contentType)) + .build() + } + + private fun unscrambleWithAwt( + imageBytes: ByteArray, + chapterKey: ByteArray, + pageIndex: Int, + gridSize: Int, + originalWidth: Int, + originalHeight: Int, + formatName: String + ): ByteArray { + return try { + val source = ImageIO.read(ByteArrayInputStream(imageBytes)) + ?: throw RuntimeException("Cannot decode image – format may be unsupported") + val tileCount = gridSize * gridSize + val order = IntArray(tileCount) { it } + + val tilesSignature = hmacSha256(chapterKey, "tiles:$pageIndex") + var counter = 0 + var wordIndex = 8 + var randomBlock = ByteArray(0) + fun nextRandom(): Long { + if (wordIndex >= 8) { + randomBlock = hmacSha256(tilesSignature, "perm:${counter++}") + wordIndex = 0 + } + val value = ByteBuffer.wrap(randomBlock, wordIndex * 4, 4) + .order(ByteOrder.LITTLE_ENDIAN).int.toLong() and 0xFFFFFFFFL + wordIndex++ + return value + } + for (i in tileCount - 1 downTo 1) { + val j = (nextRandom() % (i + 1)).toInt() + val tmp = order[i]; order[i] = order[j]; order[j] = tmp + } + + val inverse = IntArray(tileCount) + for (i in 0 until tileCount) inverse[order[i]] = i + + val tileWidth = source.width / gridSize + val tileHeight = source.height / gridSize + val outW = if (originalWidth > 0) originalWidth else source.width + val outH = if (originalHeight > 0) originalHeight else source.height + + val output = BufferedImage(outW, outH, BufferedImage.TYPE_INT_ARGB) + val g = output.createGraphics() + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR) + + for (target in 0 until tileCount) { + val from = inverse[target] + val srcX = (from % gridSize) * tileWidth + val srcY = (from / gridSize) * tileHeight + val dstX = (target % gridSize) * tileWidth + val dstY = (target / gridSize) * tileHeight + g.drawImage( + source, + dstX, dstY, dstX + tileWidth, dstY + tileHeight, + srcX, srcY, srcX + tileWidth, srcY + tileHeight, + null + ) + } + g.dispose() + + val bos = ByteArrayOutputStream() + ImageIO.write(output, formatName, bos) + bos.toByteArray() + } catch (e: Exception) { + System.err.println("PhiliaScans: unscrambleWithAwt internal error") + e.printStackTrace() + throw e + } + } + + private fun formatNameFromMime(mime: String): String = when { + mime.contains("webp", ignoreCase = true) -> "webp" + mime.contains("jpeg") || mime.contains("jpg") -> "jpeg" + mime.contains("png") -> "png" + else -> "webp" + } + + private fun resolveChapterKey(keyB64: String, payloadA: String, payloadB: String): ByteArray? { + if (payloadA.isNotBlank() && payloadB.isNotBlank()) { + val a = runCatching { context.decodeBase64(payloadA) }.getOrNull() + val b = runCatching { context.decodeBase64(payloadB) }.getOrNull() + if (a != null && b != null && a.size >= 32 && b.size >= 32) + return ByteArray(32) { i -> (a[i].toInt() xor b[i].toInt()).toByte() } + } + return runCatching { context.decodeBase64(keyB64) }.getOrNull()?.takeIf { it.isNotEmpty() } + } + + private fun hmacSha256(key: ByteArray, message: String): ByteArray { + val mac = Mac.getInstance("HmacSHA256") + mac.init(SecretKeySpec(key, "HmacSHA256")) + return mac.doFinal(message.toByteArray(Charsets.UTF_8)) + } + + private fun aesCtrDecrypt(data: ByteArray, chapterKey: ByteArray, pageIndex: Int, prefix: String): ByteArray { + val derived = hmacSha256(chapterKey, "$prefix$pageIndex") + val cipher = Cipher.getInstance("AES/CTR/NoPadding") + cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(derived, "AES"), IvParameterSpec(ByteArray(16))) + return cipher.doFinal(data) + } + + private fun xorKeystream(data: ByteArray, chapterKey: ByteArray, pageIndex: Int): ByteArray { + val result = data.copyOf() + var blockIndex = 0 + var blockOffset = 0 + var block = hmacSha256(chapterKey, "page:$pageIndex:${0}") + for (i in result.indices) { + if (blockOffset == block.size) { + blockIndex++ + block = hmacSha256(chapterKey, "page:$pageIndex:$blockIndex") + blockOffset = 0 + } + result[i] = (result[i].toInt() xor block[blockOffset].toInt()).toByte() + blockOffset++ + } + return result + } + + private fun chaCha20(data: ByteArray, key: ByteArray): ByteArray { + val result = data.copyOf() + val nonce = ByteArray(12) + var counter = 0 + var block = chaCha20Block(key, nonce, counter) + var offset = 0 + for (i in result.indices) { + if (offset == block.size) { + counter++ + block = chaCha20Block(key, nonce, counter) + offset = 0 + } + result[i] = (result[i].toInt() xor block[offset].toInt()).toByte() + offset++ + } + return result + } + + private fun chaCha20Block(key: ByteArray, nonce: ByteArray, counter: Int): ByteArray { + val state = IntArray(16) + state[0] = 0x61707865; state[1] = 0x3320646e; state[2] = 0x79622d32; state[3] = 0x6b206574 + for (i in 0 until 8) state[4 + i] = key.readIntLe(i * 4) + state[12] = counter; state[13] = nonce.readIntLe(0); state[14] = nonce.readIntLe(4); state[15] = nonce.readIntLe(8) + val working = state.copyOf() + repeat(10) { + quarterRound(working, 0, 4, 8, 12); quarterRound(working, 1, 5, 9, 13) + quarterRound(working, 2, 6, 10, 14); quarterRound(working, 3, 7, 11, 15) + quarterRound(working, 0, 5, 10, 15); quarterRound(working, 1, 6, 11, 12) + quarterRound(working, 2, 7, 8, 13); quarterRound(working, 3, 4, 9, 14) + } + val block = ByteArray(64) + for (i in 0 until 16) block.writeIntLe(i * 4, working[i] + state[i]) + return block + } + + private fun quarterRound(state: IntArray, a: Int, b: Int, c: Int, d: Int) { + state[a] += state[b]; state[d] = Integer.rotateLeft(state[d] xor state[a], 16) + state[c] += state[d]; state[b] = Integer.rotateLeft(state[b] xor state[c], 12) + state[a] += state[b]; state[d] = Integer.rotateLeft(state[d] xor state[a], 8) + state[c] += state[d]; state[b] = Integer.rotateLeft(state[b] xor state[c], 7) + } + + private fun ByteArray.readIntLe(offset: Int) = (this[offset].toInt() and 0xFF) or + ((this[offset + 1].toInt() and 0xFF) shl 8) or + ((this[offset + 2].toInt() and 0xFF) shl 16) or + ((this[offset + 3].toInt() and 0xFF) shl 24) + + private fun ByteArray.writeIntLe(offset: Int, value: Int) { + this[offset] = value.toByte(); this[offset + 1] = (value ushr 8).toByte() + this[offset + 2] = (value ushr 16).toByte(); this[offset + 3] = (value ushr 24).toByte() + } + + private fun readerHeaders() = getRequestHeaders().newBuilder() + .set("Accept", "application/json") + .set("Referer", "https://$domain/") + .set("X-Requested-With", "XMLHttpRequest") + .build() + + private val Manga.slug get() = url.trim('/').substringAfterLast('/') + + private fun JSONArray.toTags(): Set<MangaTag> = mapJSONNotNull { item -> + val key = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val title = item.getStringOrNull("name") ?: return@mapJSONNotNull null + MangaTag(key = key, title = title, source = source) + }.toSet() + + private fun JSONArray?.toNames(): Set<String> { + if (this == null) return emptySet() + return mapJSONNotNull { it.getStringOrNull("name") }.toSet() + } + + private fun parseState(value: String?): MangaState? = when (value?.uppercase(Locale.ROOT)) { + "ON_GOING", "ONGOING", "RELEASING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "ON_HOLD", "HIATUS" -> MangaState.PAUSED + "CANCELED", "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + private fun stateToApi(state: MangaState): String? = when (state) { + MangaState.ONGOING -> "on_going" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on_hold" + MangaState.ABANDONED -> "canceled" + else -> null + } + + private fun typeToApi(type: ContentType): String? = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + else -> null + } + + private fun parseContentRating(value: String?): ContentRating? = when (value?.lowercase(Locale.ROOT)) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "adult", "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + + private companion object { + private const val HEADER_READER_TOKEN = "X-Reader-Access-Token" + private val PROTECTED_IMAGE_REGEX = Regex(""".*_s\.[^.]+$""") + private const val MAGIC_HIGH = 0xFF.toByte() + private const val MAGIC_AES = 0x02.toByte() + private const val MAGIC_CHACHA = 0x03.toByte() + private const val MAGIC_AES4 = 0x04.toByte() + } +} From 8ffb04b20c3591acaff8ce03d9a07311bdcf57f8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 5 Aug 2026 13:32:34 +0200 Subject: [PATCH 175/434] Add Toonily.kt to nsfw directory --- src/main/kotlin/tsuki/site/en/{ => nsfw}/Toonily.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) rename src/main/kotlin/tsuki/site/en/{ => nsfw}/Toonily.kt (73%) diff --git a/src/main/kotlin/tsuki/site/en/Toonily.kt b/src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt similarity index 73% rename from src/main/kotlin/tsuki/site/en/Toonily.kt rename to src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt index 145f349a..ea7b085a 100644 --- a/src/main/kotlin/tsuki/site/en/Toonily.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt @@ -1,12 +1,13 @@ -package tsuki.site.en +package tsuki.site.en.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser +import tsuki.model.ContentType import tsuki.model.MangaParserSource -@MangaSourceParser("TOONILY", "Toonily", "en") +@MangaSourceParser("TOONILY", "Toonily", "en", ContentType.HENTAI) internal class Toonily(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.TOONILY, "toonily.com") { override val listUrl = "webtoon/" From ac7f4e97c7d8389a9c0bee8caf64e444c9c1f735 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:01:52 +0200 Subject: [PATCH 176/434] Create TempleScan.kt --- .../kotlin/tsuki/site/en/nsfw/TempleScan.kt | 371 ++++++++++++++++++ 1 file changed, 371 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt diff --git a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt new file mode 100644 index 00000000..874defab --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt @@ -0,0 +1,371 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.json.extractNextJs +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.safety.Safelist +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("TEMPLESCAN", "Temple Scan", "en", ContentType.HENTAI) +internal class TempleScan(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.TEMPLESCAN, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("templetoons.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Origin", "https://$domain") + .set("User-Agent", config[userAgentKey]) + return builder.build() + } + + private val rscHeaders get() = getRequestHeaders().newBuilder() + .set("rsc", "1") + .build() + + @Volatile + private var seriesCache: List<SeriesItem>? = null + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (seriesCache == null) { + val response = webClient.httpGet("https://$domain/comics".toHttpUrl(), rscHeaders) + seriesCache = parseBrowseSeries(response) + } + + val allSeries = seriesCache ?: return emptyList() + + val statusFilter = filter.states.firstOrNull() + val filtered = allSeries.filter { series -> + val query = filter.query?.trim()?.takeIf(String::isNotBlank) + val matchesQuery = if (query != null) { + series.title.contains(query, ignoreCase = true) || + (series.alternativeNames?.contains(query, ignoreCase = true) == true) + } else true + val matchesStatus = statusFilter == null || series.status == stateToApi(statusFilter) + matchesQuery && matchesStatus + } + + val sorted = when (order) { + SortOrder.UPDATED -> filtered.sortedByDescending { it.updated } + SortOrder.NEWEST -> filtered.sortedByDescending { it.created } + SortOrder.POPULARITY -> filtered.sortedByDescending { it.views } + SortOrder.ALPHABETICAL -> filtered.sortedByDescending { it.title } + else -> filtered + } + + val startIndex = (page - 1) * pageSize + val endIndex = minOf(startIndex + pageSize, sorted.size) + if (startIndex >= sorted.size) return emptyList() + + return sorted.subList(startIndex, endIndex).map { item -> + Manga( + id = generateUid("/comic/${item.slug}"), + url = "/comic/${item.slug}", + publicUrl = "https://$domain/comic/${item.slug}", + title = item.title, + altTitles = emptySet(), + coverUrl = item.thumbnail?.toAbsoluteUrl(domain), + authors = emptySet(), + state = when (item.status) { + "Ongoing" -> MangaState.ONGOING + "Hiatus" -> MangaState.PAUSED + "Completed" -> MangaState.FINISHED + "Canceled", "Dropped" -> MangaState.ABANDONED + else -> null + }, + contentRating = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removePrefix("/comic/") + val response = webClient.httpGet("https://$domain/comic/$slug".toHttpUrl(), rscHeaders) + val details = parseSeriesDetails(response) + + return manga.copy( + title = details.title, + altTitles = details.alternativeNames?.let { setOf(it) } ?: emptySet(), + coverUrl = details.thumbnail?.toAbsoluteUrl(domain), + description = details.description?.let { desc -> + val cleanDesc = if (desc.contains("#")) desc.substringBefore("#") + .replace(TRAILING_WORD_REGEX, "").trim() + else desc + Jsoup.clean(cleanDesc, Safelist.none()) + }?.trim().orEmpty(), + authors = details.author?.let { setOf(it) } ?: emptySet(), + state = when (details.status) { + "Ongoing" -> MangaState.ONGOING + "Hiatus" -> MangaState.PAUSED + "Completed" -> MangaState.FINISHED + "Canceled", "Dropped" -> MangaState.ABANDONED + else -> null + }, + contentRating = if (details.adult) ContentRating.ADULT else ContentRating.SAFE, + tags = buildSet { + details.badge?.let { add(MangaTag(it.lowercase(), it, source)) } + details.year?.let { add(MangaTag(it, it, source)) } + if (details.adult) add(MangaTag("adult", "Adult", source)) + details.tags?.forEach { tag -> + add(MangaTag(tag.tag.name.lowercase(), tag.tag.name, source)) + } + details.description?.let { desc -> + TAG_REGEX.findAll(desc).forEach { match -> + val tagName = match.groupValues[1] + add(MangaTag(tagName.lowercase(), tagName, source)) + } + } + }, + chapters = getChapterList(slug), + ) + } + + private suspend fun getChapterList(mangaSlug: String): List<MangaChapter> { + val response = webClient.httpGet("https://$domain/comic/$mangaSlug".toHttpUrl(), rscHeaders) + val chapterData = parseChapterList(response) ?: return emptyList() + + return chapterData.seasons.flatMap { season -> + season.chapters.filter { it.price == 0 }.map { chapter -> + MangaChapter( + id = generateUid("$mangaSlug/${chapter.slug}"), + title = buildString { + append(chapter.name) + if (!chapter.title.isNullOrBlank()) append(": ${chapter.title}") + }, + number = chapter.name.toFloatOrNull() ?: 0f, + url = "/comic/$mangaSlug/${chapter.slug}", + uploadDate = chapter.created, + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + } + }.sortedByDescending { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val response = webClient.httpGet("https://$domain${chapter.url}".toHttpUrl(), rscHeaders) + val pagesData = parsePagesList(response) + return pagesData.pages.mapIndexed { _, url -> + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } + } + + private fun parseBrowseSeries(response: Response): List<SeriesItem> { + val jsonArray: JSONArray = response.extractNextJs { + it is JSONArray && it.length() > 0 && it.optJSONObject(0)?.has("series_slug") == true + } as? JSONArray + ?: throw RuntimeException("Browse series array not found") + + return (0 until jsonArray.length()).map { i -> + jsonArray.getJSONObject(i).toSeriesItem() + } + } + + private fun parseSeriesDetails(response: Response): SeriesDetails { + val obj: JSONObject = response.extractNextJs { + it is JSONObject && it.has("series_slug") && it.has("title") + } as? JSONObject + ?: throw RuntimeException("Series details not found") + return obj.toSeriesDetails() + } + + private fun parseChapterList(response: Response): ChapterList? { + val obj: JSONObject? = response.extractNextJs { + it is JSONObject && it.has("Season") + } as? JSONObject + return obj?.toChapterList() + } + + private fun parsePagesList(response: Response): PagesList { + val obj: JSONObject = response.extractNextJs { + it is JSONObject && it.has("pages") + } as? JSONObject + ?: throw RuntimeException("Pages list not found") + return obj.toPagesList() + } + + private fun JSONObject.toSeriesItem(): SeriesItem { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + return SeriesItem( + slug = getString("series_slug"), + title = getString("title"), + alternativeNames = optString("alternative_names", null), + thumbnail = optString("thumbnail", null), + status = optString("status", null), + updated = dateFormat.parseSafe(optString("update_chapter", null)), + created = dateFormat.parseSafe(optString("created_at", null)), + views = optLong("total_views", 0), + ) + } + + private fun JSONObject.toSeriesDetails(): SeriesDetails { + return SeriesDetails( + slug = getString("series_slug"), + title = getString("title"), + thumbnail = optString("thumbnail", null), + author = optString("author", null), + studio = optString("studio", null), + year = optString("release_year", null), + alternativeNames = optString("alternative_names", null), + adult = optBoolean("adult", false), + badge = optString("badge", null), + status = optString("status", null), + description = optString("description", null), + tags = optJSONArray("tag_series")?.let { arr -> + (0 until arr.length()).map { i -> + val tagObj = arr.getJSONObject(i).getJSONObject("tag") + TagWrapper(Tag(tagObj.getString("name"))) + } + }, + ) + } + + private fun JSONObject.toChapterList(): ChapterList { + val seasonsArray = getJSONArray("Season") + val seasons = (0 until seasonsArray.length()).map { i -> + val seasonObj = seasonsArray.getJSONObject(i) + val chaptersArray = seasonObj.getJSONArray("Chapter") + val chapters = (0 until chaptersArray.length()).map { j -> + val chap = chaptersArray.getJSONObject(j) + ChapterItem( + name = chap.getString("chapter_name"), + title = chap.optString("chapter_title", null), + slug = chap.getString("chapter_slug"), + price = chap.getInt("price"), + created = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + .parseSafe(chap.optString("created_at", null)), + ) + } + Season(chapters) + } + return ChapterList(seasons) + } + + private fun JSONObject.toPagesList(): PagesList { + val pagesArray = getJSONArray("pages") + val pages = (0 until pagesArray.length()).map { pagesArray.getString(it) } + return PagesList(pages) + } + + private fun stateToApi(state: MangaState): String = when (state) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + MangaState.ABANDONED -> "Canceled" + else -> "" + } + + data class SeriesItem( + val slug: String, + val title: String, + val alternativeNames: String?, + val thumbnail: String?, + val status: String?, + val updated: Long, + val created: Long, + val views: Long, + ) + + data class SeriesDetails( + val slug: String, + val title: String, + val thumbnail: String?, + val author: String?, + val studio: String?, + val year: String?, + val alternativeNames: String?, + val adult: Boolean, + val badge: String?, + val status: String?, + val description: String?, + val tags: List<TagWrapper>?, + ) + + data class TagWrapper(val tag: Tag) + data class Tag(val name: String) + + data class ChapterList(val seasons: List<Season>) + data class Season(val chapters: List<ChapterItem>) + data class ChapterItem( + val name: String, + val title: String?, + val slug: String, + val price: Int, + val created: Long, + ) + + data class PagesList(val pages: List<String>) + + companion object { + private val TAG_REGEX = Regex("""(?i)#(\w+)""") + private val TRAILING_WORD_REGEX = Regex("""[\w\s]+:?\s*$""") + } +} From 9769535b1b48c24742f850a12dc3e83298c6e7bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:09:54 +0200 Subject: [PATCH 177/434] Update sorting order and chapter number extraction Refactor sorting and chapter number extraction in TempleScan. --- src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt index 874defab..90dc94df 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt @@ -23,6 +23,7 @@ import tsuki.util.generateUid import tsuki.util.json.extractNextJs import tsuki.util.parseSafe import tsuki.util.toAbsoluteUrl +import tsuki.util.extractChapterNumber import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Response @@ -107,7 +108,7 @@ internal class TempleScan(context: MangaLoaderContext) : SortOrder.UPDATED -> filtered.sortedByDescending { it.updated } SortOrder.NEWEST -> filtered.sortedByDescending { it.created } SortOrder.POPULARITY -> filtered.sortedByDescending { it.views } - SortOrder.ALPHABETICAL -> filtered.sortedByDescending { it.title } + SortOrder.ALPHABETICAL -> filtered.sortedBy { it.title } else -> filtered } @@ -189,11 +190,8 @@ internal class TempleScan(context: MangaLoaderContext) : season.chapters.filter { it.price == 0 }.map { chapter -> MangaChapter( id = generateUid("$mangaSlug/${chapter.slug}"), - title = buildString { - append(chapter.name) - if (!chapter.title.isNullOrBlank()) append(": ${chapter.title}") - }, - number = chapter.name.toFloatOrNull() ?: 0f, + title = chapter.name, + number = chapter.name.extractChapterNumber(), url = "/comic/$mangaSlug/${chapter.slug}", uploadDate = chapter.created, source = source, @@ -202,7 +200,7 @@ internal class TempleScan(context: MangaLoaderContext) : branch = null, ) } - }.sortedByDescending { it.number } + }.sortedBy { it.number } } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { From ded16537359ea7bca20beb0c6be562f59268585b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:18:14 +0200 Subject: [PATCH 178/434] fix for Manhwa18.com --- .../kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt | 115 ++++++++++++------ 1 file changed, 75 insertions(+), 40 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt index 4821e237..220bd7a0 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt @@ -5,8 +5,28 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -15,7 +35,7 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale -@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) +@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", ContentType.HENTAI) internal class Manhwa18Com(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { @@ -31,8 +51,13 @@ internal class Manhwa18Com(context: MangaLoaderContext) : .build() override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, - SortOrder.NEWEST, SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.POPULARITY, // most viewed + SortOrder.RELEVANCE, // most liked + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.ADDED, // new manga + SortOrder.RATING, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -42,8 +67,6 @@ internal class Manhwa18Com(context: MangaLoaderContext) : isSearchWithFiltersSupported = true, ) - private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) - private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) private val numericRegex = Regex("""\d+""") @@ -76,21 +99,6 @@ internal class Manhwa18Com(context: MangaLoaderContext) : }.toSet() } - private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { - doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } - doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } - val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") - ?: doc.selectFirst(".au-cover-art")?.attr("style") - if (styleUrl != null) { - val url = styleUrl.substringAfter("url('").substringBefore("')") - .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } - if (url.isNotEmpty()) { - return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) - } - } - return fallback - } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } val url = buildString { @@ -111,9 +119,11 @@ internal class Manhwa18Com(context: MangaLoaderContext) : append( when (order) { SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" SortOrder.POPULARITY -> "top" + SortOrder.RELEVANCE -> "like" SortOrder.UPDATED -> "update" - SortOrder.NEWEST -> "new" + SortOrder.ADDED -> "new" SortOrder.RATING -> "like" else -> "update" } @@ -167,35 +177,59 @@ internal class Manhwa18Com(context: MangaLoaderContext) : val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title - val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) - val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() + val title = doc.selectFirst("span.series-name a")?.text()?.trim() ?: manga.title - val statusText = doc.select(".au-stat") - .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } - ?.selectFirst(".v")?.text()?.trim() + val coverUrl = doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content") + ?: doc.selectFirst(".img-in-ratio")?.attr("style")?.let { style -> + val url = style.substringAfter("url('").substringBefore("')") + .ifEmpty { style.substringAfter("url(\"").substringBefore("\"") } + if (url.isNotEmpty()) url.toAbsoluteUrl(domain) else null + } ?: manga.coverUrl.orEmpty() + + val descElement = doc.selectFirst(".summary-content") + val description = descElement?.let { + it.selectFirst(".summary-more")?.remove() + it.text().trim() + } + + val statusText = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Status:", ignoreCase = true) == true } + ?.selectFirst(".info-value a")?.text()?.trim() val state = when (statusText?.lowercase()) { - "ongoing" -> MangaState.ONGOING + "on going" -> MangaState.ONGOING "completed" -> MangaState.FINISHED "on hold" -> MangaState.PAUSED else -> null } - val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> + val altTitle = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Other name:", ignoreCase = true) == true } + ?.selectFirst(".info-value")?.text()?.trim() + ?.removeSurrounding("(", ")") + + val genreItem = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Genre:", ignoreCase = true) == true } + val tags = genreItem?.select("a[href*='/genre/']")?.mapNotNull { a -> val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } ?: return@mapNotNull null - MangaTag(a.text().trim(), slug, source) - }.toSet() + val name = a.text().trim() + if (slug.isNotEmpty() && name.isNotEmpty()) MangaTag(name, slug, source) else null + }?.toSet() ?: emptySet() + + val author = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Author:", ignoreCase = true) == true } + ?.selectFirst(".info-value a")?.text()?.trim() val chapters = parseChapters(doc) val result = manga.copy( title = title, + altTitles = if (altTitle != null) setOf(altTitle) else emptySet(), coverUrl = coverUrl, description = description, state = state, tags = tags, - authors = emptySet(), + authors = if (author != null) setOf(author) else emptySet(), chapters = chapters, ) synchronized(detailsCacheLock) { @@ -205,12 +239,13 @@ internal class Manhwa18Com(context: MangaLoaderContext) : } private fun parseChapters(doc: org.jsoup.nodes.Document): List<MangaChapter> { - return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> + val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) + return doc.select("ul.list-chapters.at-series > a").mapNotNull { a -> val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) - val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null - val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() - val uploadDate = dateStr?.let { - runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + val chTitle = a.selectFirst(".chapter-name")?.text()?.trim() ?: return@mapNotNull null + val dateTimeStr = a.selectFirst(".chapter-time")?.text()?.trim() + val uploadDate = dateTimeStr?.substringAfter(" - ")?.let { + runCatching { dateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() @@ -244,4 +279,4 @@ internal class Manhwa18Com(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 2c7ddf780bb92fda1468841566e219f45870c830 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 11:59:34 +0200 Subject: [PATCH 179/434] Update NatsuParser.kt --- src/main/kotlin/tsuki/parsers/NatsuParser.kt | 36 +++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt index 8c69696e..66442a11 100644 --- a/src/main/kotlin/tsuki/parsers/NatsuParser.kt +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -33,7 +33,6 @@ import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.MultipartBody -import okhttp3.OkHttpClient import org.json.JSONArray import org.jsoup.HttpStatusException import java.text.SimpleDateFormat @@ -69,6 +68,7 @@ internal abstract class NatsuParser( SortOrder.POPULARITY, SortOrder.ALPHABETICAL, SortOrder.RATING, + SortOrder.RELEVANCE, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -83,10 +83,18 @@ internal abstract class NatsuParser( override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = getOrFetchGenres(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED + ), availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, - ContentType.COMICS, ContentType.NOVEL, + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, ), ) @@ -111,9 +119,6 @@ internal abstract class NatsuParser( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - val author = filter.author - val formParts = mutableMapOf<String, String>() formParts["nonce"] = getNonce() @@ -129,9 +134,11 @@ internal abstract class NatsuParser( formParts["page"] = page.toString() - formParts["author"] = if (!author.isNullOrEmpty()) - JSONArray(author).toString() - else "[]" + formParts["author"] = if (!filter.author.isNullOrEmpty()) { + JSONArray(filter.author!!).toString() + } else { + "[]" + } formParts["artist"] = "[]" formParts["project"] = "0" @@ -154,6 +161,7 @@ internal abstract class NatsuParser( when (state) { MangaState.ONGOING -> "ongoing" MangaState.FINISHED -> "completed" + MangaState.ABANDONED -> "cancelled" MangaState.PAUSED -> "on-hiatus" else -> null } @@ -166,10 +174,13 @@ internal abstract class NatsuParser( SortOrder.POPULARITY -> "popular" SortOrder.ALPHABETICAL -> "title" SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "bookmarked" else -> "popular" } - if (!query.isNullOrEmpty()) formParts["query"] = query + if (!filter.query.isNullOrEmpty()) { + formParts["query"] = filter.query!! + } val doc = httpPost("https://${domain}/wp-admin/admin-ajax.php?action=advanced_search", formParts) return parseMangaList(doc) @@ -383,7 +394,6 @@ internal abstract class NatsuParser( return tags } - private val multipartHttpClient by lazy { OkHttpClient.Builder().build() } protected open suspend fun httpPost(url: String, form: Map<String, String>): Document { val body = MultipartBody.Builder().setType(MultipartBody.FORM) form.forEach { (k, v) -> body.addFormDataPart(k, v) } @@ -392,7 +402,7 @@ internal abstract class NatsuParser( .header("Referer", "https://${domain}/advanced-search/") .header("Origin", "https://${domain}") .build() - val response = multipartHttpClient.newCall(request).await() + val response = context.httpClient.newCall(request).await() return response.parseHtml() } } From de159ec38a108b1eaab5bfe80ad75160266d29bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 12:53:34 +0200 Subject: [PATCH 180/434] Create DynastyScans.kt --- src/main/kotlin/tsuki/site/en/DynastyScans.kt | 455 ++++++++++++++++++ 1 file changed, 455 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/DynastyScans.kt diff --git a/src/main/kotlin/tsuki/site/en/DynastyScans.kt b/src/main/kotlin/tsuki/site/en/DynastyScans.kt new file mode 100644 index 00000000..328a415a --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/DynastyScans.kt @@ -0,0 +1,455 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +private const val SERIES_TYPE = "Series" +private const val ANTHOLOGY_TYPE = "Anthology" +private const val DOUJIN_TYPE = "Doujin" +private const val ISSUE_TYPE = "Issue" + +private const val SERIES_DIR = "series" +private const val CHAPTERS_DIR = "chapters" +private const val ANTHOLOGIES_DIR = "anthologies" +private const val DOUJINS_DIR = "doujins" +private const val ISSUES_DIR = "issues" + +private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") +} + +@MangaSourceParser("DYNASTYSCANS", "Dynasty Scans", "en") +internal class DynastyScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DYNASTYSCANS, 20) { + + override val configKeyDomain = ConfigKey.Domain("dynasty-scans.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + + private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { + if (tagsCache != null) return@coroutineScope tagsCache!! + val tags = (1..3).map { page -> async { getTags(page) } }.awaitAll().flattenTo(LinkedHashSet()) + tagsCache = tags + tags + } + + private suspend fun getTags(page: Int): Set<MangaTag> { + val url = "https://$domain/tags?page=$page" + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".tag-list a").mapToSet { + MangaTag( + key = it.attr("href").removeSuffix('/').substringAfterLast('/'), + title = it.text().toTitleCase(sourceLocale), + source = source, + ) + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrEmpty()) { + return searchList(page, filter) + } + + if (filter.tags.isNotEmpty()) { + val tag = filter.tags.oneOrThrowIfMany() + val url = "https://$domain/tags/${tag?.key}?view=groupings&page=$page" + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + val url = "https://$domain/chapters/added.json?page=$page" + val json = webClient.httpGet(url).parseJson() + val chaptersArray = json.getJSONArray("chapters") + val totalPages = json.getInt("total_pages") + + val entries = LinkedHashSet<MangaEntry>() + for (i in 0 until chaptersArray.length()) { + val chapter = chaptersArray.getJSONObject(i) + val tags = chapter.getJSONArray("tags") + var isSeries = false + + for (j in 0 until tags.length()) { + val tag = tags.getJSONObject(j) + val type = tag.getString("type") + val name = tag.getString("name") + val permalink = tag.getString("permalink") + + if (type in setOf(SERIES_TYPE, ANTHOLOGY_TYPE, DOUJIN_TYPE, ISSUE_TYPE)) { + entries.add(MangaEntry(url = "/${type.directory()}/$permalink", title = name)) + isSeries = isSeries || type == SERIES_TYPE + } + } + + if (!isSeries) { + entries.add( + MangaEntry( + url = "/$CHAPTERS_DIR/${chapter.getString("permalink")}", + title = chapter.getString("title") + ) + ) + } + } + + return entries.map { entry -> + Manga( + id = generateUid(entry.url), + url = entry.url, + publicUrl = entry.url.toAbsoluteUrl(domain), + title = entry.title, + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.take(if (page < totalPages) pageSize else entries.size) + } + + private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("li.span2").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.selectFirstOrThrow("div.caption").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun searchList(page: Int, filter: MangaListFilter): List<Manga> { + val query = filter.query.orEmpty() + val url = buildString { + append("https://$domain/search?q=") + append(query.urlEncoded()) + append("&classes[]=Series&page=$page") + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("dl.chapter-list dd").mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = a.text(), + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val (directory, permalink) = manga.url.trim('/').split('/').let { it[0] to it[1] } + val jsonUrl = "https://$domain/$directory/$permalink.json" + val json = webClient.httpGet(jsonUrl).parseJson() + + if (directory == CHAPTERS_DIR) { + return chapterDetails(json, manga) + } + + val data = parseMangaResponse(json) + val authors = LinkedHashSet<String>() + val tagNames = LinkedHashSet<String>() + val statuses = LinkedHashSet<String>() + + data.tags.forEach { tag -> + when (tag.type) { + "Author" -> authors.add(tag.name) + "General" -> tagNames.add(tag.name) + "Status" -> statuses.add(tag.name) + } + } + + data.taggings.filterIsInstance<ChapterItem.Chapter>().forEach { chapter -> + chapter.tags.forEach { tag -> + when (tag.type) { + "Author" -> authors.add(tag.name) + "General" -> tagNames.add(tag.name) + } + } + } + + val chapters = fetchAllChapters(data) + + return manga.copy( + title = data.name, + coverUrl = data.cover?.let { "https://$domain$it" }, + description = buildString { + data.description?.let { desc -> + append(org.jsoup.Jsoup.parseBodyFragment(desc).wholeText().trim()) + append("\n\n") + } + append("Type: ${data.type}\n") + if (authors.isNotEmpty()) { + append("Authors: ${authors.joinToString(", ")}\n") + } + }.trim(), + authors = authors.toSet(), + tags = tagNames.map { MangaTag(it, it.lowercase().replace(" ", "-"), source) }.toSet(), + state = when { + statuses.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING + statuses.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED + statuses.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED + statuses.any { it == "Dropped" || it == "Cancelled" || it == "Abandoned" } -> MangaState.ABANDONED + else -> null + }, + chapters = chapters, + ) + } + + private fun chapterDetails(json: JSONObject, manga: Manga): Manga { + val data = parseChapterResponse(json) + return manga.copy( + title = data.title, + coverUrl = data.pages.firstOrNull()?.let { "https://$domain${it.url}" }, + description = "Type: Chapter\nReleased: ${data.releasedOn}", + state = MangaState.FINISHED, + chapters = listOf( + MangaChapter( + id = generateUid("/$CHAPTERS_DIR/${data.permalink}"), + title = "Chapter", + number = 1f, + url = "/$CHAPTERS_DIR/${data.permalink}", + uploadDate = dateFormat.parseSafe(data.releasedOn), + source = source, + scanlator = data.tags.filter { it.type == "Scanlator" }.joinToString { it.name }.nullIfEmpty(), + volume = 0, + branch = null, + ) + ), + ) + } + + private suspend fun fetchAllChapters(data: MangaResponse): List<MangaChapter> = coroutineScope { + val firstPage = data.taggings + val totalPages = data.totalPages + + val deferred = (2..totalPages).map { page -> + async { + val url = "https://$domain/${data.directory}/${data.permalink}.json?page=$page" + parseMangaResponse(webClient.httpGet(url).parseJson()).taggings + } + } + val allTaggings = firstPage + deferred.awaitAll().flatten() + + var header: String? = null + val result = mutableListOf<MangaChapter>() + + allTaggings.forEach { item -> + when (item) { + is ChapterItem.Header -> header = item.header + is ChapterItem.Chapter -> { + var name = header?.let { "$it ${item.title}" } ?: item.title + if (data.type != SERIES_TYPE) { + name += item.tags.filter { it.type == "Author" } + .joinToString(prefix = " by ", separator = " and ") { it.name } + } + result.add( + MangaChapter( + id = generateUid("/$CHAPTERS_DIR/${item.permalink}"), + title = name, + number = (result.size + 1).toFloat(), + url = "/$CHAPTERS_DIR/${item.permalink}", + uploadDate = dateFormat.parseSafe(item.releasedOn), + source = source, + scanlator = item.tags.filter { it.type == "Scanlator" }.joinToString { it.name }.nullIfEmpty(), + volume = 0, + branch = null, + ) + ) + } + } + } + + if (data.type != DOUJIN_TYPE) result.sortedBy { it.number } else result + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val permalink = chapter.url.substringAfterLast('/') + val jsonUrl = "https://$domain/$CHAPTERS_DIR/$permalink.json" + val json = webClient.httpGet(jsonUrl).parseJson() + val data = parseChapterResponse(json) + return data.pages.mapIndexed { _, page -> + MangaPage( + id = generateUid(page.url), + url = "https://$domain${page.url}", + preview = null, + source = source, + ) + } + } + + private data class MangaResponse( + val name: String, + val type: String, + val permalink: String, + val directory: String, + val cover: String?, + val description: String?, + val tags: List<BrowseTag>, + val taggings: List<ChapterItem>, + val totalPages: Int, + ) + + private sealed class ChapterItem { + data class Header(val header: String) : ChapterItem() + data class Chapter( + val title: String, + val permalink: String, + val releasedOn: String, + val tags: List<BrowseTag>, + ) : ChapterItem() + } + + private data class ChapterResponse( + val title: String, + val permalink: String, + val releasedOn: String, + val tags: List<BrowseTag>, + val pages: List<Page>, + ) + + private data class Page(val url: String) + private data class BrowseTag(val type: String, val name: String, val permalink: String) + private data class MangaEntry(val url: String, val title: String) + + private fun parseMangaResponse(json: JSONObject): MangaResponse { + val name = json.getString("name") + val type = json.getString("type") + val permalink = json.getString("permalink") + val directory = when (type) { + SERIES_TYPE -> SERIES_DIR + ANTHOLOGY_TYPE -> ANTHOLOGIES_DIR + DOUJIN_TYPE -> DOUJINS_DIR + ISSUE_TYPE -> ISSUES_DIR + else -> throw Exception("Unknown type: $type") + } + val cover = json.optString("cover", null) + val description = json.optString("description", null) + val tags = json.getJSONArray("tags").parseTags() + val taggings = json.optJSONArray("taggings")?.parseTaggings() ?: emptyList() + val totalPages = json.optInt("total_pages", 1) + return MangaResponse(name, type, permalink, directory, cover, description, tags, taggings, totalPages) + } + + private fun parseChapterResponse(json: JSONObject): ChapterResponse { + val title = json.getString("title") + val permalink = json.getString("permalink") + val releasedOn = json.getString("released_on") + val tags = json.getJSONArray("tags").parseTags() + val pagesArray = json.getJSONArray("pages") + val pages = (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + Page(pageObj.getString("url")) + } + return ChapterResponse(title, permalink, releasedOn, tags, pages) + } + + private fun JSONArray.parseTags(): List<BrowseTag> = (0 until length()).map { i -> + val obj = getJSONObject(i) + BrowseTag(obj.getString("type"), obj.getString("name"), obj.getString("permalink")) + } + + private fun JSONArray.parseTaggings(): List<ChapterItem> = (0 until length()).map { i -> + val obj = getJSONObject(i) + if (obj.has("header")) { + ChapterItem.Header(obj.getString("header")) + } else { + ChapterItem.Chapter( + title = obj.getString("title"), + permalink = obj.getString("permalink"), + releasedOn = obj.getString("released_on"), + tags = obj.getJSONArray("tags").parseTags(), + ) + } + } + + private fun String.directory() = when (this) { + SERIES_TYPE -> SERIES_DIR + ANTHOLOGY_TYPE -> ANTHOLOGIES_DIR + DOUJIN_TYPE -> DOUJINS_DIR + ISSUE_TYPE -> ISSUES_DIR + else -> throw IllegalArgumentException("Unknown type: $this") + } +} From 018ae78c06681185576e8e3c0219df6e8ce7d82e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:28:51 +0200 Subject: [PATCH 181/434] Update MangaCrazy.kt --- src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt index cf2b8e77..aab9bbb0 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt @@ -2,9 +2,10 @@ package tsuki.site.all.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + import tsuki.model.ContentType import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser import java.util.Locale @@ -12,4 +13,4 @@ import java.util.Locale internal class MangaCrazy(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { override val sourceLocale: Locale = Locale.ENGLISH -} \ No newline at end of file +} From 2afa5dd9c42673a4573c44fafc95c2ff26910634 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:32:21 +0200 Subject: [PATCH 182/434] Update SwatManga.kt --- src/main/kotlin/tsuki/site/ar/SwatManga.kt | 45 +++++++++++----------- 1 file changed, 23 insertions(+), 22 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ar/SwatManga.kt b/src/main/kotlin/tsuki/site/ar/SwatManga.kt index 0ed2ab0f..ad1b9a21 100644 --- a/src/main/kotlin/tsuki/site/ar/SwatManga.kt +++ b/src/main/kotlin/tsuki/site/ar/SwatManga.kt @@ -1,18 +1,32 @@ package tsuki.site.ar +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser - -import tsuki.model.* -import tsuki.util.* - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONObject +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.urlEncoded import java.text.SimpleDateFormat -import java.util.Locale +import java.util.* @MangaSourceParser("SWATMANGA", "Swat Manga", "ar", ContentType.MANGA) internal class SwatManga(context: MangaLoaderContext) : @@ -56,26 +70,18 @@ internal class SwatManga(context: MangaLoaderContext) : override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val url = buildString { append("https://appswat.com/v2/api/v2/series/?page_size=20") - - // Add page offset if (page > 1) { val offset = (page - 1) * 20 append("&offset=$offset") } - - // Add search query if (!filter.query.isNullOrEmpty()) { append("&search=${filter.query.urlEncoded()}") } - - // Add sort order when (order) { SortOrder.POPULARITY -> append("&order_by=followers_count") SortOrder.RATING -> append("&order_by=-rating") - else -> {} // Default relevance + else -> {} } - - // Add genre filters if (filter.tags.isNotEmpty()) { filter.tags.forEach { tag -> append("&genres=${tag.key}") @@ -122,7 +128,6 @@ internal class SwatManga(context: MangaLoaderContext) : )) } - // Extract translator/editor info for authors val authors = mutableSetOf<String>() json.optJSONObject("translator")?.let { translator -> authors.add(translator.getString("name")) @@ -151,12 +156,10 @@ internal class SwatManga(context: MangaLoaderContext) : val seriesId = manga.url.substringAfter("/series/") val chaptersDeferred = async { getChapters(seriesId) } - // Get detailed manga info from the same API endpoint val detailUrl = "https://appswat.com/v2/api/v2/series/?page_size=100" val response = webClient.httpGet(detailUrl).parseJson() val results = response.getJSONArray("results") - // Find the specific manga by ID var updatedManga = manga for (i in 0 until results.length()) { val item = results.getJSONObject(i) @@ -196,7 +199,6 @@ internal class SwatManga(context: MangaLoaderContext) : val allChapters = mutableListOf<JSONObject>() var page = 1 - // Fetch all chapters with pagination while (true) { val chaptersUrl = "https://appswat.com/v2/api/v2/chapters/?page=$page&serie=$seriesId&order_by=-order&page_size=20" val response = webClient.httpGet(chaptersUrl).parseJson() @@ -208,7 +210,6 @@ internal class SwatManga(context: MangaLoaderContext) : allChapters.add(results.getJSONObject(i)) } - // Check if there's a next page - properly handle null case val hasNext = !response.isNull("next") if (!hasNext) break page++ From fc7e90ac59bafb410d2a8ebcff986a369ddf8ede Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:33:12 +0200 Subject: [PATCH 183/434] Update SwatManga.kt --- src/main/kotlin/tsuki/site/ar/SwatManga.kt | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ar/SwatManga.kt b/src/main/kotlin/tsuki/site/ar/SwatManga.kt index ad1b9a21..8de12950 100644 --- a/src/main/kotlin/tsuki/site/ar/SwatManga.kt +++ b/src/main/kotlin/tsuki/site/ar/SwatManga.kt @@ -1,12 +1,10 @@ package tsuki.site.ar -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONObject import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser + import tsuki.model.ContentRating import tsuki.model.ContentType import tsuki.model.Manga @@ -20,13 +18,18 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder + import tsuki.util.generateUid import tsuki.util.nullIfEmpty import tsuki.util.parseJson import tsuki.util.parseJsonArray import tsuki.util.urlEncoded + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject import java.text.SimpleDateFormat -import java.util.* +import java.util.Locale @MangaSourceParser("SWATMANGA", "Swat Manga", "ar", ContentType.MANGA) internal class SwatManga(context: MangaLoaderContext) : From d8105f02ced006cdd4cdef4cd2a1ee4eea85d547 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:36:03 +0200 Subject: [PATCH 184/434] Update TeamXNovel.kt --- src/main/kotlin/tsuki/site/ar/TeamXNovel.kt | 36 ++++++++++++++++++--- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt index 7e126923..699e97c9 100644 --- a/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt +++ b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt @@ -5,8 +5,34 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -144,8 +170,8 @@ internal class TeamXNovel(context: MangaLoaderContext) : val mangaUrl = manga.url.toAbsoluteUrl(domain) val maxPageChapterSelect = doc.select(".pagination .page-item a") var maxPageChapter = 1 - if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { + if (maxPageChapterSelect.isNotEmpty()) { + maxPageChapterSelect.forEach { val i = it.attr("href").substringAfterLast("=").toInt() if (i > maxPageChapter) { maxPageChapter = i @@ -272,4 +298,4 @@ internal class TeamXNovel(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From d59452b1667d1ce85467ffb6bc7906b5a04bc81b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:52:24 +0200 Subject: [PATCH 185/434] Update ComicLand.kt --- .../kotlin/tsuki/site/en/nsfw/ComicLand.kt | 35 +++++++++++++------ 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt index e738c63e..34fda989 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt @@ -5,8 +5,21 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.util.* -import tsuki.model.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded import okhttp3.Interceptor import okhttp3.Response @@ -30,13 +43,13 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( override val availableSortOrders = setOf(SortOrder.UPDATED) override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = false, - isTagsExclusionSupported = false, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - isAuthorSearchSupported = false - ) + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + isAuthorSearchSupported = false + ) private val api = "https://api.comicland.org/api" @@ -111,7 +124,7 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( title = chapter.optString("title"), number = index.toFloat(), volume = 0, - url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", + url = "/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", scanlator = null, uploadDate = 0, branch = null, @@ -190,4 +203,4 @@ internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( availableContentRating = emptySet() ) } -} \ No newline at end of file +} From 49e5c932c723fa7e47c83f2fce390345575d5db5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:52:40 +0200 Subject: [PATCH 186/434] Update HentaiXComic.kt --- src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt index b9df12d4..72b65da2 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt @@ -11,9 +11,6 @@ import tsuki.model.MangaTag import tsuki.util.parseHtml import tsuki.util.toTitleCase - - - @MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) internal class HentaixComic(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.HENTAIXCOMIC, "hentaixcomic.com", 16) { @@ -31,4 +28,4 @@ internal class HentaixComic(context: MangaLoaderContext) : } return set } -} \ No newline at end of file +} From 0970e5d3baa233920aacce066cba855aa903c173 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:53:54 +0200 Subject: [PATCH 187/434] Update HeyToon.kt --- src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt | 35 +++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt index b5dd4e77..8d85090c 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt @@ -5,13 +5,36 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import okhttp3.Headers +import org.jsoup.nodes.Document +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseJsonArray +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded import tsuki.util.json.getStringOrNull import tsuki.util.json.mapJSON -import okhttp3.Headers -import org.jsoup.nodes.Document import java.util.Calendar import java.util.EnumSet @@ -266,8 +289,8 @@ internal class HeyToon(context: MangaLoaderContext) : set(Calendar.SECOND, 0) set(Calendar.MILLISECOND, 0) }.timeInMillis - } catch (e: Exception) { + } catch (_: Exception) { 0 } } -} \ No newline at end of file +} From df03510ce77056e9b49da68bd99b582849581f53 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:55:01 +0200 Subject: [PATCH 188/434] Update MangaDistrict.kt --- .../kotlin/tsuki/site/en/nsfw/MangaDistrict.kt | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt index b2df4c8d..017b39c7 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt @@ -4,8 +4,19 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toTitleCase import org.jsoup.nodes.Document import java.text.SimpleDateFormat @@ -100,4 +111,4 @@ internal class MangaDistrict(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 2c00197badf374a3be2eeeb5e743165eec4a2bcf Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:55:58 +0200 Subject: [PATCH 189/434] Update Manhwa18CC.kt --- .../kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt index da6ccd3e..40a283cc 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt @@ -4,8 +4,29 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -141,4 +162,4 @@ internal class Manhwa18CC(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 0cf7cbec57782b9ab0450363dbd3f544343655a5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:56:56 +0200 Subject: [PATCH 190/434] Update Manhwa210.kt --- .../kotlin/tsuki/site/en/nsfw/Manhwa210.kt | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt index aa24fe8a..f2b0fafa 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt @@ -5,8 +5,32 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.selectLast +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -239,4 +263,4 @@ internal class Manhwa210(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From 80a93d7c7b5aeb044f44cba257915e291439fe03 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:57:59 +0200 Subject: [PATCH 191/434] Update ManhwaRead.kt --- .../kotlin/tsuki/site/en/nsfw/ManhwaRead.kt | 25 ++++++++++++++++--- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt index 66f0bdcb..2184b76e 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt @@ -5,8 +5,24 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import java.util.Base64 import org.json.JSONArray @@ -14,7 +30,8 @@ import org.json.JSONObject import org.jsoup.nodes.Document import java.nio.charset.StandardCharsets import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("MANHWAREAD", "ManhwaRead", "en", ContentType.HENTAI) internal class ManhwaRead(context: MangaLoaderContext) : @@ -291,4 +308,4 @@ internal class ManhwaRead(context: MangaLoaderContext) : return if (sb.isNotEmpty()) "?$sb" else "" } private val CHAPTER_DATA_REGEX = Regex("""var\s+chapterData\s*=\s*(\{.*\})""") -} \ No newline at end of file +} From 2d71af37ae791306067a722da1c00c83de122ab6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:59:06 +0200 Subject: [PATCH 192/434] Update AquaReader.kt --- src/main/kotlin/tsuki/site/en/AquaReader.kt | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/AquaReader.kt b/src/main/kotlin/tsuki/site/en/AquaReader.kt index d1a9bf21..0ce6e8ec 100644 --- a/src/main/kotlin/tsuki/site/en/AquaReader.kt +++ b/src/main/kotlin/tsuki/site/en/AquaReader.kt @@ -4,8 +4,21 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl import org.jsoup.nodes.Document import java.text.SimpleDateFormat From 151e4293794f7fd897b6017e825767e30bb531fa Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:01:01 +0200 Subject: [PATCH 193/434] Update AsuraScan.kt --- src/main/kotlin/tsuki/site/en/AsuraScan.kt | 35 ++++++++++++++++++---- 1 file changed, 30 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/AsuraScan.kt b/src/main/kotlin/tsuki/site/en/AsuraScan.kt index d9ecc879..e2223bd9 100644 --- a/src/main/kotlin/tsuki/site/en/AsuraScan.kt +++ b/src/main/kotlin/tsuki/site/en/AsuraScan.kt @@ -5,8 +5,33 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectOrThrow +import tsuki.util.src +import tsuki.util.substringBetween +import tsuki.util.toAbsoluteUrl import tsuki.util.json.toJSONArrayOrNull import tsuki.util.json.toJSONObjectOrNull @@ -18,7 +43,7 @@ import java.util.EnumSet import java.util.Locale import java.util.TreeMap -@MangaSourceParser("ASURASCANS", "AsuraComic", "en") +@MangaSourceParser("ASURASCANS", "Asura Scans", "en") internal class AsuraScansParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.ASURASCANS, pageSize = 20) { @@ -204,7 +229,7 @@ internal class AsuraScansParser(context: MangaLoaderContext) : ?.groupValues ?.getOrNull(1) ?.toFloatOrNull() - ?: i + 1f + ?: (i + 1f) val dateText = a.selectFirst("span.text-sm.text-white\\/40") ?.text() ?.trim() @@ -351,4 +376,4 @@ internal class AsuraScansParser(context: MangaLoaderContext) : ) private val asuraGenreKeyRegex = Regex("[^a-z0-9]+") } -} \ No newline at end of file +} From 57b8f599febac7652a4ebc3153befadf32d1fed1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:03:19 +0200 Subject: [PATCH 194/434] Update Atsumaru.kt --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 45 ++++++++++++++++++----- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index dca36f0d..e5c724f7 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -5,14 +5,24 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONArray -import org.json.JSONObject -import java.util.EnumSet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.delay @@ -20,6 +30,11 @@ import kotlin.math.pow import kotlin.math.roundToLong import kotlin.random.Random import kotlin.time.Duration.Companion.milliseconds +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.util.EnumSet suspend fun <T> retryWithBackoff( maxRetries: Int = 3, @@ -190,11 +205,21 @@ internal class Atsumaru(context: MangaLoaderContext) : override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = (allGenres.map { (name, id) -> MangaTag(name, "genre:$id", source) } + allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), ) - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter, ): List<Manga> = getSearchPage(page, order, filter) + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> = getSearchPage(page, order, filter) private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query ?: "*" From 0d60cd90c6cde6b1620706d62ea75e2b78702f05 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:06:53 +0200 Subject: [PATCH 195/434] Update Batcave.kt --- src/main/kotlin/tsuki/site/en/Batcave.kt | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index 5ace266b..d04757e7 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -7,8 +7,25 @@ import tsuki.core.PagedMangaParser import tsuki.exception.ParseException import tsuki.network.OkHttpWebClient -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.FormBody import okhttp3.MediaType.Companion.toMediaTypeOrNull From 5574fdc301af72d5848bb3466bffed2bdaf21dc8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:07:17 +0200 Subject: [PATCH 196/434] Delete src/main/kotlin/tsuki/site/en/CoffeeManga.kt --- src/main/kotlin/tsuki/site/en/CoffeeManga.kt | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/en/CoffeeManga.kt diff --git a/src/main/kotlin/tsuki/site/en/CoffeeManga.kt b/src/main/kotlin/tsuki/site/en/CoffeeManga.kt deleted file mode 100644 index d3dbb49e..00000000 --- a/src/main/kotlin/tsuki/site/en/CoffeeManga.kt +++ /dev/null @@ -1,11 +0,0 @@ -package tsuki.site.en - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("COFFEEMANGA", "CoffeeManga", "en") -internal class CoffeeManga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.COFFEEMANGA, "coffeemanga.ink") \ No newline at end of file From f25f0be7ed3c410a4cf9f9509afb1c543e26a2af Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:18:52 +0200 Subject: [PATCH 197/434] Update DemonicScan.kt --- src/main/kotlin/tsuki/site/en/DemonicScan.kt | 31 ++++++++++++++++---- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/DemonicScan.kt b/src/main/kotlin/tsuki/site/en/DemonicScan.kt index 06c59f4b..1028bc3e 100644 --- a/src/main/kotlin/tsuki/site/en/DemonicScan.kt +++ b/src/main/kotlin/tsuki/site/en/DemonicScan.kt @@ -5,8 +5,28 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.textOrNull +import tsuki.util.toTitleCase import androidx.collection.arraySetOf import org.jsoup.nodes.Element @@ -163,16 +183,17 @@ internal class DemonicScans(context: MangaLoaderContext) : else -> null } - val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { i, el -> + val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { _, el -> val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) val href = el.attr("href").let { if (it.startsWith("http")) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" } val date = el.selectFirst("span")?.text() + MangaChapter( id = generateUid(href), title = el.ownText(), - number = i + 1f, + number = el.ownText().extractChapterNumber(), volume = 0, url = href, scanlator = null, @@ -244,4 +265,4 @@ internal class DemonicScans(context: MangaLoaderContext) : MangaTag("Tragedy", "18", source), MangaTag("Webtoons", "24", source), ) -} \ No newline at end of file +} From e0f128f4628dd02350a598304118330e0d82abf0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:20:03 +0200 Subject: [PATCH 198/434] Update FlameComics.kt --- src/main/kotlin/tsuki/site/en/FlameComics.kt | 23 ++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/FlameComics.kt b/src/main/kotlin/tsuki/site/en/FlameComics.kt index 349ccd51..9692fb89 100644 --- a/src/main/kotlin/tsuki/site/en/FlameComics.kt +++ b/src/main/kotlin/tsuki/site/en/FlameComics.kt @@ -5,8 +5,27 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.SinglePageMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.findGroupValue +import tsuki.util.generateUid +import tsuki.util.longOf +import tsuki.util.mapToSet +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder import tsuki.util.json.asTypedList import tsuki.util.json.entries import tsuki.util.json.getFloatOrDefault From 17219bd82f74c9daa000b7540fc03ca9f0fcbda2 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:20:54 +0200 Subject: [PATCH 199/434] Update HiveToons.kt --- src/main/kotlin/tsuki/site/en/HiveToons.kt | 24 +++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/HiveToons.kt b/src/main/kotlin/tsuki/site/en/HiveToons.kt index 9a236ef1..017e0d43 100644 --- a/src/main/kotlin/tsuki/site/en/HiveToons.kt +++ b/src/main/kotlin/tsuki/site/en/HiveToons.kt @@ -4,15 +4,33 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray + import org.json.JSONArray import org.json.JSONObject import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import kotlin.collections.isNotEmpty @MangaSourceParser("HIVETOONS", "HiveToons", "en") internal class HiveToons(context: MangaLoaderContext): From 08d8da6d1e9c6d9d10d1566b2ff29c681cba9032 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:21:42 +0200 Subject: [PATCH 200/434] Update KaliScan.kt --- src/main/kotlin/tsuki/site/en/KaliScan.kt | 28 ++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/KaliScan.kt b/src/main/kotlin/tsuki/site/en/KaliScan.kt index 4efd9148..d656b6e9 100644 --- a/src/main/kotlin/tsuki/site/en/KaliScan.kt +++ b/src/main/kotlin/tsuki/site/en/KaliScan.kt @@ -4,8 +4,30 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadthemeParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.removeSuffix import org.jsoup.nodes.Document import java.text.SimpleDateFormat @@ -208,4 +230,4 @@ internal class KaliScan(context: MangaLoaderContext) : ) } ?: emptySet() } -} \ No newline at end of file +} From 432e76ddb56367494645d63808404ee7d48a32c6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:22:41 +0200 Subject: [PATCH 201/434] Update MangaFox.kt --- src/main/kotlin/tsuki/site/en/MangaFox.kt | 27 +++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaFox.kt b/src/main/kotlin/tsuki/site/en/MangaFox.kt index 36dafa32..4c08337f 100644 --- a/src/main/kotlin/tsuki/site/en/MangaFox.kt +++ b/src/main/kotlin/tsuki/site/en/MangaFox.kt @@ -6,8 +6,24 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock @@ -39,7 +55,10 @@ internal class MangaFox(context: MangaLoaderContext) : OkHttpWebClient(httpClient, source) } - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED) + override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true @@ -237,4 +256,4 @@ internal class MangaFox(context: MangaLoaderContext) : override suspend fun getPageUrl(page: MangaPage): String = page.url override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() -} \ No newline at end of file +} From 7a7b90578bf2158b91b598660fc267cc536e059d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:24:05 +0200 Subject: [PATCH 202/434] Update MangaGG.kt --- src/main/kotlin/tsuki/site/en/MangaGG.kt | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaGG.kt b/src/main/kotlin/tsuki/site/en/MangaGG.kt index 58d27c86..4cbede7e 100644 --- a/src/main/kotlin/tsuki/site/en/MangaGG.kt +++ b/src/main/kotlin/tsuki/site/en/MangaGG.kt @@ -4,8 +4,15 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -13,7 +20,6 @@ import org.jsoup.nodes.Document import java.text.SimpleDateFormat import kotlinx.coroutines.awaitAll - @MangaSourceParser("MANGAGG", "MangaGG", "en") internal class MangaGG(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MANGAGG, "mangagg.com") { @@ -150,4 +156,4 @@ internal class MangaGG(context: MangaLoaderContext) : branch = null, ) } -} \ No newline at end of file +} From 4392a06b2cb2ff2cb65d029dd9d0eaaae4f9d2b6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:28:43 +0200 Subject: [PATCH 203/434] Update MangaGeko.kt --- src/main/kotlin/tsuki/site/en/MangaGeko.kt | 33 +++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaGeko.kt b/src/main/kotlin/tsuki/site/en/MangaGeko.kt index bfe960a3..8f518e2d 100644 --- a/src/main/kotlin/tsuki/site/en/MangaGeko.kt +++ b/src/main/kotlin/tsuki/site/en/MangaGeko.kt @@ -5,8 +5,28 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject @@ -62,7 +82,12 @@ internal class MangaGeko(context: MangaLoaderContext) : return MangaListFilterOptions( availableTags = tags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS + ), availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), ) } @@ -244,6 +269,6 @@ internal class MangaGeko(context: MangaLoaderContext) : source = source, ) } - + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() } From 209d7eece3bdbd4bfead1871f08a1a41a98008cb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:29:10 +0200 Subject: [PATCH 204/434] Update MangaJinx.kt --- src/main/kotlin/tsuki/site/en/MangaJinx.kt | 29 +++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaJinx.kt b/src/main/kotlin/tsuki/site/en/MangaJinx.kt index c64acdfc..c4573d7f 100644 --- a/src/main/kotlin/tsuki/site/en/MangaJinx.kt +++ b/src/main/kotlin/tsuki/site/en/MangaJinx.kt @@ -4,13 +4,36 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadthemeParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.removeSuffix import org.jsoup.nodes.Document import java.text.SimpleDateFormat import java.util.EnumSet + @MangaSourceParser("MANGAJINX", "MangaJinx", "en") internal class MangaJinx(context: MangaLoaderContext) : MadthemeParser(context, MangaParserSource.MANGAJINX, "mgjinx.com", pageSize=24) { @@ -208,4 +231,4 @@ internal class MangaJinx(context: MangaLoaderContext) : ) } ?: emptySet() } -} \ No newline at end of file +} From 1848c182c803736268cd1bf17a4ab7f82b63e00c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:30:13 +0200 Subject: [PATCH 205/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index d04b0401..8812d8ac 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -6,8 +6,29 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import tsuki.util.json.mapJSON import tsuki.util.json.mapJSONNotNullToSet import tsuki.util.json.mapJSONToSet From e3991908e11b7dcc106f33eb69370d8e249a7fac Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:39:01 +0200 Subject: [PATCH 206/434] Update MangaKatana.kt --- src/main/kotlin/tsuki/site/en/MangaKatana.kt | 82 ++++++-------------- 1 file changed, 23 insertions(+), 59 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaKatana.kt b/src/main/kotlin/tsuki/site/en/MangaKatana.kt index 109c04b7..473b5a61 100644 --- a/src/main/kotlin/tsuki/site/en/MangaKatana.kt +++ b/src/main/kotlin/tsuki/site/en/MangaKatana.kt @@ -5,13 +5,28 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* - -import org.jsoup.nodes.Document +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import java.text.SimpleDateFormat + import java.util.EnumSet import java.util.Locale +import kotlin.collections.isNotEmpty +import org.jsoup.nodes.Document private const val two = "?sv=mk" private const val three = "?sv=3" @@ -69,28 +84,18 @@ internal class MangaKatana(context: MangaLoaderContext): private val dateFormat = SimpleDateFormat("MMM-dd-yyyy", Locale.US) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter, ): List<Manga> { val url = buildUrl(page, order, filter) val doc = webClient.httpGet(url).parseHtml() - return doc.select("div#book_list > div.item").map { element -> - val link = element.selectFirst("div.text > h3 > a")!! val thumb = element.selectFirst("img")!! - Manga( id = generateUid(link.absUrl("href")), title = link.ownText().ifEmpty { link.text() }, url = link.absUrl("href"), publicUrl = link.absUrl("href"), - coverUrl = thumb.absUrl("src"), - rating = RATING_UNKNOWN, contentRating = null, tags = emptySet(), @@ -99,21 +104,15 @@ internal class MangaKatana(context: MangaLoaderContext): altTitles = emptySet(), largeCoverUrl = null, description = null, - source = source ) } } - private fun buildUrl( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): String { + private fun buildUrl(page: Int, order: SortOrder, filter: MangaListFilter, ): String { val query = filter.query?.trim().orEmpty() val tags = filter.tags.map { it.key }.filter { it.isNotBlank() } val status = filter.states.firstOrNull() - return when { query.isNotEmpty() -> { if (page == 1) { @@ -122,7 +121,6 @@ internal class MangaKatana(context: MangaLoaderContext): "$baseUrl/page/$page?search=${query.urlEncoded()}&search_by=m_name" } } - status != null -> { "$baseUrl/genres/?" + "filter=1" + @@ -133,10 +131,7 @@ internal class MangaKatana(context: MangaLoaderContext): "&status=${status.toMKStatus()}" + "&page=$page" } - - tags.isNotEmpty() -> - "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" - + tags.isNotEmpty() -> "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" else -> when (order) { SortOrder.NEWEST -> "$baseUrl/new-manga?page=$page" SortOrder.UPDATED -> "$baseUrl/latest/page/$page" @@ -146,16 +141,12 @@ internal class MangaKatana(context: MangaLoaderContext): } override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() - val chapters = doc.select("tr:has(.chapter)") .mapNotNull { tr -> val a = tr.selectFirst("a") ?: return@mapNotNull null val href = a.absUrl("href") - if (href.isBlank() || href == "#" || !href.startsWith("http")) return@mapNotNull null - MangaChapter( id = generateUid(href), title = a.text(), @@ -174,9 +165,7 @@ internal class MangaKatana(context: MangaLoaderContext): title = doc.selectFirst("h1")?.text() ?: doc.selectFirst("h1.heading")?.text() ?: manga.title, - authors = doc.select(".author").eachText().toSet(), - tags = doc.select(".genres a").map { MangaTag( key = it.text().lowercase(), @@ -184,7 +173,6 @@ internal class MangaKatana(context: MangaLoaderContext): source = source ) }.toSet(), - description = buildString { append(doc.select(".summary > p").text()) val alt = doc.select(".alt_name").text() @@ -192,9 +180,7 @@ internal class MangaKatana(context: MangaLoaderContext): append("\n\nAlt name(s): $alt") } }.trim(), - state = parseStatus(doc.select(".value.status").text()), - coverUrl = doc.select("div.media div.cover img").attr("abs:src"), chapters = chapters ) @@ -223,8 +209,7 @@ internal class MangaKatana(context: MangaLoaderContext): dateFormat.parse(text)?.time ?: 0L }.getOrDefault(0L) } - - + private fun availableTags() = setOf( MangaTag("4 koma", "4-koma", source), MangaTag("Action", "action", source), @@ -283,10 +268,8 @@ internal class MangaKatana(context: MangaLoaderContext): override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val url = applyServer(chapter.url) val doc = webClient.httpGet(url).parseHtml() - parseThzq(doc)?.let { if (it.isNotEmpty()) return it } @@ -297,7 +280,6 @@ internal class MangaKatana(context: MangaLoaderContext): parseThzq(fallbackDoc)?.let { if (it.isNotEmpty()) return it } - return emptyList() } @@ -307,22 +289,17 @@ internal class MangaKatana(context: MangaLoaderContext): } private fun parseThzq(doc: Document): List<MangaPage>? { - val script = doc.select("script") .asSequence() .map { it.data() } .firstOrNull { it.contains("thzq") } ?: return null - val start = script.indexOf("var thzq") if (start == -1) return null - val arrStart = script.indexOf('[', start) if (arrStart == -1) return null - var depth = 0 var end = -1 - for (i in arrStart until script.length) { when (script[i]) { '[' -> depth++ @@ -334,23 +311,17 @@ internal class MangaKatana(context: MangaLoaderContext): } } } - if (end == -1) return null - val raw = script.substring(arrStart, end) - val urls = ArrayList<String>(32) - var i = 0 while (i < raw.length) { if (raw[i] == '\'' || raw[i] == '"') { val quote = raw[i] val startIdx = ++i - while (i < raw.length && raw[i] != quote) { i++ } - if (i > startIdx) { val url = raw.substring(startIdx, i) if (url.startsWith("http")) { @@ -360,9 +331,7 @@ internal class MangaKatana(context: MangaLoaderContext): } i++ } - if (urls.isEmpty()) return null - return urls.map { url -> MangaPage( id = generateUid(url), @@ -374,16 +343,12 @@ internal class MangaKatana(context: MangaLoaderContext): } private fun parseHtmlImages(doc: Document): List<MangaPage>? { - val pages = doc.select("div.wrap_img img") .asSequence() .mapNotNull { img -> - val url = img.attr("data-src") .ifEmpty { img.attr("src") } - if (url.isEmpty() || !url.startsWith("http")) return@mapNotNull null - MangaPage( id = generateUid(url), url = url, @@ -392,7 +357,6 @@ internal class MangaKatana(context: MangaLoaderContext): ) } .toList() - return pages.ifEmpty { null } } } From b18efee16b67436977b6d127e8dcda80ac5d32e2 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:44:43 +0200 Subject: [PATCH 207/434] Update MangaPill.kt --- src/main/kotlin/tsuki/site/en/MangaPill.kt | 35 ++++++++++++++++++---- 1 file changed, 29 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaPill.kt b/src/main/kotlin/tsuki/site/en/MangaPill.kt index 83c0db82..db1e1da3 100644 --- a/src/main/kotlin/tsuki/site/en/MangaPill.kt +++ b/src/main/kotlin/tsuki/site/en/MangaPill.kt @@ -5,8 +5,26 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl import okhttp3.HttpUrl.Companion.toHttpUrl import java.util.EnumSet @@ -36,7 +54,12 @@ internal class MangaPill(context: MangaLoaderContext): override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = fetchTags(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.UPCOMING + ), availableContentTypes = EnumSet.of( ContentType.MANGA, ContentType.MANHWA, @@ -47,7 +70,7 @@ internal class MangaPill(context: MangaLoaderContext): ContentType.OTHER, ), ) - + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { if (!filter.query.isNullOrEmpty() || filter.tags.isNotEmpty() || filter.types.isNotEmpty() || filter.states.isNotEmpty()) { return fetchSearch(page, filter) @@ -208,7 +231,7 @@ internal class MangaPill(context: MangaLoaderContext): chapters = chapters, ) } - + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() return doc.select("picture img").map { img -> @@ -221,7 +244,7 @@ internal class MangaPill(context: MangaLoaderContext): ) } } - + private suspend fun fetchTags(): Set<MangaTag> { val doc = webClient.httpGet("https://$domain/search").parseHtml() return doc.select("div.m-1 label input").mapNotNull { input -> From 652387d7cd4a8692fec6e863545f03dc3b3a3422 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:47:48 +0200 Subject: [PATCH 208/434] Refactor MangaTown class with updated methods --- src/main/kotlin/tsuki/site/en/MangaTown.kt | 55 +++++++++++++++------- 1 file changed, 39 insertions(+), 16 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaTown.kt b/src/main/kotlin/tsuki/site/en/MangaTown.kt index 4cc000b8..212af601 100644 --- a/src/main/kotlin/tsuki/site/en/MangaTown.kt +++ b/src/main/kotlin/tsuki/site/en/MangaTown.kt @@ -5,13 +5,44 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import java.text.SimpleDateFormat -import java.util.* +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import kotlin.collections.toList +import kotlin.collections.toSet @MangaSourceParser("MANGATOWN", "MangaTown", "en") internal class MangaTown(context: MangaLoaderContext) : @@ -73,8 +104,7 @@ internal class MangaTown(context: MangaLoaderContext) : ContentType.MANHWA, ), ) - - + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query?.trim() val hasFilters = filter.types.isNotEmpty() || filter.demographics.isNotEmpty() || @@ -225,7 +255,6 @@ internal class MangaTown(context: MangaLoaderContext) : } } - override suspend fun getDetails(manga: Manga): Manga { val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() val root = doc.body().selectFirst("section.main") @@ -279,8 +308,7 @@ internal class MangaTown(context: MangaLoaderContext) : } ?: emptyList(), ) } - - + private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> { val subdomain = "m." + domain.removePrefix("www.") val doc = webClient.httpGet(manga.url.toAbsoluteUrl(subdomain), getRequestHeaders()).parseHtml() @@ -306,8 +334,7 @@ internal class MangaTown(context: MangaLoaderContext) : ) } } - - + private fun parseChapterDate(dateFormat: SimpleDateFormat, date: String?): Long { return when { date.isNullOrEmpty() -> 0L @@ -317,7 +344,6 @@ internal class MangaTown(context: MangaLoaderContext) : } } - private suspend fun fetchAvailableTags(): Set<MangaTag> { val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() val root = doc.body().selectFirst("aside.right") @@ -335,7 +361,6 @@ internal class MangaTown(context: MangaLoaderContext) : } } - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl, getRequestHeaders()).parseHtml() @@ -364,14 +389,12 @@ internal class MangaTown(context: MangaLoaderContext) : } } - override suspend fun getPageUrl(page: MangaPage): String { if (page.url.startsWith("http")) return page.url val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() return doc.selectFirst("div#viewer img")?.attrAsAbsoluteUrl("src") ?: throw Exception("Could not find image") } - - + private fun String.nullIfEmpty(): String? = ifEmpty { null } -} \ No newline at end of file +} From 116d8124bc0e9a5004528b5287c75c94d29f8d87 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 00:51:16 +0200 Subject: [PATCH 209/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/en/Mangadotnet.kt | 35 +++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt index 4003469b..e670206d 100644 --- a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/en/Mangadotnet.kt @@ -5,8 +5,25 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray @@ -114,8 +131,18 @@ internal class Mangadotnet(context: MangaLoaderContext) : return MangaListFilterOptions( availableTags = tags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableDemographics = EnumSet.of(Demographic.JOSEI, Demographic.SEINEN, Demographic.SHOUJO, Demographic.SHOUNEN), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), + availableDemographics = EnumSet.of( + Demographic.JOSEI, + Demographic.SEINEN, + Demographic.SHOUJO, + Demographic.SHOUNEN + ), ) } From 35cb75e12e8f084ae1494077212c0f9904f2088e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:02:52 +0200 Subject: [PATCH 210/434] Update Mangago.kt --- src/main/kotlin/tsuki/site/en/Mangago.kt | 40 ++++++++++++++++++------ 1 file changed, 31 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangago.kt b/src/main/kotlin/tsuki/site/en/Mangago.kt index f99b5778..4c94b896 100644 --- a/src/main/kotlin/tsuki/site/en/Mangago.kt +++ b/src/main/kotlin/tsuki/site/en/Mangago.kt @@ -7,8 +7,29 @@ import tsuki.core.PagedMangaParser import tsuki.bitmap.Bitmap import tsuki.bitmap.Rect -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.requireElementById +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.Interceptor import okhttp3.Response @@ -94,7 +115,7 @@ internal class MangagoParser(context: MangaLoaderContext) : override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query.orEmpty() - if (!query.isNullOrEmpty()) { + if (query.isNotEmpty()) { val url = buildString { append("https://") append(domain) @@ -265,6 +286,7 @@ internal class MangagoParser(context: MangaLoaderContext) : return buildChapterList(rawChapters) } + @Suppress("ControlFlowWithEmptyBody") private fun buildChapterList(chapters: List<ChapterParseData>): List<MangaChapter> { val scanlatorCounts = mutableMapOf<String, Int>() for (chapter in chapters) { @@ -352,7 +374,7 @@ internal class MangagoParser(context: MangaLoaderContext) : val regex = Regex("""(?:ch\.?|chapter|vol\.?\s*\d+\s+ch\.?)\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) val match = regex.find(title) match?.groupValues?.get(1)?.toFloat() - } catch (e: Exception) { + } catch (_: Exception) { null } } @@ -399,7 +421,7 @@ internal class MangagoParser(context: MangaLoaderContext) : while (allImages.size < pagesCount) { val batchUrl = buildBatchUrl(batchStart) val batchDoc = if (batchStart == 1) doc else webClient.httpGet(batchUrl).parseHtml() - val batchImages = decryptImageList(batchDoc, batchUrl) + val batchImages = decryptImageList(batchDoc) if (batchImages.isEmpty()) { break @@ -433,7 +455,7 @@ internal class MangagoParser(context: MangaLoaderContext) : val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() if (imgsrcsScript != null) { - val images = decryptImageList(doc, fullUrl) + val images = decryptImageList(doc) val cols = getColsFromDoc(doc) ?: "" val js = getDeobfuscatedJS(doc) ?: throw Exception("Could not get JS") @@ -470,7 +492,7 @@ internal class MangagoParser(context: MangaLoaderContext) : override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - private suspend fun decryptImageList(doc: Document, sourceUrl: String): List<String> { + private suspend fun decryptImageList(doc: Document): List<String> { val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() ?: throw Exception("Could not find imgsrcs") @@ -618,7 +640,7 @@ internal class MangagoParser(context: MangaLoaderContext) : } imgList[loc].toString().toInt() } - } catch (e: NumberFormatException) { + } catch (_: NumberFormatException) { return imgList } @@ -685,4 +707,4 @@ internal class MangagoParser(context: MangaLoaderContext) : private val KEY_LOCATION_REGEX = Regex("""str\.charAt\(\s*(\d+)\s*\)""") private val JS_FILTERS = listOf("jQuery", "document", "getContext", "toDataURL", "getImageData", "width", "height") } -} \ No newline at end of file +} From 345539f580d35c348a316771ae2c6bb67ef1b4fd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 01:10:25 +0200 Subject: [PATCH 211/434] Update Mangataro.kt --- src/main/kotlin/tsuki/site/en/Mangataro.kt | 39 +++++++++++++++++++--- 1 file changed, 34 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangataro.kt b/src/main/kotlin/tsuki/site/en/Mangataro.kt index 96937325..23cb56e9 100644 --- a/src/main/kotlin/tsuki/site/en/Mangataro.kt +++ b/src/main/kotlin/tsuki/site/en/Mangataro.kt @@ -6,8 +6,27 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.Headers import org.json.JSONArray @@ -25,6 +44,7 @@ import java.util.EnumSet import java.util.HashSet import java.util.LinkedHashSet import java.util.Locale +import kotlin.collections.isNotEmpty @MangaSourceParser("MANGATAROORG", "Mangataro", "en", ContentType.MANGA) internal class Mangataro(context: MangaLoaderContext) : @@ -58,8 +78,16 @@ internal class Mangataro(context: MangaLoaderContext) : override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = genreTags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), ) @@ -182,6 +210,7 @@ internal class Mangataro(context: MangaLoaderContext) : return result } + @Suppress("SameParameterValue") private fun buildChapterUrl( mangaId: String, offset: Int, @@ -842,4 +871,4 @@ internal class Mangataro(context: MangaLoaderContext) : tag("1342", "Zombies"), ) } -} \ No newline at end of file +} From 57496668815f22a3acc9a1a481345e9d35909311 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:21:04 +0200 Subject: [PATCH 212/434] Update Mgread.kt --- src/main/kotlin/tsuki/site/en/Mgread.kt | 152 ++++++++++++++++++++++-- 1 file changed, 140 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mgread.kt b/src/main/kotlin/tsuki/site/en/Mgread.kt index 31f21338..491a1e5b 100644 --- a/src/main/kotlin/tsuki/site/en/Mgread.kt +++ b/src/main/kotlin/tsuki/site/en/Mgread.kt @@ -6,8 +6,24 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.UserAgents -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray @@ -19,9 +35,8 @@ import okhttp3.Interceptor import okhttp3.Response import java.util.Locale import java.util.TimeZone - -// TODO -// add filter options +import kotlin.collections.filterNot +import kotlin.text.isNotEmpty @MangaSourceParser("MGREADIO", "Mgread.io", "en") internal class MgreadIo(context: MangaLoaderContext) : @@ -46,10 +61,34 @@ internal class MgreadIo(context: MangaLoaderContext) : override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.UPDATED, SortOrder.POPULARITY, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, ) override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, + isTagsExclusionSupported = false, + isMultipleTagsSupported = true, + isOriginalLocaleSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), ) private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US) @@ -57,8 +96,6 @@ internal class MgreadIo(context: MangaLoaderContext) : timeZone = TimeZone.getTimeZone("GMT+7") } - override suspend fun getFilterOptions() = MangaListFilterOptions() - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } @@ -90,14 +127,60 @@ internal class MgreadIo(context: MangaLoaderContext) : }.filterNot { it.isAnime() } } - val slug = if (order == SortOrder.POPULARITY) "manga-ranking" else "recently-updated" - val url = if (page == 1) "https://$domain/$slug/" else "https://$domain/$slug/page/$page/" + val url = buildAdvancedFilterUrl(page, order, filter) val doc = webClient.httpGet(url).parseHtml() return parseMangaGrid(doc).filterNot { it.isAnime() } } + private fun buildAdvancedFilterUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + val builder = "https://$domain/advanced-filter".toHttpUrl().newBuilder() + if (page > 1) { + builder.addPathSegment("page") + builder.addPathSegment(page.toString()) + } + + val sortParam = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "views" + SortOrder.POPULARITY_TODAY -> "views_day" + SortOrder.POPULARITY_WEEK -> "views_week" + SortOrder.POPULARITY_MONTH -> "view_month" + SortOrder.RATING -> "rating" + SortOrder.NEWEST -> "new" + SortOrder.NEWEST_ASC -> "old" + else -> "updated" + } + builder.addQueryParameter("sort", sortParam) + + val statusParam = filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "season_end" + MangaState.ABANDONED -> "dropped" + else -> null + } + } + statusParam?.let { builder.addQueryParameter("status", it) } + + val ageRatingParam = filter.contentRating.firstOrNull()?.let { cr -> + when (cr) { + ContentRating.SAFE -> "all" + ContentRating.SUGGESTIVE -> "16+" + ContentRating.ADULT -> "18+" + } + } + ageRatingParam?.let { builder.addQueryParameter("age_rating", it) } + + filter.tags.forEach { tag -> + builder.addQueryParameter("genre[]", tag.key) + } + + return builder.build().toString() + } + private fun parseMangaGrid(doc: Document): List<Manga> { - return doc.select(".manga-item-grid").map { element -> + return doc.select(".manga-item-grid, .manga-item-details").map { element -> val titleElement = element.selectFirst("h2 a[href*='/manga/']") ?: element.selectFirst("a[href*='/manga/']:not([href*='/chapter-'])") ?: return@map null @@ -179,7 +262,7 @@ internal class MgreadIo(context: MangaLoaderContext) : }.trim().takeIf(String::isNotEmpty) val tags = genre.split(",").mapNotNull { it.trim().takeIf(String::isNotEmpty) }.map { name -> - MangaTag(name, name.lowercase(), source) + MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() val mangaId = doc.selectFirst("#manga-title[data-id], #chapter-search-input[data-manga-id]") @@ -296,4 +379,49 @@ internal class MgreadIo(context: MangaLoaderContext) : private fun SimpleDateFormat.parseSafe(date: String): Long = runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} \ No newline at end of file + + private fun fetchTags(): Set<MangaTag> { + return GENRES.map { (name, slug) -> + MangaTag( + title = name, + key = slug, + source = source, + ) + }.toSet() + } + + companion object { + private val GENRES = listOf( + "Action" to "action", + "Adaptation" to "adaptation", + "Adventure" to "adventure", + "Anime" to "anime", + "Comedy" to "comedy", + "Cooking" to "cooking", + "Crime" to "crime", + "Drama" to "drama", + "Ecchi" to "ecchi", + "Fantasy" to "fantasy", + "Harem" to "harem", + "Historical" to "historical", + "Horror" to "horror", + "Isekai" to "isekai", + "Josei" to "josei", + "Martial Arts" to "martial-arts", + "Mature" to "mature", + "Mecha" to "mecha", + "Medical" to "medical", + "Music" to "music", + "Mystery" to "mystery", + "Romance" to "romance", + "School Life" to "school-life", + "Shoujo" to "shoujo", + "Shounen" to "shounen", + "Slice of life" to "slice-of-life", + "Smut" to "smut", + "Sports" to "sports", + "Supernatural" to "supernatural", + "Webtoons" to "webtoons", + ) + } +} From 97d2e4b4e1b6f2e70007d8d3928ffe572b6028c6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:28:16 +0200 Subject: [PATCH 213/434] Update MistScans.kt --- src/main/kotlin/tsuki/site/en/MistScans.kt | 35 +++++++++++++++------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MistScans.kt b/src/main/kotlin/tsuki/site/en/MistScans.kt index 473832fc..f931a982 100644 --- a/src/main/kotlin/tsuki/site/en/MistScans.kt +++ b/src/main/kotlin/tsuki/site/en/MistScans.kt @@ -6,19 +6,30 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import java.text.SimpleDateFormat +import java.util.Calendar import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.Calendar import java.util.EnumSet import java.util.Locale - -// Todo: preloading chapter image fix +import kotlin.collections.reversed @MangaSourceParser("MISTSCANS", "Mist Scans", "en") internal class MistScans(context: MangaLoaderContext) : @@ -37,6 +48,7 @@ internal class MistScans(context: MangaLoaderContext) : .set("Referer", "$baseUrl/") .build() + @Volatile private var cachedGenres: List<Genre>? = null private data class Genre( @@ -83,9 +95,10 @@ internal class MistScans(context: MangaLoaderContext) : } return when (order) { - SortOrder.UPDATED -> fetchLatest() - SortOrder.ADDED -> fetchAdded() - else -> fetchPopular() + SortOrder.UPDATED -> fetchLatest() + SortOrder.ADDED -> fetchAdded() + SortOrder.POPULARITY -> fetchPopular() + else -> fetchPopular() } } @@ -185,7 +198,7 @@ internal class MistScans(context: MangaLoaderContext) : authors = listOfNotNull(author, artist).toSet(), tags = (genres + listOfNotNull(type)).map { MangaTag(it.lowercase(), it, source) }.toSet(), state = status, - chapters = chapters, + chapters = chapters.reversed(), ) } @@ -223,7 +236,7 @@ internal class MistScans(context: MangaLoaderContext) : branch = null, source = source, ) - }.filterNotNull().sortedBy { it.number } + }.filterNotNull() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { From 2c779d7657911f48b75f5b04ba246e8d5660c969 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:30:48 +0200 Subject: [PATCH 214/434] Update RoliaScan.kt --- src/main/kotlin/tsuki/site/en/RoliaScan.kt | 26 +++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/RoliaScan.kt b/src/main/kotlin/tsuki/site/en/RoliaScan.kt index 6348dba3..4efb5a01 100644 --- a/src/main/kotlin/tsuki/site/en/RoliaScan.kt +++ b/src/main/kotlin/tsuki/site/en/RoliaScan.kt @@ -6,8 +6,25 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.Headers import okhttp3.HttpUrl @@ -22,9 +39,8 @@ import java.time.format.DateTimeFormatter import java.util.EnumSet import java.util.Locale import java.util.Calendar - -// year filter broken -// filter does not work in other sortorder mode except updated +import kotlin.collections.forEach +import kotlin.collections.mapNotNull @MangaSourceParser("ROLIASCAN", "Rolia Scan", "en") internal class RoliaScan(context: MangaLoaderContext) : From 37e1624954b9e9eaf9079770e8d72e5927c22699 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:31:10 +0200 Subject: [PATCH 215/434] Update ScytheScans.kt --- src/main/kotlin/tsuki/site/en/ScytheScans.kt | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/ScytheScans.kt b/src/main/kotlin/tsuki/site/en/ScytheScans.kt index 5829ce10..7b9f7cb7 100644 --- a/src/main/kotlin/tsuki/site/en/ScytheScans.kt +++ b/src/main/kotlin/tsuki/site/en/ScytheScans.kt @@ -3,13 +3,15 @@ package tsuki.site.en import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MangaThemesia -import tsuki.model.Manga + import tsuki.model.MangaChapter import tsuki.model.MangaPage import tsuki.model.MangaParserSource + import tsuki.util.generateUid import tsuki.util.parseHtml import tsuki.util.toAbsoluteUrl + import java.util.Base64 @MangaSourceParser("SYCTHESCANS", "Scythe Scans", "en") From d398ad0e3848c872c32bea04b2cf3cdfee80a534 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:34:39 +0200 Subject: [PATCH 216/434] Update VortexScans.kt --- src/main/kotlin/tsuki/site/en/VortexScans.kt | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/VortexScans.kt b/src/main/kotlin/tsuki/site/en/VortexScans.kt index 53b501c5..f8bef529 100644 --- a/src/main/kotlin/tsuki/site/en/VortexScans.kt +++ b/src/main/kotlin/tsuki/site/en/VortexScans.kt @@ -6,8 +6,20 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson import org.json.JSONArray import org.json.JSONObject @@ -18,6 +30,8 @@ import java.util.EnumSet import java.util.Locale import java.util.TimeZone +// Todo dedup + @MangaSourceParser("VORTEXSCANS", "Vortex Scans", "en") internal class VortexScans(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.VORTEXSCANS, pageSize = 20) { From 82aabcd763d479633d28a73202b90f67922075dc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:35:24 +0200 Subject: [PATCH 217/434] Update WeebCentral.kt --- src/main/kotlin/tsuki/site/en/WeebCentral.kt | 31 +++++++++++++++++--- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/WeebCentral.kt b/src/main/kotlin/tsuki/site/en/WeebCentral.kt index f5a106f8..d09a8abf 100644 --- a/src/main/kotlin/tsuki/site/en/WeebCentral.kt +++ b/src/main/kotlin/tsuki/site/en/WeebCentral.kt @@ -1,13 +1,36 @@ package tsuki.site.en import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.AbstractMangaParser -import tsuki.MangaParserAuthProvider -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectFirstOrThrow import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -355,4 +378,4 @@ internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(co return resolver.resolveManga(this, mangaId) } -} \ No newline at end of file +} From 4bb4089af2588d0d95050bc1aadf7a5b052f1a65 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:35:52 +0200 Subject: [PATCH 218/434] Update ZinManga.kt --- src/main/kotlin/tsuki/site/en/ZinManga.kt | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/ZinManga.kt b/src/main/kotlin/tsuki/site/en/ZinManga.kt index f2bf309c..ee58ca94 100644 --- a/src/main/kotlin/tsuki/site/en/ZinManga.kt +++ b/src/main/kotlin/tsuki/site/en/ZinManga.kt @@ -4,8 +4,12 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseJson import tsuki.util.json.getFloatOrDefault import tsuki.util.json.getIntOrDefault import tsuki.util.json.getStringOrNull @@ -59,4 +63,4 @@ internal class Zinmanga(context: MangaLoaderContext) : } return collected.sortedBy { it.number } } -} \ No newline at end of file +} From f30a8aa5bb98b54fa0d4f9da9902525e8905297c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:37:06 +0200 Subject: [PATCH 219/434] Refactor imports in OlympusScanlation.kt --- .../kotlin/tsuki/site/es/OlympusScanlation.kt | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt index 3045c159..5ae2c923 100644 --- a/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt +++ b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt @@ -4,13 +4,25 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder -import tsuki.model.* -import tsuki.util.* +import tsuki.util.generateUid +import tsuki.util.parseJson import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock -import tsuki.network.OkHttpWebClient import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale From 81759c0313f96f77cbf4e0d94d64d510d0b739cc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:38:10 +0200 Subject: [PATCH 220/434] Update ShadowManga.kt --- src/main/kotlin/tsuki/site/es/ShadowManga.kt | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/es/ShadowManga.kt b/src/main/kotlin/tsuki/site/es/ShadowManga.kt index dbb41480..37c1a842 100644 --- a/src/main/kotlin/tsuki/site/es/ShadowManga.kt +++ b/src/main/kotlin/tsuki/site/es/ShadowManga.kt @@ -5,8 +5,21 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson import kotlinx.coroutines.delay import kotlinx.coroutines.runBlocking @@ -124,14 +137,12 @@ internal class ShadowManga(context: MangaLoaderContext) : else -> url = "$baseApi/novedades" } - // Fetch and manually parse as JSONArray val bodyString = webClient.httpGet(url, getRequestHeaders()).body?.string() ?: return emptyList() val rawJson = runCatching { JSONArray(bodyString) }.getOrNull() ?: return emptyList() val allSeries = if (!isSearch) { - // Auto‑detect format for popular/novedades val isGenreWrapped = rawJson.length() > 0 && rawJson.optJSONObject(0)?.has("genero") == true if (isGenreWrapped) { val seen = HashSet<Int>() @@ -151,7 +162,6 @@ internal class ShadowManga(context: MangaLoaderContext) : (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } } } else { - // Search always returns a flat array (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } } From 197276c557fcf3b6f4afa8368802a732a571c281 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:39:01 +0200 Subject: [PATCH 221/434] Update Submanhwa.kt --- src/main/kotlin/tsuki/site/es/Submanhwa.kt | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/es/Submanhwa.kt b/src/main/kotlin/tsuki/site/es/Submanhwa.kt index 02000e48..75346a29 100644 --- a/src/main/kotlin/tsuki/site/es/Submanhwa.kt +++ b/src/main/kotlin/tsuki/site/es/Submanhwa.kt @@ -5,8 +5,22 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import org.jsoup.nodes.Document import org.jsoup.nodes.Element From fd308991ce2397b77dd3da16be3e6047edc2c3f1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:40:19 +0200 Subject: [PATCH 222/434] Update AnimeSama.kt --- src/main/kotlin/tsuki/site/fr/AnimeSama.kt | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt index ae8a972c..b104e5b4 100644 --- a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt +++ b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt @@ -5,11 +5,24 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson -import tsuki.network.OkHttpWebClient import org.jsoup.nodes.Document import org.jsoup.nodes.Element import okhttp3.HttpUrl.Companion.toHttpUrlOrNull @@ -64,7 +77,7 @@ internal class AnimeSama(context: MangaLoaderContext) : ) override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( - availableTags = fetchGenres(), + availableTags = fetchGenres(), availableStates = emptySet() ) From 75aefe04f6d4ab10fa94810af47f93bf5882ddb8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:41:40 +0200 Subject: [PATCH 223/434] Refactor BacaKomikpics class for improved structure --- .../tsuki/site/id/nsfw/BacaKomikpics.kt | 27 +++++++++++++++---- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt index e6b9cbba..1820086e 100644 --- a/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt +++ b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt @@ -5,8 +5,25 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.Headers import org.json.JSONArray @@ -15,7 +32,7 @@ import java.util.EnumSet @MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) internal class BacaKomikpics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance + PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") @@ -35,7 +52,7 @@ internal class BacaKomikpics(context: MangaLoaderContext) : override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = false, // weird handling of website + isSearchSupported = false, isMultipleTagsSupported = false, ) @@ -183,4 +200,4 @@ internal class BacaKomikpics(context: MangaLoaderContext) : } private fun String.nullIfEmpty() = ifBlank { null } -} \ No newline at end of file +} From 25431ab89cb8c7d5749656db6b74c5cd7bc82b43 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:42:47 +0200 Subject: [PATCH 224/434] Update AinzScans.kt --- src/main/kotlin/tsuki/site/id/AinzScans.kt | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/AinzScans.kt b/src/main/kotlin/tsuki/site/id/AinzScans.kt index 75ba679c..8f804cdb 100644 --- a/src/main/kotlin/tsuki/site/id/AinzScans.kt +++ b/src/main/kotlin/tsuki/site/id/AinzScans.kt @@ -5,8 +5,23 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.urlEncoded import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.json.JSONArray @@ -18,6 +33,7 @@ import java.util.TimeZone import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.jsoup.Jsoup +import kotlin.collections.toSet @MangaSourceParser("AINZSCANS", "AinzScans", "id") internal class AinzScans(context: MangaLoaderContext) : From b7366e1a56eecdf6e2329c7fcd0254c5a7ebdce3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:51:46 +0200 Subject: [PATCH 225/434] Update Kiryuu.kt --- src/main/kotlin/tsuki/site/id/Kiryuu.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/Kiryuu.kt b/src/main/kotlin/tsuki/site/id/Kiryuu.kt index b1703213..8bc65fa1 100644 --- a/src/main/kotlin/tsuki/site/id/Kiryuu.kt +++ b/src/main/kotlin/tsuki/site/id/Kiryuu.kt @@ -2,9 +2,10 @@ package tsuki.site.id import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource import tsuki.parsers.NatsuParser +import tsuki.model.MangaParserSource + @MangaSourceParser("KIRYUU", "Kiryuu", "id") internal class Kiryuu(context: MangaLoaderContext) : NatsuParser(context, MangaParserSource.KIRYUU, "v7.kiryuu.to") From 26afb6d268cf7efc8dc2663634260c13190fe4d2 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:52:39 +0200 Subject: [PATCH 226/434] Add model imports to KomikCast.kt Added multiple model imports to enhance functionality. --- src/main/kotlin/tsuki/site/id/KomikCast.kt | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/KomikCast.kt b/src/main/kotlin/tsuki/site/id/KomikCast.kt index 276c8bd7..72c6d264 100644 --- a/src/main/kotlin/tsuki/site/id/KomikCast.kt +++ b/src/main/kotlin/tsuki/site/id/KomikCast.kt @@ -5,7 +5,19 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentType +import tsuki.model.SortOrder + import tsuki.util.generateUid import tsuki.util.oneOrThrowIfMany @@ -15,6 +27,7 @@ import java.text.DecimalFormatSymbols import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale +import kotlin.collections.toSet @MangaSourceParser("KOMIKCAST", "KomikCast", "id") internal class Komikcast(context: MangaLoaderContext) : From 770175e7580f4a476394457437a9b383c8d558d5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:57:26 +0200 Subject: [PATCH 227/434] Update Komiku.kt --- src/main/kotlin/tsuki/site/id/Komiku.kt | 29 ++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Komiku.kt b/src/main/kotlin/tsuki/site/id/Komiku.kt index 5f7e5bb9..55924925 100644 --- a/src/main/kotlin/tsuki/site/id/Komiku.kt +++ b/src/main/kotlin/tsuki/site/id/Komiku.kt @@ -4,8 +4,31 @@ import okhttp3.Headers import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MangaReaderParser -import tsuki.model.* -import tsuki.util.* + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import org.jsoup.nodes.Document import java.text.SimpleDateFormat @@ -16,7 +39,7 @@ import java.util.EnumSet internal class Komiku(context: MangaLoaderContext) : MangaReaderParser(context, MangaParserSource.KOMIKU, "komiku.org", 10, 10) { - private val apiDomain = "api.komiku.org" + private val apiDomain = "api.$domain" override val datePattern = "dd/MM/yyyy" override val selectPage = "#Baca_Komik img" override val selectTestScript = "script:containsData(thisIsNeverFound)" From 219f9294afe692755b107203080a51ac1b4cc0ac Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 12:59:47 +0200 Subject: [PATCH 228/434] Update MGKomik.kt --- src/main/kotlin/tsuki/site/id/MGKomik.kt | 48 ++++++++++++------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/MGKomik.kt b/src/main/kotlin/tsuki/site/id/MGKomik.kt index 83d3cb54..a4b54960 100644 --- a/src/main/kotlin/tsuki/site/id/MGKomik.kt +++ b/src/main/kotlin/tsuki/site/id/MGKomik.kt @@ -5,14 +5,19 @@ import tsuki.MangaSourceParser import tsuki.MangaParserAuthProvider import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.parseHtml import okhttp3.Interceptor import androidx.collection.arraySetOf import okhttp3.Response - @MangaSourceParser("MGKOMIK", "MGKomik", "id") internal class MGKomik(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MGKOMIK, "id.mgkomik.cc"), Interceptor, MangaParserAuthProvider { @@ -30,28 +35,23 @@ internal class MGKomik(context: MangaLoaderContext) : path.contains("wp-json") || path.endsWith("/ajax/chapters") - return if (isAjax) { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .header("X-Requested-With", "XMLHttpRequest") - .header("Sec-Fetch-Dest", "empty") - .header("Sec-Fetch-Mode", "cors") - .header("Sec-Fetch-Site", "same-origin") - .header("Origin", "https://$domain") - .header("Priority", "u=1, i") - .removeHeader("Sec-Fetch-User") - .removeHeader("Upgrade-Insecure-Requests") - .build() - ) - } else { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .build() - ) + val builder = request.newBuilder() + .header("Sec-CH-UA-Model", "\"\"") + + if (isAjax) { + builder.header("X-Requested-With", "XMLHttpRequest") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Origin", "https://$domain") + .header("Priority", "u=1, i") + .removeHeader("Sec-Fetch-User") + .removeHeader("Upgrade-Insecure-Requests") } + + return chain.proceed(builder.build()) } + override suspend fun getListPage( page: Int, order: SortOrder, @@ -179,4 +179,4 @@ internal class MGKomik(context: MangaLoaderContext) : MangaTag("Yuri", "yuri", source), ) -} \ No newline at end of file +} From 4df5bf4e74f70c90c7451c68f8e4769a98a5114f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:02:22 +0200 Subject: [PATCH 229/434] Update ReYume.kt --- src/main/kotlin/tsuki/site/id/ReYume.kt | 26 ++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/ReYume.kt b/src/main/kotlin/tsuki/site/id/ReYume.kt index 6c73e9b1..1dd7eb2b 100644 --- a/src/main/kotlin/tsuki/site/id/ReYume.kt +++ b/src/main/kotlin/tsuki/site/id/ReYume.kt @@ -4,14 +4,30 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.ZeistMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import tsuki.util.json.asTypedList import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document import java.text.SimpleDateFormat +import kotlin.collections.dropLast @MangaSourceParser("REYUME", "ReYume", "id") internal class ReYume(context: MangaLoaderContext) : @@ -124,14 +140,14 @@ internal class ReYume(context: MangaLoaderContext) : val mangaTitle = doc.selectFirst("h1#post-title")?.text().orEmpty() return json.mapIndexedNotNull { i, j -> - val name = j.getJSONObject("title").getString("\$t") + val name = j.getJSONObject("title").getString($$"$t") val chapterName = if (mangaTitle.isNotEmpty() && name.contains(mangaTitle, ignoreCase = true)) { name.replace(mangaTitle, "", ignoreCase = true).trim().trim('-').trim().takeIf { it.isNotEmpty() } ?: name } else { name } val href = j.getJSONArray("link").asTypedList<JSONObject>().first { it.getString("rel") == "alternate" }.getString("href") - val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") + val dateText = j.getJSONObject("published").getString($$"$t").substringBefore("T") val slug = mangaUrl.substringAfterLast('/') val slugChapter = href.substringAfterLast('/') if (slug == slugChapter) return@mapIndexedNotNull null @@ -176,4 +192,4 @@ internal class ReYume(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} From f9d5b1195ac82fff3f2cb874397e70a3efc7eb0d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:06:55 +0200 Subject: [PATCH 230/434] Update Shinigami.kt --- src/main/kotlin/tsuki/site/id/Shinigami.kt | 26 +++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Shinigami.kt b/src/main/kotlin/tsuki/site/id/Shinigami.kt index 2f80aaf7..29901a9c 100644 --- a/src/main/kotlin/tsuki/site/id/Shinigami.kt +++ b/src/main/kotlin/tsuki/site/id/Shinigami.kt @@ -4,9 +4,29 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.* + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import okhttp3.Headers import java.text.SimpleDateFormat From 6267121b5657cd0bb56ea06d3ba012271547f42e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:08:27 +0200 Subject: [PATCH 231/434] Update Softkomik.kt --- src/main/kotlin/tsuki/site/id/Softkomik.kt | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Softkomik.kt b/src/main/kotlin/tsuki/site/id/Softkomik.kt index 218ed40f..5c0713b4 100644 --- a/src/main/kotlin/tsuki/site/id/Softkomik.kt +++ b/src/main/kotlin/tsuki/site/id/Softkomik.kt @@ -6,8 +6,25 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import kotlinx.coroutines.runBlocking import kotlinx.coroutines.sync.Mutex From 8ad4deebfb3e594b74bc461deb5f7bd7eb657976 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:09:41 +0200 Subject: [PATCH 232/434] Update Westmanga.kt --- src/main/kotlin/tsuki/site/id/Westmanga.kt | 30 +++++++++++++++++----- 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Westmanga.kt b/src/main/kotlin/tsuki/site/id/Westmanga.kt index 7e413752..3339c00e 100644 --- a/src/main/kotlin/tsuki/site/id/Westmanga.kt +++ b/src/main/kotlin/tsuki/site/id/Westmanga.kt @@ -5,8 +5,22 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson import tsuki.util.json.getLongOrDefault import tsuki.util.json.getStringOrNull import tsuki.util.json.mapJSON @@ -146,11 +160,13 @@ internal class WestmangaParser(context: MangaLoaderContext) : data.optJSONArray("genres")?.let { genres -> for (i in 0 until genres.length()) { val genre = genres.getJSONObject(i) - add(MangaTag( - title = genre.getString("name"), - key = genre.getInt("id").toString(), - source = source, - )) + add( + MangaTag( + title = genre.getString("name"), + key = genre.getInt("id").toString(), + source = source, + ) + ) } } } From 94032f9ec76847dc966b33580281ec3164453aa6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:10:58 +0200 Subject: [PATCH 233/434] Update Roxinha.kt --- src/main/kotlin/tsuki/site/pt/Roxinha.kt | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/pt/Roxinha.kt b/src/main/kotlin/tsuki/site/pt/Roxinha.kt index 02a3d1b2..82efa58b 100644 --- a/src/main/kotlin/tsuki/site/pt/Roxinha.kt +++ b/src/main/kotlin/tsuki/site/pt/Roxinha.kt @@ -5,8 +5,21 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl @@ -34,7 +47,7 @@ internal class Roxinha(context: MangaLoaderContext) : isSearchWithFiltersSupported = true, isMultipleTagsSupported = false, - ) + ) private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") @@ -219,4 +232,4 @@ internal class Roxinha(context: MangaLoaderContext) : private fun String.nullIfEmpty() = ifBlank { null } private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} \ No newline at end of file +} From 65fcaedb2ade31df629743a535465df91b760d16 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:14:06 +0200 Subject: [PATCH 234/434] Update GalleryAdultsParser.kt --- .../kotlin/tsuki/parsers/GalleryAdultsParser.kt | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt index 40984908..7514ac1d 100644 --- a/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt @@ -89,22 +89,16 @@ internal abstract class GalleryAdultsParser( ), ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val url = buildString { append("https://") append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?q=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) append("&") } - else -> { val tag = filter.tags.oneOrThrowIfMany() val lang = filter.locale @@ -158,9 +152,7 @@ internal abstract class GalleryAdultsParser( ) } } - - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. + private suspend fun fetchAvailableTags(): Set<MangaTag> { return coroutineScope { (1..3).map { page -> From efe9eded50e990bfeff729a24120ce8e3d4f2a1f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:14:40 +0200 Subject: [PATCH 235/434] Update Multporn.kt --- src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt index 395e25dc..cc1f14c4 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt @@ -78,14 +78,13 @@ internal class Multporn(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?search_api_views_fulltext=") - val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> + val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "+") { part -> part.urlEncoded() } append(encodedQuery) From 61d092a6b833d8475a907c0aa75470185ef0f4a3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:15:38 +0200 Subject: [PATCH 236/434] Update GalleryParser.kt --- src/main/kotlin/tsuki/parsers/GalleryParser.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/GalleryParser.kt b/src/main/kotlin/tsuki/parsers/GalleryParser.kt index f873b1d9..d2957e4b 100644 --- a/src/main/kotlin/tsuki/parsers/GalleryParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryParser.kt @@ -58,10 +58,9 @@ internal abstract class GalleryParser( override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = urlBuilder().apply { when { - !query.isNullOrEmpty() -> addQueryParameter("search", query) + !filter.query.isNullOrEmpty() -> addQueryParameter("search", filter.query) filter.tags.isNotEmpty() -> addPathSegments(filter.tags.first().key) order == SortOrder.POPULARITY -> addPathSegment("hot") } From 1a1d7f6eff79a46f373cd213267554ddabf4fd34 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:16:04 +0200 Subject: [PATCH 237/434] Update HeanCmsParser.kt --- src/main/kotlin/tsuki/parsers/HeanCmsParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt index 2a222c84..d4b4e8ea 100644 --- a/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt +++ b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt @@ -83,14 +83,13 @@ internal abstract class HeanCmsParser( protected open val paramsUpdated = "latest" override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(apiPath) append("/query?query_string=") - query?.let { - append(query.urlEncoded()) + filter.query?.let { + append(filter.query.urlEncoded()) } append("&series_type=Comic&perPage=$pageSize") From 82d02539453742f8a76267d3aa395e332b40cbc7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:16:46 +0200 Subject: [PATCH 238/434] Update LikeMangaParser.kt --- src/main/kotlin/tsuki/parsers/LikeMangaParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt index fea3a4fb..127b408e 100644 --- a/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt @@ -85,17 +85,16 @@ internal abstract class LikeMangaParser( ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) append("/?act=search") - query?.let { + filter.query?.let { append("&f") append("[keyword]".urlEncoded()) append("=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&f") From a9feb706dc4c3a4fce172c133a1dc39651bf0242 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:17:21 +0200 Subject: [PATCH 239/434] Refactor Manhwa18CC class and update imports --- src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt index 40a283cc..e2883830 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt @@ -94,15 +94,14 @@ internal class Manhwa18CC(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page.toString()) } From f4e05dda90bbf5ce560cf113a9d81d182743fac8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:17:54 +0200 Subject: [PATCH 240/434] Update NHentaiXXXParser.kt --- src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index a33673ee..064e54fa 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -68,7 +68,6 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) @@ -90,7 +89,7 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : append("/search/?key=") val parts = tags.map { it.title }.toMutableList() - if (!query.isNullOrEmpty()) parts.add(query.urlEncoded()) + if (!filter.query.isNullOrEmpty()) parts.add(filter.query.urlEncoded()) append(parts.joinToString("+")) } } @@ -174,7 +173,7 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : webClient.httpHead(newUrl) newUrl } catch (_: Exception) { - null // Skip errors and continue checking other formats + null } } } From aee26562d03a68de663542321e98e6b0916584e5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:18:18 +0200 Subject: [PATCH 241/434] Update MadthemeParser.kt --- src/main/kotlin/tsuki/parsers/MadthemeParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadthemeParser.kt b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt index 0336e3cf..6838cd81 100644 --- a/src/main/kotlin/tsuki/parsers/MadthemeParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt @@ -136,7 +136,6 @@ internal abstract class MadthemeParser( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) @@ -146,9 +145,9 @@ internal abstract class MadthemeParser( append("?page=") append(page.toString()) - query?.let { + filter.query?.let { append("&q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&sort=") From aff5e3f68eeef63b59271f01a04ce1157ee3835f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:19:24 +0200 Subject: [PATCH 242/434] Update FlameComics.kt --- src/main/kotlin/tsuki/site/en/FlameComics.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/FlameComics.kt b/src/main/kotlin/tsuki/site/en/FlameComics.kt index 9692fb89..a275c8dc 100644 --- a/src/main/kotlin/tsuki/site/en/FlameComics.kt +++ b/src/main/kotlin/tsuki/site/en/FlameComics.kt @@ -75,8 +75,7 @@ internal class FlameComics(context: MangaLoaderContext) : } override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query - val hasSearchQuery = !query.isNullOrEmpty() + val hasSearchQuery = !filter.query.isNullOrEmpty() val hasTagFilter = filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() val rawMangas = if (hasSearchQuery || hasTagFilter || order != SortOrder.UPDATED) { @@ -118,7 +117,7 @@ internal class FlameComics(context: MangaLoaderContext) : val filteredByTags = rawMangas.filter { (manga, _, _) -> manga.tags.matches(filter) } val filteredManga = if (hasSearchQuery) { - val normalizedQuery = removeSpecialCharsRegex.replace(query.lowercase(), "") + val normalizedQuery = removeSpecialCharsRegex.replace(filter.query!!.lowercase(), "") filteredByTags.filter { (manga, _, _) -> val titles = mutableListOf(manga.title) titles.addAll(manga.altTitles) From c4e0ec156bb64329ba7abc8f0bdb4f991ac1c6e7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:20:34 +0200 Subject: [PATCH 243/434] Update Manhwa18Com.kt --- src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt index 220bd7a0..ad005d59 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt @@ -100,12 +100,11 @@ internal class Manhwa18Com(context: MangaLoaderContext) : } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } val url = buildString { append("https://$domain/tim-kiem?page=$page") - if (query != null) { + if (filter.query != null) { append("&q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&accept_genres=") if (filter.tags.isNotEmpty()) { From c8cca458b69d8150772ea795eaa693287fe79220 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:20:59 +0200 Subject: [PATCH 244/434] Update HentaiFox.kt --- src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt index 4d309389..8f5f1efb 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt @@ -45,9 +45,8 @@ internal class HentaiFox(context: MangaLoaderContext) : append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?q=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) if (page > 1) { append("&page=") append(page.toString()) @@ -133,4 +132,4 @@ internal class HentaiFox(context: MangaLoaderContext) : } return joiner.complete() } -} \ No newline at end of file +} From 6aa6d4e66045f1510f40d0363f8e5b354a0697c1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:21:18 +0200 Subject: [PATCH 245/434] Update ReYume.kt --- src/main/kotlin/tsuki/site/id/ReYume.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/ReYume.kt b/src/main/kotlin/tsuki/site/id/ReYume.kt index 1dd7eb2b..8265b539 100644 --- a/src/main/kotlin/tsuki/site/id/ReYume.kt +++ b/src/main/kotlin/tsuki/site/id/ReYume.kt @@ -42,20 +42,19 @@ internal class ReYume(context: MangaLoaderContext) : override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query val url = buildString { append("https://") append(domain) append("/feeds/posts/default/-/") when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("Series") append("?alt=json&orderby=published&max-results=") append((maxMangaResults + 1).toString()) append("&start-index=") append(startIndex.toString()) append("&q=label:Series+") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } else -> { if (filter.tags.isNotEmpty()) { From 81778e65ca0ebd3f2d43dc16ff7c4c3705dbf427 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:23:36 +0200 Subject: [PATCH 246/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 33 ++++++++----------------- 1 file changed, 10 insertions(+), 23 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index 8812d8ac..c8152d06 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -119,13 +119,8 @@ internal class MangaK(context: MangaLoaderContext) : } } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val allExcludedTags = filter.tagsExclude.map { it.key }.toMutableSet() - val query = filter.query val url = buildString { append(apiUrl) append("/titles/search?page=") @@ -144,11 +139,10 @@ internal class MangaK(context: MangaLoaderContext) : else -> "latest" }, ) - - if (!query.isNullOrEmpty()) { + + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { q -> append("&q=") - // api does not like commans and character limits to 60-70 - append(query.replace(",", "").urlEncoded()) + append(q.replace(",", "").urlEncoded()) } if (filter.tags.isNotEmpty()) { @@ -233,16 +227,14 @@ internal class MangaK(context: MangaLoaderContext) : source = source, ) } - - + private fun String?.toContentRating() = when (this) { "safe" -> ContentRating.SAFE "suggestive" -> ContentRating.SUGGESTIVE "erotica", "pornographic" -> ContentRating.ADULT else -> null } - - + private fun JSONObject.parseAltTitles(): Set<String> { val result = mutableSetOf<String>() optJSONArray("alt_names")?.let { arr -> @@ -253,15 +245,13 @@ internal class MangaK(context: MangaLoaderContext) : optString("alt_name").nullIfEmpty()?.let { result.add(it) } return result } - - + private fun JSONObject.parseTags(): Set<MangaTag> = optJSONArray("genres")?.mapJSONNotNullToSet { genre -> val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) }.orEmpty() - - + override suspend fun getDetails(manga: Manga): Manga { val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() .getJSONObject("data").getJSONObject("title") @@ -283,8 +273,7 @@ internal class MangaK(context: MangaLoaderContext) : chapters = fetchChapters(manga.url, cv), ) } - - + private suspend fun fetchChapters(id: String, cv: Long): List<MangaChapter> { val json = webClient.httpGet("$apiUrl/titles/$id/chapters?cv=$cv").parseJson() val chapters = json.getJSONObject("data").getJSONArray("chapters").mapJSON { it } @@ -304,8 +293,7 @@ internal class MangaK(context: MangaLoaderContext) : ) }.reversed() } - - + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val doc = webClient.httpGet("https://$domain${chapter.url}").parseHtml() val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() @@ -326,7 +314,6 @@ internal class MangaK(context: MangaLoaderContext) : } } - private fun Double.toRating() = if (this > 0.0) (this / 5.0).toFloat() else RATING_UNKNOWN private fun String?.toMangaState() = when (this?.lowercase(Locale.US)) { From e7eaf17d559e552727cefeaa0c89bc98c9b08922 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:33:20 +0200 Subject: [PATCH 247/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index c8152d06..6472ee2a 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -139,10 +139,11 @@ internal class MangaK(context: MangaLoaderContext) : else -> "latest" }, ) - + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { q -> append("&q=") - append(q.replace(",", "").urlEncoded()) + // API rejects queries longer than ~50 chars + append(q.take(50).replace(",", "").urlEncoded()) } if (filter.tags.isNotEmpty()) { @@ -227,14 +228,14 @@ internal class MangaK(context: MangaLoaderContext) : source = source, ) } - + private fun String?.toContentRating() = when (this) { "safe" -> ContentRating.SAFE "suggestive" -> ContentRating.SUGGESTIVE "erotica", "pornographic" -> ContentRating.ADULT else -> null } - + private fun JSONObject.parseAltTitles(): Set<String> { val result = mutableSetOf<String>() optJSONArray("alt_names")?.let { arr -> @@ -245,13 +246,13 @@ internal class MangaK(context: MangaLoaderContext) : optString("alt_name").nullIfEmpty()?.let { result.add(it) } return result } - + private fun JSONObject.parseTags(): Set<MangaTag> = optJSONArray("genres")?.mapJSONNotNullToSet { genre -> val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) }.orEmpty() - + override suspend fun getDetails(manga: Manga): Manga { val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() .getJSONObject("data").getJSONObject("title") @@ -273,7 +274,7 @@ internal class MangaK(context: MangaLoaderContext) : chapters = fetchChapters(manga.url, cv), ) } - + private suspend fun fetchChapters(id: String, cv: Long): List<MangaChapter> { val json = webClient.httpGet("$apiUrl/titles/$id/chapters?cv=$cv").parseJson() val chapters = json.getJSONObject("data").getJSONArray("chapters").mapJSON { it } @@ -293,7 +294,7 @@ internal class MangaK(context: MangaLoaderContext) : ) }.reversed() } - + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val doc = webClient.httpGet("https://$domain${chapter.url}").parseHtml() val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() From 45f87a6033c71ce7364708aba2e55ddf18635c9b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:34:16 +0200 Subject: [PATCH 248/434] Update MangaReaderParser.kt --- src/main/kotlin/tsuki/parsers/MangaReaderParser.kt | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt index 133aa5af..24f19b9b 100644 --- a/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt +++ b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt @@ -113,23 +113,18 @@ internal abstract class MangaReaderParser( protected val mutex = Mutex() override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) - when { - - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/page/") append(page.toString()) append("/?s=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } - else -> { append(listUrl) - append("/?order=") append( when (order) { @@ -141,21 +136,18 @@ internal abstract class MangaReaderParser( else -> "" }, ) - filter.tags.forEach { append("&") append("genre[]".urlEncoded()) append("=") append(it.key) } - filter.tagsExclude.forEach { append("&") append("genre[]".urlEncoded()) append("=-") append(it.key) } - if (filter.states.isNotEmpty()) { filter.states.oneOrThrowIfMany()?.let { append("&status=") @@ -167,7 +159,6 @@ internal abstract class MangaReaderParser( } } } - filter.types.oneOrThrowIfMany()?.let { append("&type=") append( @@ -181,7 +172,6 @@ internal abstract class MangaReaderParser( }, ) } - append("&page=") append(page.toString()) } From 35b6a2be9b99e650f4631c261d9c9ae87312c5db Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:34:36 +0200 Subject: [PATCH 249/434] Update HentaiEnvy.kt --- src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt index c560534a..b33087df 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt @@ -52,9 +52,8 @@ internal class HentaiEnvy(context: MangaLoaderContext) : append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?s_key=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) append("&") } @@ -87,4 +86,4 @@ internal class HentaiEnvy(context: MangaLoaderContext) : } return parseMangaList(webClient.httpGet(url).parseHtml()) } -} \ No newline at end of file +} From 9e017780374bc5c7962273659a0e53a844c8e211 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:35:18 +0200 Subject: [PATCH 250/434] Update Manga18Parser.kt --- src/main/kotlin/tsuki/parsers/Manga18Parser.kt | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt index 5d1a40dd..50d2bcf5 100644 --- a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -126,7 +126,6 @@ internal abstract class Manga18Parser( protected open val selectChapter = "div.chapter_box li" override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim().orEmpty() val tag = filter.tags.firstOrNull() val url = buildString(120) { @@ -135,7 +134,7 @@ internal abstract class Manga18Parser( append('/') when { - tag != null && query.isNotEmpty() -> { + tag != null && filter.query.isNotEmpty() -> { throw IllegalArgumentException("Search is not supported with tags") } tag != null -> { @@ -144,11 +143,11 @@ internal abstract class Manga18Parser( append('/') append(page) } - query.isNotEmpty() -> { + filter.query.isNotEmpty() -> { append(listUrl) append(page) append("?search=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&order_by=latest") } else -> { @@ -156,7 +155,7 @@ internal abstract class Manga18Parser( append(page) } } - if (query.isEmpty()) { + if (filter.query!!.isEmpty()) { append("?order_by=") when (order) { SortOrder.POPULARITY -> append("views") From 3aa847dbf874bd38e13440d1027eba9a8dcfe428 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 13:35:40 +0200 Subject: [PATCH 251/434] Update Manga18Parser.kt --- src/main/kotlin/tsuki/parsers/Manga18Parser.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt index 50d2bcf5..467bfffe 100644 --- a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -134,7 +134,7 @@ internal abstract class Manga18Parser( append('/') when { - tag != null && filter.query.isNotEmpty() -> { + tag != null && filter.query!!.isNotEmpty() -> { throw IllegalArgumentException("Search is not supported with tags") } tag != null -> { @@ -143,7 +143,7 @@ internal abstract class Manga18Parser( append('/') append(page) } - filter.query.isNotEmpty() -> { + filter.query!!.isNotEmpty() -> { append(listUrl) append(page) append("?search=") From 638afdf7c12d827d7abf5b5778106192b562c1bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:12:29 +0200 Subject: [PATCH 252/434] Update Atsumaru.kt --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 270 +++++++++++----------- 1 file changed, 132 insertions(+), 138 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index e5c724f7..f3372a98 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -25,59 +25,12 @@ import tsuki.util.parseJson import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.delay -import kotlin.math.pow -import kotlin.math.roundToLong -import kotlin.random.Random -import kotlin.time.Duration.Companion.milliseconds import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject import java.util.EnumSet -suspend fun <T> retryWithBackoff( - maxRetries: Int = 3, - baseDelay: Long = 800L, - maxDelay: Long = 10_000L, - factor: Double = 2.0, - jitter: Double = 0.3, // ± 30% - isTransient: (Exception) -> Boolean = { isTransientHttpException(it) }, - block: suspend () -> T -): T { - var lastException: Exception? = null - repeat(maxRetries) { attempt -> - try { - return block() - } catch (e: Exception) { - if (!isTransient(e)) throw e - lastException = e - if (attempt < maxRetries - 1) { - val delayMs = (baseDelay * factor.pow(attempt)) - .toLong() - .coerceAtMost(maxDelay) - .let { it + (it * jitter * (Random.nextDouble() * 2 - 1)).roundToLong() } - .coerceAtLeast(0) - delay(delayMs.milliseconds) - } - } - } - throw lastException!! -} - -fun isTransientHttpException(e: Exception): Boolean { - val msg = e.message.orEmpty() - return when { - msg.contains("502", ignoreCase = true) -> true - msg.contains("503", ignoreCase = true) -> true - msg.contains("504", ignoreCase = true) -> true - msg.contains("timeout", ignoreCase = true) -> true - msg.contains("connection reset", ignoreCase = true) -> true - msg.contains("unreachable", ignoreCase = true) -> true - else -> false - } -} - @MangaSourceParser("ATSUMARU", "Atsumaru", "en") internal class Atsumaru(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.ATSUMARU, pageSize = 40) { @@ -104,7 +57,9 @@ internal class Atsumaru(context: MangaLoaderContext) : SortOrder.POPULARITY, // trending (views) SortOrder.UPDATED, // recently added (dateAdded) SortOrder.ADDED, // recently new added mangas - SortOrder.RELEVANCE // chapters count + SortOrder.RELEVANCE, // rising + SortOrder.ALPHABETICAL, + SortOrder.RATING, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -206,9 +161,9 @@ internal class Atsumaru(context: MangaLoaderContext) : availableTags = (allGenres.map { (name, id) -> MangaTag(name, "genre:$id", source) } + allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, MangaState.ABANDONED ), availableContentTypes = EnumSet.of( @@ -219,13 +174,60 @@ internal class Atsumaru(context: MangaLoaderContext) : ), ) - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> = getSearchPage(page, order, filter) + private suspend fun fetchRising(page: Int) = fetchList("rising", page) + private suspend fun fetchTopRated(page: Int) = fetchList("topRated", page) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + return when (order) { + SortOrder.RELEVANCE -> fetchRising(page) + SortOrder.RATING -> fetchTopRated(page) + else -> getSearchPage(page, order, filter) + } + } + + private suspend fun fetchList(path: String, page: Int): List<Manga> { + val offset = (page - 1) * pageSize + val url = "$baseUrl/api/home2/$path".toHttpUrl().newBuilder() + .addQueryParameter("offset", offset.toString()) + .addQueryParameter("limit", pageSize.toString()) + .build() + val json = webClient.httpGet(url, apiHeaders).parseJson() + val items = json.optJSONArray("items") ?: return emptyList() + val showAdult = config[showAdultKey] + return (0 until items.length()).mapNotNull { i -> + val obj = items.getJSONObject(i) + val isAdult = obj.optBoolean("isAdult", false) + if (!showAdult && isAdult) return@mapNotNull null + val id = obj.getString("id") + val title = obj.getString("title") + val image = obj.optString("mediumImage").takeIf { it.isNotEmpty() } ?: obj.optString("image") + val coverUrl = if (image.isNotEmpty()) { + if (image.startsWith("http") || image.startsWith("//")) image + else "https://$domain/static/$image" + } else null + val type = obj.optString("type") + val rating = obj.optDouble("mbRating", -1.0).takeIf { it >= 0 }?.let { (it / 10.0).toFloat() } ?: RATING_UNKNOWN + Manga( + id = generateUid(id), + title = title, + altTitles = emptySet(), + url = "/manga/$id", + publicUrl = "$baseUrl/manga/$id", + rating = rating, + coverUrl = coverUrl, + tags = setOfNotNull(type.takeIf { it.isNotEmpty() }?.let { MangaTag(it, "type:$it", source) }), + source = source, + contentRating = if (isAdult) ContentRating.ADULT else null, + state = null, + authors = emptySet(), + ) + } + } private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query ?: "*" val url = "$baseUrl/collections/manga/documents/search".toHttpUrl().newBuilder().apply { - addQueryParameter("q", query) - if (query != "*") { + addQueryParameter("q", filter.query) + if (filter.query != "*") { addQueryParameter("query_by", "title,englishTitle,otherNames,authors") addQueryParameter("query_by_weights", "4,3,2,1") addQueryParameter("num_typos", "4,3,2,1") @@ -313,15 +315,15 @@ internal class Atsumaru(context: MangaLoaderContext) : SortOrder.POPULARITY -> "trending:desc" SortOrder.UPDATED -> "released:desc" SortOrder.ADDED -> "dateAdded:desc" - SortOrder.RELEVANCE -> "chapterCount:desc" + SortOrder.RELEVANCE -> "rising:desc" + SortOrder.ALPHABETICAL -> "title:desc" + SortOrder.RATING -> "topRated:desc" else -> "released:desc" } addQueryParameter("sort_by", sort) }.build() - val json = retryWithBackoff { - webClient.httpGet(url, apiHeaders).parseJson() - } + val json = webClient.httpGet(url, apiHeaders).parseJson() val hits = json.optJSONArray("hits") ?: return emptyList() return (0 until hits.length()).map { i -> hits.getJSONObject(i).getJSONObject("document").toManga() @@ -338,90 +340,86 @@ internal class Atsumaru(context: MangaLoaderContext) : synchronized(detailsCacheLock) { detailsCache[manga.url]?.let { return it } } - val result = try { - retryWithBackoff { - coroutineScope { - val mangaId = manga.url.substringAfterLast("/") - - val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } - val pageJson = pageDeferred.await() - val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga - - val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } - val description = mangaPage.optString("synopsis") ?: manga.description - - val rawRating = mangaPage.optDouble("avgRating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val posterObj = mangaPage.optJSONObject("poster") - val posterImage = posterObj?.optString("mediumImage") - val coverUrl = if (!posterImage.isNullOrEmpty()) { - "https://$domain/static/$posterImage" - } else manga.coverUrl - - val authors = mangaPage.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() - } ?: emptySet() - - val statusText = mangaPage.optString("status").orEmpty() - val state = when (statusText.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "canceled" -> MangaState.ABANDONED - else -> manga.state - } + coroutineScope { + val mangaId = manga.url.substringAfterLast("/") + + val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } + val pageJson = pageDeferred.await() + val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga + + val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } + val description = mangaPage.optString("synopsis") ?: manga.description + + val rawRating = mangaPage.optDouble("avgRating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val posterObj = mangaPage.optJSONObject("poster") + val posterImage = posterObj?.optString("mediumImage") + val coverUrl = if (!posterImage.isNullOrEmpty()) { + "https://$domain/static/$posterImage" + } else manga.coverUrl + + val authors = mangaPage.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() + } ?: emptySet() + + val statusText = mangaPage.optString("status").orEmpty() + val state = when (statusText.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> manga.state + } - val tagSet = mutableSetOf<MangaTag>() - mangaPage.optJSONArray("genres")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) - } + val tagSet = mutableSetOf<MangaTag>() + mangaPage.optJSONArray("genres")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) } - mangaPage.optJSONArray("tags")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) - } + } + mangaPage.optJSONArray("tags")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) } + } - val scanlators = mutableMapOf<String, String>() - mangaPage.optJSONArray("scanlators")?.let { scanArr -> - for (i in 0 until scanArr.length()) { - val sc = scanArr.optJSONObject(i) ?: continue - val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue - val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue - scanlators[id] = name - } + val scanlators = mutableMapOf<String, String>() + mangaPage.optJSONArray("scanlators")?.let { scanArr -> + for (i in 0 until scanArr.length()) { + val sc = scanArr.optJSONObject(i) ?: continue + val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue + val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue + scanlators[id] = name } - - val chapters = loadChapters(mangaId, scanlators) - - val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() - } ?: emptySet() - - manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - state = state, - tags = tagSet, - rating = rating, - chapters = chapters, - altTitles = altTitles, - ) } + + val chapters = loadChapters(mangaId, scanlators) + + val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() + } ?: emptySet() + + manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + state = state, + tags = tagSet, + rating = rating, + chapters = chapters, + altTitles = altTitles, + ) } } catch (_: Exception) { manga } - synchronized(detailsCacheLock) { detailsCache[manga.url] = result } @@ -429,9 +427,7 @@ internal class Atsumaru(context: MangaLoaderContext) : } private suspend fun loadChapters(mangaId: String, scanlators: Map<String, String>): List<MangaChapter> { - val chaptersJson = retryWithBackoff { - webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() - } + val chaptersJson = webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() val chapters = (0 until chaptersArray.length()).map { i -> @@ -480,9 +476,7 @@ internal class Atsumaru(context: MangaLoaderContext) : .addQueryParameter("mangaId", mangaId) .addQueryParameter("chapterId", chapterId) .build() - val json = retryWithBackoff { - webClient.httpGet(url, apiHeaders).parseJson() - } + val json = webClient.httpGet(url, apiHeaders).parseJson() val pagesArray = json.getJSONObject("readChapter").getJSONArray("pages") return (0 until pagesArray.length()).map { i -> val page = pagesArray.getJSONObject(i) From 8d93ca75298586805a1795b7139da569de481b5a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:13:05 +0200 Subject: [PATCH 253/434] Update ManhwaZ.kt --- src/main/kotlin/tsuki/site/en/ManhwaZ.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/ManhwaZ.kt b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt index 9d28b2cd..af8d3061 100644 --- a/src/main/kotlin/tsuki/site/en/ManhwaZ.kt +++ b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt @@ -46,14 +46,13 @@ internal class Manhwaz(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?s=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page.toString()) } From 9c6b0e94729a82738cfb65a38ef338f970505b64 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:14:55 +0200 Subject: [PATCH 254/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index f4d90686..75533438 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -120,13 +120,12 @@ abstract class MangaThemesia( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim().orEmpty() val url = buildString { append("https://$domain/$mangaDirectory/") - if (query.isNotEmpty()) { - append("?s=${query.urlEncoded()}") - } else { + if (filter.query.isNullOrEmpty()) { append("?page=$page") + } else { + append("?s=${filter.query.urlEncoded()}") } filter.states.firstOrNull()?.let { append("&status=") From 9ac658146efe4c7227b4fdc72f2ac8306fb2d91e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:15:22 +0200 Subject: [PATCH 255/434] Update ZeistMangaParser.kt --- src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt index 9b8ce3dc..a9058f70 100644 --- a/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt @@ -120,14 +120,13 @@ internal abstract class ZeistMangaParser( override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query val url = buildString { append("https://") append(domain) append("/feeds/posts/default/-/") when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append(mangaCategory) append("?alt=json&orderby=published&max-results=") append((maxMangaResults + 1).toString()) @@ -136,7 +135,7 @@ internal abstract class ZeistMangaParser( append("&q=label:") append(mangaCategory) append("+") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } else -> { From 403f6edfb826fb340fa32c70410caf313774f425 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:16:31 +0200 Subject: [PATCH 256/434] Update Mangago.kt --- src/main/kotlin/tsuki/site/en/Mangago.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangago.kt b/src/main/kotlin/tsuki/site/en/Mangago.kt index 4c94b896..7e7a2015 100644 --- a/src/main/kotlin/tsuki/site/en/Mangago.kt +++ b/src/main/kotlin/tsuki/site/en/Mangago.kt @@ -114,13 +114,12 @@ internal class MangagoParser(context: MangaLoaderContext) : private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query.orEmpty() - if (query.isNotEmpty()) { + if (filter.query != null) { val url = buildString { append("https://") append(domain) append("/r/l_search?name=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page) } From 634dc6e9dc47b1f65b32587e53d9d75cc18f5cc4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:18:07 +0200 Subject: [PATCH 257/434] Update Mangataro.kt --- src/main/kotlin/tsuki/site/en/Mangataro.kt | 21 +++------------------ 1 file changed, 3 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangataro.kt b/src/main/kotlin/tsuki/site/en/Mangataro.kt index 23cb56e9..e7ff1e32 100644 --- a/src/main/kotlin/tsuki/site/en/Mangataro.kt +++ b/src/main/kotlin/tsuki/site/en/Mangataro.kt @@ -79,7 +79,7 @@ internal class Mangataro(context: MangaLoaderContext) : override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = genreTags, availableStates = EnumSet.of( - MangaState.ONGOING, + MangaState.ONGOING, MangaState.FINISHED ), availableContentTypes = EnumSet.of( @@ -90,25 +90,17 @@ internal class Mangataro(context: MangaLoaderContext) : ), ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val payload = JSONObject().apply { put("page", page) put("search", filter.query.orEmpty()) - // Year val yearArray = JSONArray() if (filter.year > 0) yearArray.put(filter.year.toString()) put("years", yearArray.toString()) - // Genres (tags) put("genres", filter.tags.toGenrePayload()) - // Types val typeArray = JSONArray() filter.types.forEach { ct -> val apiType = when (ct) { @@ -122,7 +114,6 @@ internal class Mangataro(context: MangaLoaderContext) : } put("types", typeArray.toString()) - // Statuses val statusArray = JSONArray() filter.states.forEach { state -> val apiStatus = when (state) { @@ -211,13 +202,7 @@ internal class Mangataro(context: MangaLoaderContext) : } @Suppress("SameParameterValue") - private fun buildChapterUrl( - mangaId: String, - offset: Int, - limit: Int, - token: String, - timestamp: String, - ): String { + private fun buildChapterUrl(mangaId: String, offset: Int, limit: Int, token: String, timestamp: String): String { return HttpUrl.Builder() .scheme("https") .host(domain) From 727676abdfc5d77d40d92cc7e3e1a76eaf2bd9b8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:18:47 +0200 Subject: [PATCH 258/434] Update IMHentai.kt --- src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt index f53ab83b..b12d9cf4 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt @@ -84,11 +84,7 @@ internal class ImHentai(context: MangaLoaderContext) : keys.add(userAgentKey) } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val url = buildString { append("https://") append(domain) @@ -96,9 +92,8 @@ internal class ImHentai(context: MangaLoaderContext) : append(page.toString()) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("&key=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) } else -> { From e0d36673b83e6539d06cf5341960c2130afef988 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:20:24 +0200 Subject: [PATCH 259/434] Update Webtoons.kt --- src/main/kotlin/tsuki/site/all/Webtoons.kt | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt index 820e9221..bab400cb 100644 --- a/src/main/kotlin/tsuki/site/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -278,11 +278,10 @@ internal abstract class WebtoonsParser( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query.orEmpty() val currentContent = config[contentKey] ?: webtoons - if (query.startsWith("https://") || query.startsWith(ID_SEARCH_PREFIX)) { - return handleDirectSearch(query, currentContent) + if (filter.query!!.startsWith("https://") || filter.query!!.startsWith(ID_SEARCH_PREFIX)) { + return handleDirectSearch(filter.query!!, currentContent) } val selectedGenre = filter.tags.firstOrNull() @@ -291,12 +290,12 @@ internal abstract class WebtoonsParser( append("https://$domain/$languageCode") when { - query.isNotBlank() -> { + filter.query!!.isNotBlank() -> { append("/search") - var cleanQuery = query + var cleanQuery = filter.query var searchType: String? = null when { - cleanQuery.startsWith("type:originals ") -> { + cleanQuery!!.startsWith("type:originals ") -> { searchType = "originals" cleanQuery = cleanQuery.removePrefix("type:originals ").trim() } From 26eced616ca1fb1dd1e980542c3855f5c55fe97d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:23:04 +0200 Subject: [PATCH 260/434] revert --- src/main/kotlin/tsuki/site/all/Webtoons.kt | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt index bab400cb..820e9221 100644 --- a/src/main/kotlin/tsuki/site/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -278,10 +278,11 @@ internal abstract class WebtoonsParser( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query.orEmpty() val currentContent = config[contentKey] ?: webtoons - if (filter.query!!.startsWith("https://") || filter.query!!.startsWith(ID_SEARCH_PREFIX)) { - return handleDirectSearch(filter.query!!, currentContent) + if (query.startsWith("https://") || query.startsWith(ID_SEARCH_PREFIX)) { + return handleDirectSearch(query, currentContent) } val selectedGenre = filter.tags.firstOrNull() @@ -290,12 +291,12 @@ internal abstract class WebtoonsParser( append("https://$domain/$languageCode") when { - filter.query!!.isNotBlank() -> { + query.isNotBlank() -> { append("/search") - var cleanQuery = filter.query + var cleanQuery = query var searchType: String? = null when { - cleanQuery!!.startsWith("type:originals ") -> { + cleanQuery.startsWith("type:originals ") -> { searchType = "originals" cleanQuery = cleanQuery.removePrefix("type:originals ").trim() } From c5d9bc90667017cb05abc3d892d33d61e2226dc5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 8 Aug 2026 16:19:03 +0200 Subject: [PATCH 261/434] Create Wurmz.kt --- src/main/kotlin/tsuki/site/id/Wurmz.kt | 289 +++++++++++++++++++++++++ 1 file changed, 289 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Wurmz.kt diff --git a/src/main/kotlin/tsuki/site/id/Wurmz.kt b/src/main/kotlin/tsuki/site/id/Wurmz.kt new file mode 100644 index 00000000..02d2a92a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Wurmz.kt @@ -0,0 +1,289 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("WURMZ", "Wurmz", "id") +class Wurmz(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.WURMZ, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("wurmz.net") + + override val availableSortOrders: Set<SortOrder> = setOf( + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val allGenres = listOf( + "Fantasy", "Romance", "Drama", "Action", "Comedy", "Shounen", "Adventure", "Shoujo", + "School", "Slice of Life", "Seinen", "Supernatural", "Historical", "Josei", "Isekai", + "Webtoons", "Martial Art", "Harem", "Ecchi", "Reincarnation", "Magic", "Mystery", + "Psychological", "Yaoi", "Mature", "Horror", "Tragedy", "Shounen Ai", "Sci-fi", + "Demons", "Adaptation", "Gender Bender", "Game", "Yuri", "Shoujo Ai", "Thriller", + "Sport", "Cooking", "Crime", "Gore", "Royal Family", "Super Power", "Regression", + "Military", "Royalty", "Vampire", "Office Workers", "Transmigration", "Medical", + "Time Travel", "Childhood Friends", "Music", "Monsters", "Revenge", "College Life", + "Villainess", "Kids", "Mecha", "One Shot", "Police", "Omegaverse", "Girls", + "Reverse Harem", "Animals", "Ghosts", "Project", "Age Gap", "Showbiz", "Violence", + "Boys", "Shotacon", "Survival", "Beasts", "Bloody", "Crossdressing", "Delinguents", + "Dungeons", "Gyaru", "Non-human", "Post-Apocalyptic", "Zombies", "Doujinshi", + "18+", "Bodyswap", "NTR", "Yakuzas", "BDSM", "Lolicon", "Philosophical", "System", + "Aliens", "Hentai", "Anthology", "Fetish", "Incest", "Virtual Reality", "Dementia", + "Nakadashi", "Samurai", "4-Koma", "Cheating", "Femdom", "Mafia", "Milf", "Ninja", + "Cunnilingus", "Guideverse", "Infidelity", "Parodi", "Reverse Isekai", "Villain" + ) + + return MangaListFilterOptions( + availableTags = allGenres.map { genre -> + MangaTag(key = genre, title = genre, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + + val cards = doc.select("article.comic-card") + if (cards.isEmpty()) { + return emptyList() + } + return cards.mapNotNull { parseMangaCard(it) } + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return buildString { + append("https://") + append(domain) + append("/semua-komik") + + val params = mutableMapOf<String, String>() + + filter.query?.takeIf { it.isNotEmpty() }?.let { rawQuery -> + val cleaned = cleanQuery(rawQuery) + if (cleaned.isNotEmpty()) { + params["q"] = cleaned.urlEncoded() + } + } + + filter.types.firstOrNull()?.let { type -> + params["type"] = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + } + + filter.states.firstOrNull()?.let { state -> + params["status"] = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "tamat" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "drop" + else -> "" + } + } + + filter.tags.take(3).forEach { tag -> + params["genres[]"] = tag.key + } + + params["sort"] = sortParamForOrder(order) + + if (page > 1) { + params["page"] = page.toString() + } + + if (params.isNotEmpty()) { + append("?") + append(params.entries.joinToString("&") { (key, value) -> + val encodedKey = if (key.endsWith("[]")) key else key.urlEncoded() + "$encodedKey=${value.urlEncoded()}" + }) + } + } + } + + private fun cleanQuery(query: String): String { + return query + .replace(Regex("Bahasa Indonesia", RegexOption.IGNORE_CASE), "") + .trim() + .replace(Regex("\\s+"), " ") + } + + private fun sortParamForOrder(order: SortOrder): String { + return when (order) { + SortOrder.UPDATED -> "update" + SortOrder.ADDED -> "new" + SortOrder.ADDED_ASC -> "old" + SortOrder.POPULARITY_TODAY -> "popular_today" + SortOrder.POPULARITY_WEEK -> "popular_7d" + SortOrder.POPULARITY_MONTH -> "popular_30d" + SortOrder.POPULARITY -> "popular_all" + else -> "update" + } + } + + private fun parseMangaCard(card: Element): Manga? { + val link = card.selectFirst("a[href]") ?: return null + val href = link.attrAsRelativeUrl("href") + val titleEl = card.selectFirst("h2.comic-title") + val title = titleEl?.text() ?: return null + val coverEl = card.selectFirst("img") + val coverUrl = coverEl?.src() + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + authors = emptySet(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.selectFirst(".cover-frame img")?.src() ?: manga.coverUrl + + val desc = doc.selectFirst(".mt-5 p.text-muted")?.text() ?: "" + + val genreElements = doc.select(".flex.flex-wrap.gap-1.5 .chip") + val tags = genreElements.mapNotNull { a -> + val genreName = a.text() + MangaTag(key = genreName, title = genreName, source = source) + }.toSet() + + val statusText = doc.selectFirst(".status-badge")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + } + + val authorText = doc.selectFirst(".text-xs.text-muted.mt-2.5")?.text() + val authors = authorText?.substringAfter("Oleh ")?.substringBefore(" ·")?.split('|') + ?.map { it.trim() } + ?.filter { it.isNotEmpty() && it != "-?" } + ?.toSet() ?: emptySet() + + val altText = doc.selectFirst(".text-sm.text-faint.mt-1")?.text() + val altTitles = altText?.split("·")?.map { it.trim() }?.toSet() ?: emptySet() + + val chapterLinks = doc.select(".chap-cell") + val chapters = chapterLinks.map { a -> + val href = a.attrAsRelativeUrl("href") + val label = a.text().trim() + val number = label.extractChapterNumber() + MangaChapter( + id = generateUid(href), + url = href, + title = label, + number = number, + volume = 0, + uploadDate = 0, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + publicUrl = fullUrl, + coverUrl = cover, + description = desc, + tags = tags, + state = state, + authors = authors, + altTitles = altTitles, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val imgElements = doc.select(".reader-page img") + if (imgElements.isEmpty()) { + val lazyImgs = doc.select("img[data-src]") + if (lazyImgs.isNotEmpty()) { + return lazyImgs.map { img -> + val url = img.attr("data-src").toRelativeUrl(domain) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + throw ParseException("No images found on chapter page", fullUrl) + } + + return imgElements.map { img -> + val url = img.attr("src").ifEmpty { img.attr("data-src") }.toRelativeUrl(domain) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } +} From 510a39366daf5aba2b7a06add356f14ce7931142 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 8 Aug 2026 20:52:33 +0200 Subject: [PATCH 262/434] Update Wurmz.kt --- src/main/kotlin/tsuki/site/id/Wurmz.kt | 80 +++++++++++--------------- 1 file changed, 34 insertions(+), 46 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Wurmz.kt b/src/main/kotlin/tsuki/site/id/Wurmz.kt index 02d2a92a..a286eff6 100644 --- a/src/main/kotlin/tsuki/site/id/Wurmz.kt +++ b/src/main/kotlin/tsuki/site/id/Wurmz.kt @@ -55,27 +55,8 @@ class Wurmz(context: MangaLoaderContext) : ) override suspend fun getFilterOptions(): MangaListFilterOptions { - val allGenres = listOf( - "Fantasy", "Romance", "Drama", "Action", "Comedy", "Shounen", "Adventure", "Shoujo", - "School", "Slice of Life", "Seinen", "Supernatural", "Historical", "Josei", "Isekai", - "Webtoons", "Martial Art", "Harem", "Ecchi", "Reincarnation", "Magic", "Mystery", - "Psychological", "Yaoi", "Mature", "Horror", "Tragedy", "Shounen Ai", "Sci-fi", - "Demons", "Adaptation", "Gender Bender", "Game", "Yuri", "Shoujo Ai", "Thriller", - "Sport", "Cooking", "Crime", "Gore", "Royal Family", "Super Power", "Regression", - "Military", "Royalty", "Vampire", "Office Workers", "Transmigration", "Medical", - "Time Travel", "Childhood Friends", "Music", "Monsters", "Revenge", "College Life", - "Villainess", "Kids", "Mecha", "One Shot", "Police", "Omegaverse", "Girls", - "Reverse Harem", "Animals", "Ghosts", "Project", "Age Gap", "Showbiz", "Violence", - "Boys", "Shotacon", "Survival", "Beasts", "Bloody", "Crossdressing", "Delinguents", - "Dungeons", "Gyaru", "Non-human", "Post-Apocalyptic", "Zombies", "Doujinshi", - "18+", "Bodyswap", "NTR", "Yakuzas", "BDSM", "Lolicon", "Philosophical", "System", - "Aliens", "Hentai", "Anthology", "Fetish", "Incest", "Virtual Reality", "Dementia", - "Nakadashi", "Samurai", "4-Koma", "Cheating", "Femdom", "Mafia", "Milf", "Ninja", - "Cunnilingus", "Guideverse", "Infidelity", "Parodi", "Reverse Isekai", "Villain" - ) - return MangaListFilterOptions( - availableTags = allGenres.map { genre -> + availableTags = GENRES.map { genre -> MangaTag(key = genre, title = genre, source = source) }.toSet(), availableStates = EnumSet.of( @@ -111,11 +92,8 @@ class Wurmz(context: MangaLoaderContext) : val params = mutableMapOf<String, String>() - filter.query?.takeIf { it.isNotEmpty() }?.let { rawQuery -> - val cleaned = cleanQuery(rawQuery) - if (cleaned.isNotEmpty()) { - params["q"] = cleaned.urlEncoded() - } + filter.query?.takeIf { it.isNotEmpty() }?.let { + params["q"] = it.replace("Bahasa Indonesia", "").trim() } filter.types.firstOrNull()?.let { type -> @@ -141,7 +119,16 @@ class Wurmz(context: MangaLoaderContext) : params["genres[]"] = tag.key } - params["sort"] = sortParamForOrder(order) + params["sort"] = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.ADDED -> "new" + SortOrder.ADDED_ASC -> "old" + SortOrder.POPULARITY_TODAY -> "popular_today" + SortOrder.POPULARITY_WEEK -> "popular_7d" + SortOrder.POPULARITY_MONTH -> "popular_30d" + SortOrder.POPULARITY -> "popular_all" + else -> "update" + } if (page > 1) { params["page"] = page.toString() @@ -157,26 +144,6 @@ class Wurmz(context: MangaLoaderContext) : } } - private fun cleanQuery(query: String): String { - return query - .replace(Regex("Bahasa Indonesia", RegexOption.IGNORE_CASE), "") - .trim() - .replace(Regex("\\s+"), " ") - } - - private fun sortParamForOrder(order: SortOrder): String { - return when (order) { - SortOrder.UPDATED -> "update" - SortOrder.ADDED -> "new" - SortOrder.ADDED_ASC -> "old" - SortOrder.POPULARITY_TODAY -> "popular_today" - SortOrder.POPULARITY_WEEK -> "popular_7d" - SortOrder.POPULARITY_MONTH -> "popular_30d" - SortOrder.POPULARITY -> "popular_all" - else -> "update" - } - } - private fun parseMangaCard(card: Element): Manga? { val link = card.selectFirst("a[href]") ?: return null val href = link.attrAsRelativeUrl("href") @@ -286,4 +253,25 @@ class Wurmz(context: MangaLoaderContext) : MangaPage(id = generateUid(url), url = url, preview = null, source = source) } } + + companion object { + val GENRES = listOf( + "Fantasy", "Romance", "Drama", "Action", "Comedy", "Shounen", "Adventure", "Shoujo", + "School", "Slice of Life", "Seinen", "Supernatural", "Historical", "Josei", "Isekai", + "Webtoons", "Martial Art", "Harem", "Ecchi", "Reincarnation", "Magic", "Mystery", + "Psychological", "Yaoi", "Mature", "Horror", "Tragedy", "Shounen Ai", "Sci-fi", + "Demons", "Adaptation", "Gender Bender", "Game", "Yuri", "Shoujo Ai", "Thriller", + "Sport", "Cooking", "Crime", "Gore", "Royal Family", "Super Power", "Regression", + "Military", "Royalty", "Vampire", "Office Workers", "Transmigration", "Medical", + "Time Travel", "Childhood Friends", "Music", "Monsters", "Revenge", "College Life", + "Villainess", "Kids", "Mecha", "One Shot", "Police", "Omegaverse", "Girls", + "Reverse Harem", "Animals", "Ghosts", "Project", "Age Gap", "Showbiz", "Violence", + "Boys", "Shotacon", "Survival", "Beasts", "Bloody", "Crossdressing", "Delinguents", + "Dungeons", "Gyaru", "Non-human", "Post-Apocalyptic", "Zombies", "Doujinshi", + "18+", "Bodyswap", "NTR", "Yakuzas", "BDSM", "Lolicon", "Philosophical", "System", + "Aliens", "Hentai", "Anthology", "Fetish", "Incest", "Virtual Reality", "Dementia", + "Nakadashi", "Samurai", "4-Koma", "Cheating", "Femdom", "Mafia", "Milf", "Ninja", + "Cunnilingus", "Guideverse", "Infidelity", "Parodi", "Reverse Isekai", "Villain" + ) + } } From eb7f2e2d605314775a66c9897c0b8e06ff87cea8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 8 Aug 2026 23:59:52 +0200 Subject: [PATCH 263/434] Create CGBUM.kt --- src/main/kotlin/tsuki/site/id/CGBUM.kt | 306 +++++++++++++++++++++++++ 1 file changed, 306 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/CGBUM.kt diff --git a/src/main/kotlin/tsuki/site/id/CGBUM.kt b/src/main/kotlin/tsuki/site/id/CGBUM.kt new file mode 100644 index 00000000..e6cb9dda --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/CGBUM.kt @@ -0,0 +1,306 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("CGBUM", "CGBUM", "id") +class CGBUM(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CGBUM, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("cgbum.com") + + override val availableSortOrders: Set<SortOrder> = setOf( + SortOrder.UPDATED, // latest + SortOrder.ADDED, // newest + SortOrder.ADDED_ASC, // oldest + SortOrder.POPULARITY, // views + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = GENRES.map { genre -> + MangaTag(key = genre, title = genre, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.HENTAI + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = if (!filter.query.isNullOrBlank()) { + buildSearchUrl(page, filter.query.urlEncoded()) + } else { + buildBrowseUrl(page, order, filter) + } + val doc = webClient.httpGet(url).parseHtml() + + val cards = doc.select("article.comic-card") + return if (cards.isEmpty()) emptyList() else cards.mapNotNull { parseMangaCard(it) } + } + + private fun buildSearchUrl(page: Int, query: String): String { + return buildString { + append("https://$domain/cari?q=") + append(query.urlEncoded().replace("%20", "+")) + if (page > 1) { + append("&page=") + append(page) + } + } + } + + private fun buildBrowseUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return buildString { + append("https://$domain/daftar-komik?") + append("type=") + filter.types.firstOrNull()?.let { type -> + append( + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.HENTAI -> "pornhwa" + else -> {} + } + ) + } + append("&status=") + filter.states.firstOrNull()?.let { state -> + append( + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "tamat" + else -> {} + } + ) + } + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.ADDED -> "newest" + SortOrder.ADDED_ASC -> "oldest" + SortOrder.POPULARITY -> "views" + else -> "latest" + } + ) + filter.tags.take(3).forEach { tag -> + append("&genres%5B%5D=") + append(tag.key) + } + if (page > 1) { + append("&page=") + append(page) + } + } + } + + private fun parseMangaCard(card: Element): Manga? { + val coverLink = card.selectFirst("a.comic-card-cover") ?: return null + val href = coverLink.attrAsRelativeUrl("href") + val titleEl = card.selectFirst(".comic-card-title a") + val title = titleEl?.text() ?: return null + val coverUrl = card.selectFirst("img")?.src()?.takeIf { it.isNotBlank() } + + val typeBadge = card.selectFirst(".badge-type")?.text() ?: "Unknown" + val typeTag = when (typeBadge.lowercase()) { + "manga" -> MangaTag("type:manga", "Manga", source) + "manhwa" -> MangaTag("type:manhwa", "Manhwa", source) + "manhua" -> MangaTag("type:manhua", "Manhua", source) + else -> null + } + + val isAdult = card.selectFirst(".badge-pornhwa") != null + + val statusText = card.selectFirst(".badge-status")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "tamat" -> MangaState.FINISHED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (typeTag != null) tags.add(typeTag) + if (isAdult) tags.add(MangaTag("adult", "Adult", source)) + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + authors = emptySet(), + tags = tags, + rating = RATING_UNKNOWN, + state = state, + contentRating = if (isAdult) ContentRating.ADULT else null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.selectFirst(".comic-cover img")?.src() ?: manga.coverUrl + + val desc = doc.selectFirst(".comic-synopsis .synopsis-content")?.text() ?: "" + + val genreElements = doc.select(".comic-genres .genre-pill") + val tags = genreElements.mapNotNull { a -> + val genreName = a.text().trim() + if (genreName.isNotEmpty()) MangaTag(key = genreName, title = genreName, source = source) + else null + }.toSet() + + val statusText = doc.selectFirst(".badge-status")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "tamat" -> MangaState.FINISHED + else -> null + } + + val isAdult = doc.selectFirst(".badge-pornhwa") != null || + tags.any { it.title.equals("adult", ignoreCase = true) } + val contentRating = if (isAdult) ContentRating.ADULT else null + + val authorRow = doc.select(".meta-row").firstOrNull { it.selectFirst(".meta-label")?.text() == "Author" } + val author = authorRow?.selectFirst(".meta-value")?.text()?.trim()?.takeIf { it.isNotBlank() && it != "-" } + val authors = if (author != null) setOf(author) else emptySet() + + val chapterLinks = doc.select(".chapter-grid a.ch-grid-item") + val chapters = chapterLinks.map { a -> + val href = a.attrAsRelativeUrl("href") + val label = a.text().trim() + val number = label.extractChapterNumber() + MangaChapter( + id = generateUid(href), + url = href, + title = label, + number = number, + volume = 0, + uploadDate = 0, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + publicUrl = fullUrl, + coverUrl = cover, + description = desc, + tags = tags, + state = state, + authors = authors, + contentRating = contentRating, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val containers = doc.select(".page-container") + if (containers.isEmpty()) { + throw ParseException("No image containers found", fullUrl) + } + + return containers.mapNotNull { container -> + val url = container.attr("data-url").takeIf { it.isNotBlank() } + ?: container.selectFirst("img")?.attr("src")?.takeIf { it.isNotBlank() } + if (url == null) return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val fullUrl = seed.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val section = doc.selectFirst("section:has(.section-header h2:contains(Baca Juga))") + ?: return emptyList() + + return section.select("article.comic-card").mapNotNull { card -> + parseMangaCard(card) + } + } + + companion object { + val GENRES = listOf( + "adaptation", "adult", "adultmature", "age gap", "ahegao", "aliens", "anal", + "animals", "anthology", "bdsm", "beasts", "big ass", "big breast", "big breasts", + "big penis", "bisexual", "blackmail", "bloody", "blowjob", "body swap", "bodyswap", + "bondage", "business suit", "cheating", "cheating infidelity", "childhood friends", + "collar", "college life", "comedy", "condom", "contest winning", "cooking", "crime", + "cunnilingus", "curse", "dark skin", "defloration", "delinquents", "demon girl", + "demons", "double penetration", "doujinshi", "drama", "dungeons", "ecchi", "elf", + "emperor s daughter", "exhibitionism", "fantasy", "femdom", "fetish", "ffm threesome", + "filming", "fingering", "footjob", "full color", "futanari", "game", "gender bender", + "genderswap", "ghost", "ghosts", "glasses", "gore", "group", "gyaru", "handjob", + "harem", "hentai", "historical", "horns", "huge breast", "humiliation", + "inverted nipples", "isekai", "josei", "josei w", "kemomimi", "lingerie", "lolicon", + "milf", "magic", "maid", "manhua", "manhwa", "martial arts", "masturbation", + "mature", "medical", "military", "mind break", "mind control", "mmf threesome", + "monster girls", "monsters", "music", "mystery", "ntr", "nakadashi", "netorare", + "non human", "obsessive male lead", "office workers", "omegaverse", "oneshot", + "paizuri", "police", "pregnant", "psychological", "rape", "regression", + "reincarnation", "revenge", "reverse harem", "romance", "royal family", "royalty", + "runaway", "school life", "sci fi", "seinen", "seinen m", "sex toys", "shoujo ai", + "shoujo g", "shounen ai", "shounen b", "showbiz", "slice of life", "small breast", + "smut", "space", "sports", "stocking", "story arc", "super power", "supernatural", + "survival", "thriller", "time travel", "tomboy", "tower climbing", "traditional games", + "tragedy", "transmigration", "twintails", "unusual pupils", "vampires", "video games", + "villainess", "violence", "virginity", "virtual reality", "webtoon", "wuxia", + "yakuzas", "yaoi bl", "yuri gl", "zombies", "action", "adventure", "boys", "girl", + "hairy", "horror", "manga", "rofan" + ) + } +} From 273d07c49c2f327a0aeeb4c4de3e403141835a47 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:11:55 +0200 Subject: [PATCH 264/434] Create HentaiEra.kt --- .../kotlin/tsuki/site/all/nsfw/HentaiEra.kt | 153 ++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt new file mode 100644 index 00000000..4a83e09b --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt @@ -0,0 +1,153 @@ +package tsuki.site.all.nsfw + +import org.jsoup.nodes.Element +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("HENTAIERA", "HentaiEra", type = ContentType.HENTAI) +internal class HentaiEra(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIERA, "hentaiera.com", 25) { + override val selectTags = ".tags_section" + override val selectTag = ".galleries_info li:contains(Tags) div.info_tags" + override val selectAuthor = ".galleries_info li:contains(Artists) span.item_name" + override val selectLanguageChapter = ".galleries_info li:contains(Languages) div.info_tags .item_name" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.JAPANESE, + Locale("es"), + Locale("ru"), + Locale("ko"), + Locale.GERMAN, + ), + ) + + override fun Element.parseTags() = select("a.tag, .gallery_title a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".item_name")?.text() ?: it.text() + MangaTag( + key = key, + title = name, + source = source, + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/search/?key=") + append(filter.query.urlEncoded()) + append("&") + } + else -> { + if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { + append("/search/?key=") + append(buildQuery(filter.tags, filter.locale)) + + append("&") + } else if (filter.tags.isNotEmpty()) { + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + } + append("/") + + append("?") + } else if (filter.locale != null) { + append("/language/") + append(filter.locale?.toLanguagePath()) + append("/") + + append("?") + } else { + append("/?") + } + } + } + append("page=") + append(page.toString()) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun buildQuery(tags: Collection<MangaTag>, locale: Locale?): String { + val queryDefault = + "&search=&mg=1&dj=1&ws=1&is=1&ac=1&gc=1&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0<=1&dl=0&pp=0&tr=0" + val tag = tags.joinToString(" ", postfix = " ") { it.key } + val queryMod = when (val lp = locale?.toLanguagePath()) { + "english" -> queryDefault.replace("en=0", "en=1") + "japanese" -> queryDefault.replace("jp=0", "jp=1") + "spanish" -> queryDefault.replace("es=0", "es=1") + "french" -> queryDefault.replace("fr=0", "fr=1") + "korean" -> queryDefault.replace("kr=0", "kr=1") + "russian" -> queryDefault.replace("ru=0", "ru=1") + "german" -> queryDefault.replace("de=0", "de=1") + null -> "&search=&mg=1&dj=1&ws=1&is=1&ac=1&gc=1&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1<=1&dl=0&pp=0&tr=0" + else -> throw IllegalArgumentException("Unsupported locale: $lp") + } + return tag + queryMod + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val urlChapters = doc.selectFirstOrThrow("#cover a, .cover a, .left_cover a").attr("href") + val tag = doc.selectFirst(selectTag)?.parseTags() + val branch = doc.select(selectLanguageChapter).joinToString(separator = " / ") { + it.text() + } + val author = doc.selectFirst(selectAuthor)?.text() + return manga.copy( + tags = tag.orEmpty(), + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 1f, + volume = 0, + url = urlChapters, + scanlator = null, + uploadDate = 0, + branch = branch, + source = source, + ), + ), + ) + } +} From 33e31042f150f5e1bfb4ebb2bbc9c26687a60dbb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:29:43 +0200 Subject: [PATCH 265/434] Create Danbooru.kt --- .../kotlin/tsuki/site/all/nsfw/Danbooru.kt | 223 ++++++++++++++++++ 1 file changed, 223 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt new file mode 100644 index 00000000..b59c008d --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt @@ -0,0 +1,223 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("DANBOORU", "Danbooru") +internal class Danbooru(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DANBOORU, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("danbooru.donmai.us") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient: WebClient by lazy { + OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(this) + .build(), + source + ) + } + + override fun getRequestHeaders(): Headers { + return super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, // "updated_at" + SortOrder.ALPHABETICAL, // "name" + SortOrder.NEWEST, // "created_at" + SortOrder.POPULARITY, // "post_count" + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.startsWith("http://") || query.startsWith("https://")) { + val url = query.toHttpUrlOrNull() + if (url != null && url.host == domain.toHttpUrl().host && url.pathSegments.size >= 2 && url.pathSegments[0] == "pools") { + val poolId = url.pathSegments[1] + return listOf( + Manga( + id = generateUid(poolId), + url = "/pools/$poolId", + publicUrl = "https://$domain/pools/$poolId", + title = poolId, + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + } + } + + val sortParam = when (order) { + SortOrder.UPDATED -> "updated_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.NEWEST -> "created_at" + SortOrder.POPULARITY -> "post_count" + else -> "updated_at" + } + + val url = "https://$domain/pools/gallery".toHttpUrl().newBuilder() + .addQueryParameter("search[category]", "series") + .addQueryParameter("search[order]", sortParam) + .addQueryParameter("page", page.toString()) + .apply { + if (query.isNotBlank()) { + addQueryParameter("search[name_contains]", query) + } + } + .build() + + val doc = webClient.httpGet(url).parseHtml() + val entries = doc.select("article.post-preview") + return entries.mapNotNull { element -> + val link = element.selectFirst(".post-preview-link") ?: return@mapNotNull null + val href = link.attr("href") + val title = element.selectFirst("div.text-center")?.text() ?: return@mapNotNull null + val cover = element.selectFirst("source")?.attr("srcset") + ?.substringAfterLast(',')?.trim() + ?.substringBeforeLast(' ')?.trimStart() + Manga( + id = generateUid(href), + url = href, + publicUrl = "https://$domain$href", + title = title, + altTitles = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.publicUrl + val doc = webClient.httpGet(fullUrl).parseHtml() + val title = doc.selectFirst(".pool-category-series, .pool-category-collection")?.text() + ?: doc.selectFirst("h1")?.text() ?: manga.title + val description = doc.getElementById("description")?.wholeText() + val author = doc.selectFirst("#description a[href*=artists]")?.ownText() + val artists = setOfNotNull(author) + + val poolJson = webClient.httpGet("$fullUrl.json").parseJson() + val postIds = poolJson.getJSONArray("post_ids") + val updatedAt = poolJson.optString("updated_at", null) + + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until postIds.length()) { + val postId = postIds.getInt(i) + chapters.add( + MangaChapter( + id = generateUid("/posts/$postId"), + title = "Post ${i + 1}", + number = (i + 1).toFloat(), + volume = 0, + url = "/posts/$postId", + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + ) + } + if (chapters.isNotEmpty() && updatedAt != null) { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) + val date = dateFormat.parseSafe(updatedAt) + chapters[0] = chapters[0].copy(uploadDate = date) + } + + return manga.copy( + title = title, + description = description, + authors = artists, + chapters = chapters.sortedBy { it.number }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val postId = chapter.url.substringAfterLast("/") + val json = webClient.httpGet("https://$domain/posts/$postId.json").parseJson() + val imageUrl = json.optString("file_url", null) + ?: json.optString("large_file_url", null) + ?: json.optString("preview_file_url", null) + ?: throw Exception("Image URL not found for post $postId") + val fullImageUrl = if (imageUrl.startsWith("http")) imageUrl else "https://$domain$imageUrl" + return listOf( + MangaPage( + id = generateUid(fullImageUrl), + url = fullImageUrl, + preview = null, + source = source, + ) + ) + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (request.url.host == "cdn.donmai.us") { + val newRequest = request.newBuilder() + .removeHeader("Cookie") + .header("Accept", "image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5") + .header("Sec-Fetch-Dest", "image") + .header("Sec-Fetch-Mode", "no-cors") + .header("Sec-Fetch-Site", "same-site") + .build() + return chain.proceed(newRequest) + } + return chain.proceed(request) + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } +} From be52e7237c416a1d642476a38418b725f1362fd2 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 00:44:24 +0200 Subject: [PATCH 266/434] Update MangaDistrict.kt --- .../tsuki/site/en/nsfw/MangaDistrict.kt | 137 +++++++++++++----- 1 file changed, 100 insertions(+), 37 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt index 017b39c7..6366b0dc 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt @@ -3,13 +3,18 @@ package tsuki.site.en.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException import tsuki.model.ContentType import tsuki.model.Manga import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter import tsuki.model.MangaParserSource import tsuki.model.MangaTag import tsuki.model.SortOrder +import tsuki.model.ContentRating +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaState import tsuki.util.attrAsRelativeUrl import tsuki.util.generateUid @@ -17,8 +22,13 @@ import tsuki.util.mapNotNullToSet import tsuki.util.parseHtml import tsuki.util.selectFirstOrThrow import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.extractChapterNumber +import tsuki.util.oneOrThrowIfMany +import org.jsoup.HttpStatusException import org.jsoup.nodes.Document +import java.net.HttpURLConnection import java.text.SimpleDateFormat import java.util.EnumSet @@ -40,27 +50,105 @@ internal class MangaDistrict(context: MangaLoaderContext) : SortOrder.UPDATED, ) + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = false, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val pages = page + 1 + fun sortParam(): String = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "new-manga" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "" + else -> "" + } + val url = if (filter.tags.isNotEmpty()) { + val genreSlug = filter.tags.first().key + buildString { + append("https://") + append(domain) + append("/publication-genre/$genreSlug/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + append("?m_orderby=") + append(sortParam()) + } + } else { + buildString { + append("https://") + append(domain) + if (pages > 1) { + append("/page/") + append(pages.toString()) + } + append("/?s=") + append(filter.query?.urlEncoded() ?: "") + append("&post_type=wp-manga") + filter.states.forEach { + append("&status[]=") + when (it) { + MangaState.ONGOING -> append("on-going") + MangaState.FINISHED -> append("end") + MangaState.ABANDONED -> append("canceled") + MangaState.PAUSED -> append("on-hold") + MangaState.UPCOMING -> append("upcoming") + else -> throw IllegalArgumentException("$it not supported") + } + } + filter.contentRating.oneOrThrowIfMany()?.let { + append("&adult=") + append( + when (it) { + ContentRating.SAFE -> "0" + ContentRating.ADULT -> "1" + else -> "" + } + ) + } + if (filter.year != 0) { + append("&release=") + append(filter.year.toString()) + } + filter.author?.takeIf { it.isNotEmpty() }?.let { + append("&author=") + append(it.lowercase().replace(" ", "-")) + } + append("&m_orderby=") + append(sortParam()) + } + } + val html = try { + webClient.httpGet(url).parseHtml() + } catch (e: HttpStatusException) { + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() + else throw ParseException("Can't fetch data from source!", url) + } + return parseMangaList(html) + } + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val chapters = doc.body() .select("li.wp-manga-chapter") .mapNotNull { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage - val name = a.selectFirst("p")?.text() ?: a.ownText() - - val dateText = - li.selectFirst("a.c-new-tag")?.attr("title") + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst("span.chapter-release-date i")?.text() - MangaChapter( id = generateUid(href), title = name, - number = extractChapterNumber(name) ?: 0f, + number = name.extractChapterNumber(), volume = 0, url = link, uploadDate = parseChapterDate(dateFormat, dateText), @@ -69,44 +157,19 @@ internal class MangaDistrict(context: MangaLoaderContext) : source = source, ) } - .sortedWith( - compareBy<MangaChapter> { it.number } - .thenBy { it.title } - ) - + .sortedWith(compareBy<MangaChapter> { it.number }.thenBy { it.title }) return chapters } - private fun extractChapterNumber(name: String): Float? { - val match = Regex( - """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", - RegexOption.IGNORE_CASE - ).find(name) - - return match - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - override suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/series/").parseHtml() - val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") - + val doc = webClient.httpGet("https://$domain/").parseHtml() + val elements = doc.select("div.genres_wrap ul li a") return elements.mapNotNullToSet { a -> - val href = a.attr("href") - .removeSuffix("/") - .substringAfterLast(tagPrefix, "") - + val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") if (href.isBlank()) return@mapNotNullToSet null - MangaTag( key = href, - title = a.text() - .replace(Regex("""\s*\(\d+\)"""), "") - .replace(Regex("""[^\x20-\x7E]"""), "") - .trim() - .toTitleCase(), + title = a.text().trim().toTitleCase(), source = source, ) } From 1d009958e97bb43d16684350c7f211a5cdcd1830 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:14:15 +0200 Subject: [PATCH 267/434] Update Webtoons.kt --- src/main/kotlin/tsuki/site/all/Webtoons.kt | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt index 820e9221..9b906000 100644 --- a/src/main/kotlin/tsuki/site/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -553,24 +553,24 @@ internal abstract class WebtoonsParser( it.title.lowercase() to it.key } - @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", ContentType.MANGA) class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_EN) - @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", ContentType.MANGA) class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ID) - @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es",ContentType.MANGA) class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ES) - @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", ContentType.MANGA) class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_FR) - @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", ContentType.MANGA) class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_TH) - @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", ContentType.MANGA) class Chinese(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ZH) - @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", ContentType.MANGA) class German(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_DE) } From bfe953df0cdc287dc4f7c2e478b224bad565e916 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:14:45 +0200 Subject: [PATCH 268/434] Update HentaiNexus.kt --- src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt index aa0c4d90..a4eb82b4 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt @@ -37,7 +37,7 @@ import org.jsoup.nodes.Document private const val SERVER_PNG = "png" // old method uses now image_fallback private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format -@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) +@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", ContentType.HENTAI) internal class HentaiNexus(context: MangaLoaderContext) : GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { override val selectGallery = "div.container div.columns div.column" From e65e73ec993950ac58d25cefde797fb10eb250ed Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:15:04 +0200 Subject: [PATCH 269/434] Update Manhwa210.kt --- src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt index f2b0fafa..8cca2cce 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt @@ -39,7 +39,7 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale -@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) +@MangaSourceParser("MANHWA210", "Manhwa210", "en", ContentType.HENTAI) internal class Manhwa210(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { From 188fa81b15a0efb521d12b1335fab5fd2108779e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 12:20:21 +0200 Subject: [PATCH 270/434] Update Danbooru.kt --- src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt index b59c008d..2de9a82c 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt @@ -17,6 +17,7 @@ import tsuki.model.MangaPage import tsuki.model.MangaParserSource import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.ContentType import tsuki.util.generateUid import tsuki.util.parseHtml @@ -31,7 +32,7 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale -@MangaSourceParser("DANBOORU", "Danbooru") +@MangaSourceParser("DANBOORU", "Danbooru", type = ContentType.HENTAI) internal class Danbooru(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.DANBOORU, pageSize = 20), Interceptor { From bf8813a5cb451979757fa19d2c1b278eed9d6470 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:00:47 +0200 Subject: [PATCH 271/434] Create KomikDewasaART.kt --- src/main/kotlin/tsuki/site/id/KomikDewasaART.kt | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/KomikDewasaART.kt diff --git a/src/main/kotlin/tsuki/site/id/KomikDewasaART.kt b/src/main/kotlin/tsuki/site/id/KomikDewasaART.kt new file mode 100644 index 00000000..c2137725 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/KomikDewasaART.kt @@ -0,0 +1,15 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("KOMIKDEWASAART", "Komik Dewasa.art", "id", ContentType.HENTAI) +internal class KomikDewasaART(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KOMIKDEWASAART, "komikdewasa.art") { + override val mangaDirectory = "komik" + override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" +} From 9e7fe5210c020b1854ae720d51be650767432d17 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:25:34 +0200 Subject: [PATCH 272/434] Create EZManhwaParser.kt --- src/main/kotlin/tsuki/parsers/EZManhwaParser.kt | 1 + 1 file changed, 1 insertion(+) create mode 100644 src/main/kotlin/tsuki/parsers/EZManhwaParser.kt diff --git a/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt new file mode 100644 index 00000000..8b137891 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt @@ -0,0 +1 @@ + From b50ddfc2151c0853fc16f96c21b4aa2090e4211b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 13:29:05 +0200 Subject: [PATCH 273/434] Create QiScans.kt --- src/main/kotlin/tsuki/site/en/QiScans.kt | 165 +++++++++++++++++++++++ 1 file changed, 165 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/QiScans.kt diff --git a/src/main/kotlin/tsuki/site/en/QiScans.kt b/src/main/kotlin/tsuki/site/en/QiScans.kt new file mode 100644 index 00000000..390a8223 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/QiScans.kt @@ -0,0 +1,165 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.EZManhwaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.parseJson +import tsuki.util.json.mapJSON + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.util.EnumSet + +@MangaSourceParser("QISCANS", "QiScans", "en") +internal class QiScans(context: MangaLoaderContext) : + EZManhwaParser(context, MangaParserSource.QISCANS, "qimanga.com", "https://api.qimanga.com/api/v1") { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override fun getRequestHeaders(): Headers { + return super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Sec-Fetch-Dest", "empty") + .set("Sec-Fetch-Mode", "cors") + .set("Sec-Fetch-Site", "same-site") + .build() + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val base = super.getFilterOptions() + return base.copy( + availableTags = GENRES + .filter { it.first.isNotEmpty() } + .map { (key, name) -> MangaTag(key = key, title = name, source = source) } + .toSet() + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val endpoint = if (filter.query.isNullOrBlank()) "$apiUrl/series" else "$apiUrl/series/search" + val url = endpoint.toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + + if (filter.query.isNullOrBlank()) { + val sort = when (order) { + SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "latest" + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popular" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "newest" + SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "alphabetical" + else -> "latest" + } + addQueryParameter("sort", sort) + + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + MangaState.ABANDONED -> "DROPPED" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + }) + } + filter.tags.firstOrNull()?.key?.let { genreSlug -> + if (genreSlug.isNotBlank()) { + addQueryParameter("genre", genreSlug) + } + } + } else { + addQueryParameter("q", filter.query) + } + }.build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: org.json.JSONObject): List<Manga> { + val data = json.getJSONArray("data") + return data.mapJSON { obj -> + obj.toManga(domain, source, isNsfwSource) + } + } + + companion object { + val GENRES = listOf( + "" to "All", + "acting" to "Acting", + "action" to "Action", + "action-582" to "Action (Alt)", + "adventure" to "Adventure", + "adventure-589" to "Adventure (Alt)", + "apocalypce" to "Apocalypse", + "comedy" to "Comedy", + "cooking" to "Cooking", + "crazy-mc" to "Crazy MC", + "cultivation" to "Cultivation", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "fantasy-747" to "Fantasy (Alt)", + "fight" to "Fight", + "gender-bender" to "Gender Bender", + "harem" to "Harem", + "hidden" to "Hidden", + "historical" to "Historical", + "horror" to "Horror", + "josei" to "Josei", + "live" to "Live", + "magic" to "Magic", + "manhua" to "Manhua", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medieval-area" to "Medieval Area", + "munchkin" to "Munchkin", + "murim" to "Murim", + "mystery" to "Mystery", + "myth" to "Myth", + "politics" to "Politics", + "psychological" to "Psychological", + "reincarnation" to "Reincarnation", + "revenge" to "Revenge", + "romance" to "Romance", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "shounen" to "Shounen", + "slice-of-life" to "Slice of Life", + "sports" to "Sports", + "supernatural" to "Supernatural", + "superpower" to "Superpower", + "system" to "System", + "taming" to "Taming", + "tower" to "Tower", + "tragedy" to "Tragedy", + "urban" to "Urban", + "vampiers" to "Vampires", + "virtual-reality" to "Virtual Reality", + "wuxia" to "Wuxia", + ) + } +} From cc81e4a91156defa92434e9674231a9a55c2e9ad Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 14:31:10 +0200 Subject: [PATCH 274/434] Create MangaOni.kt --- src/main/kotlin/tsuki/site/es/MangaOni.kt | 360 ++++++++++++++++++++++ 1 file changed, 360 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/es/MangaOni.kt diff --git a/src/main/kotlin/tsuki/site/es/MangaOni.kt b/src/main/kotlin/tsuki/site/es/MangaOni.kt new file mode 100644 index 00000000..b893c139 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/MangaOni.kt @@ -0,0 +1,360 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.nio.charset.Charset +import java.text.SimpleDateFormat +import java.util.Base64 +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAONI", "MangaOni", "es") +internal class MangaOni(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAONI, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("manga-oni.com") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) + + private val hideNsfwKey = ConfigKey.ShowSuspiciousContent(false) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(hideNsfwKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Origin", "https://$domain") + .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") + .set("Accept-Language", "es-ES,es;q=0.9,en;q=0.8") + .set("User-agent", config[userAgentKey]) + return builder.build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, // visitas + SortOrder.UPDATED, // recientes + SortOrder.ALPHABETICAL, // nombre + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (_, name, id) -> + MangaTag(key = id, title = name, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, // "En desarrollo" -> 1 + MangaState.FINISHED, // "Completo" -> 0 + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, // 0 + ContentType.MANHWA, // 1 + ContentType.OTHER, // "One Shot" 2 + ContentType.MANHUA, // 3 + ContentType.NOVEL, // 4 + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + if (query.isNotEmpty()) { + return searchManga(query, page) + } + + val url = when (order) { + SortOrder.UPDATED -> "https://$domain/recientes?p=$page" + else -> buildDirectoryUrl(page, order, filter) + } + + val doc = webClient.httpGet(url).parseHtml() + return when (order) { + SortOrder.UPDATED -> parseLatestListing(doc) + else -> parseDirectoryListing(doc) + } + } + + private fun buildDirectoryUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return StringBuilder("https://$domain/directorio?").apply { + append("genero=") + append(filter.tags.firstOrNull()?.key ?: "false") + + append("&estado=") + append( + filter.states.firstOrNull()?.let { + when (it) { + MangaState.ONGOING -> "1" + MangaState.FINISHED -> "0" + else -> null + } + } ?: "false" + ) + + append("&tipo=") + append( + filter.types.firstOrNull()?.let { + when (it) { + ContentType.MANGA -> "0" + ContentType.MANHWA -> "1" + ContentType.OTHER -> "2" + ContentType.MANHUA -> "3" + ContentType.NOVEL -> "4" + else -> null + } + } ?: "false" + ) + + append("&adulto=") + if (config[hideNsfwKey]) { + append("0") + } else { + when (filter.contentRating.oneOrThrowIfMany()) { + ContentRating.ADULT -> append("1") + ContentRating.SAFE -> append("0") + else -> append("false") + } + } + + append("&filtro=") + append( + when (order) { + SortOrder.POPULARITY -> "visitas" + SortOrder.ALPHABETICAL -> "nombre" + else -> "visitas" + } + ) + append("&orden=") + append(if (order == SortOrder.ALPHABETICAL) "asc" else "desc") + + append("&p=$page") + }.toString() + } + + private suspend fun searchManga(query: String, page: Int): List<Manga> { + val url = "https://$domain/buscar?q=${query.urlEncoded()}&p=$page" + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResult(doc) + } + + private fun parseLatestListing(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("div._1bJU3").map { element -> + val a = element.selectFirst("a._2dU-m") ?: return@map null + val href = a.attrAsRelativeUrl("href") + val cover = a.selectFirst("img")?.attr("data-src") ?: a.selectFirst("img")?.attr("src") + val title = element.selectFirst("a[data-test=latest-update-name]")?.text() ?: return@map null + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover?.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + private fun parseDirectoryListing(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("#article-div a").map { element -> + val href = element.attrAsRelativeUrl("href") + val cover = element.select("img").attr("abs:data-src").ifEmpty { + element.select("img").attr("abs:src") + } + val title = element.select("div:eq(1)").text() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private fun parseSearchResult(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("#article-div > div").map { element -> + val a = element.selectFirst("a") ?: return@map null + val href = a.attrAsRelativeUrl("href") + val title = a.text().ifEmpty { element.select("a").lastOrNull()?.text() } ?: return@map null + val cover = element.select("img").attr("abs:src") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + // ───────────────────────── DETAILS ──────────────────────────── + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.select("img[src*=cover]").attr("abs:src") + val description = doc.select("div#sinopsis").lastOrNull()?.ownText() + val author = doc.select("div#info-i").text().let { info -> + if (info.contains("Autor", true)) { + info.substringAfter("Autor:").substringBefore("Fecha:").trim() + } else null + } ?: "N/A" + + val genres = doc.select("div#categ a").eachText() + val tags = genres.map { g -> MangaTag(key = g.lowercase(), title = g, source = source) }.toSet() + + val status = when (doc.selectFirst("strong:contains(Estado) + span")?.text()) { + "En desarrollo" -> MangaState.ONGOING + "Finalizado" -> MangaState.FINISHED + else -> null + } + + val chapters = doc.select("div#c_list a").mapIndexed { _, element -> + val chHref = element.attrAsRelativeUrl("href") + val name = element.text() + val chNum = element.select("span").attr("data-num").toFloatOrNull() ?: -1f + val dateStr = element.select("span").attr("datetime") + val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L + MangaChapter( + id = generateUid(chHref), + url = chHref, + title = name, + number = chNum, + volume = 0, + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = description, + authors = setOf(author), + tags = tags, + state = status, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val script = doc.selectFirst("script:containsData(unicap)") + ?.data() ?: throw Exception("No se encontró unicap") + val encoded = script.substringAfter("'").substringBefore("'") + val decoded = Base64.getDecoder().decode(encoded) + .toString(Charset.defaultCharset()) + val path = decoded.substringBefore("||") + val filesStr = decoded.substringAfter("[").substringBefore("]") + val files = filesStr.split(",").map { it.trim().removeSurrounding("\"") } + return files.mapIndexed { _, file -> + val imageUrl = path + file + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } + + companion object { + val GENRES = listOf( + Triple("Todos", "Todos", "false"), + Triple("Comedia", "Comedia", "1"), + Triple("Drama", "Drama", "2"), + Triple("Acción", "Acción", "3"), + Triple("Escolar", "Escolar", "4"), + Triple("Romance", "Romance", "5"), + Triple("Ecchi", "Ecchi", "6"), + Triple("Aventura", "Aventura", "7"), + Triple("Shōnen", "Shōnen", "8"), + Triple("Shōjo", "Shōjo", "9"), + Triple("Deportes", "Deportes", "10"), + Triple("Psicológico", "Psicológico", "11"), + Triple("Fantasía", "Fantasía", "12"), + Triple("Mecha", "Mecha", "13"), + Triple("Gore", "Gore", "14"), + Triple("Yaoi", "Yaoi", "15"), + Triple("Yuri", "Yuri", "16"), + Triple("Misterio", "Misterio", "17"), + Triple("Sobrenatural", "Sobrenatural", "18"), + Triple("Seinen", "Seinen", "19"), + Triple("Ficción", "Ficción", "20"), + Triple("Harem", "Harem", "21"), + Triple("Webtoon", "Webtoon", "25"), + Triple("Histórico", "Histórico", "27"), + Triple("Músical", "Músical", "30"), + Triple("Ciencia ficción", "Ciencia ficción", "31"), + Triple("Shōjo-ai", "Shōjo-ai", "32"), + Triple("Josei", "Josei", "33"), + Triple("Magia", "Magia", "34"), + Triple("Artes Marciales", "Artes Marciales", "35"), + Triple("Horror", "Horror", "36"), + Triple("Demonios", "Demonios", "37"), + Triple("Supervivencia", "Supervivencia", "38"), + Triple("Recuentos de la vida", "Recuentos de la vida", "39"), + Triple("Shōnen ai", "Shōnen ai", "40"), + Triple("Militar", "Militar", "41"), + Triple("Eroge", "Eroge", "42"), + Triple("Isekai", "Isekai", "43"), + ) + } +} From 50cc949b8f73552fa56b11292a09d31ce880d0c7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:13:41 +0200 Subject: [PATCH 275/434] Create ScansGG.kt --- src/main/kotlin/tsuki/site/en/ScansGG.kt | 297 +++++++++++++++++++++++ 1 file changed, 297 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/ScansGG.kt diff --git a/src/main/kotlin/tsuki/site/en/ScansGG.kt b/src/main/kotlin/tsuki/site/en/ScansGG.kt new file mode 100644 index 00000000..fa871905 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ScansGG.kt @@ -0,0 +1,297 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("SCANSGG", "ScansGG", "en") +internal class ScansGG(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SCANSGG, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("scans.gg") + + private val apiUrl = "https://api.scans.gg" + private val cdnUrl = "https://cdn.scans.gg/uploads" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + return super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Referer", "https://$domain/") + .build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val tagsMap = mapOf( + 49 to "Regression", 48 to "Male Protagonist", 47 to "Survival", + 46 to "Avant Garde", 45 to "Award Winning", 44 to "Lolicon", + 43 to "Mahou Shoujo", 42 to "Doujinshi", 41 to "Girls Love", + 40 to "Hentai", 39 to "Mecha", 38 to "Shotacon", 37 to "Ecchi", + 36 to "Music", 35 to "Smut", 34 to "Erotica", 33 to "Adult", + 32 to "Gourmet", 31 to "Yuri", 30 to "Shoujo Ai", 29 to "Yaoi", + 28 to "Shounen Ai", 27 to "Boys Love", 26 to "Harem", 25 to "Tragedy", + 24 to "Gender Bender", 23 to "Suspense", 22 to "Psychological", + 21 to "Mature", 20 to "Horror", 19 to "Mystery", 18 to "Martial Arts", + 17 to "Sci-fi", 16 to "Adventure", 15 to "Supernatural", 14 to "Sports", + 13 to "Shounen", 12 to "Historical", 11 to "Seinen", 10 to "Action", + 9 to "Josei", 8 to "Thriller", 7 to "School Life", 6 to "Slice Of Life", + 5 to "Drama", 4 to "Comedy", 3 to "Shoujo", 2 to "Romance", 1 to "Fantasy" + ) + + private val typeToApiId = mapOf( + ContentType.MANGA to 2, ContentType.MANHWA to 3, ContentType.MANHUA to 4, + ContentType.COMICS to 1, ContentType.OTHER to 5 + ) + + private val stateToApiId = mapOf( + MangaState.ONGOING to 1, MangaState.FINISHED to 2, + MangaState.PAUSED to 3, MangaState.ABANDONED to 4 + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tagsMap.map { (id, name) -> MangaTag(key = id.toString(), title = name, source = source) } + .toSet(), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, + ContentType.COMICS, ContentType.OTHER + ), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (order == SortOrder.UPDATED && query.isEmpty()) return fetchLatest(page) + + val limit = if (order == SortOrder.UPDATED) 14 else 21 + val offset = (page - 1) * limit + val url = "$apiUrl/series".toHttpUrl().newBuilder() + .addQueryParameter("limit", limit.toString()) + .addQueryParameter("offset", offset.toString()) + .apply { + if (query.isNotEmpty()) addQueryParameter("q", query) + val typeIds = filter.types.mapNotNull { typeToApiId[it] } + if (typeIds.isNotEmpty()) addQueryParameter("q_type", typeIds.joinToString(",", "[", "]")) + val statusIds = filter.states.mapNotNull { stateToApiId[it] } + if (statusIds.isNotEmpty()) addQueryParameter("q_status", statusIds.joinToString(",", "[", "]")) + val tagIds = filter.tags.mapNotNull { it.key.toIntOrNull() } + if (tagIds.isNotEmpty()) addQueryParameter("q_tags", tagIds.joinToString(",", "[", "]")) + } + .build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private suspend fun fetchLatest(page: Int): List<Manga> { + val url = "$apiUrl/chapters".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("limit", "14") + .addQueryParameter("chapters", "true") + .addQueryParameter("series_details", "true") + .addQueryParameter("group_details", "true") + .addQueryParameter("sort", "date") + .build() + + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + val meta = json.optJSONObject("meta") + val hasMore = meta?.optBoolean("has_more", false) ?: false + return (0 until data.length()).map { i -> data.getJSONObject(i).toManga(cdnUrl) } + .take(if (hasMore) 14 else data.length()) + } + + private fun parseSeriesList(json: JSONObject): List<Manga> { + val data = json.getJSONArray("data") + return (0 until data.length()).map { i -> data.getJSONObject(i).toManga(cdnUrl) } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val seriesUrl = "$apiUrl/series".toHttpUrl().newBuilder() + .addQueryParameter("id", manga.url) + .addQueryParameter("trackers", "true") + .addQueryParameter("sources", "true") + .build() + val seriesJson = webClient.httpGet(seriesUrl).parseJson() + val seriesObj = seriesJson.getJSONObject("data") + val updatedManga = seriesObj.toMangaDetail(cdnUrl, tagsMap) + .copy(id = manga.id, source = source) + + val chapters = fetchChaptersPaginated(manga.url) + updatedManga.copy(chapters = chapters) + } + + private suspend fun fetchChaptersPaginated(seriesId: String): List<MangaChapter> { + val allChapters = mutableListOf<MangaChapter>() + val groupTitles = mutableMapOf<Int, String>() + var page = 1 + var hasMore = true + + while (hasMore) { + val url = "$apiUrl/chapters".toHttpUrl().newBuilder() + .addQueryParameter("series_id", seriesId) + .addQueryParameter("limit", "100") + .addQueryParameter("page", page.toString()) + .addQueryParameter("group_details", "true") + .addQueryParameter("sort_order", "desc") + .build() + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + for (i in 0 until data.length()) { + val ch = data.getJSONObject(i) + val chId = ch.getInt("id") + val number = ch.getDouble("number").toFloat() + val title = ch.optString("title", null) + val createdAt = ch.optString("created_at", null) + val groupId = ch.optInt("group_id", 0) + val groupObj = ch.optJSONObject("group") + val groupTitle = groupObj?.optString("title", null) + if (groupTitle != null) { + groupTitles[groupId] = groupTitle + } + + val name = buildString { + append("Chapter ${number.toString().removeSuffix(".0")}") + if (!title.isNullOrEmpty()) append(" - $title") + } + + allChapters.add( + MangaChapter( + id = generateUid(chId.toString()), + title = name, + number = number, + volume = 0, + url = "$apiUrl/chapter-navigation?series_id=$seriesId&chapter_id=$chId&group_id=$groupId", + uploadDate = dateFormat.parseSafe(createdAt), + scanlator = groupTitle, + branch = null, + source = source, + ) + ) + } + val meta = json.optJSONObject("meta") + hasMore = meta?.optBoolean("has_more", false) ?: false + page++ + } + + return if (groupTitles.size > 1) { + allChapters.map { ch -> + val branch = ch.scanlator ?: "Unknown" + ch.copy(branch = branch) + }.sortedBy { it.number } + } else { + allChapters.sortedBy { it.number } + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val json = webClient.httpGet(chapter.url).parseJson() + val data = json.optJSONObject("data") ?: return emptyList() + val chapterObj = data.optJSONObject("chapter") ?: data + val chapterId = chapterObj.optInt("id", 0) + val pages = chapterObj.optJSONArray("pages") ?: return emptyList() + return (0 until pages.length()).map { i -> + val page = pages.getJSONObject(i) + val path = page.getString("path") + val imageUrl = "$cdnUrl/pages/$chapterId/$path" + MangaPage(id = generateUid(imageUrl), url = imageUrl, preview = null, source = source) + } + } + + private fun JSONObject.toManga(cdnUrl: String): Manga { + val id = getInt("id").toString() + val title = getString("title") + val cover = optString("cover", null) + return Manga( + id = generateUid(id), url = id, + publicUrl = "https://$domain/series/$id", title = title, + altTitles = emptySet(), coverUrl = cover?.let { "$cdnUrl/covers/$it" }, + rating = RATING_UNKNOWN, contentRating = null, + tags = emptySet(), state = null, authors = emptySet(), source = source, + ) + } + + private fun JSONObject.toMangaDetail(cdnUrl: String, tagsMap: Map<Int, String>): Manga { + val id = getInt("id").toString() + val title = getString("title") + val summary = optString("summary", null) + val cover = optString("cover", null) + val authorList = optJSONArray("author")?.let { arr -> (0 until arr.length()).map { arr.getString(it) } } ?: emptyList() + val artistList = optJSONArray("artist")?.let { arr -> (0 until arr.length()).map { arr.getString(it) } } ?: emptyList() + val altTitlesArray = optJSONArray("alternative_titles") + val altTitles = if (altTitlesArray != null) { + (0 until altTitlesArray.length()).map { altTitlesArray.getJSONObject(it).getString("title") }.toSet() + } else emptySet() + val tagsIds = optJSONArray("tags")?.let { arr -> (0 until arr.length()).map { arr.getInt(it) } } ?: emptyList() + val genres = tagsIds.mapNotNull { tagsMap[it] } + val typeId = optInt("type", -1) + val typeName = when (typeId) { + 1 -> "Comic"; 2 -> "Manga"; 3 -> "Manhwa"; 4 -> "Manhua"; 5 -> "Webtoon" + else -> null + } + val tags = mutableSetOf<MangaTag>().apply { + addAll(genres.map { MangaTag(key = it, title = it, source = source) }) + typeName?.let { add(MangaTag(key = "type:$typeId", title = it, source = source)) } + } + val statusId = optInt("status", -1) + val state = when (statusId) { + 1 -> MangaState.ONGOING; 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED; 4, 5 -> MangaState.ABANDONED + else -> null + } + return Manga( + id = generateUid(id), url = id, + publicUrl = "https://$domain/series/$id", title = title, + altTitles = altTitles, coverUrl = cover?.let { "$cdnUrl/covers/$it" }, + description = summary, authors = (authorList + artistList).toSet(), + tags = tags, state = state, rating = RATING_UNKNOWN, + source = source, contentRating = null, + ) + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } +} From 8bfe3381cbb324e2b0012ea4932a135c231c7344 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:17:45 +0200 Subject: [PATCH 276/434] Update MangaDexParser.kt --- src/main/kotlin/tsuki/site/all/MangaDexParser.kt | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaDexParser.kt b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt index d24ceffe..12c32b75 100644 --- a/src/main/kotlin/tsuki/site/all/MangaDexParser.kt +++ b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt @@ -7,7 +7,7 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.FlexibleMangaParser import tsuki.exception.ParseException -import tsuki.network.CommonHeaders +import tsuki.network.UserAgents import tsuki.model.ContentRating import tsuki.model.Demographic @@ -62,8 +62,6 @@ import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.Response import org.json.JSONArray import org.json.JSONObject import java.text.SimpleDateFormat @@ -78,21 +76,12 @@ private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks lik private const val LOCALE_FALLBACK = "en" private const val SERVER_DATA = "data" private const val SERVER_DATA_SAVER = "data-saver" -private const val USER_AGENT = "Usagi/1 (Android)" @MangaSourceParser("MANGADEXORG", "MangaDex") internal class MangaDexParser(context: MangaLoaderContext) : FlexibleMangaParser(context, MangaParserSource.MANGADEXORG) { override val configKeyDomain = ConfigKey.Domain("mangadex.org") - override val userAgentKey = ConfigKey.UserAgent(USER_AGENT) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = request.newBuilder() - .header(CommonHeaders.USER_AGENT, USER_AGENT) - .build() - return chain.proceed(newRequest) - } + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) private val preferredServerKey = ConfigKey.PreferredImageServer( presetValues = mapOf( From d9db4a9c8684db3bb2b11f0444b9c260549985bf Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 9 Aug 2026 15:23:57 +0200 Subject: [PATCH 277/434] Implement EZManhwaParser for manga parsing --- .../kotlin/tsuki/parsers/EZManhwaParser.kt | 269 ++++++++++++++++++ 1 file changed, 269 insertions(+) diff --git a/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt index 8b137891..262320fd 100644 --- a/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt +++ b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt @@ -1 +1,270 @@ +package tsuki.parsers +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet + +import org.json.JSONObject +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +abstract class EZManhwaParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + api: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + protected open val apiUrl = api + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isAuthorSearchSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.NOVEL, + ), + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val endpoint = if (filter.query.isNullOrBlank()) "$apiUrl/series" else "$apiUrl/series/search" + val url = endpoint.toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + + if (filter.query.isNullOrBlank()) { + val sort = when (order) { + SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "latest" + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popular" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "newest" + SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "alphabetical" + else -> "latest" + } + addQueryParameter("sort", sort) + + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + MangaState.ABANDONED -> "DROPPED" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + }) + } + } else { + addQueryParameter("q", filter.query) + } + }.build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: JSONObject): List<Manga> { + val data = json.getJSONArray("data") + return data.mapJSON { obj -> + obj.toManga(domain, source, isNsfwSource) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val url = "$apiUrl/series/${manga.url}" + val json = webClient.httpGet(url).parseJson() + val updatedManga = json.toManga(domain, source, isNsfwSource).copy( + id = manga.id, + source = source, + ) + val chapters = fetchChapters(manga.url) + return updatedManga.copy(chapters = chapters) + } + + private suspend fun fetchChapters(seriesSlug: String): List<MangaChapter> { + val allChapters = mutableListOf<MangaChapter>() + var page = 1 + var hasMore = true + + while (hasMore) { + val url = "$apiUrl/series/$seriesSlug/chapters?page=$page&perPage=100&sort=desc" + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + val totalPages = json.getInt("totalPages") + val current = json.getInt("current") + + data.mapJSONToSet { chObj -> + chObj.toSChapter(seriesSlug, dateFormat) + }.let { allChapters.addAll(it) } + + hasMore = current < totalPages + page++ + } + return allChapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = "$apiUrl/${chapter.url}" + val json = webClient.httpGet(url).parseJson() + + if (json.optBoolean("requiresPurchase", false)) { + throw AuthRequiredException( + source, + RuntimeException("This chapter requires coins. Purchase it via the website to read.") + ) + } + + val images = json.getJSONArray("images") + return (0 until images.length()).map { i -> + val imgUrl = images.getJSONObject(i).getString("url") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + fun JSONObject.toManga(domain: String, source: MangaSource, isNsfw: Boolean): Manga { + val slug = getString("slug") + val title = getString("title") + val cover = optString("cover", "").takeIf { it.isNotBlank() && it != "null" } + val status = optString("status", "").takeIf { it.isNotBlank() && it != "null" } + val altTitles = optString("alternativeTitles", "").takeIf { it.isNotBlank() && it != "null" } + val description = optString("description", "").takeIf { it.isNotBlank() && it != "null" } + val author = optString("author", "").takeIf { it.isNotBlank() && it != "null" } + val artist = optString("artist", "").takeIf { it.isNotBlank() && it != "null" } + val genresArray = optJSONArray("genres") + val genres = genresArray?.mapJSON { it.getString("name") } ?: emptyList() + + val state = when (status?.uppercase(Locale.ROOT)) { + "ONGOING", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "DROPPED" -> MangaState.ABANDONED + "HIATUS" -> MangaState.PAUSED + else -> null + } + + val altSet = altTitles?.split(",") + ?.map { it.trim() } + ?.filter { it.isNotBlank() && it != "null" } + ?.toSet() ?: emptySet() + val authors = setOfNotNull(author, artist).filter { it.isNotBlank() && it != "null" }.toSet() + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "/series/$slug".toAbsoluteUrl(domain), + title = title, + altTitles = altSet, + coverUrl = cover, + description = description, + authors = authors, + tags = genres.map { MangaTag(key = it, title = it, source = source) }.toSet(), + state = state, + rating = RATING_UNKNOWN, + source = source, + contentRating = if (isNsfw) ContentRating.ADULT else null, + ) + } + + private fun JSONObject.toSChapter(seriesSlug: String, dateFormat: SimpleDateFormat): MangaChapter { + val slug = getString("slug") + val number = optDouble("number", -1.0).takeIf { it != -1.0 }?.toFloat() ?: -1f + val title = optString("title", "").takeIf { it.isNotBlank() && it != "null" } + val createdAt = optString("createdAt", "").takeIf { it.isNotBlank() && it != "null" } + + val prefix = if (optBoolean("requiresPurchase", false)) "🔒 " else "" + val numStr = if (number % 1.0 == 0.0) number.toInt().toString() else number.toString() + val chapterName = when { + title == null || title == numStr -> "Chapter $numStr" + title.matches(Regex("^(?i)(chapter|ch\\.?|episode|ep\\.?)\\s*.*")) -> title + title.startsWith("-") || title.startsWith(":") -> "Chapter $numStr $title" + else -> "Chapter $numStr - $title" + } + + return MangaChapter( + id = generateUid(slug), + url = "series/$seriesSlug/chapters/$slug", + title = prefix + chapterName, + number = number, + volume = 0, + uploadDate = dateFormat.parseSafe(createdAt), + scanlator = null, + branch = null, + source = source, + ) + } + + private fun SimpleDateFormat.parseSafe(dateStr: String?): Long { + return dateStr?.let { runCatching { parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L + } +} From db5f1c16ccae15ea097d12084e1f9d20bde5fa7e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 14:01:30 +0200 Subject: [PATCH 278/434] Update Mangago.kt --- src/main/kotlin/tsuki/site/en/Mangago.kt | 26 ++++++++++++------------ 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Mangago.kt b/src/main/kotlin/tsuki/site/en/Mangago.kt index 7e7a2015..9f6b90dd 100644 --- a/src/main/kotlin/tsuki/site/en/Mangago.kt +++ b/src/main/kotlin/tsuki/site/en/Mangago.kt @@ -30,6 +30,7 @@ import tsuki.util.requireElementById import tsuki.util.selectFirstOrThrow import tsuki.util.toAbsoluteUrl import tsuki.util.urlEncoded +import tsuki.util.extractChapterNumber import okhttp3.Interceptor import okhttp3.Response @@ -252,7 +253,7 @@ internal class MangagoParser(context: MangaLoaderContext) : authors = author?.let { setOf(it) } ?: emptySet(), tags = genres, state = state, - chapters = chapters, + chapters = chapters.sortedBy { it.number }, ) } @@ -278,9 +279,13 @@ internal class MangagoParser(context: MangaLoaderContext) : val scanlator = element.selectFirst("td.no a, td.uk-table-shrink a")?.text()?.trim() ?.ifEmpty { null } - ChapterParseData(name, url, dateUpload, scanlator) + ChapterParseData( + name, + url, + dateUpload, + scanlator + ) } - .reversed() return buildChapterList(rawChapters) } @@ -363,21 +368,16 @@ internal class MangagoParser(context: MangaLoaderContext) : return result } + private fun extractChapterNumber(title: String): Float? { + val num = title.extractChapterNumber() + return if (num == 0f) null else num + } + private fun extractTitleSuffix(title: String): String? { val regex = Regex("""(?:ch\.?|chapter)\s*\d+(?:\.\d+)?\s*[:\-]\s*(.+)""", RegexOption.IGNORE_CASE) return regex.find(title)?.groupValues?.get(1)?.trim()?.takeIf { it.isNotEmpty() } } - private fun extractChapterNumber(title: String): Float? { - return try { - val regex = Regex("""(?:ch\.?|chapter|vol\.?\s*\d+\s+ch\.?)\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - val match = regex.find(title) - match?.groupValues?.get(1)?.toFloat() - } catch (_: Exception) { - null - } - } - private val jsCache = mutableMapOf<String, Pair<String, Long>>() private val maxCacheTime = 1000 * 60 * 5 // 5 minutes From 5035f96167972a7aa8ae6c16a8f1afe5db9a9d8c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:54:09 +0200 Subject: [PATCH 279/434] Update QiScans.kt --- src/main/kotlin/tsuki/site/en/QiScans.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/QiScans.kt b/src/main/kotlin/tsuki/site/en/QiScans.kt index 390a8223..e8d27571 100644 --- a/src/main/kotlin/tsuki/site/en/QiScans.kt +++ b/src/main/kotlin/tsuki/site/en/QiScans.kt @@ -105,7 +105,7 @@ internal class QiScans(context: MangaLoaderContext) : } companion object { - val GENRES = listOf( + private val GENRES = listOf( "" to "All", "acting" to "Acting", "action" to "Action", From 971b7b424881c73dedc782b4d9af4251501d8d47 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:54:56 +0200 Subject: [PATCH 280/434] Update MangaOni.kt --- src/main/kotlin/tsuki/site/es/MangaOni.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/es/MangaOni.kt b/src/main/kotlin/tsuki/site/es/MangaOni.kt index b893c139..08ad9124 100644 --- a/src/main/kotlin/tsuki/site/es/MangaOni.kt +++ b/src/main/kotlin/tsuki/site/es/MangaOni.kt @@ -316,7 +316,7 @@ internal class MangaOni(context: MangaLoaderContext) : } companion object { - val GENRES = listOf( + private val GENRES = listOf( Triple("Todos", "Todos", "false"), Triple("Comedia", "Comedia", "1"), Triple("Drama", "Drama", "2"), From b888b6730e5017cbafaada293a4f7a635f0d7278 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:56:31 +0200 Subject: [PATCH 281/434] Update CGBUM.kt --- src/main/kotlin/tsuki/site/id/CGBUM.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/CGBUM.kt b/src/main/kotlin/tsuki/site/id/CGBUM.kt index e6cb9dda..401162f8 100644 --- a/src/main/kotlin/tsuki/site/id/CGBUM.kt +++ b/src/main/kotlin/tsuki/site/id/CGBUM.kt @@ -274,7 +274,7 @@ class CGBUM(context: MangaLoaderContext) : } companion object { - val GENRES = listOf( + private val GENRES = listOf( "adaptation", "adult", "adultmature", "age gap", "ahegao", "aliens", "anal", "animals", "anthology", "bdsm", "beasts", "big ass", "big breast", "big breasts", "big penis", "bisexual", "blackmail", "bloody", "blowjob", "body swap", "bodyswap", From 881889a7e10f96423a0b8b72b7b205e6b0943beb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 22:56:42 +0200 Subject: [PATCH 282/434] Update Wurmz.kt --- src/main/kotlin/tsuki/site/id/Wurmz.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/Wurmz.kt b/src/main/kotlin/tsuki/site/id/Wurmz.kt index a286eff6..7dd29c24 100644 --- a/src/main/kotlin/tsuki/site/id/Wurmz.kt +++ b/src/main/kotlin/tsuki/site/id/Wurmz.kt @@ -255,7 +255,7 @@ class Wurmz(context: MangaLoaderContext) : } companion object { - val GENRES = listOf( + private val GENRES = listOf( "Fantasy", "Romance", "Drama", "Action", "Comedy", "Shounen", "Adventure", "Shoujo", "School", "Slice of Life", "Seinen", "Supernatural", "Historical", "Josei", "Isekai", "Webtoons", "Martial Art", "Harem", "Ecchi", "Reincarnation", "Magic", "Mystery", From 20d733c0797f92b9e5f380a35b837ef084674714 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 10 Aug 2026 23:36:07 +0200 Subject: [PATCH 283/434] Create AstraManga.kt --- src/main/kotlin/tsuki/site/ru/AstraManga.kt | 367 ++++++++++++++++++++ 1 file changed, 367 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ru/AstraManga.kt diff --git a/src/main/kotlin/tsuki/site/ru/AstraManga.kt b/src/main/kotlin/tsuki/site/ru/AstraManga.kt new file mode 100644 index 00000000..ae3edac1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/AstraManga.kt @@ -0,0 +1,367 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Favicon +import tsuki.model.Favicons +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("ASTRAMANGA", "AstraManga", "ru") +internal class AstraManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ASTRAMANGA, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("astramanga.org") + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/favicon.ico", 256, null), + ), + domain, + ) + } + + private val apiUrl = "https://api.$domain/api/v1" + private val mediaUrl = "https://$domain/media" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (name, id) -> MangaTag(name, id, source) }.toSet(), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildSearchUrl(page, order, filter) + val json = webClient.httpGet(url).parseJson() + val data = json.optJSONObject("data") ?: return emptyList() + val titles = data.optJSONArray("titles") ?: return emptyList() + return (0 until titles.length()).map { i -> + parseSearchManga(titles.getJSONObject(i)) + } + } + + private fun buildSearchUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return "$apiUrl/search".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", "30") + .apply { + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { + addQueryParameter("q", it) + } + + val sortParam = when (order) { + SortOrder.POPULARITY -> "-popularity" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.RATING -> "-rating" + SortOrder.ALPHABETICAL -> "name" + else -> "-popularity" + } + addQueryParameter("sort", sortParam) + + filter.types.mapNotNull { type -> + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comics" + else -> null + } + }.takeIf { it.isNotEmpty() }?.let { + addQueryParameter("types", it.joinToString(",")) + } + + filter.tags.map { it.key } + .takeIf { it.isNotEmpty() } + ?.let { addQueryParameter("genres", it.joinToString(",")) } + + filter.tagsExclude.map { it.key } + .takeIf { it.isNotEmpty() } + ?.let { addQueryParameter("exclude_genres", it.joinToString(",")) } + + if (filter.yearFrom > 0) addQueryParameter("year_from", filter.yearFrom.toString()) + if (filter.yearTo > 0) addQueryParameter("year_to", filter.yearTo.toString()) + } + .build() + .toString() + } + + private fun parseSearchManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("name") + val coverUrl = resolveCoverUrl(obj) + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + largeCoverUrl = null, + description = null, + tags = emptySet(), + authors = emptySet(), + state = null, + rating = RATING_UNKNOWN, + contentRating = null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url + val titleUrl = "$apiUrl/titles/$slug" + val json = webClient.httpGet(titleUrl).parseJson() + val data = json.getJSONObject("data") + val updated = parseDetailManga(data) + + val titleId = data.getInt("id") + val branchesUrl = "$apiUrl/titles/$titleId/branches" + val branchesJson = webClient.httpGet(branchesUrl).parseJson() + val branches = branchesJson.getJSONObject("data").getJSONArray("branches") + var branchId = -1 + var maxChapters = 0 + for (i in 0 until branches.length()) { + val branch = branches.getJSONObject(i) + val isMain = branch.optBoolean("is_main", false) + val count = branch.optInt("count_chapters", 0) + if (isMain || count > maxChapters) { + branchId = branch.getInt("id") + maxChapters = count + } + } + if (branchId == -1) return updated.copy(chapters = emptyList()) + + val chaptersUrl = "$apiUrl/branches/$branchId/chapters" + val chaptersJson = webClient.httpGet( + chaptersUrl.toHttpUrl().newBuilder() + .addQueryParameter("size", maxChapters.coerceAtLeast(100).toString()) + .build() + ).parseJson() + val items = chaptersJson.getJSONObject("data").getJSONArray("items") + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until items.length()) { + val ch = items.getJSONObject(i) + val chId = ch.getLong("id") + val number = ch.getDouble("number").toFloat() + val volume = ch.optInt("volume_number", 0) + val name = ch.optString("name", null) + val publishedAt = ch.optString("published_at", null) + val uploadDate = dateFormat.parseSafe(publishedAt) + val title = buildString { + if (volume > 0) append("Том $volume ") + append("Глава ${number.toString().removeSuffix(".0")}") + if (!name.isNullOrBlank()) append(" — $name") + } + chapters.add( + MangaChapter( + id = generateUid(chId.toString()), + title = title, + number = number, + volume = volume, + url = "$slug/${number.toString().removeSuffix(".0")}/$chId", + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + ) + } + return updated.copy(chapters = chapters.sortedBy { it.number }) + } + + private fun parseDetailManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("name") + val coverUrl = resolveCoverUrl(obj) + val description = obj.optString("description", null) + val status = obj.optString("status", null) + val state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "paused" -> MangaState.PAUSED + "frozen", "discontinued" -> MangaState.ABANDONED + else -> null + } + + val rawRating = obj.optDouble("rating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val genresArray = obj.optJSONArray("genres") + val tagsArray = obj.optJSONArray("tags") + val allTags = mutableSetOf<MangaTag>() + genresArray?.let { arr -> + for (i in 0 until arr.length()) { + val genre = arr.getJSONObject(i) + val id = genre.getString("id") + val name = genre.getString("name") + allTags.add(MangaTag(id, name, source)) + } + } + tagsArray?.let { arr -> + for (i in 0 until arr.length()) { + val tag = arr.getJSONObject(i) + val id = tag.getString("id") + val name = tag.getString("name") + if (allTags.none { it.title == name }) { + allTags.add(MangaTag(id, name, source)) + } + } + } + + val authors = mutableSetOf<String>() + obj.optJSONObject("publishing_house")?.optString("name")?.let { authors.add(it) } + obj.optJSONArray("publishers")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.optString("name")?.let { authors.add(it) } + } + } + + val altTitles = mutableSetOf<String>() + obj.optString("secondary_name", null)?.let { altTitles.add(it) } + obj.optJSONArray("alternative_names")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optString(i)?.let { altTitles.add(it) } + } + } + + val isNsfw = obj.optBoolean("is_erotic", false) || + obj.optBoolean("is_yaoi", false) || + obj.optBoolean("is_lgbt", false) + val contentRating = if (isNsfw) ContentRating.ADULT else ContentRating.SAFE + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = altTitles, + coverUrl = coverUrl, + largeCoverUrl = null, + description = description, + tags = allTags, + authors = authors, + state = state, + rating = rating, + contentRating = contentRating, + source = source, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfterLast("/") + val url = "$apiUrl/chapters/$chapterId/pages" + val json = webClient.httpGet(url).parseJson() + val pages = json.getJSONObject("data").getJSONArray("pages") + return (0 until pages.length()).map { i -> + val imageUrl = pages.getJSONObject(i).getString("image_url") + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun resolveCoverUrl(obj: JSONObject): String? { + val coverVersions = obj.optJSONObject("cover_versions") + coverVersions?.optString("mid")?.takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + coverVersions?.optString("high")?.takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + obj.optString("cover_image").takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + return null + } + + private fun resolveUrl(path: String): String { + return if (path.startsWith("http://") || path.startsWith("https://")) { + path + } else { + "$mediaUrl/$path" + } + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L + } + + companion object { + private val GENRES = mapOf( + "Аниме" to "1", "Антиутопия" to "2", "Апокалиптический" to "3", "Арт" to "4", + "Безумие" to "5", "Боевик" to "6", "Боевые искусства" to "7", "Вестерн" to "8", + "Военное" to "9", "Выживание" to "10", "Гарем" to "11", "Героическое фэнтези" to "12", + "Гуро" to "13", "Гэг-юмор" to "14", "Детектив" to "15", "Детское" to "16", + "Дзёсей" to "17", "Драма" to "18", "Завоевание мира" to "19", "Исекай" to "20", + "Искусство" to "21", "Исторический" to "22", "Киберпанк" to "23", "Кодомо" to "24", + "Комедия" to "25", "Космос" to "26", "Криминал / Преступники" to "27", + "Кулинария" to "28", "Культивация" to "29", "Литрес" to "30", + "Махо-сёдзё" to "31", "Меха" to "32", "Мистика" to "33", "Мифология" to "34", + "Мурим" to "35", "Научная фантастика" to "36", "Обратный Гарем" to "37", + "Омегаверс" to "38", "Пародия" to "39", "Повседневность" to "40", + "Постапокалипсис" to "41", "Приключения" to "42", "Психология" to "43", + "Регрессия" to "44", "Рисование" to "45", "Романтика" to "46", + "Самурайский боевик" to "47", "Сверхъестественное" to "48", "Сёдзе" to "49", + "Сёнен" to "50", "Спорт" to "51", "Средневековье" to "52", "Стимпанк" to "53", + "Сэйнэн" to "54", "Сянься" to "55", "Трагедия" to "56", "Триллер" to "57", + "Ужасы" to "58", "Фантастика" to "59", "Философия" to "60", "Фэнтези" to "61", + "Школьная жизнь" to "62", "Экшен" to "63", "Элементы юмора" to "64", + "Юмор" to "65", "Образовательная литература" to "131" + ) + } +} From 5dc0ea2b3f40055dbef41b64cba722bb4505d259 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:55:11 +0200 Subject: [PATCH 284/434] Add siteMangadotnet.kt file --- .../Mangadotnet.kt => all/siteMangadotnet.kt} | 207 ++++++++++-------- 1 file changed, 117 insertions(+), 90 deletions(-) rename src/main/kotlin/tsuki/{site/en/Mangadotnet.kt => all/siteMangadotnet.kt} (73%) diff --git a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt b/src/main/kotlin/tsuki/all/siteMangadotnet.kt similarity index 73% rename from src/main/kotlin/tsuki/site/en/Mangadotnet.kt rename to src/main/kotlin/tsuki/all/siteMangadotnet.kt index e670206d..00fe8790 100644 --- a/src/main/kotlin/tsuki/site/en/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/all/siteMangadotnet.kt @@ -1,4 +1,4 @@ -package tsuki.site.en +package tsuki.site.all import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -25,6 +25,8 @@ import tsuki.util.parseJson import tsuki.util.parseJsonArray import tsuki.util.parseSafe +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject @@ -46,30 +48,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) - private val genreNames = setOf( - "Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Historical", "Horror", "Mecha", "Mystery", "Psychological", - "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", - "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", - "Medical", "Military", "Music", "School Life", "Webtoon", - "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", - "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", - "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", - "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", - "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", - "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", - "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", - "Magical Girls", "Mahou Shoujo", "Mature", "Medieval Area", - "Monster Girls", "Monsters", "Ninja", "Nobility", - "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", - "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", - "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", - "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", - "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", - "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", - "Wuxia", "Yaoi", "Yuri", "Zombies" - ) - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { super.onCreateConfig(keys) keys.add(userAgentKey) @@ -116,6 +94,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : isMultipleTagsSupported = true, isTagsExclusionSupported = true, isSearchWithFiltersSupported = true, + isOriginalLocaleSupported = true, ) override val availableSortOrders: Set<SortOrder> = EnumSet.of( @@ -127,7 +106,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = genreNames.map { name -> MangaTag(name, name, source) }.toSet() + val tags = GENRES.map { name -> MangaTag(name, name, source) }.toSet() return MangaListFilterOptions( availableTags = tags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), @@ -143,10 +122,21 @@ internal class Mangadotnet(context: MangaLoaderContext) : Demographic.SHOUJO, Demographic.SHOUNEN ), + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.GERMAN, + Locale("id"), + Locale("pt", "BR"), + Locale("es"), + Locale("th"), + Locale("vi"), + ), ) } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + selectedLanguage = filter.locale?.let { localeToLang(it) } ?: "en" if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.states.isNotEmpty() || filter.types.isNotEmpty() || filter.demographics.isNotEmpty()) { return getSearchPage(page, order, filter) @@ -256,22 +246,17 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } - private val detailsCacheLock = Any() - - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + private var selectedLanguage: String? = null + private fun localeToLang(locale: Locale): String { + if (locale.language == "pt" && locale.country == "BR") return "pt-br" + return locale.language } - @Suppress("UNCHECKED_CAST") - override suspend fun getDetails(manga: Manga): Manga { - synchronized(detailsCacheLock) { - detailsCache[manga.url]?.let { return it } - } - + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return manga + val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return@coroutineScope manga - val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return manga + val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return@coroutineScope manga val title = mangaInfo["title"] as? String ?: manga.title val description = mangaInfo["description"] as? String val photo = mangaInfo["photo"] as? String @@ -321,18 +306,28 @@ internal class Mangadotnet(context: MangaLoaderContext) : "CN" -> tagSet.add(MangaTag("Manhua", "Manhua", source)) } - val tagsArray = mangaInfo["tags"] as? List<Map<String, Any?>> - tagsArray?.flatMap { category -> - (category["tags"] as? List<Map<String, Any?>> ?: emptyList()) - .mapNotNull { it["name"] as? String } - }?.distinct()?.take(20)?.forEach { tagSet.add(MangaTag(it, it, source)) } + val tagsArray = mangaInfo["tags"] + if (tagsArray is List<*>) { + for (category in tagsArray) { + if (category !is Map<*, *>) continue + val innerTags = category["tags"] + if (innerTags !is List<*>) continue + for (tag in innerTags) { + if (tag !is Map<*, *>) continue + val name = tag["name"] as? String ?: continue + val key = name.lowercase().replace(" ", "-") + tagSet.add(MangaTag(key, name, source)) + } + } + } val rawRating = (mangaInfo["avg_rating"] as? Number)?.toDouble() ?: -1.0 val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - val chapters = fetchChapters(manga.url) + val chapterLang = selectedLanguage ?: "en" + val chaptersDeferred = async { fetchChaptersAndVolumes(manga.url, chapterLang) } - val result = manga.copy( + manga.copy( title = title, coverUrl = coverUrl, largeCoverUrl = largeCoverUrl, @@ -342,12 +337,8 @@ internal class Mangadotnet(context: MangaLoaderContext) : tags = tagSet, state = state, authors = setOfNotNull(author).filterTo(mutableSetOf()) { it.isNotBlank() }, - chapters = chapters, + chapters = chaptersDeferred.await(), ) - synchronized(detailsCacheLock) { - detailsCache[manga.url] = result - } - return result } private fun parseJsonArrayString(raw: String?): List<String>? { @@ -358,8 +349,20 @@ internal class Mangadotnet(context: MangaLoaderContext) : } catch (_: Exception) { null } } - private suspend fun fetchChapters(mangaId: String): List<MangaChapter> { - val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=en").parseJsonArray() + private suspend fun fetchChaptersAndVolumes(mangaId: String, lang: String): List<MangaChapter> = coroutineScope { + val chaptersDeferred = async { fetchChaptersFromApi(mangaId, lang) } + val volumesDeferred = async { fetchVolumesFromApi(mangaId, lang) } + + val chapters = chaptersDeferred.await() + val volumes = volumesDeferred.await() + val builder = ChaptersListBuilder(chapters.size + volumes.size) + builder.addAll(chapters) + builder.addAll(volumes) + builder.toList() + } + + private suspend fun fetchChaptersFromApi(mangaId: String, lang: String): List<MangaChapter> { + val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=$lang").parseJsonArray() val allChapters = (0 until response.length()).map { response.getJSONObject(it) } val chaptersByTeam = mutableMapOf<String, MutableList<JSONObject>>() @@ -373,7 +376,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : } val allNumbers = allChapters.map { it.optDouble("chapter_number", 0.0).toFloat() }.distinct().sorted() - val builder = ChaptersListBuilder(allChapters.size * chaptersByTeam.size) + val result = mutableListOf<MangaChapter>() for ((team, teamChapters) in chaptersByTeam) { val mapByNum = teamChapters.associateBy { it.optDouble("chapter_number", 0.0).toFloat() } @@ -395,7 +398,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : put("id", chId) put("source", src) }.toString() - builder.add( + result.add( MangaChapter( id = generateUid("$team-$chId"), title = title, @@ -410,45 +413,42 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } } + return result + } + private suspend fun fetchVolumesFromApi(mangaId: String, lang: String): List<MangaChapter> { val volumesResponse = runCatching { - webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=en").parseJsonArray() - }.getOrNull() - - if (volumesResponse != null) { - val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } - .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } - - for (vol in volumes) { - if (vol.optString("language", "en") != "en") continue - val volId = vol.getInt("id") - val volNum = vol.optDouble("volume_number", 0.0).toFloat() - val volDate = vol.optString("date_added", "").nullIfEmpty() - val volSource = vol.optString("source", "user") - val volGroup = vol.optString("group_name", "").nullIfEmpty() - ?: vol.optString("scanlator_name", "").nullIfEmpty() - ?: "Unknown" - val volumeUrl = JSONObject().apply { - put("id", volId) - put("source", volSource) - }.toString() - builder.add( - MangaChapter( - id = generateUid("vol-$volId"), - title = "Volume ${volNum.toString().removeSuffix(".0")}", - number = volNum, - volume = 0, - url = volumeUrl, - uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = volGroup, - branch = "Volume", - ) - ) - } + webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=$lang").parseJsonArray() + }.getOrNull() ?: return emptyList() + + val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } + .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } + + return volumes.mapNotNull { vol -> + if (vol.optString("language", "en") != lang) return@mapNotNull null + val volId = vol.getInt("id") + val volNum = vol.optDouble("volume_number", 0.0).toFloat() + val volDate = vol.optString("date_added", "").nullIfEmpty() + val volSource = vol.optString("source", "user") + val volGroup = vol.optString("group_name", "").nullIfEmpty() + ?: vol.optString("scanlator_name", "").nullIfEmpty() + ?: "Unknown" + val volumeUrl = JSONObject().apply { + put("id", volId) + put("source", volSource) + }.toString() + MangaChapter( + id = generateUid("vol-$volId"), + title = "Volume ${volNum.toString().removeSuffix(".0")}", + number = volNum, + volume = 0, + url = volumeUrl, + uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = volGroup, + branch = "Volume", + ) } - - return builder.toList() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { @@ -501,4 +501,31 @@ internal class Mangadotnet(context: MangaLoaderContext) : @Suppress("UNCHECKED_CAST") private fun Map<String, Any?>.asMap(key: String): Map<String, Any?>? = this[key] as? Map<String, Any?> private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this + + companion object { + private val GENRES = setOf( + "Action", "Adventure", "Comedy", "Drama", "Fantasy", + "Historical", "Horror", "Mecha", "Mystery", "Psychological", + "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", + "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", + "Medical", "Military", "Music", "School Life", "Webtoon", + "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", + "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", + "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", + "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", + "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", + "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", + "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", + "Magical Girls", "Mahou Shoujo", "Mature", "Medieval Area", + "Monster Girls", "Monsters", "Ninja", "Nobility", + "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", + "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", + "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", + "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", + "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", + "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", + "Wuxia", "Yaoi", "Yuri", "Zombies" + ) + + } } From 0b216d953032b79c1b197bb6a84eb50328dbca25 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 11 Aug 2026 12:55:27 +0200 Subject: [PATCH 285/434] Rename src/main/kotlin/tsuki/all/siteMangadotnet.kt to src/main/kotlin/tsuki/site/all/Mangadotnet.kt --- .../tsuki/{all/siteMangadotnet.kt => site/all/Mangadotnet.kt} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/kotlin/tsuki/{all/siteMangadotnet.kt => site/all/Mangadotnet.kt} (100%) diff --git a/src/main/kotlin/tsuki/all/siteMangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt similarity index 100% rename from src/main/kotlin/tsuki/all/siteMangadotnet.kt rename to src/main/kotlin/tsuki/site/all/Mangadotnet.kt From 4e43beaf7dcfed9bcc53bc74e4d2e6c8e757ebbd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:00:38 +0200 Subject: [PATCH 286/434] Update HentaiFox.kt --- src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt index 8f5f1efb..106ddd27 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt @@ -17,6 +17,8 @@ import tsuki.model.Manga import tsuki.model.MangaListFilter import tsuki.model.MangaListFilterCapabilities import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import org.jsoup.internal.StringUtil import org.jsoup.nodes.Element @@ -37,6 +39,15 @@ internal class HentaiFox(context: MangaLoaderContext) : isMultipleTagsSupported = true, ) + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/images/favicon.ico", 16, null), + ), + domain, + ) + } + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { From c9a3dc931e22dc01262db29ada3ba09cc8af85b3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:25:23 +0200 Subject: [PATCH 287/434] Update Softkomik.kt --- src/main/kotlin/tsuki/site/id/Softkomik.kt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/kotlin/tsuki/site/id/Softkomik.kt b/src/main/kotlin/tsuki/site/id/Softkomik.kt index 5c0713b4..8e407288 100644 --- a/src/main/kotlin/tsuki/site/id/Softkomik.kt +++ b/src/main/kotlin/tsuki/site/id/Softkomik.kt @@ -40,6 +40,8 @@ import java.net.URLDecoder import java.util.concurrent.ConcurrentHashMap import java.util.concurrent.TimeUnit +import tsuki.Broken +@Broken @MangaSourceParser("SOFTKOMIK", "Softkomik", "id") internal class Softkomik(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.SOFTKOMIK, pageSize = 24), MangaParserAuthProvider { From 358301f1b85b63f94a170c60839668170771845a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 11 Aug 2026 13:36:48 +0200 Subject: [PATCH 288/434] Update MistScans.kt --- src/main/kotlin/tsuki/site/en/MistScans.kt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MistScans.kt b/src/main/kotlin/tsuki/site/en/MistScans.kt index f931a982..4a497058 100644 --- a/src/main/kotlin/tsuki/site/en/MistScans.kt +++ b/src/main/kotlin/tsuki/site/en/MistScans.kt @@ -20,16 +20,16 @@ import tsuki.model.SortOrder import tsuki.util.generateUid import tsuki.util.mapToSet +import tsuki.util.extractChapterNumber + import java.text.SimpleDateFormat import java.util.Calendar - import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.jsoup.Jsoup import org.jsoup.nodes.Document import org.jsoup.nodes.Element import java.util.EnumSet import java.util.Locale -import kotlin.collections.reversed @MangaSourceParser("MISTSCANS", "Mist Scans", "en") internal class MistScans(context: MangaLoaderContext) : @@ -198,7 +198,7 @@ internal class MistScans(context: MangaLoaderContext) : authors = listOfNotNull(author, artist).toSet(), tags = (genres + listOfNotNull(type)).map { MangaTag(it.lowercase(), it, source) }.toSet(), state = status, - chapters = chapters.reversed(), + chapters = chapters.sortedBy { it.number }, ) } @@ -228,7 +228,7 @@ internal class MistScans(context: MangaLoaderContext) : MangaChapter( id = generateUid(url), title = name, - number = 0f, + number = name.extractChapterNumber(), volume = 0, url = url, uploadDate = uploadDate, From 3c19fc3d24e4a9a1fec806461513c87c6fb2ac5e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:35:59 +0200 Subject: [PATCH 289/434] Update MadaraParser.kt --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index e9bb7c75..1d74ac0a 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -60,6 +60,10 @@ import java.util.Base64 import java.util.Calendar import java.util.EnumSet +/** + * Todo: make ContentType filter. + */ + internal abstract class MadaraParser( context: MangaLoaderContext, source: MangaParserSource, From 5c5ae3af3a9282a5432f4a89ec93079ec621d6ac Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 00:36:40 +0200 Subject: [PATCH 290/434] Create MangasOriginesFr.kt --- .../kotlin/tsuki/site/fr/MangasOriginesFr.kt | 191 ++++++++++++++++++ 1 file changed, 191 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt new file mode 100644 index 00000000..1f694b86 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -0,0 +1,191 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import java.time.LocalDate +import java.time.ZoneId +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGASORIGINESFR", "Mangas-Origines.fr", "fr") +internal class MangasOriginesFr(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGASORIGINESFR, "mangas-origines.fr") { + + override val datePattern = "dd/MM/yyyy" + override val tagPrefix = "manga-genres/" + override val listUrl = "oeuvre/" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = setOf( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), +// availableContentTypes = EnumSet.of( +// ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA +// ), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED + ), + ) + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val slug = manga.publicUrl.toMangaSlug() + val fullUrl = "https://$domain/oeuvre/$slug/" + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1.ori-sr-title")?.text() ?: manga.title + val cover = doc.selectFirst("div.ori-sr-cover img")?.attr("abs:src") ?: manga.coverUrl.orEmpty() + val description = buildString { + doc.select("div.ori-sr-syn-texte p").eachText().forEach { appendLine(it) } + }.trim() + + val infos = doc.select("div.ori-sr-infos dt").associate { dt -> + dt.text().lowercase(Locale.FRENCH) to dt.nextElementSibling()?.text().orEmpty() + } + + val state = when (infos["statut"]?.lowercase(Locale.FRENCH)) { + "en cours" -> MangaState.ONGOING + "terminé" -> MangaState.FINISHED + "en pause" -> MangaState.PAUSED + "abandonné", "annulé" -> MangaState.ABANDONED + else -> null + } + + val altTitle = infos["nom alternatif"]?.takeIf { it.isNotBlank() } + val authors = listOfNotNull( + infos["auteur"]?.takeIf { it.isNotBlank() }, + infos["scénario"]?.takeIf { it.isNotBlank() }, + infos["artiste"]?.takeIf { it.isNotBlank() }, + infos["dessin"]?.takeIf { it.isNotBlank() } + ).distinct().toSet() + + val tags = doc.select("div.ori-sr-genres a.ori-sr-genre").map { + MangaTag(it.text(), it.text(), source) + }.toSet() + + val chaptersDeferred = async { fetchChapters(slug) } + + manga.copy( + url = slug, + title = title, + altTitles = setOfNotNull(altTitle), + coverUrl = cover, + description = description, + authors = authors, + tags = tags, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + private suspend fun fetchChapters(slug: String): List<MangaChapter> { + val doc = webClient.httpPost( + "https://$domain/oeuvre/$slug/ajax/chapters/", + emptyMap() + ).parseHtml() + return doc.select("div.ori-chl-row").map { element -> + val link = element.selectFirst("a.ori-chl-corps")!! + val href = link.attr("href") + val name = element.selectFirst("span.ori-chl-nom")?.text() ?: link.text() + val dateText = element.selectFirst("span.ori-chl-date")?.text() + MangaChapter( + id = generateUid(href.toChapterSlug()), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = href.toChapterSlug(), + uploadDate = parseChapterDate(dateText), + source = source, + scanlator = null, + branch = null, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = "https://$domain/oeuvre/${chapter.url}/" + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.reading-content img.wp-manga-chapter-img").mapIndexed { _, img -> + val url = img.attr("src").trim() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun String.toMangaSlug(): String = trimEnd('/').substringAfterLast("/").substringBefore("?") + + private fun String.toChapterSlug(): String { + val parts = split("/").filter { it.isNotBlank() } + return if (parts.size >= 2) parts.takeLast(2).joinToString("/") else this + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val regex = Regex("""(\d{1,2})\s+(\p{L}+)\.?\s+(\d{4})""") + val match = regex.find(date) ?: return 0L + val (day, month, year) = match.destructured + val monthNumber = monthNumber(month) ?: return 0L + return try { + LocalDate.of(year.toInt(), monthNumber, day.toInt()) + .atStartOfDay(ZoneId.of("Europe/Paris")) + .toInstant() + .toEpochMilli() + } catch (_: Exception) { 0L } + } + + private fun monthNumber(month: String): Int? { + val name = month.lowercase(Locale.FRENCH) + return when { + name.startsWith("jan") -> 1 + name.startsWith("fev") || name.startsWith("fév") -> 2 + name.startsWith("mar") -> 3 + name.startsWith("avr") -> 4 + name.startsWith("mai") -> 5 + name.startsWith("juin") -> 6 + name.startsWith("juil") -> 7 + name.startsWith("ao") -> 8 + name.startsWith("sep") -> 9 + name.startsWith("oct") -> 10 + name.startsWith("nov") -> 11 + name.startsWith("dec") || name.startsWith("déc") -> 12 + else -> null + } + } +} From 4bb9a79966f8b6c0d288a867cc777d2f4995ed89 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 12:49:21 +0200 Subject: [PATCH 291/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 28 ++++++++++++++----------- 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index 6472ee2a..41ad3c0d 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -127,18 +127,22 @@ internal class MangaK(context: MangaLoaderContext) : append(page) append("&limit=") append(pageSize) - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.NEWEST -> "newest" - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "rating" - SortOrder.POPULARITY_WEEK -> "views_7days" - SortOrder.POPULARITY_MONTH -> "views_30days" - else -> "latest" - }, - ) + + // To have "Best match" results + if (filter.query.isNullOrBlank()) { + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "newest" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "rating" + SortOrder.POPULARITY_WEEK -> "views_7days" + SortOrder.POPULARITY_MONTH -> "views_30days" + else -> "latest" + } + ) + } filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { q -> append("&q=") From 60250c6d8e71bfe98938bf6fcd7957c9a43bdde7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 22:52:45 +0200 Subject: [PATCH 292/434] Update MangasOriginesFr.kt --- src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt index 1f694b86..7799755b 100644 --- a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -33,12 +33,7 @@ internal class MangasOriginesFr(context: MangaLoaderContext) : override val datePattern = "dd/MM/yyyy" override val tagPrefix = "manga-genres/" override val listUrl = "oeuvre/" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - + override val availableSortOrders: Set<SortOrder> = setOf( SortOrder.UPDATED, SortOrder.POPULARITY, From 844505415d1734ab73006b820a9aac7c0f1b8324 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:17:07 +0200 Subject: [PATCH 293/434] fix chapter images --- src/main/kotlin/tsuki/site/fr/AnimeSama.kt | 52 ++++++++++++++++------ 1 file changed, 39 insertions(+), 13 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt index b104e5b4..b8cf751b 100644 --- a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt +++ b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt @@ -1,6 +1,5 @@ package tsuki.site.fr -import kotlinx.coroutines.Dispatchers import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey @@ -23,6 +22,7 @@ import tsuki.util.generateUid import tsuki.util.parseHtml import tsuki.util.parseJson +import kotlinx.coroutines.Dispatchers import org.jsoup.nodes.Document import org.jsoup.nodes.Element import okhttp3.HttpUrl.Companion.toHttpUrlOrNull @@ -35,6 +35,11 @@ import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.withContext import java.net.URLEncoder +/** + * I dont like this website. + * Todo: fix cover image + */ + @MangaSourceParser("ANIMESAMA", "AnimeSama", "fr") internal class AnimeSama(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.ANIMESAMA, pageSize = 48) { @@ -138,10 +143,18 @@ internal class AnimeSama(context: MangaLoaderContext) : val title = el.selectFirst("h2.card-title")?.text() ?: return null val cover = proxyImage(el.selectFirst("img")?.absUrl("src")) return Manga( - id = generateUid(url), url = url, publicUrl = url, - title = title, coverUrl = cover, altTitles = emptySet(), - rating = RATING_UNKNOWN, contentRating = null, tags = emptySet(), - state = null, authors = emptySet(), source = source + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source ) } @@ -167,10 +180,18 @@ internal class AnimeSama(context: MangaLoaderContext) : val tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet() return Manga( - id = generateUid(url), url = url, publicUrl = url, - title = title, coverUrl = cover, altTitles = emptySet(), - rating = RATING_UNKNOWN, contentRating = null, tags = tags, - state = state, authors = setOfNotNull(author), description = description, + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = tags, + state = state, + authors = setOfNotNull(author), + description = description, source = source ) } @@ -266,9 +287,14 @@ internal class AnimeSama(context: MangaLoaderContext) : .build() return MangaChapter( id = generateUid(chapterUrl.toString()), - title = "Chapitre $chapterName", number = id.toFloat(), volume = 0, - url = chapterUrl.toString(), uploadDate = 0, scanlator = scanlator, - branch = null, source = source + title = "Chapitre $chapterName", + number = id.toFloat(), + volume = 0, + url = chapterUrl.toString(), + uploadDate = 0, + scanlator = scanlator, + branch = null, + source = source ) } @@ -280,7 +306,7 @@ internal class AnimeSama(context: MangaLoaderContext) : val rawTitle = url.queryParameter("title") ?: return emptyList() val encodedTitle = withContext(Dispatchers.IO) { URLEncoder.encode(rawTitle, "UTF-8") - } + }.replace("+", "%20") val apiUrl = "$baseUrl/s2/scans/get_nb_chap_et_img.php".toHttpUrl().newBuilder() .addQueryParameter("oeuvre", rawTitle) .build() From 305a5d9de735a8d15de6cd57a87f910b0e867162 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:47:19 +0200 Subject: [PATCH 294/434] Create ManhwaListID.kt --- src/main/kotlin/tsuki/site/id/ManhwaListID.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/ManhwaListID.kt diff --git a/src/main/kotlin/tsuki/site/id/ManhwaListID.kt b/src/main/kotlin/tsuki/site/id/ManhwaListID.kt new file mode 100644 index 00000000..e7e4a853 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/ManhwaListID.kt @@ -0,0 +1,11 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWALISTID", "ManhwaList ID", "id") +internal class ManhwaListID(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.MANHWALISTID, "manhwalist02.asia") From a58273575228c048ff7864832d74b3c8c5a482ee Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:51:04 +0200 Subject: [PATCH 295/434] Create Sasangeyou.kt --- .../kotlin/tsuki/site/id/nsfw/Sasangeyou.kt | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt b/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt new file mode 100644 index 00000000..b456847e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt @@ -0,0 +1,17 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SASANGEYOUNET", "Sasangeyou.net", "id", ContentType.HENTAI) +internal class Sasangeyou(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SASANGEYOUNET, "sasangeyou.net") { + override val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale("id")) +} From 276a56eb9f37c9a89a7f0b8b7e603a0561572b02 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:53:49 +0200 Subject: [PATCH 296/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 75533438..01081d36 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -302,7 +302,7 @@ abstract class MangaThemesia( branch = null, source = source, ) - }.sortedBy { it.number } + }.reversed() } protected open fun parseChapterDate(date: String?): Long { From f6cd1f829c1368994fdba69bf23cad7aa72308bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:59:59 +0200 Subject: [PATCH 297/434] Update print statement from 'Hello' to 'Goodbye' --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index 1d74ac0a..a7d88535 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -45,6 +45,7 @@ import tsuki.util.selectLast import tsuki.util.toMutableMap import tsuki.util.toRelativeUrl import tsuki.util.extractChapterNumber +import tsuki.util.mapChapters import androidx.collection.scatterSetOf import kotlinx.coroutines.async @@ -682,7 +683,7 @@ internal abstract class MadaraParser( protected open suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).map { li -> + return doc.body().select(selectChapter).mapChapters(reversed = true) { _, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -699,10 +700,9 @@ internal abstract class MadaraParser( scanlator = null, branch = null, ) - }.sortedBy { it.number } + } } - protected open suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { val doc = if (postReq) { val mangaId = document.select("div#manga-chapters-holder").attr("data-id") @@ -714,7 +714,7 @@ internal abstract class MadaraParser( webClient.httpPost(url, emptyMap()).parseHtml() } val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.select(selectChapter).map { li -> + return doc.select(selectChapter).mapChapters(reversed = true) { _, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage @@ -731,7 +731,7 @@ internal abstract class MadaraParser( scanlator = null, source = source, ) - }.sortedBy { it.number } + } } override suspend fun getRelatedManga(seed: Manga): List<Manga> { From f8a83ee111136159d9a10845235063b1453830a7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 00:47:09 +0200 Subject: [PATCH 298/434] Create Ikiru.kt --- src/main/kotlin/tsuki/site/id/Ikiru.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Ikiru.kt diff --git a/src/main/kotlin/tsuki/site/id/Ikiru.kt b/src/main/kotlin/tsuki/site/id/Ikiru.kt new file mode 100644 index 00000000..dd6f63c1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Ikiru.kt @@ -0,0 +1,11 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.NatsuParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("IKIRU", "Ikiru", "id") +internal class Ikiru(context: MangaLoaderContext) : + NatsuParser(context, MangaParserSource.IKIRU,"07.ikiru.wtf") From be4f651bef9c88524f8f56c2308af51984494bea Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:30:10 +0200 Subject: [PATCH 299/434] Create CosmicScans.kt --- src/main/kotlin/tsuki/site/id/CosmicScans.kt | 446 +++++++++++++++++++ 1 file changed, 446 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/CosmicScans.kt diff --git a/src/main/kotlin/tsuki/site/id/CosmicScans.kt b/src/main/kotlin/tsuki/site/id/CosmicScans.kt new file mode 100644 index 00000000..213b3e79 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/CosmicScans.kt @@ -0,0 +1,446 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseHtml + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("COSMICSCANSID", "CosmicScans", "id") +internal class CosmicScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.COSMICSCANSID, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("02.cosmicscans.to") + private val apiUrl = "https://cdncid.csmcscns.id/v1/manga" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Referer", "https://$domain/") + .build() + + private val cursorCache = mutableMapOf<String, String>() + private val lastPage = mutableMapOf<String, Int>() + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = GENRES.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val endpoint = if (query.isNotBlank()) "search" else "filter" + val key = buildSearchKey(endpoint, query, order, filter) + lastPage[key] = page + + val url = "$apiUrl/$endpoint".toHttpUrl().newBuilder() + .addQueryParameter("limit", pageSize.toString()) + .apply { + if (query.isNotBlank()) { + addQueryParameter("q", query) + } + if (endpoint == "filter") { + addQueryParameter("order_by", mapSortOrder(order)) + } + if (endpoint != "search") { + if (page > 1) { + cursorCache["$key:$page"]?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("after", it) + } + } + } + } + .build() + + val response = try { + webClient.httpGet(url, getRequestHeaders()) + } catch (e: Exception) { + throw ParseException("Failed to load list page $page (URL: $url): ${e.message}", url.toString(), e) + } + + val responseBody = try { + response.parseJson() + } catch (e: Exception) { + throw ParseException("Invalid JSON from list API (URL: $url): ${e.message}", url.toString(), e) + } + + val result = try { + parseMangaListResponse(responseBody, key) + } catch (e: Exception) { + throw ParseException("Failed to parse manga list (URL: $url): ${e.message}", url.toString(), e) + } + + return applyClientFilters(result, filter) + } + + private fun mapSortOrder(order: SortOrder): String = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" + SortOrder.NEWEST -> "added" + else -> "update" + } + + private fun buildSearchKey(endpoint: String, query: String, order: SortOrder, filter: MangaListFilter): + String = listOf( + endpoint, query, mapSortOrder(order), + filter.states.joinToString(",") { it.name }, + filter.types.joinToString(",") { it.name }, + filter.tags.joinToString(",") { it.key }, + ).joinToString(":") + + private fun parseMangaListResponse(json: JSONObject, key: String): List<Manga> { + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown API error") + throw ParseException("CosmicScans API error: $message", "https://$domain") + } + + val data = json.optJSONArray("data") + ?: throw ParseException("Missing 'data' array in list response", "https://$domain") + + val cursor = json.optJSONObject("cursor") + val page = lastPage[key] ?: 1 + + if (cursor != null) { + val nextCursor = cursor.optString("nextCursor", "") + if (nextCursor.isNotBlank()) { + cursorCache["$key:${page + 1}"] = nextCursor + } + } + + val mangas = mutableListOf<Manga>() + for (i in 0 until data.length()) { + val obj = data.getJSONObject(i) + mangas += parseMangaFromListJson(obj) + } + return mangas + } + + private fun parseMangaFromListJson(obj: JSONObject): Manga { + val title = obj.optString("title", "").ifBlank { + throw ParseException("Missing 'title' in manga entry", "https://$domain") + } + val slug = obj.optString("slug", "").ifBlank { + throw ParseException("Missing 'slug' in manga entry: $title", "https://$domain") + } + val cover = obj.optString("cover", "") + val status = obj.optString("status", "") + val type = obj.optString("type", "").takeIf { it.isNotBlank() && it != "null" } + val genresArray = obj.optJSONArray("genres") ?: JSONArray() + val genres = (0 until genresArray.length()).map { genresArray.getString(it) } + + val state = when (status.lowercase(Locale.ROOT)) { + "ongoing" -> MangaState.ONGOING + "completed", "complete" -> MangaState.FINISHED + "hiatus", "on hiatus", "on-hold", "on hold" -> MangaState.PAUSED + else -> null + } + + return Manga( + id = generateUid("/series/$slug"), + url = "/series/$slug", + publicUrl = "https://$domain/series/$slug/", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + authors = emptySet(), + state = state, + description = type?.let { "Type: $it" }, + source = source, + ) + } + + private fun applyClientFilters(mangas: List<Manga>, filter: MangaListFilter): List<Manga> { + return mangas.filter { manga -> + if (filter.states.isNotEmpty() && manga.state !in filter.states) { + return@filter false + } + + if (filter.types.isNotEmpty()) { + val mangaType = manga.description + ?.substringAfter("Type: ", "") + ?.lowercase(Locale.ROOT) + val matchesType = filter.types.any { type -> + when (type) { + ContentType.MANGA -> mangaType == "manga" + ContentType.MANHWA -> mangaType == "manhwa" + ContentType.MANHUA -> mangaType == "manhua" + else -> false + } + } + if (!matchesType) return@filter false + } + + if (filter.tags.isNotEmpty()) { + val mangaTagKeys = manga.tags.map { it.key }.toSet() + val allMatch = filter.tags.all { tag -> tag.key in mangaTagKeys } + if (!allMatch) return@filter false + } + + true + } + } + + override suspend fun getDetails(manga: Manga): Manga { + suspend fun fetchDetailFromSlug(slug: String): JSONObject? { + val apiUrl = "$apiUrl/mangaDetail/$slug" + return try { + val response = webClient.httpGet(apiUrl, getRequestHeaders()) + val json = response.parseJson() + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown error") + throw ParseException("Details API error for slug '$slug': $message", apiUrl) + } + json.optJSONObject("data") + } catch (e: ParseException) { + throw e + } catch (_: Exception) { + null + } + } + + val initialSlug = manga.url + .removePrefix("/series/") + .removeSuffix("/") + .ifBlank { throw ParseException("Manga URL is blank: ${manga.url}", manga.url) } + + var data = fetchDetailFromSlug(initialSlug) + + if (data == null) { + val pageUrl = manga.publicUrl + if (pageUrl.isBlank()) { + throw ParseException("Public URL missing for manga: ${manga.title}", manga.url) + } + + val doc = try { + webClient.httpGet(pageUrl).parseHtml() + } catch (e: Exception) { + throw ParseException("Failed to load manga page for slug resolution: $pageUrl", pageUrl, e) + } + + val canonicalUrl = doc.selectFirst("link[rel=canonical]")?.attr("href") + ?: doc.selectFirst("meta[property=og:url]")?.attr("content") + ?: pageUrl + + val canonicalSlug = canonicalUrl + .substringAfterLast('/') + .substringBefore('?') + .trim() + .ifBlank { throw ParseException("Could not extract canonical slug from page: $pageUrl", pageUrl) } + + if (canonicalSlug != initialSlug) { + data = fetchDetailFromSlug(canonicalSlug) + } + } + + if (data == null) { + throw ParseException( + "Details not found for manga '${manga.title}' (tried slugs: $initialSlug)", + manga.publicUrl, + ) + } + + val title = data.optString("title", manga.title).ifBlank { manga.title } + val cover = data.optString("cover", manga.coverUrl) + val sinopsis = data.optString("sinopsis", "").takeIf { it.isNotBlank() && it != "null" } + val type = data.optString("type", "").takeIf { it.isNotBlank() && it != "null" } + val author = data.optString("author", "").takeIf { it.isNotBlank() && it != "null" } + val artist = data.optString("artist", "").takeIf { it.isNotBlank() && it != "null" } + val genresArray = data.optJSONArray("genre") ?: data.optJSONArray("genres") ?: JSONArray() + val genres = (0 until genresArray.length()).map { genresArray.getString(it) } + val status = data.optString("status", "") + val state = when (status.lowercase(Locale.ROOT)) { + "ongoing" -> MangaState.ONGOING + "completed", "complete" -> MangaState.FINISHED + "hiatus", "on hiatus", "on-hold", "on hold" -> MangaState.PAUSED + else -> null + } + + val rating = data.optString("rating", "").toFloatOrNull() + val normalizedRating = if (rating != null && rating > 0f) rating / 2f else RATING_UNKNOWN + + val description = buildString { + if (sinopsis != null) append(sinopsis).append("\n\n") + if (type != null) append("Type: $type").append("\n") + if (author != null) append("Author: $author").append("\n") + if (artist != null) append("Artist: $artist") + }.trim().ifBlank { null } + + val chaptersJson = data.optJSONArray("chapters") ?: JSONArray() + val chapters = parseChapters(chaptersJson) + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = setOfNotNull(author), + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + state = state, + rating = normalizedRating, + chapters = chapters, + contentRating = null, + ) + } + + private fun parseChapters(chaptersJson: JSONArray): List<MangaChapter> { + if (chaptersJson.length() == 0) { + throw ParseException("No chapters found in manga detail response", "https://$domain") + } + + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until chaptersJson.length()) { + val obj = chaptersJson.getJSONObject(i) + val slug = obj.optString("slug", "").ifBlank { + throw ParseException("Chapter entry $i missing slug", "https://$domain") + } + val chapterNumRaw = obj.optString("chapterNum", "").ifBlank { + throw ParseException("Chapter entry $i missing chapterNum", "https://$domain") + } + val time = obj.optString("time", "") + val number = Regex("""^(\d+(?:\.\d+)?)""") + .find(chapterNumRaw) + ?.groupValues + ?.get(1) + ?.toFloatOrNull() + ?: throw ParseException("Invalid chapter number format: $chapterNumRaw", "https://$domain") + + val uploadDate = runCatching { dateFormat.parse(time)?.time }.getOrNull() ?: 0L + + chapters += MangaChapter( + id = generateUid("/chapter/$slug"), + title = "Chapter $chapterNumRaw".trim(), + url = "/chapter/$slug", + number = number, + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + + if (chapters.isEmpty()) { + throw ParseException("No valid chapters found", "https://$domain") + } + + return chapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterSlug = chapter.url.substringAfterLast('/') + if (chapterSlug.isBlank()) { + throw ParseException("Chapter URL missing slug: ${chapter.url}", chapter.url) + } + + val url = "$apiUrl/readingPage/$chapterSlug" + val response = try { + webClient.httpGet(url, getRequestHeaders()) + } catch (e: Exception) { + throw ParseException("Failed to load pages (URL: $url): ${e.message}", url, e) + } + + val json = try { + response.parseJson() + } catch (e: Exception) { + throw ParseException("Invalid JSON from reading page API (URL: $url): ${e.message}", url, e) + } + + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown error") + throw ParseException("Reading page API error: $message", url) + } + + val data = json.optJSONObject("data") + ?: throw ParseException("Missing 'data' object in reading page response", url) + + val chaptersArray = data.optJSONArray("chapters") + ?: throw ParseException("Missing 'chapters' array in reading page data", url) + + if (chaptersArray.length() == 0) { + throw ParseException("No pages found in reading page data", url) + } + + val pages = mutableListOf<MangaPage>() + for (i in 0 until chaptersArray.length()) { + val html = chaptersArray.getString(i) + val img = Jsoup.parse(html).selectFirst("img") + ?: throw ParseException("Page $i does not contain an <img> tag", url) + val imageUrl = img.attr("src").takeIf { it.isNotBlank() } + ?: throw ParseException("Page $i has empty src attribute", url) + pages += MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + + return pages + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + companion object { + private val GENRES = listOf( + "Action", "Adventure", "Comedy", "Drama", "Fantasy", "Martial Arts", "Romance", + "School Life", "Shounen", "Supernatural", "System", "Thriller", "Murim", + ) + } +} From 4a665377be02efb81fd0d0a17286e671069f93e8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 01:37:29 +0200 Subject: [PATCH 300/434] Update CosmicScans.kt --- src/main/kotlin/tsuki/site/id/CosmicScans.kt | 33 +++++--------------- 1 file changed, 8 insertions(+), 25 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/CosmicScans.kt b/src/main/kotlin/tsuki/site/id/CosmicScans.kt index 213b3e79..81d6a3cb 100644 --- a/src/main/kotlin/tsuki/site/id/CosmicScans.kt +++ b/src/main/kotlin/tsuki/site/id/CosmicScans.kt @@ -179,7 +179,6 @@ internal class CosmicScans(context: MangaLoaderContext) : } val cover = obj.optString("cover", "") val status = obj.optString("status", "") - val type = obj.optString("type", "").takeIf { it.isNotBlank() && it != "null" } val genresArray = obj.optJSONArray("genres") ?: JSONArray() val genres = (0 until genresArray.length()).map { genresArray.getString(it) } @@ -202,7 +201,6 @@ internal class CosmicScans(context: MangaLoaderContext) : tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), authors = emptySet(), state = state, - description = type?.let { "Type: $it" }, source = source, ) } @@ -300,12 +298,11 @@ internal class CosmicScans(context: MangaLoaderContext) : val title = data.optString("title", manga.title).ifBlank { manga.title } val cover = data.optString("cover", manga.coverUrl) val sinopsis = data.optString("sinopsis", "").takeIf { it.isNotBlank() && it != "null" } - val type = data.optString("type", "").takeIf { it.isNotBlank() && it != "null" } + val status = data.optString("status", "") val author = data.optString("author", "").takeIf { it.isNotBlank() && it != "null" } - val artist = data.optString("artist", "").takeIf { it.isNotBlank() && it != "null" } val genresArray = data.optJSONArray("genre") ?: data.optJSONArray("genres") ?: JSONArray() val genres = (0 until genresArray.length()).map { genresArray.getString(it) } - val status = data.optString("status", "") + val state = when (status.lowercase(Locale.ROOT)) { "ongoing" -> MangaState.ONGOING "completed", "complete" -> MangaState.FINISHED @@ -316,20 +313,13 @@ internal class CosmicScans(context: MangaLoaderContext) : val rating = data.optString("rating", "").toFloatOrNull() val normalizedRating = if (rating != null && rating > 0f) rating / 2f else RATING_UNKNOWN - val description = buildString { - if (sinopsis != null) append(sinopsis).append("\n\n") - if (type != null) append("Type: $type").append("\n") - if (author != null) append("Author: $author").append("\n") - if (artist != null) append("Artist: $artist") - }.trim().ifBlank { null } - val chaptersJson = data.optJSONArray("chapters") ?: JSONArray() val chapters = parseChapters(chaptersJson) return manga.copy( title = title, coverUrl = cover, - description = description, + description = sinopsis, authors = setOfNotNull(author), tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), state = state, @@ -341,25 +331,22 @@ internal class CosmicScans(context: MangaLoaderContext) : private fun parseChapters(chaptersJson: JSONArray): List<MangaChapter> { if (chaptersJson.length() == 0) { - throw ParseException("No chapters found in manga detail response", "https://$domain") + return emptyList() } val chapters = mutableListOf<MangaChapter>() for (i in 0 until chaptersJson.length()) { val obj = chaptersJson.getJSONObject(i) - val slug = obj.optString("slug", "").ifBlank { - throw ParseException("Chapter entry $i missing slug", "https://$domain") - } - val chapterNumRaw = obj.optString("chapterNum", "").ifBlank { - throw ParseException("Chapter entry $i missing chapterNum", "https://$domain") - } + val slug = obj.optString("slug", "").ifBlank { continue } + val chapterNumRaw = obj.optString("chapterNum", "").ifBlank { continue } val time = obj.optString("time", "") + val number = Regex("""^(\d+(?:\.\d+)?)""") .find(chapterNumRaw) ?.groupValues ?.get(1) ?.toFloatOrNull() - ?: throw ParseException("Invalid chapter number format: $chapterNumRaw", "https://$domain") + ?: continue val uploadDate = runCatching { dateFormat.parse(time)?.time }.getOrNull() ?: 0L @@ -376,10 +363,6 @@ internal class CosmicScans(context: MangaLoaderContext) : ) } - if (chapters.isEmpty()) { - throw ParseException("No valid chapters found", "https://$domain") - } - return chapters.sortedBy { it.number } } From 3e5a9e48b1547b58d1113d9f4f994beb5aa04d91 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:28:37 +0200 Subject: [PATCH 301/434] Create Manga18fx.kt --- .../kotlin/tsuki/site/en/nsfw/Manga18fx.kt | 210 ++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt new file mode 100644 index 00000000..d31cd509 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt @@ -0,0 +1,210 @@ +package tsuki.site.en.nsfw + + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.MangaPage +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat + +/** + * Not a real Madara website, it just imitates it good. + */ + +@MangaSourceParser("MANGA18FX", "Manga18fx", "en", ContentType.HENTAI) +internal class Manga18fx(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGA18FX, "manga18fx.com") { + + override val datePattern = "dd MMM yy" + override val selectDesc = ".dsct" + + override val selectDate = "span.chapter-time" + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = false, + isMultipleTagsSupported = false, + ) + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val scraped = doc.select(".header-bottom li a").mapNotNull { a -> + val href = a.attr("href") + val url = if (href.startsWith("http")) href else "https://$domain/$href" + val name = a.text().trim().ifBlank { return@mapNotNull null } + MangaTag(name, url, source) + }.toSet() + + val hardcoded = listOf( + MangaTag("Manhwa", "https://$domain/manga-genre/manhwa", source), + MangaTag("Manhua", "https://$domain/manga-genre/manhua", source), + MangaTag("Raw", "https://$domain/manga-genre/raw", source), + ) + + return scraped + hardcoded + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.isEmpty() && filter.tags.isNotEmpty()) { + val genreUrl = filter.tags.first().key + val doc = webClient.httpGet(genreUrl).parseHtml() + return parseLatestOrSearch(doc) + } + if (query.isNotEmpty()) { + val url = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("q", query) + .addQueryParameter("page", page.toString()) + .build() + val doc = webClient.httpGet(url).parseHtml() + return parseLatestOrSearch(doc) + } + + return when (order) { + SortOrder.POPULARITY -> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + parsePopular(doc) + } + else -> { + val url = "https://$domain/page/$page" + val doc = webClient.httpGet(url).parseHtml() + parseLatestOrSearch(doc) + } + } + } + + private fun parsePopular(doc: Document): List<Manga> { + val block = doc.selectFirst(".trending-block") ?: return emptyList() + return block.select("a").map { a -> mangaFromElement(a) } + } + + private fun parseLatestOrSearch(doc: Document): List<Manga> { + return doc.select(".bsx-item").mapNotNull { item -> + val a = item.selectFirst("a") ?: return@mapNotNull null + mangaFromElement(a) + } + } + + private fun mangaFromElement(a: Element): Manga { + val href = a.attrAsRelativeUrl("href").removeSuffix("/") + val title = a.attr("title").ifBlank { a.text() } + val img = a.selectFirst("img") + val cover = img?.attr("data-src").orEmpty().ifBlank { img?.attr("src").orEmpty() } + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> = parseChapters(doc) + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = parseChapters(document) + + private fun parseChapters(doc: Document): List<MangaChapter> { + val container = doc.selectFirst(".row-content-chapter") ?: return emptyList() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + + return container.children().mapNotNull { child -> + val a = child.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.text().trim() + val dateText = child.selectFirst("span.chapter-time")?.text() + val number = name.extractChapterNumber() + + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = parseChapterDate(dateFormat, dateText), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + var images = doc.select("div.page-break img") + if (images.isEmpty()) { + images = doc.select("div.read-content img, div.reading-content img") + } + return images.mapIndexed { index, img -> + val url = img.attr("data-src") + .ifBlank { img.attr("src") } + .ifBlank { throw ParseException("Image URL missing for page $index", fullUrl) } + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val fullUrl = seed.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select(".related-items .item").mapNotNull { item -> + val a = item.selectFirst("a[href*=/manga/]") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href").removeSuffix("/") + val title = item.selectFirst("h5 a")?.text()?.trim() + ?: a.attr("title").ifBlank { a.text() } + val img = item.selectFirst("img") + val cover = img?.attr("data-src").orEmpty().ifBlank { img?.attr("src").orEmpty() } + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } +} From 3714f0602bf7f87ddfec87d2663d6e54bdf08e79 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 12:59:55 +0200 Subject: [PATCH 302/434] Create EXHentai.kt --- .../kotlin/tsuki/site/all/nsfw/EXHentai.kt | 537 ++++++++++++++++++ 1 file changed, 537 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt diff --git a/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt new file mode 100644 index 00000000..2adb3d60 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt @@ -0,0 +1,537 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.bitmap.Rect +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.exception.TooManyRequestExceptions + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.copyCookies +import tsuki.util.cssUrl +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.headersContentLength +import tsuki.util.insertCookies +import tsuki.util.isNumeric +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.ownTextOrNull +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.selectFirstOrThrow +import tsuki.util.styleValueOrNull +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder + +import androidx.collection.ArraySet +import androidx.collection.MutableIntLongMap +import androidx.collection.MutableIntObjectMap +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Collections.emptyList +import java.util.EnumSet +import java.util.Locale +import java.util.concurrent.TimeUnit +import kotlin.collections.orEmpty + +private const val DOMAIN_UNAUTHORIZED = "e-hentai.org" +private const val DOMAIN_AUTHORIZED = "exhentai.org" +private val TAG_PREFIXES = arrayOf("male:", "female:", "other:") +private const val BANNED_RESPONSE_LENGTH = 256L + +@MangaSourceParser("EXHENTAI", "EXHentai", type = ContentType.HENTAI) +internal class EHentai(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor { + + override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) + + override val configKeyDomain: ConfigKey.Domain + get() { + val isAuthorized = checkAuth() + return ConfigKey.Domain( + if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, + if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED, + ) + } + + override val authUrl: String + get() = "https://${domain}/bounce_login.php" + + private val ratingPattern = Regex("-?[0-9]+px") + private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))") + private val spacesCleanupPattern = Regex("(^\\s+|\\s+$|\\s+(?=\\s))") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) + private val nextPages = MutableIntObjectMap<MutableIntLongMap>() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun isAuthorized(): Boolean = checkAuth() + + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = mapTags(), + availableContentTypes = EnumSet.of( + ContentType.DOUJINSHI, + ContentType.MANGA, + ContentType.ARTIST_CG, + ContentType.GAME_CG, + ContentType.COMICS, + ContentType.IMAGE_SET, + ContentType.OTHER, + ), + availableLocales = setOf( + Locale.JAPANESE, + Locale.ENGLISH, + Locale.CHINESE, + Locale("nl"), + Locale.FRENCH, + Locale.GERMAN, + Locale("hu"), + Locale.ITALIAN, + Locale("kr"), + Locale("pl"), + Locale("pt"), + Locale("ru"), + Locale("es"), + Locale("th"), + Locale("vi"), + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + return getListPage(page, order, filter, updateDm = false) + } + + private suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + updateDm: Boolean, + ): List<Manga> { + val next = synchronized(nextPages) { + nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L + } + + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + + val url = urlBuilder() + url.addEncodedQueryParameter("next", next.toString()) + url.addQueryParameter("f_search", filter.toSearchQuery()) + + val fCats = filter.types.toFCats() + if (fCats != 0) { + url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString()) + } + if (updateDm) { + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + url.addQueryParameter("inline_set", "dm_e") + } + url.addQueryParameter("advsearch", "1") + if (config[suspiciousContentKey]) { + url.addQueryParameter("f_sh", "on") + } + val body = webClient.httpGet(url.build()).parseHtml().body() + val root = body.selectFirst("table.itg")?.selectFirst("tbody") ?: if (updateDm) { + if (body.getElementsContainingText("No hits found").isNotEmpty()) { + return emptyList() + } else { + body.parseFailed("Cannot find root") + } + } else { + return getListPage(page, order, filter, updateDm = true) + } + val nextTimestamp = getNextTimestamp(body) + synchronized(nextPages) { + nextPages.getOrPut(filter.hashCode()) { + MutableIntLongMap() + }.put(page + 1, nextTimestamp) + } + + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val gLink = td2.selectFirstOrThrow("div.glink") + val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found") + val rawTitle = gLink.text() + val author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.textOrNull() + Manga( + id = generateUid(href), + title = rawTitle.cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + tags = tagsDiv.parseTags(), + state = when { + rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING + else -> null + }, + authors = setOfNotNull(author), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val tagList = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + val gd3 = root.getElementById("gd3") + val lang = gd3 + ?.selectFirst("tr:contains(Language)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + val uploadDate = gd3 + ?.selectFirst("tr:contains(Posted)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + .let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) } + val uploader = gd3 + ?.getElementsByAttributeValueContaining("href", "/uploader/") + ?.firstOrNull() + ?.ownTextOrNull() + val tags = tagList?.parseTags().orEmpty() + + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()), + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + tags = manga.tags + tags, + description = tagList?.select("tr")?.joinToString("<br>") { tr -> + val (tc, td) = tr.children() + val subTags = td.select("a").joinToString { it.html() } + "<b>${tc.html()}</b> $subTags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + title = null, + number = i.toFloat(), + volume = 0, + url = url, + uploadDate = uploadDate, + source = source, + scanlator = uploader, + branch = lang, + ) + } + chapters.toList() + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + preview = a.children().firstOrNull()?.extractPreview(), + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } + + @Suppress("SpellCheckingInspection") + private val tags: String + get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," + + "big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," + + "catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," + + "dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," + + "females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," + + "glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," + + "horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," + + "latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," + + "mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," + + "pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," + + "sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," + + "swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," + + "unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," + + "tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis" + + private fun mapTags(): Set<MangaTag> { + val tagElements = tags.split(",") + val result = ArraySet<MangaTag>(tagElements.size) + for (tag in tagElements) { + val el = tag.trim() + if (el.isEmpty()) continue + result += MangaTag( + title = el.toTitleCase(Locale.ENGLISH), + key = el, + source = source, + ) + } + return result + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.headersContentLength() <= BANNED_RESPONSE_LENGTH) { + val text = response.peekBody(BANNED_RESPONSE_LENGTH).use { it.string() } + if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { + val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + response.closeQuietly() + throw TooManyRequestExceptions( + url = response.request.url.toString(), + retryAfter = TimeUnit.HOURS.toMillis(hours) + + TimeUnit.MINUTES.toMillis(minutes) + + TimeUnit.SECONDS.toMillis(seconds), + ) + } + } + val imageRect = response.request.url.fragment?.split(',') + if (imageRect != null && imageRect.size == 4) { + // rect: top,left,right,bottom + return context.redrawImageResponse(response) { bitmap -> + val srcRect = Rect( + left = imageRect[0].toInt(), + top = imageRect[1].toInt(), + right = imageRect[2].toInt(), + bottom = imageRect[3].toInt(), + ) + val dstRect = Rect(0, 0, srcRect.width, srcRect.height) + val result = context.createBitmap(dstRect.width, dstRect.height) + result.drawBitmap(bitmap, srcRect, dstRect) + result + } + } + return response + } + + private fun Locale.toLanguagePath() = when (language) { + else -> getDisplayLanguage(Locale.ENGLISH).lowercase() + } + + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(suspiciousContentKey) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val query = seed.title + return getListPage( + page = 0, + order = defaultSortOrder, + filter = MangaListFilter(query = query), + ) + } + + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } + + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.findAll(style).toList() + var p1 = v1.groupValues.first().dropLast(2).toInt() + val p2 = v2.groupValues.first().dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } + + private fun String.cleanupTitle(): String { + return replace(titleCleanupPattern, "") + .replace(spacesCleanupPattern, "") + } + + private fun Element.parseTags(): Set<MangaTag> { + + fun Element.parseTag() = textOrNull()?.let { + MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source) + } + + val result = ArraySet<MangaTag>() + for (prefix in TAG_PREFIXES) { + getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag) + getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag) + } + return result + } + + private fun Element.extractPreview(): String? { + val style = attr("style") + val attrs = style.split(';').mapNotNull { part -> + val trimmed = part.trim() + if (trimmed.isEmpty()) { + null + } else { + val idx = trimmed.indexOf(':') + if (idx < 0) null else { + trimmed.substring(0, idx) to trimmed.substring(idx + 1).trim() + } + } + }.toMap() + + val width = attrs["width"]?.removeSuffix("px")?.toIntOrNull() ?: return null + val height = attrs["height"]?.removeSuffix("px")?.toIntOrNull() ?: return null + + val bgPart = attrs["background"] ?: return null + val bgTokens = bgPart.substringAfter("url").split(Regex("\\s+")).filter { it.isNotBlank() } + val url = bgTokens.firstOrNull()?.removeSurrounding("(", ")") ?: return null + val x = bgTokens.getOrNull(1)?.removeSuffix("px")?.toIntOrNull() ?: 0 + val y = bgTokens.getOrNull(2)?.removeSuffix("px")?.toIntOrNull() ?: 0 + + val left = -x + val right = left + width + val bottom = y + height + + return "$url#$left,$y,$right,$bottom" + } + + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } + + private fun MangaListFilter.toSearchQuery(): String? { + if (isEmpty()) return null + + val parts = mutableListOf<String>() + + query?.let { parts.add(it) } + + for (tag in tags) { + if (tag.key.isNumeric()) continue + parts.add("tag:\"${tag.key}\"") + } + + for (tag in tagsExclude) { + if (tag.key.isNumeric()) continue + parts.add("-tag:\"${tag.key}\"") + } + + locale?.let { lc -> + parts.add("language:\"${lc.toLanguagePath()}\"") + } + + author?.let { + parts.add("artist:\"${it}\"") + } + + return parts.joinToString(" ").nullIfEmpty() + } + + private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct -> + val cat: Int = when (ct) { + ContentType.DOUJINSHI -> 2 + ContentType.MANGA -> 4 + ContentType.ARTIST_CG -> 8 + ContentType.GAME_CG -> 16 + ContentType.COMICS -> 512 + ContentType.IMAGE_SET -> 32 + else -> 449 // 1 or 64 or 128 or 256 + } + acc or cat + } + + private fun checkAuth(): Boolean { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } +} From 60c3fb9cb89942e77e9d929c10c0d6a01b885310 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 19:32:48 +0300 Subject: [PATCH 303/434] feat(source): Add Kagane EN parser with Cloudflare challenge handling New Kagane (kagane.to) source: - Search, details, pages, chapters via webClient API - Token refresh retry on 401/403/507 for image pages - Cloudflare retry (6s) on protected/blocked responses - Routes verification to site root instead of POST-only API URL - Strips host-app 'Content-Encoding: gzip' header that Cloudflare blocks --- src/main/kotlin/tsuki/site/en/Kagane.kt | 793 ++++++++++++++++++++++++ 1 file changed, 793 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/Kagane.kt diff --git a/src/main/kotlin/tsuki/site/en/Kagane.kt b/src/main/kotlin/tsuki/site/en/Kagane.kt new file mode 100644 index 00000000..84bd70ef --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Kagane.kt @@ -0,0 +1,793 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.urlBuilder + +import okhttp3.Interceptor +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import kotlinx.coroutines.delay +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.HttpStatusException +import java.io.IOException +import java.net.URI +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("KAGANE", "Kagane", "en") +internal class Kagane(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KAGANE, pageSize = 35) { + + override val configKeyDomain = ConfigKey.Domain("kagane.to") + private val apiUrl = "https://kagane.to" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + private var genresCache: Set<MangaTag>? = null + private val UUID_REGEX = Regex( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + ) + + private companion object { + const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L + val KAGANE_LANGS = arrayOf( + "en", + "ja", + "ko", + "zh-Hans", + "zh-Hant", + "es", + "es-419", + "fr", + "de", + "pt", + "pt-BR", + "ru", + "it", + "id", + "vi", + "th", + "pl", + "hi", + "ar", + ) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = genresCache ?: fetchGenres().also { genresCache = it } + return MangaListFilterOptions( + availableTags = genres, + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), + ) + } + + private suspend fun fetchGenres(): Set<MangaTag> { + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + return try { + val raw = webClient.httpGet("$apiUrl/api/v2/genres/list", headers).parseRaw() + val genres = runCatching { JSONArray(raw) }.getOrElse { + val wrapper = runCatching { JSONObject(raw) }.getOrNull() + wrapper?.optJSONArray("content") + ?: wrapper?.optJSONArray("genres") + ?: JSONArray() + } + buildSet { + for (i in 0 until genres.length()) { + val item = genres.optJSONObject(i) ?: continue + val id = item.optString("genre_id").ifBlank { item.optString("id") } + val title = item.optString("genre_name") + .ifBlank { item.optString("genreName") } + .ifBlank { item.optString("name") } + .ifBlank { item.optString("title") } + if (id.isNotBlank() && title.isNotBlank() && UUID_REGEX.matches(id)) { + add(MangaTag(title, id, source)) + } + } + } + } catch (_: Exception) { + emptySet() + } + } + + private fun parseContentRating(value: String?): ContentRating? { + return when (value?.lowercase(Locale.ROOT)) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "adult", "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.UPDATED -> "updated_at,desc" + SortOrder.POPULARITY -> "total_views,desc" + SortOrder.NEWEST -> "created_at,desc" + SortOrder.ALPHABETICAL -> "series_name,asc" + else -> "updated_at,desc" + } + + val url = "$apiUrl/api/v2/search/series?page=${page - 1}&size=$pageSize&sort=$sortParam" + val jsonBody = JSONObject() + if (!filter.query.isNullOrEmpty()) { + jsonBody.put("title", filter.query) + } + jsonBody.put("source_type", JSONArray().apply { + put("Official") + put("Unofficial") + put("Mixed") + }) + jsonBody.put("content_lang", JSONArray().apply { + KAGANE_LANGS.forEach(::put) + }) + + val genreIds = filter.tags.map { it.key }.filter { UUID_REGEX.matches(it) } + if (genreIds.isNotEmpty()) { + val genresArr = JSONArray() + genreIds.forEach { genresArr.put(it) } + val genresObj = JSONObject() + genresObj.put("values", genresArr) + genresObj.put("match_all", false) + jsonBody.put("genres", genresObj) + } + if (filter.tagsExclude.isNotEmpty()) { + val excludedGenreIds = filter.tagsExclude.map { it.key }.filter { UUID_REGEX.matches(it) } + if (excludedGenreIds.isNotEmpty()) { + val genresObj = jsonBody.optJSONObject("genres") ?: JSONObject().also { + jsonBody.put("genres", it) + } + genresObj.put("exclude", JSONArray().apply { + excludedGenreIds.forEach(::put) + }) + } + } + jsonBody.put("content_rating", JSONArray().apply { + val ratings = filter.contentRating.ifEmpty { + EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + } + if (ContentRating.SAFE in ratings) put("Safe") + if (ContentRating.SUGGESTIVE in ratings) put("Suggestive") + if (ContentRating.ADULT in ratings) { + put("Erotica") + put("Pornographic") + } + }) + + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val responseBody = try { + requestWithCloudflareRetry(url) { + webClient.httpPost(url.toHttpUrl(), jsonBody, headers).parseRaw() + } + } catch (e: HttpStatusException) { + // Always surface 403 via the Cloudflare verification path — the interceptor below + // identifies block pages *and* challenge pages, not just "Just a moment…" challenges. + if (e.statusCode == 403 || e.statusCode == 429 || e.statusCode == 503) { + requestCloudflareVerification(url, e) + } else { + throw e + } + } catch (e: ParseException) { + // Proxy/network-level errors carrying the same markers should also relaunch the + // verification rather than presenting as a bare parse failure. + val causeMessage = e.message.orEmpty() + " " + (e.cause?.message.orEmpty()) + if (causeMessage.contains("CloudFlare", ignoreCase = true) || + causeMessage.contains("cf-mitigated", ignoreCase = true) || + causeMessage.contains("cf-error-details", ignoreCase = true) + ) { + requestCloudflareVerification(url, e) + } else { + throw e + } + } + + if (responseBody.isCloudflareChallenge()) { + requestCloudflareVerification(url) + } + + val response = try { + JSONObject(responseBody) + } catch (e: Exception) { + throw Exception("Invalid JSON search response: $responseBody") + } + + val content = response.optJSONArray("content") + ?: response.optJSONObject("result")?.optJSONArray("items") + ?: return emptyList() + + return (0 until content.length()).mapNotNull { i -> + val item = content.getJSONObject(i) + val id = item.optString("id").ifBlank { item.optString("series_id") } + if (id.isBlank()) return@mapNotNull null + val name = item.optString("name").ifBlank { item.optString("title") }.ifBlank { return@mapNotNull null } + val src = item.optString("source").ifBlank { item.optString("source_name") } + val title = if (src.isNotEmpty()) "$name [$src]" else name + val coverImageId = item.optString("cover_image_id").ifBlank { item.optString("coverImageId") } + val coverUrl = if (coverImageId.isNotBlank()) { + "$apiUrl/api/v2/image/$coverImageId" + } else { + "$apiUrl/api/v2/series/$id/thumbnail" + } + + Manga( + id = generateUid(id), + url = id, + publicUrl = "https://$domain/series/$id", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = parseContentRating(item.optString("content_rating")), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val seriesId = manga.url + val url = "$apiUrl/api/v2/series/$seriesId" + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val resp = requestWithCloudflareRetry(url) { + webClient.httpGet(url, headers) + } + val respBody = resp.body?.string() ?: "" + if (!resp.isSuccessful) throw Exception("Details error ${resp.code}: $respBody") + val json = try { + JSONObject(respBody) + } catch (e: Exception) { + throw Exception("Invalid JSON details: $respBody") + } + + val state = when ( + json.optString("publication_status") + .ifBlank { json.optString("upload_status") } + .ifBlank { json.optString("status") } + .uppercase(Locale.ROOT) + ) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED", "ENDED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + "ABANDONED", "CANCELLED", "CANCELED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val genres = json.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).mapNotNull { i -> + when (val item = arr.opt(i)) { + is String -> { + if (UUID_REGEX.matches(item)) { + MangaTag(item, item, source) + } else { + null + } + } + is JSONObject -> { + val key = item.optString("genre_id").ifBlank { item.optString("id") } + val name = item.optString("genre_name") + .ifBlank { item.optString("genreName") } + .ifBlank { item.optString("name") } + .ifBlank { item.optString("title") } + if (key.isNotBlank() && name.isNotBlank()) { + MangaTag(name, key, source) + } else { + null + } + } + else -> null + } + }.toSet() + } ?: emptySet() + + val authors = linkedSetOf<String>() + json.optJSONArray("authors")?.let { arr -> + for (i in 0 until arr.length()) { + when (val item = arr.opt(i)) { + is String -> item.takeIf { it.isNotBlank() }?.let(authors::add) + is JSONObject -> item.optString("name") + .ifBlank { item.optString("title") } + .takeIf { it.isNotBlank() } + ?.let(authors::add) + } + } + } + if (authors.isEmpty()) { + json.optJSONArray("series_staff")?.let { arr -> + for (i in 0 until arr.length()) { + val staff = arr.optJSONObject(i) ?: continue + val role = staff.optString("role") + if ( + role.contains("author", ignoreCase = true) || + role.contains("story", ignoreCase = true) || + role.contains("artist", ignoreCase = true) || + role.contains("art", ignoreCase = true) + ) { + staff.optString("name").takeIf { it.isNotBlank() }?.let(authors::add) + } + } + } + } + + val altTitles = json.optJSONArray("series_alternate_titles")?.let { arr -> + buildSet { + for (i in 0 until arr.length()) { + val item = arr.optJSONObject(i) ?: continue + item.optString("title").takeIf { it.isNotBlank() }?.let(::add) + } + } + } ?: emptySet() + + val description = buildString { + json.optString("description") + .ifBlank { json.optString("summary") } + .takeIf { it.isNotBlank() } + ?.let { + append(it.trim()) + } + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Associated Name(s):\n") + altTitles.forEach { + append(it) + append('\n') + } + } + }.trim().ifBlank { null } + + val coverUrl = json.optJSONArray("series_covers") + ?.optJSONObject(0) + ?.optString("image_id") + ?.takeIf { it.isNotBlank() } + ?.let { "$apiUrl/api/v2/image/$it" } + ?: manga.coverUrl + + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH) + fun parseChapters(content: JSONArray): List<MangaChapter> { + val chapters = ArrayList<MangaChapter>(content.length()) + for (i in 0 until content.length()) { + val ch = content.optJSONObject(i) ?: continue + val chId = ch.optString("book_id") + .ifBlank { ch.optString("id") } + .ifBlank { ch.optString("bookId") } + if (chId.isBlank()) continue + val chapterNo = ch.optString("chapter_no") + val chapterNumber = chapterNo.toChapterNumberOrNull() + ?: ch.optDouble("number", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + val sortNumber = ch.optDouble("sort_no", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + ?: ch.optDouble("number_sort", ch.optDouble("numberSort", Double.NaN)).takeIf { !it.isNaN() }?.toFloat() + val number = when { + sortNumber != null && chapterNumber != null && sortNumber >= chapterNumber -> sortNumber + sortNumber != null && chapterNumber == null -> sortNumber + chapterNumber != null -> chapterNumber + else -> 0f + } + val rawTitle = ch.optString("title").ifBlank { ch.optString("name") }.trim() + val chTitle = rawTitle.ifBlank { + chapterNo.takeIf { it.isNotBlank() }?.let { "Ch.$it" }.orEmpty() + }.ifBlank { "Chapter $number" } + val volume = ch.optString("volume_no") + .ifBlank { ch.optString("volume") } + .toIntOrNull() ?: 0 + val dateStr = ch.optString("published_on") + .ifBlank { ch.optString("release_date") } + .ifBlank { ch.optString("releaseDate") } + .ifBlank { ch.optString("created_at") } + val groups = ch.optJSONArray("groups") + chapters.add( + MangaChapter( + id = generateUid("$seriesId:$chId"), + title = chTitle, + number = number, + volume = volume, + url = "/series/$seriesId/reader/$chId", + uploadDate = dateFormat.parseSafe(dateStr), + source = source, + scanlator = groups?.let { arr -> + buildList { + for (j in 0 until arr.length()) { + arr.optJSONObject(j)?.optString("title")?.takeIf { it.isNotBlank() }?.let(::add) + } + }.joinToString().ifBlank { null } + }, + branch = null, + ), + ) + } + return chapters.sortedWith( + compareBy<MangaChapter> { it.number <= 0f } + .thenBy { it.number } + .thenBy { it.volume } + .thenBy { it.title.orEmpty() }, + ) + } + + var chapters = parseChapters( + json.optJSONArray("series_books") + ?: json.optJSONArray("seriesBooks") + ?: json.optJSONArray("books") + ?: json.optJSONArray("content") + ?: JSONArray(), + ) + if (chapters.isEmpty()) { + val chaptersUrl = "$apiUrl/api/v2/series/$seriesId/books/list" + val chapterResp = webClient.httpGet(chaptersUrl, headers).parseJson() + chapters = parseChapters( + chapterResp.optJSONArray("series_books") + ?: chapterResp.optJSONArray("seriesBooks") + ?: chapterResp.optJSONArray("content") + ?: JSONArray(), + ) + } + + return manga.copy( + title = json.optString("title").ifBlank { manga.title }, + altTitles = altTitles, + coverUrl = coverUrl, + description = description, + state = state, + authors = authors, + tags = genres, + chapters = chapters, + contentRating = parseContentRating(json.optString("content_rating")) ?: manga.contentRating, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + // Disable related/suggested manga feature + return emptyList() + } + + private var cacheUrl = "https://kstatic.to" + private var accessToken: String = "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val uri = URI(chapter.url) + val pathParts = uri.path.split("/").filter { it.isNotEmpty() } + if (pathParts.size < 4) throw Exception("Invalid chapter URL format: ${chapter.url}") + + val chapterId = pathParts.last() + val challenge = getChallengeResponse(chapterId) + accessToken = challenge.optString("access_token").ifBlank { + challenge.optString("accessToken") + }.ifBlank { + throw Exception("Invalid challenge response: missing access token") + } + cacheUrl = challenge.optString("cache_url").ifBlank { + challenge.optString("cacheUrl") + }.ifBlank { + throw Exception("Invalid challenge response: missing cache url") + } + + val pages = parseManifestPages(challenge) + if (pages.isEmpty()) { + throw Exception("Invalid challenge response: missing pages manifest") + } + + return pages.sortedBy { it.pageNumber }.map { page -> + val ext = page.ext?.takeIf { it.isNotBlank() } ?: "jxl" + val imageUrl = "$cacheUrl/api/v2/books/page".toHttpUrl().newBuilder() + .addPathSegment(chapterId) + .addPathSegment("${page.pageUuid}.$ext") + .addQueryParameter("token", accessToken) + .addQueryParameter("is_datasaver", "false") + .build() + .toString() + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun <T> requestWithCloudflareRetry( + url: String, + block: suspend () -> T, + ): T { + try { + return block() + } catch (e: Exception) { + if (!e.isCloudflareProtectionError()) { + throw e + } + delay(CLOUDFLARE_RETRY_DELAY_MS) + try { + return block() + } catch (retryError: Exception) { + // A Cloudflare block page (stale/expired clearance) is not resolvable by the + // app on its own — the "blocked" exception has no resolver action. Route it + // through the verification flow so the user gets a working continuation + // instead of a dead-end "Try again" card. + if (e.isCloudflareBlock() || retryError.isCloudflareBlock()) { + requestCloudflareVerification(url, retryError) + } + throw retryError + } + } + } + + private fun Throwable.isCloudflareBlock(): Boolean { + val name = javaClass.simpleName + return name.contains("CloudflareBlocked", ignoreCase = true) || + name.contains("CloudFlareBlocked") || + (isCloudflareProtectionError() && message.orEmpty().contains("blocked", ignoreCase = true)) + } + + private fun Throwable.isCloudflareProtectionError(): Boolean { + val name = javaClass.simpleName + if (name.contains("Cloudflare", ignoreCase = true) || name.contains("CloudFlare")) { + return true + } + val message = "${message.orEmpty()} ${cause?.message.orEmpty()}" + return message.contains("cf-mitigated", ignoreCase = true) || + message.contains("Just a moment", ignoreCase = true) || + message.contains("challenges.cloudflare.com", ignoreCase = true) || + message.contains("cf-error-details", ignoreCase = true) || + message.contains("cf-chl-bypass", ignoreCase = true) + } + + private fun requestCloudflareVerification(url: String, cause: Throwable? = null): Nothing { + try { + // Always open the site root (not the POST-only API endpoint): Cloudflare serves the + // challenge on any browsable page, and solving it writes cf_clearance into the + // shared cookie jar for the app's OkHttp requests to reuse. + context.requestBrowserAction(this, "https://$domain/") + } catch (e: UnsupportedOperationException) { + throw ParseException( + "Cloudflare verification required. Open Kagane in WebView and retry.", + url, + cause ?: e, + ) + } + throw ParseException("Retry after Cloudflare verification.", url, cause) + } + + private fun String.isCloudflareChallenge(): Boolean { + // Match any of the CF block/challenge markers *after* the page has finished loading — + // including the block-page banner, the challenge-platform script path, and the new + // "cf-chl-bypass"-style resumption marker DOMs that a stale clearance cookie can land on. + return contains("cf-mitigated", ignoreCase = true) + || contains("Just a moment", ignoreCase = true) + || contains("challenges.cloudflare.com", ignoreCase = true) + || contains("/cdn-cgi/challenge-platform/", ignoreCase = true) + // CF block-page (Ray ID + "Sorry, you have been blocked") — handled via verification path. + || contains("Sorry, you have been blocked", ignoreCase = true) + || contains("cf-error-details", ignoreCase = true) + || contains("cf-chl-bypass", ignoreCase = true) + } + + private data class ManifestPage( + val pageNumber: Int, + val pageUuid: String, + val ext: String?, + ) + + private fun parseManifestPages(challenge: JSONObject): List<ManifestPage> { + val pagesJson = challenge.optJSONObject("manifest")?.optJSONArray("pages") + ?: challenge.optJSONArray("pages") + ?: JSONArray() + return buildList { + for (i in 0 until pagesJson.length()) { + val page = pagesJson.optJSONObject(i) ?: continue + val pageUuid = page.optString("page_id") + .ifBlank { page.optString("pageId") } + .ifBlank { page.optString("page_uuid") } + .ifBlank { page.optString("pageUuid") } + if (pageUuid.isBlank()) continue + add( + ManifestPage( + pageNumber = page.optInt( + "page_no", + page.optInt("pageNo", page.optInt("page_number", i + 1)), + ), + pageUuid = pageUuid, + ext = page.optString("ext").ifBlank { null }, + ), + ) + } + } + } + + private var integrityToken: String = "" + private var integrityTokenExp: Long = 0L + + private suspend fun getIntegrityToken(): String { + val now = System.currentTimeMillis() + if (integrityToken.isNotBlank() && now < integrityTokenExp) { + return integrityToken + } + + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val response = requestWithCloudflareRetry("$apiUrl/api/integrity") { + webClient.httpPost( + urlBuilder().addPathSegments("api/integrity").build(), + JSONObject(), + headers, + ).parseJson() + } + + val token = response.optString("token") + if (token.isBlank()) { + throw Exception("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = response.optLong("exp", 0L) * 1000L + return integrityToken + } + + private suspend fun getChallengeResponse(chapterId: String): JSONObject { + val integrityToken = getIntegrityToken() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=false" + return requestWithCloudflareRetry(challengeUrl) { + webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers).parseJson() + } + } + + private fun String.toChapterNumberOrNull(): Float? = trim() + .replace(',', '.') + .toFloatOrNull() + + private fun getIntegrityTokenBlocking(): String { + val now = System.currentTimeMillis() + if (integrityToken.isNotBlank() && now < integrityTokenExp) { + return integrityToken + } + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val request = Request.Builder() + .url(urlBuilder().addPathSegments("api/integrity").build()) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Integrity token request failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + val token = json.optString("token") + if (token.isBlank()) { + throw IOException("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = json.optLong("exp", 0L) * 1000L + } + return integrityToken + } + + private fun refreshAccessTokenBlocking(chapterId: String) { + val integrityToken = getIntegrityTokenBlocking() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=false".toHttpUrl() + val request = Request.Builder() + .url(challengeUrl) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Challenge refresh failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + accessToken = json.optString("access_token").ifBlank { + json.optString("accessToken") + }.ifBlank { + throw IOException("Invalid challenge response: missing access token") + } + cacheUrl = json.optString("cache_url").ifBlank { + json.optString("cacheUrl") + }.ifBlank { + throw IOException("Invalid challenge response: missing cache url") + } + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + val host = url.host + var requestBuilder = request.newBuilder() + // The host GZipInterceptor stamps a lying "Content-Encoding: gzip" on non-multipart + // bodies (the JSON body is sent plain, never gzipped). Cloudflare treats that as an + // anomaly on a flagged network and serves a hard block page instead of a challenge. + // Drop it so the request matches a clean browser/OkHttp shape. + .removeHeader("Content-Encoding") + .removeHeader("cf-connecting-ip") + if (host == domain || host.endsWith(".$domain")) { + requestBuilder = requestBuilder + .header("Origin", "https://$domain") + .header("Referer", "https://$domain/") + } + val newRequest = requestBuilder.build() + + var response = chain.proceed(newRequest) + + // Token refresh retry: image page requests carry a `token` query param, so a request + // running into an expired/invalid token (401/403/507) is retried once with a fresh + // challenge. Mirrors the behaviour of the legacy Keiyoushi Kagane_EN interceptor. + if (url.queryParameterNames.contains("token") && + (response.code == 401 || response.code == 403 || response.code == 507) + ) { + response.close() + val segments = url.pathSegments + val chapterId = segments.getOrNull(4) + if (chapterId != null) { + runCatching { refreshAccessTokenBlocking(chapterId) }.onSuccess { + val retryRequest = newRequest.newBuilder() + .url(url.newBuilder().setQueryParameter("token", accessToken).build()) + .build() + response = chain.proceed(retryRequest) + } + } + } + return response + } +} From 3b8a6a7736197d7e43ba343edf696325750c2205 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:47:06 +0300 Subject: [PATCH 304/434] feat(source): Make Kagane available for all languages --- src/main/kotlin/tsuki/site/en/Kagane.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/Kagane.kt b/src/main/kotlin/tsuki/site/en/Kagane.kt index 84bd70ef..f0d7e8bd 100644 --- a/src/main/kotlin/tsuki/site/en/Kagane.kt +++ b/src/main/kotlin/tsuki/site/en/Kagane.kt @@ -41,7 +41,7 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale -@MangaSourceParser("KAGANE", "Kagane", "en") +@MangaSourceParser("KAGANE", "Kagane") internal class Kagane(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.KAGANE, pageSize = 35) { From 3a1748742d8286cbcb46e3d640f908cb92d59d60 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:48:33 +0300 Subject: [PATCH 305/434] refactor(source): Move Kagane to all languages group --- src/main/kotlin/tsuki/site/{en => all}/Kagane.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename src/main/kotlin/tsuki/site/{en => all}/Kagane.kt (99%) diff --git a/src/main/kotlin/tsuki/site/en/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt similarity index 99% rename from src/main/kotlin/tsuki/site/en/Kagane.kt rename to src/main/kotlin/tsuki/site/all/Kagane.kt index f0d7e8bd..7e55453d 100644 --- a/src/main/kotlin/tsuki/site/en/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -1,4 +1,4 @@ -package tsuki.site.en +package tsuki.site.all import tsuki.MangaLoaderContext import tsuki.MangaSourceParser From d05fb30373b064c1ceb639501b35bb2e096eda39 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 22:58:48 +0200 Subject: [PATCH 306/434] Update Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 192 ++++++++++++++++------- 1 file changed, 137 insertions(+), 55 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index 7e55453d..1506e344 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -18,6 +18,7 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.ContentType import tsuki.util.generateUid import tsuki.util.parseJson @@ -40,6 +41,12 @@ import java.net.URI import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale +import kotlin.time.Duration.Companion.milliseconds + +/** + * Created from: https://github.com/glitch-228 + * Modified from: InvalidDavid + */ @MangaSourceParser("KAGANE", "Kagane") internal class Kagane(context: MangaLoaderContext) : @@ -48,11 +55,34 @@ internal class Kagane(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("kagane.to") private val apiUrl = "https://kagane.to" + private val dataSaverKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "false" to "Normal", + "true" to "Data saving" + ), + defaultValue = "false", + ) + + private val isDataSaver: Boolean + get() = config[dataSaverKey] == "true" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(dataSaverKey) + } + override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, + SortOrder.RELEVANCE, SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, SortOrder.NEWEST, + SortOrder.NEWEST_ASC, SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -91,6 +121,16 @@ internal class Kagane(context: MangaLoaderContext) : ) } + private fun Locale.toKaganeLangCode(): String { + return when (toLanguageTag()) { + "zh-Hans" -> "zh-Hans" + "zh-Hant" -> "zh-Hant" + "es-419" -> "es-419" + "pt-BR" -> "pt-BR" + else -> language + } + } + override suspend fun getFilterOptions(): MangaListFilterOptions { val genres = genresCache ?: fetchGenres().also { genresCache = it } return MangaListFilterOptions( @@ -100,6 +140,22 @@ internal class Kagane(context: MangaLoaderContext) : ContentRating.SUGGESTIVE, ContentRating.ADULT, ), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.OTHER, + ), + availableLocales = KAGANE_LANGS.mapNotNull { lang -> + runCatching { Locale.forLanguageTag(lang) }.getOrNull() + }.toSet(), ) } @@ -145,14 +201,28 @@ internal class Kagane(context: MangaLoaderContext) : override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val sortParam = when (order) { - SortOrder.UPDATED -> "updated_at,desc" + SortOrder.RELEVANCE -> "" SortOrder.POPULARITY -> "total_views,desc" + SortOrder.POPULARITY_ASC -> "total_views,asc" + SortOrder.RATING -> "avg_views,desc" + SortOrder.RATING_ASC -> "avg_views,asc" + SortOrder.UPDATED -> "updated_at,desc" + SortOrder.UPDATED_ASC -> "updated_at,asc" SortOrder.NEWEST -> "created_at,desc" + SortOrder.NEWEST_ASC -> "created_at,asc" SortOrder.ALPHABETICAL -> "series_name,asc" + SortOrder.ALPHABETICAL_DESC -> "series_name,desc" else -> "updated_at,desc" } - val url = "$apiUrl/api/v2/search/series?page=${page - 1}&size=$pageSize&sort=$sortParam" + val urlBuilder = "$apiUrl/api/v2/search/series".toHttpUrl().newBuilder() + .addQueryParameter("page", (page - 1).toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("sort", sortParam) + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("exact", "1") + } + val url = urlBuilder.build().toString() val jsonBody = JSONObject() if (!filter.query.isNullOrEmpty()) { jsonBody.put("title", filter.query) @@ -163,7 +233,12 @@ internal class Kagane(context: MangaLoaderContext) : put("Mixed") }) jsonBody.put("content_lang", JSONArray().apply { - KAGANE_LANGS.forEach(::put) + val selectedLocale = filter.locale + if (selectedLocale != null) { + put(selectedLocale.toKaganeLangCode()) + } else { + KAGANE_LANGS.forEach(::put) + } }) val genreIds = filter.tags.map { it.key }.filter { UUID_REGEX.matches(it) } @@ -198,6 +273,39 @@ internal class Kagane(context: MangaLoaderContext) : } }) + if (filter.states.isNotEmpty()) { + val statuses = JSONArray() + filter.states.forEach { state -> + when (state) { + MangaState.ONGOING -> statuses.put("Ongoing") + MangaState.FINISHED -> statuses.put("Completed") + MangaState.PAUSED -> statuses.put("Hiatus") + MangaState.ABANDONED -> statuses.put("Abandoned") + else -> Unit + } + } + if (statuses.length() > 0) { + jsonBody.put("upload_status", statuses) + } + } + + if (filter.types.isNotEmpty()) { + val formats = JSONArray() + filter.types.forEach { type -> + when (type) { + ContentType.MANGA -> formats.put("Manga") + ContentType.MANHWA -> formats.put("Manhwa") + ContentType.MANHUA -> formats.put("Manhua") + ContentType.COMICS -> formats.put("Comic") + ContentType.OTHER -> formats.put("Other") + else -> Unit + } + } + if (formats.length() > 0) { + jsonBody.put("format", formats) + } + } + val headers = getRequestHeaders().newBuilder() .add("Origin", "https://$domain") .add("Referer", "https://$domain/") @@ -208,16 +316,12 @@ internal class Kagane(context: MangaLoaderContext) : webClient.httpPost(url.toHttpUrl(), jsonBody, headers).parseRaw() } } catch (e: HttpStatusException) { - // Always surface 403 via the Cloudflare verification path — the interceptor below - // identifies block pages *and* challenge pages, not just "Just a moment…" challenges. if (e.statusCode == 403 || e.statusCode == 429 || e.statusCode == 503) { requestCloudflareVerification(url, e) } else { throw e } } catch (e: ParseException) { - // Proxy/network-level errors carrying the same markers should also relaunch the - // verification rather than presenting as a bare parse failure. val causeMessage = e.message.orEmpty() + " " + (e.cause?.message.orEmpty()) if (causeMessage.contains("CloudFlare", ignoreCase = true) || causeMessage.contains("cf-mitigated", ignoreCase = true) || @@ -235,7 +339,7 @@ internal class Kagane(context: MangaLoaderContext) : val response = try { JSONObject(responseBody) - } catch (e: Exception) { + } catch (_: Exception) { throw Exception("Invalid JSON search response: $responseBody") } @@ -288,7 +392,7 @@ internal class Kagane(context: MangaLoaderContext) : if (!resp.isSuccessful) throw Exception("Details error ${resp.code}: $respBody") val json = try { JSONObject(respBody) - } catch (e: Exception) { + } catch (_: Exception) { throw Exception("Invalid JSON details: $respBody") } @@ -486,7 +590,6 @@ internal class Kagane(context: MangaLoaderContext) : } override suspend fun getRelatedManga(seed: Manga): List<Manga> { - // Disable related/suggested manga feature return emptyList() } @@ -518,13 +621,14 @@ internal class Kagane(context: MangaLoaderContext) : return pages.sortedBy { it.pageNumber }.map { page -> val ext = page.ext?.takeIf { it.isNotBlank() } ?: "jxl" - val imageUrl = "$cacheUrl/api/v2/books/page".toHttpUrl().newBuilder() - .addPathSegment(chapterId) - .addPathSegment("${page.pageUuid}.$ext") - .addQueryParameter("token", accessToken) - .addQueryParameter("is_datasaver", "false") - .build() - .toString() + val imageUrl = "$cacheUrl/api/v2/books/page".toHttpUrl().newBuilder().apply { + if (isDataSaver) addPathSegment("datasaver") + addPathSegment(chapterId) + addPathSegment("${page.pageUuid}.$ext") + addQueryParameter("token", accessToken) + addQueryParameter("is_datasaver", isDataSaver.toString()) + }.build().toString() + MangaPage( id = generateUid(imageUrl), url = imageUrl, @@ -534,24 +638,17 @@ internal class Kagane(context: MangaLoaderContext) : } } - private suspend fun <T> requestWithCloudflareRetry( - url: String, - block: suspend () -> T, - ): T { + private suspend fun <T> requestWithCloudflareRetry(url: String, block: suspend () -> T, ): T { try { return block() } catch (e: Exception) { if (!e.isCloudflareProtectionError()) { throw e } - delay(CLOUDFLARE_RETRY_DELAY_MS) + delay(CLOUDFLARE_RETRY_DELAY_MS.milliseconds) try { return block() } catch (retryError: Exception) { - // A Cloudflare block page (stale/expired clearance) is not resolvable by the - // app on its own — the "blocked" exception has no resolver action. Route it - // through the verification flow so the user gets a working continuation - // instead of a dead-end "Try again" card. if (e.isCloudflareBlock() || retryError.isCloudflareBlock()) { requestCloudflareVerification(url, retryError) } @@ -563,8 +660,8 @@ internal class Kagane(context: MangaLoaderContext) : private fun Throwable.isCloudflareBlock(): Boolean { val name = javaClass.simpleName return name.contains("CloudflareBlocked", ignoreCase = true) || - name.contains("CloudFlareBlocked") || - (isCloudflareProtectionError() && message.orEmpty().contains("blocked", ignoreCase = true)) + name.contains("CloudFlareBlocked") || + (isCloudflareProtectionError() && message.orEmpty().contains("blocked", ignoreCase = true)) } private fun Throwable.isCloudflareProtectionError(): Boolean { @@ -574,17 +671,14 @@ internal class Kagane(context: MangaLoaderContext) : } val message = "${message.orEmpty()} ${cause?.message.orEmpty()}" return message.contains("cf-mitigated", ignoreCase = true) || - message.contains("Just a moment", ignoreCase = true) || - message.contains("challenges.cloudflare.com", ignoreCase = true) || - message.contains("cf-error-details", ignoreCase = true) || - message.contains("cf-chl-bypass", ignoreCase = true) + message.contains("Just a moment", ignoreCase = true) || + message.contains("challenges.cloudflare.com", ignoreCase = true) || + message.contains("cf-error-details", ignoreCase = true) || + message.contains("cf-chl-bypass", ignoreCase = true) } private fun requestCloudflareVerification(url: String, cause: Throwable? = null): Nothing { try { - // Always open the site root (not the POST-only API endpoint): Cloudflare serves the - // challenge on any browsable page, and solving it writes cf_clearance into the - // shared cookie jar for the app's OkHttp requests to reuse. context.requestBrowserAction(this, "https://$domain/") } catch (e: UnsupportedOperationException) { throw ParseException( @@ -593,21 +687,16 @@ internal class Kagane(context: MangaLoaderContext) : cause ?: e, ) } - throw ParseException("Retry after Cloudflare verification.", url, cause) } private fun String.isCloudflareChallenge(): Boolean { - // Match any of the CF block/challenge markers *after* the page has finished loading — - // including the block-page banner, the challenge-platform script path, and the new - // "cf-chl-bypass"-style resumption marker DOMs that a stale clearance cookie can land on. return contains("cf-mitigated", ignoreCase = true) - || contains("Just a moment", ignoreCase = true) - || contains("challenges.cloudflare.com", ignoreCase = true) - || contains("/cdn-cgi/challenge-platform/", ignoreCase = true) - // CF block-page (Ray ID + "Sorry, you have been blocked") — handled via verification path. - || contains("Sorry, you have been blocked", ignoreCase = true) - || contains("cf-error-details", ignoreCase = true) - || contains("cf-chl-bypass", ignoreCase = true) + || contains("Just a moment", ignoreCase = true) + || contains("challenges.cloudflare.com", ignoreCase = true) + || contains("/cdn-cgi/challenge-platform/", ignoreCase = true) + || contains("Sorry, you have been blocked", ignoreCase = true) + || contains("cf-error-details", ignoreCase = true) + || contains("cf-chl-bypass", ignoreCase = true) } private data class ManifestPage( @@ -680,7 +769,7 @@ internal class Kagane(context: MangaLoaderContext) : .add("Referer", "https://$domain/") .add("x-integrity-token", integrityToken) .build() - val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=false" + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver" return requestWithCloudflareRetry(challengeUrl) { webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers).parseJson() } @@ -755,10 +844,6 @@ internal class Kagane(context: MangaLoaderContext) : val url = request.url val host = url.host var requestBuilder = request.newBuilder() - // The host GZipInterceptor stamps a lying "Content-Encoding: gzip" on non-multipart - // bodies (the JSON body is sent plain, never gzipped). Cloudflare treats that as an - // anomaly on a flagged network and serves a hard block page instead of a challenge. - // Drop it so the request matches a clean browser/OkHttp shape. .removeHeader("Content-Encoding") .removeHeader("cf-connecting-ip") if (host == domain || host.endsWith(".$domain")) { @@ -770,9 +855,6 @@ internal class Kagane(context: MangaLoaderContext) : var response = chain.proceed(newRequest) - // Token refresh retry: image page requests carry a `token` query param, so a request - // running into an expired/invalid token (401/403/507) is retried once with a fresh - // challenge. Mirrors the behaviour of the legacy Keiyoushi Kagane_EN interceptor. if (url.queryParameterNames.contains("token") && (response.code == 401 || response.code == 403 || response.code == 507) ) { From ec3c17fb1ec151e8fa8c967edca829c805af4522 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:04:01 +0200 Subject: [PATCH 307/434] Create Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 878 +++++++++++++++++++++++ 1 file changed, 878 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/Kagane.kt diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt new file mode 100644 index 00000000..09d1d0e5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -0,0 +1,878 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.urlBuilder + +import okhttp3.Interceptor +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import kotlinx.coroutines.delay +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.HttpStatusException +import java.io.IOException +import java.net.URI +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import kotlin.time.Duration.Companion.milliseconds + +/** + * Created from: https://github.com/glitch-228 + * Modified from: InvalidDavid + */ + +@MangaSourceParser("KAGANE", "Kagane") +internal class Kagane(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KAGANE, pageSize = 35) { + + override val configKeyDomain = ConfigKey.Domain("kagane.to") + private val apiUrl = "https://kagane.to" + + private val dataSaverKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "false" to "Normal", + "true" to "Data saving" + ), + defaultValue = "false", + ) + + private val isDataSaver: Boolean + get() = config[dataSaverKey] == "true" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(dataSaverKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + private var genresCache: Set<MangaTag>? = null + private val UUID_REGEX = Regex( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + ) + + private companion object { + const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L + val KAGANE_LANGS = arrayOf( + "en", + "ja", + "ko", + "zh-Hans", + "zh-Hant", + "es", + "es-419", + "fr", + "de", + "pt", + "pt-BR", + "ru", + "it", + "id", + "vi", + "th", + "pl", + "hi", + "ar", + ) + } + + private fun Locale.toKaganeLangCode(): String { + return when (toLanguageTag()) { + "zh-Hans" -> "zh-Hans" + "zh-Hant" -> "zh-Hant" + "es-419" -> "es-419" + "pt-BR" -> "pt-BR" + else -> language + } + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = genresCache ?: fetchGenres().also { genresCache = it } + return MangaListFilterOptions( + availableTags = genres, + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.OTHER, + ), + availableLocales = KAGANE_LANGS.mapNotNull { lang -> + runCatching { Locale.forLanguageTag(lang) }.getOrNull() + }.toSet(), + ) + } + + private suspend fun fetchGenres(): Set<MangaTag> { + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + return try { + val raw = webClient.httpGet("$apiUrl/api/v2/genres/list", headers).parseRaw() + val genres = runCatching { JSONArray(raw) }.getOrElse { + val wrapper = runCatching { JSONObject(raw) }.getOrNull() + wrapper?.optJSONArray("content") + ?: wrapper?.optJSONArray("genres") + ?: JSONArray() + } + buildSet { + for (i in 0 until genres.length()) { + val item = genres.optJSONObject(i) ?: continue + val id = item.optString("genre_id").ifBlank { item.optString("id") } + val title = item.optString("genre_name") + .ifBlank { item.optString("genreName") } + .ifBlank { item.optString("name") } + .ifBlank { item.optString("title") } + if (id.isNotBlank() && title.isNotBlank() && UUID_REGEX.matches(id)) { + add(MangaTag(title, id, source)) + } + } + } + } catch (_: Exception) { + emptySet() + } + } + + private fun parseContentRating(value: String?): ContentRating? { + return when (value?.lowercase(Locale.ROOT)) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "adult", "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.RELEVANCE -> "" + SortOrder.POPULARITY -> "total_views,desc" + SortOrder.POPULARITY_ASC -> "total_views,asc" + SortOrder.RATING -> "avg_views,desc" + SortOrder.RATING_ASC -> "avg_views,asc" + SortOrder.UPDATED -> "updated_at,desc" + SortOrder.UPDATED_ASC -> "updated_at,asc" + SortOrder.NEWEST -> "created_at,desc" + SortOrder.NEWEST_ASC -> "created_at,asc" + SortOrder.ALPHABETICAL -> "series_name,asc" + SortOrder.ALPHABETICAL_DESC -> "series_name,desc" + else -> "updated_at,desc" + } + + val urlBuilder = "$apiUrl/api/v2/search/series".toHttpUrl().newBuilder() + .addQueryParameter("page", (page - 1).toString()) + .addQueryParameter("size", pageSize.toString()) + if (sortParam.isNotBlank()) { + urlBuilder.addQueryParameter("sort", sortParam) + } + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("exact", "1") + } + val url = urlBuilder.build().toString() + val jsonBody = JSONObject() + if (!filter.query.isNullOrEmpty()) { + jsonBody.put("title", filter.query) + } + jsonBody.put("source_type", JSONArray().apply { + put("Official") + put("Unofficial") + put("Mixed") + }) + jsonBody.put("content_lang", JSONArray().apply { + val selectedLocale = filter.locale + if (selectedLocale != null) { + put(selectedLocale.toKaganeLangCode()) + } else { + KAGANE_LANGS.forEach(::put) + } + }) + + val genreIds = filter.tags.map { it.key }.filter { UUID_REGEX.matches(it) } + if (genreIds.isNotEmpty()) { + val genresArr = JSONArray() + genreIds.forEach { genresArr.put(it) } + val genresObj = JSONObject() + genresObj.put("values", genresArr) + genresObj.put("match_all", false) + jsonBody.put("genres", genresObj) + } + if (filter.tagsExclude.isNotEmpty()) { + val excludedGenreIds = filter.tagsExclude.map { it.key }.filter { UUID_REGEX.matches(it) } + if (excludedGenreIds.isNotEmpty()) { + val genresObj = jsonBody.optJSONObject("genres") ?: JSONObject().also { + jsonBody.put("genres", it) + } + genresObj.put("exclude", JSONArray().apply { + excludedGenreIds.forEach(::put) + }) + } + } + jsonBody.put("content_rating", JSONArray().apply { + val ratings = filter.contentRating.ifEmpty { + EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + } + if (ContentRating.SAFE in ratings) put("Safe") + if (ContentRating.SUGGESTIVE in ratings) put("Suggestive") + if (ContentRating.ADULT in ratings) { + put("Erotica") + put("Pornographic") + } + }) + + if (filter.states.isNotEmpty()) { + val statuses = JSONArray() + filter.states.forEach { state -> + when (state) { + MangaState.ONGOING -> statuses.put("Ongoing") + MangaState.FINISHED -> statuses.put("Completed") + MangaState.PAUSED -> statuses.put("Hiatus") + MangaState.ABANDONED -> statuses.put("Abandoned") + else -> Unit + } + } + if (statuses.length() > 0) { + jsonBody.put("upload_status", statuses) + } + } + + if (filter.types.isNotEmpty()) { + val formats = JSONArray() + filter.types.forEach { type -> + when (type) { + ContentType.MANGA -> formats.put("Manga") + ContentType.MANHWA -> formats.put("Manhwa") + ContentType.MANHUA -> formats.put("Manhua") + ContentType.COMICS -> formats.put("Comic") + ContentType.OTHER -> formats.put("Other") + else -> Unit + } + } + if (formats.length() > 0) { + jsonBody.put("format", formats) + } + } + + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val responseBody = try { + requestWithCloudflareRetry(url) { + webClient.httpPost(url.toHttpUrl(), jsonBody, headers).parseRaw() + } + } catch (e: HttpStatusException) { + if (e.statusCode == 403 || e.statusCode == 429 || e.statusCode == 503) { + requestCloudflareVerification(url, e) + } else { + throw e + } + } catch (e: ParseException) { + val causeMessage = e.message.orEmpty() + " " + (e.cause?.message.orEmpty()) + if (causeMessage.contains("CloudFlare", ignoreCase = true) || + causeMessage.contains("cf-mitigated", ignoreCase = true) || + causeMessage.contains("cf-error-details", ignoreCase = true) + ) { + requestCloudflareVerification(url, e) + } else { + throw e + } + } + + if (responseBody.isCloudflareChallenge()) { + requestCloudflareVerification(url) + } + + val response = try { + JSONObject(responseBody) + } catch (_: Exception) { + throw Exception("Invalid JSON search response: $responseBody") + } + + val content = response.optJSONArray("content") + ?: response.optJSONObject("result")?.optJSONArray("items") + ?: return emptyList() + + return (0 until content.length()).mapNotNull { i -> + val item = content.getJSONObject(i) + val id = item.optString("id").ifBlank { item.optString("series_id") } + if (id.isBlank()) return@mapNotNull null + val name = item.optString("name").ifBlank { item.optString("title") }.ifBlank { return@mapNotNull null } + val src = item.optString("source").ifBlank { item.optString("source_name") } + val title = if (src.isNotEmpty()) "$name [$src]" else name + val coverImageId = item.optString("cover_image_id").ifBlank { item.optString("coverImageId") } + val coverUrl = if (coverImageId.isNotBlank()) { + "$apiUrl/api/v2/image/$coverImageId" + } else { + "$apiUrl/api/v2/series/$id/thumbnail" + } + + Manga( + id = generateUid(id), + url = id, + publicUrl = "https://$domain/series/$id", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = parseContentRating(item.optString("content_rating")), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val seriesId = manga.url + val url = "$apiUrl/api/v2/series/$seriesId" + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val resp = requestWithCloudflareRetry(url) { + webClient.httpGet(url, headers) + } + val respBody = resp.body?.string() ?: "" + if (!resp.isSuccessful) throw Exception("Details error ${resp.code}: $respBody") + val json = try { + JSONObject(respBody) + } catch (_: Exception) { + throw Exception("Invalid JSON details: $respBody") + } + + val state = when ( + json.optString("publication_status") + .ifBlank { json.optString("upload_status") } + .ifBlank { json.optString("status") } + .uppercase(Locale.ROOT) + ) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED", "ENDED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + "ABANDONED", "CANCELLED", "CANCELED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val genres = json.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).mapNotNull { i -> + when (val item = arr.opt(i)) { + is String -> { + if (UUID_REGEX.matches(item)) { + MangaTag(item, item, source) + } else { + null + } + } + is JSONObject -> { + val key = item.optString("genre_id").ifBlank { item.optString("id") } + val name = item.optString("genre_name") + .ifBlank { item.optString("genreName") } + .ifBlank { item.optString("name") } + .ifBlank { item.optString("title") } + if (key.isNotBlank() && name.isNotBlank()) { + MangaTag(name, key, source) + } else { + null + } + } + else -> null + } + }.toSet() + } ?: emptySet() + + val authors = linkedSetOf<String>() + json.optJSONArray("authors")?.let { arr -> + for (i in 0 until arr.length()) { + when (val item = arr.opt(i)) { + is String -> item.takeIf { it.isNotBlank() }?.let(authors::add) + is JSONObject -> item.optString("name") + .ifBlank { item.optString("title") } + .takeIf { it.isNotBlank() } + ?.let(authors::add) + } + } + } + if (authors.isEmpty()) { + json.optJSONArray("series_staff")?.let { arr -> + for (i in 0 until arr.length()) { + val staff = arr.optJSONObject(i) ?: continue + val role = staff.optString("role") + if ( + role.contains("author", ignoreCase = true) || + role.contains("story", ignoreCase = true) || + role.contains("artist", ignoreCase = true) || + role.contains("art", ignoreCase = true) + ) { + staff.optString("name").takeIf { it.isNotBlank() }?.let(authors::add) + } + } + } + } + + val altTitles = json.optJSONArray("series_alternate_titles")?.let { arr -> + buildSet { + for (i in 0 until arr.length()) { + val item = arr.optJSONObject(i) ?: continue + item.optString("title").takeIf { it.isNotBlank() }?.let(::add) + } + } + } ?: emptySet() + + val description = buildString { + json.optString("description") + .ifBlank { json.optString("summary") } + .takeIf { it.isNotBlank() } + ?.let { + append(it.trim()) + } + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Associated Name(s):\n") + altTitles.forEach { + append(it) + append('\n') + } + } + }.trim().ifBlank { null } + + val coverUrl = json.optJSONArray("series_covers") + ?.optJSONObject(0) + ?.optString("image_id") + ?.takeIf { it.isNotBlank() } + ?.let { "$apiUrl/api/v2/image/$it" } + ?: manga.coverUrl + + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH) + fun parseChapters(content: JSONArray): List<MangaChapter> { + val chapters = ArrayList<MangaChapter>(content.length()) + for (i in 0 until content.length()) { + val ch = content.optJSONObject(i) ?: continue + val chId = ch.optString("book_id") + .ifBlank { ch.optString("id") } + .ifBlank { ch.optString("bookId") } + if (chId.isBlank()) continue + val chapterNo = ch.optString("chapter_no") + val chapterNumber = chapterNo.toChapterNumberOrNull() + ?: ch.optDouble("number", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + val sortNumber = ch.optDouble("sort_no", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + ?: ch.optDouble("number_sort", ch.optDouble("numberSort", Double.NaN)).takeIf { !it.isNaN() }?.toFloat() + val number = when { + sortNumber != null && chapterNumber != null && sortNumber >= chapterNumber -> sortNumber + sortNumber != null && chapterNumber == null -> sortNumber + chapterNumber != null -> chapterNumber + else -> 0f + } + val rawTitle = ch.optString("title").ifBlank { ch.optString("name") }.trim() + val chTitle = rawTitle.ifBlank { + chapterNo.takeIf { it.isNotBlank() }?.let { "Ch.$it" }.orEmpty() + }.ifBlank { "Chapter $number" } + val volume = ch.optString("volume_no") + .ifBlank { ch.optString("volume") } + .toIntOrNull() ?: 0 + val dateStr = ch.optString("published_on") + .ifBlank { ch.optString("release_date") } + .ifBlank { ch.optString("releaseDate") } + .ifBlank { ch.optString("created_at") } + val groups = ch.optJSONArray("groups") + chapters.add( + MangaChapter( + id = generateUid("$seriesId:$chId"), + title = chTitle, + number = number, + volume = volume, + url = "/series/$seriesId/reader/$chId", + uploadDate = dateFormat.parseSafe(dateStr), + source = source, + scanlator = groups?.let { arr -> + buildList { + for (j in 0 until arr.length()) { + arr.optJSONObject(j)?.optString("title")?.takeIf { it.isNotBlank() }?.let(::add) + } + }.joinToString().ifBlank { null } + }, + branch = null, + ), + ) + } + return chapters.sortedWith( + compareBy<MangaChapter> { it.number <= 0f } + .thenBy { it.number } + .thenBy { it.volume } + .thenBy { it.title.orEmpty() }, + ) + } + + var chapters = parseChapters( + json.optJSONArray("series_books") + ?: json.optJSONArray("seriesBooks") + ?: json.optJSONArray("books") + ?: json.optJSONArray("content") + ?: JSONArray(), + ) + if (chapters.isEmpty()) { + val chaptersUrl = "$apiUrl/api/v2/series/$seriesId/books/list" + val chapterResp = webClient.httpGet(chaptersUrl, headers).parseJson() + chapters = parseChapters( + chapterResp.optJSONArray("series_books") + ?: chapterResp.optJSONArray("seriesBooks") + ?: chapterResp.optJSONArray("content") + ?: JSONArray(), + ) + } + + return manga.copy( + title = json.optString("title").ifBlank { manga.title }, + altTitles = altTitles, + coverUrl = coverUrl, + description = description, + state = state, + authors = authors, + tags = genres, + chapters = chapters, + contentRating = parseContentRating(json.optString("content_rating")) ?: manga.contentRating, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + return emptyList() + } + + private var cacheUrl = "https://kstatic.to" + private var accessToken: String = "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val uri = URI(chapter.url) + val pathParts = uri.path.split("/").filter { it.isNotEmpty() } + if (pathParts.size < 4) throw Exception("Invalid chapter URL format: ${chapter.url}") + + val chapterId = pathParts.last() + val challenge = getChallengeResponse(chapterId) + accessToken = challenge.optString("access_token").ifBlank { + challenge.optString("accessToken") + }.ifBlank { + throw Exception("Invalid challenge response: missing access token") + } + cacheUrl = challenge.optString("cache_url").ifBlank { + challenge.optString("cacheUrl") + }.ifBlank { + throw Exception("Invalid challenge response: missing cache url") + } + + val pages = parseManifestPages(challenge) + if (pages.isEmpty()) { + throw Exception("Invalid challenge response: missing pages manifest") + } + + return pages.sortedBy { it.pageNumber }.map { page -> + val ext = page.ext?.takeIf { it.isNotBlank() } ?: "jxl" + val imageUrl = "$cacheUrl/api/v2/books/page".toHttpUrl().newBuilder().apply { + if (isDataSaver) addPathSegment("datasaver") + addPathSegment(chapterId) + addPathSegment("${page.pageUuid}.$ext") + addQueryParameter("token", accessToken) + addQueryParameter("is_datasaver", isDataSaver.toString()) + }.build().toString() + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun <T> requestWithCloudflareRetry(url: String, block: suspend () -> T, ): T { + try { + return block() + } catch (e: Exception) { + if (!e.isCloudflareProtectionError()) { + throw e + } + delay(CLOUDFLARE_RETRY_DELAY_MS.milliseconds) + try { + return block() + } catch (retryError: Exception) { + if (e.isCloudflareBlock() || retryError.isCloudflareBlock()) { + requestCloudflareVerification(url, retryError) + } + throw retryError + } + } + } + + private fun Throwable.isCloudflareBlock(): Boolean { + val name = javaClass.simpleName + return name.contains("CloudflareBlocked", ignoreCase = true) || + name.contains("CloudFlareBlocked") || + (isCloudflareProtectionError() && message.orEmpty().contains("blocked", ignoreCase = true)) + } + + private fun Throwable.isCloudflareProtectionError(): Boolean { + val name = javaClass.simpleName + if (name.contains("Cloudflare", ignoreCase = true) || name.contains("CloudFlare")) { + return true + } + val message = "${message.orEmpty()} ${cause?.message.orEmpty()}" + return message.contains("cf-mitigated", ignoreCase = true) || + message.contains("Just a moment", ignoreCase = true) || + message.contains("challenges.cloudflare.com", ignoreCase = true) || + message.contains("cf-error-details", ignoreCase = true) || + message.contains("cf-chl-bypass", ignoreCase = true) + } + + private fun requestCloudflareVerification(url: String, cause: Throwable? = null): Nothing { + try { + context.requestBrowserAction(this, "https://$domain/") + } catch (e: UnsupportedOperationException) { + throw ParseException( + "Cloudflare verification required. Open Kagane in WebView and retry.", + url, + cause ?: e, + ) + } + } + + private fun String.isCloudflareChallenge(): Boolean { + return contains("cf-mitigated", ignoreCase = true) + || contains("Just a moment", ignoreCase = true) + || contains("challenges.cloudflare.com", ignoreCase = true) + || contains("/cdn-cgi/challenge-platform/", ignoreCase = true) + || contains("Sorry, you have been blocked", ignoreCase = true) + || contains("cf-error-details", ignoreCase = true) + || contains("cf-chl-bypass", ignoreCase = true) + } + + private data class ManifestPage( + val pageNumber: Int, + val pageUuid: String, + val ext: String?, + ) + + private fun parseManifestPages(challenge: JSONObject): List<ManifestPage> { + val pagesJson = challenge.optJSONObject("manifest")?.optJSONArray("pages") + ?: challenge.optJSONArray("pages") + ?: JSONArray() + return buildList { + for (i in 0 until pagesJson.length()) { + val page = pagesJson.optJSONObject(i) ?: continue + val pageUuid = page.optString("page_id") + .ifBlank { page.optString("pageId") } + .ifBlank { page.optString("page_uuid") } + .ifBlank { page.optString("pageUuid") } + if (pageUuid.isBlank()) continue + add( + ManifestPage( + pageNumber = page.optInt( + "page_no", + page.optInt("pageNo", page.optInt("page_number", i + 1)), + ), + pageUuid = pageUuid, + ext = page.optString("ext").ifBlank { null }, + ), + ) + } + } + } + + private var integrityToken: String = "" + private var integrityTokenExp: Long = 0L + + private suspend fun getIntegrityToken(): String { + val now = System.currentTimeMillis() + if (integrityToken.isNotBlank() && now < integrityTokenExp) { + return integrityToken + } + + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val response = requestWithCloudflareRetry("$apiUrl/api/integrity") { + webClient.httpPost( + urlBuilder().addPathSegments("api/integrity").build(), + JSONObject(), + headers, + ).parseJson() + } + + val token = response.optString("token") + if (token.isBlank()) { + throw Exception("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = response.optLong("exp", 0L) * 1000L + return integrityToken + } + + private suspend fun getChallengeResponse(chapterId: String): JSONObject { + val integrityToken = getIntegrityToken() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver" + return requestWithCloudflareRetry(challengeUrl) { + webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers).parseJson() + } + } + + private fun String.toChapterNumberOrNull(): Float? = trim() + .replace(',', '.') + .toFloatOrNull() + + private fun getIntegrityTokenBlocking(): String { + val now = System.currentTimeMillis() + if (integrityToken.isNotBlank() && now < integrityTokenExp) { + return integrityToken + } + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val request = Request.Builder() + .url(urlBuilder().addPathSegments("api/integrity").build()) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Integrity token request failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + val token = json.optString("token") + if (token.isBlank()) { + throw IOException("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = json.optLong("exp", 0L) * 1000L + } + return integrityToken + } + + private fun refreshAccessTokenBlocking(chapterId: String) { + val integrityToken = getIntegrityTokenBlocking() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver".toHttpUrl() + val request = Request.Builder() + .url(challengeUrl) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Challenge refresh failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + accessToken = json.optString("access_token").ifBlank { + json.optString("accessToken") + }.ifBlank { + throw IOException("Invalid challenge response: missing access token") + } + cacheUrl = json.optString("cache_url").ifBlank { + json.optString("cacheUrl") + }.ifBlank { + throw IOException("Invalid challenge response: missing cache url") + } + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + val host = url.host + var requestBuilder = request.newBuilder() + .removeHeader("Content-Encoding") + .removeHeader("cf-connecting-ip") + if (host == domain || host.endsWith(".$domain")) { + requestBuilder = requestBuilder + .header("Origin", "https://$domain") + .header("Referer", "https://$domain/") + } + val newRequest = requestBuilder.build() + + var response = chain.proceed(newRequest) + + if (url.queryParameterNames.contains("token") && + (response.code == 401 || response.code == 403 || response.code == 507) + ) { + response.close() + val segments = url.pathSegments + val chapterId = segments.getOrNull(4) + if (chapterId != null) { + runCatching { refreshAccessTokenBlocking(chapterId) }.onSuccess { + val retryRequest = newRequest.newBuilder() + .url(url.newBuilder().setQueryParameter("token", accessToken).build()) + .build() + response = chain.proceed(retryRequest) + } + } + } + return response + } +} From 41520b141d4a8059771ff956f746a5a9bbee0943 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:11:01 +0200 Subject: [PATCH 308/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 00fe8790..337715d6 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -127,7 +127,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : Locale.FRENCH, Locale.GERMAN, Locale("id"), - Locale("pt", "BR"), + Locale("pt", "br"), Locale("es"), Locale("th"), Locale("vi"), @@ -248,7 +248,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : private var selectedLanguage: String? = null private fun localeToLang(locale: Locale): String { - if (locale.language == "pt" && locale.country == "BR") return "pt-br" + if (locale.language == "pt" && locale.country == "br") return "pt-br" return locale.language } From 77b1563ac489b7c29191c4879d11849b83ca365f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:20:46 +0200 Subject: [PATCH 309/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 1 - 1 file changed, 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 337715d6..599e69cd 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -94,7 +94,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : isMultipleTagsSupported = true, isTagsExclusionSupported = true, isSearchWithFiltersSupported = true, - isOriginalLocaleSupported = true, ) override val availableSortOrders: Set<SortOrder> = EnumSet.of( From 22fed211dd4e96784e0f25b40d2e2c5a124af577 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Thu, 13 Aug 2026 23:21:03 +0200 Subject: [PATCH 310/434] Update Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 101 ++++++++++++++++------- 1 file changed, 70 insertions(+), 31 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index 09d1d0e5..46bdfa63 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -99,37 +99,78 @@ internal class Kagane(context: MangaLoaderContext) : private companion object { const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L - val KAGANE_LANGS = arrayOf( - "en", - "ja", - "ko", - "zh-Hans", - "zh-Hant", - "es", - "es-419", - "fr", - "de", - "pt", - "pt-BR", - "ru", - "it", - "id", - "vi", - "th", - "pl", - "hi", - "ar", + private val KAGANE_LANGS: Set<Locale> = setOf( + Locale("af"), + Locale("ar"), + Locale("az"), + Locale("be"), + Locale("bg"), + Locale("bn"), + Locale("ca"), + Locale("cs"), + Locale("cv"), + Locale("da"), + Locale.GERMAN, + Locale("el"), + Locale.ENGLISH, + Locale("eo"), + Locale("es"), + Locale("es","419"), + Locale("et"), + Locale("eu"), + Locale("fa"), + Locale("fi"), + Locale("fil"), + Locale.FRENCH, + Locale("ga"), + Locale("he"), + Locale("hi"), + Locale("hr"), + Locale("hu"), + Locale("id"), + Locale.ITALIAN, + Locale.JAPANESE, + Locale("jv"), + Locale("ka"), + Locale("kk"), + Locale.KOREAN, + Locale("la"), + Locale("lt"), + Locale("mn"), + Locale("ms"), + Locale("my"), + Locale("ne"), + Locale("nl"), + Locale("no"), + Locale("pl"), + Locale("pt"), + Locale("pt", "br"), + Locale("ro"), + Locale("ru"), + Locale("sk"), + Locale("sl"), + Locale("sq"), + Locale("sr"), + Locale("sv"), + Locale("ta"), + Locale("te"), + Locale("th"), + Locale("tr"), + Locale("uk"), + Locale("ur"), + Locale("uz"), + Locale("vi"), + Locale.SIMPLIFIED_CHINESE, + Locale.TRADITIONAL_CHINESE, ) } private fun Locale.toKaganeLangCode(): String { - return when (toLanguageTag()) { - "zh-Hans" -> "zh-Hans" - "zh-Hant" -> "zh-Hant" - "es-419" -> "es-419" - "pt-BR" -> "pt-BR" - else -> language - } + if (language == "pt" && country.equals("br", ignoreCase = true)) return "pt-BR" + if (language == "es" && country == "419") return "es-419" + if (language == "zh" && country.equals("CN", ignoreCase = true)) return "zh-Hans" + if (language == "zh" && country.equals("TW", ignoreCase = true)) return "zh-Hant" + return language } override suspend fun getFilterOptions(): MangaListFilterOptions { @@ -154,9 +195,7 @@ internal class Kagane(context: MangaLoaderContext) : ContentType.COMICS, ContentType.OTHER, ), - availableLocales = KAGANE_LANGS.mapNotNull { lang -> - runCatching { Locale.forLanguageTag(lang) }.getOrNull() - }.toSet(), + availableLocales = KAGANE_LANGS, ) } @@ -641,7 +680,7 @@ internal class Kagane(context: MangaLoaderContext) : } } - private suspend fun <T> requestWithCloudflareRetry(url: String, block: suspend () -> T, ): T { + private suspend fun <T> requestWithCloudflareRetry(url: String, block: suspend () -> T): T { try { return block() } catch (e: Exception) { From 5f7f01a8666002529c16221b68c3f96143c28f52 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 14 Aug 2026 13:14:37 +0200 Subject: [PATCH 311/434] Create VyManga.kt --- src/main/kotlin/tsuki/site/en/VyManga.kt | 240 +++++++++++++++++++++++ 1 file changed, 240 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/VyManga.kt diff --git a/src/main/kotlin/tsuki/site/en/VyManga.kt b/src/main/kotlin/tsuki/site/en/VyManga.kt new file mode 100644 index 00000000..1ba8d066 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/VyManga.kt @@ -0,0 +1,240 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectLast +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet + +@MangaSourceParser("VYMANGA", "VyManga", "en") +internal class VyManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VYMANGA, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("mangavyvy.net") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + append("/search?search_po=0&q=") + + filter.query?.let { + append(filter.query.urlEncoded()) + } + + append("&author_po=0&author=") + + append("&completed=") + filter.states.oneOrThrowIfMany()?.let { + append( + when (it) { + MangaState.ONGOING -> "0" + MangaState.FINISHED -> "1" + else -> "" + }, + ) + } + + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("viewed&sort_type=desc") + SortOrder.POPULARITY_ASC -> append("viewed&sort_type=asc") + SortOrder.RATING -> append("scored&sort_type=desc") + SortOrder.RATING_ASC -> append("scored&sort_type=asc") + SortOrder.NEWEST -> append("created_at&sort_type=desc") + SortOrder.NEWEST_ASC -> append("created_at&sort_type=asc") + SortOrder.UPDATED -> append("updated_at&sort_type=desc") + SortOrder.UPDATED_ASC -> append("updated_at&sort_type=asc") + else -> append("updated_at&sort_type=desc") + } + + filter.tags.forEach { tag -> + append("&genre[]=") + append(tag.key) + } + + filter.tagsExclude.forEach { tagsExclude -> + append("&exclude_genre[]=") + append(tagsExclude.key) + } + + append("&page=") + append(page.toString()) + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".comic-item").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = div.selectFirst(".comic-image img")?.src(), + title = div.selectFirst(".comic-title")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + + val tags = mutableSetOf<MangaTag>() + doc.select(".check-genre div div:has(.checkbox-genre)").forEach { element -> + val label = element.selectFirst("label")?.text()?.trim() + val input = element.selectFirst(".checkbox-genre") + val key = input?.attr("data-value")?.trim() + + if (!label.isNullOrBlank() && !key.isNullOrBlank()) { + tags += MangaTag( + title = label, + key = key, + source = source, + ) + } + } + return tags + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val simpleDateFormat = SimpleDateFormat("MMM dd, yyy", sourceLocale) + return manga.copy( + tags = doc.select("div.col-md-7 p a[href*=genre]").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('/'), + title = a.text(), + source = source, + ) + }, + description = doc.selectFirst("div.summary p.content")?.text().orEmpty(), + state = when (doc.selectLast("div.col-md-7 p:contains(Status) span")?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + rating = doc.selectFirst("div.col-md-7 p:contains(Rating)")?.text()?.substringAfterLast(':') + ?.substringBefore('/')?.toFloat() ?: RATING_UNKNOWN, + chapters = doc.select("div.list div.list-group a").mapChapters(reversed = true) { i, a -> + val url = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(url), + title = a.selectFirst("span")?.textOrNull(), + number = i + 1f, + volume = 0, + url = url, + scanlator = null, + uploadDate = parseChapterDate( + simpleDateFormat, + a.selectFirst("p")?.text(), + ), + branch = null, + source = source, + ) + }, + ) + } + + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + d.endsWith(" ago") -> parseRelativeDate(date) + else -> dateFormat.parseSafe(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis + WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("img.d-block").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} From d1e10771111f76cd2ad7769aeb6effd92be5fe2e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:38:23 +0200 Subject: [PATCH 312/434] fix TemplateScan --- src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt index 90dc94df..242b3dd5 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt @@ -30,7 +30,6 @@ import okhttp3.Response import org.json.JSONArray import org.json.JSONObject import org.jsoup.Jsoup -import org.jsoup.safety.Safelist import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -153,7 +152,7 @@ internal class TempleScan(context: MangaLoaderContext) : val cleanDesc = if (desc.contains("#")) desc.substringBefore("#") .replace(TRAILING_WORD_REGEX, "").trim() else desc - Jsoup.clean(cleanDesc, Safelist.none()) + Jsoup.parse(cleanDesc).text() }?.trim().orEmpty(), authors = details.author?.let { setOf(it) } ?: emptySet(), state = when (details.status) { From ab1801188d1b259cfa97d7b0a5c61a920fd043a8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:42:08 +0200 Subject: [PATCH 313/434] fix HentaiFox multiple tags filter --- src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt index 106ddd27..b93bd857 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt @@ -20,7 +20,6 @@ import tsuki.model.SortOrder import tsuki.model.Favicon import tsuki.model.Favicons -import org.jsoup.internal.StringUtil import org.jsoup.nodes.Element import java.util.EnumSet import java.util.Locale @@ -134,13 +133,9 @@ internal class HentaiFox(context: MangaLoaderContext) : } private fun buildQuery(tags: Collection<MangaTag>, language: Locale?): String { - val joiner = StringUtil.StringJoiner(" ") - tags.forEach { tag -> - joiner.add(tag.key) - } - language?.let { lc -> - joiner.add(lc.toLanguagePath()) - } - return joiner.complete() + val parts = mutableListOf<String>() + tags.forEach { tag -> parts += tag.key } + language?.let { parts += it.toLanguagePath() } + return parts.joinToString(" ") } } From a8b39027999d543850bf5314ec1f6b85eb939b8d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:51:39 +0200 Subject: [PATCH 314/434] dedup kagane --- src/main/kotlin/tsuki/site/all/Kagane.kt | 188 ++++++++++++----------- 1 file changed, 99 insertions(+), 89 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index 46bdfa63..b70d78d2 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -42,6 +42,8 @@ import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale import kotlin.time.Duration.Companion.milliseconds +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock /** * Created from: https://github.com/glitch-228 @@ -92,79 +94,6 @@ internal class Kagane(context: MangaLoaderContext) : isTagsExclusionSupported = true, ) - private var genresCache: Set<MangaTag>? = null - private val UUID_REGEX = Regex( - "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", - ) - - private companion object { - const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L - private val KAGANE_LANGS: Set<Locale> = setOf( - Locale("af"), - Locale("ar"), - Locale("az"), - Locale("be"), - Locale("bg"), - Locale("bn"), - Locale("ca"), - Locale("cs"), - Locale("cv"), - Locale("da"), - Locale.GERMAN, - Locale("el"), - Locale.ENGLISH, - Locale("eo"), - Locale("es"), - Locale("es","419"), - Locale("et"), - Locale("eu"), - Locale("fa"), - Locale("fi"), - Locale("fil"), - Locale.FRENCH, - Locale("ga"), - Locale("he"), - Locale("hi"), - Locale("hr"), - Locale("hu"), - Locale("id"), - Locale.ITALIAN, - Locale.JAPANESE, - Locale("jv"), - Locale("ka"), - Locale("kk"), - Locale.KOREAN, - Locale("la"), - Locale("lt"), - Locale("mn"), - Locale("ms"), - Locale("my"), - Locale("ne"), - Locale("nl"), - Locale("no"), - Locale("pl"), - Locale("pt"), - Locale("pt", "br"), - Locale("ro"), - Locale("ru"), - Locale("sk"), - Locale("sl"), - Locale("sq"), - Locale("sr"), - Locale("sv"), - Locale("ta"), - Locale("te"), - Locale("th"), - Locale("tr"), - Locale("uk"), - Locale("ur"), - Locale("uz"), - Locale("vi"), - Locale.SIMPLIFIED_CHINESE, - Locale.TRADITIONAL_CHINESE, - ) - } - private fun Locale.toKaganeLangCode(): String { if (language == "pt" && country.equals("br", ignoreCase = true)) return "pt-BR" if (language == "es" && country == "419") return "es-419" @@ -174,7 +103,9 @@ internal class Kagane(context: MangaLoaderContext) : } override suspend fun getFilterOptions(): MangaListFilterOptions { - val genres = genresCache ?: fetchGenres().also { genresCache = it } + val genres = genresMutex.withLock { + genresCache ?: fetchGenres().also { genresCache = it } + } return MangaListFilterOptions( availableTags = genres, availableContentRating = EnumSet.of( @@ -199,6 +130,10 @@ internal class Kagane(context: MangaLoaderContext) : ) } + @Volatile + private var genresCache: Set<MangaTag>? = null + private val genresMutex = Mutex() + private suspend fun fetchGenres(): Set<MangaTag> { val headers = getRequestHeaders().newBuilder() .add("Origin", "https://$domain") @@ -215,11 +150,8 @@ internal class Kagane(context: MangaLoaderContext) : buildSet { for (i in 0 until genres.length()) { val item = genres.optJSONObject(i) ?: continue - val id = item.optString("genre_id").ifBlank { item.optString("id") } - val title = item.optString("genre_name") - .ifBlank { item.optString("genreName") } - .ifBlank { item.optString("name") } - .ifBlank { item.optString("title") } + val id = item.optGenreId() + val title = item.optGenreName() if (id.isNotBlank() && title.isNotBlank() && UUID_REGEX.matches(id)) { add(MangaTag(title, id, source)) } @@ -462,11 +394,8 @@ internal class Kagane(context: MangaLoaderContext) : } } is JSONObject -> { - val key = item.optString("genre_id").ifBlank { item.optString("id") } - val name = item.optString("genre_name") - .ifBlank { item.optString("genreName") } - .ifBlank { item.optString("name") } - .ifBlank { item.optString("title") } + val key = item.optGenreId() + val name = item.optGenreName() if (key.isNotBlank() && name.isNotBlank()) { MangaTag(name, key, source) } else { @@ -776,12 +705,13 @@ internal class Kagane(context: MangaLoaderContext) : private var integrityToken: String = "" private var integrityTokenExp: Long = 0L + private val isIntegrityTokenValid: Boolean + get() = integrityToken.isNotBlank() && System.currentTimeMillis() < integrityTokenExp + private suspend fun getIntegrityToken(): String { - val now = System.currentTimeMillis() - if (integrityToken.isNotBlank() && now < integrityTokenExp) { + if (isIntegrityTokenValid) { return integrityToken } - val headers = getRequestHeaders().newBuilder() .add("Origin", "https://$domain") .add("Referer", "https://$domain/") @@ -822,8 +752,7 @@ internal class Kagane(context: MangaLoaderContext) : .toFloatOrNull() private fun getIntegrityTokenBlocking(): String { - val now = System.currentTimeMillis() - if (integrityToken.isNotBlank() && now < integrityTokenExp) { + if (isIntegrityTokenValid) { return integrityToken } val headers = getRequestHeaders().newBuilder() @@ -914,4 +843,85 @@ internal class Kagane(context: MangaLoaderContext) : } return response } + + companion object { + private const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L + + private val UUID_REGEX = Regex( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + ) + + private val KAGANE_LANGS: Set<Locale> = setOf( + Locale("af"), + Locale("ar"), + Locale("az"), + Locale("be"), + Locale("bg"), + Locale("bn"), + Locale("ca"), + Locale("cs"), + Locale("cv"), + Locale("da"), + Locale.GERMAN, + Locale("el"), + Locale.ENGLISH, + Locale("eo"), + Locale("es"), + Locale("es","419"), + Locale("et"), + Locale("eu"), + Locale("fa"), + Locale("fi"), + Locale("fil"), + Locale.FRENCH, + Locale("ga"), + Locale("he"), + Locale("hi"), + Locale("hr"), + Locale("hu"), + Locale("id"), + Locale.ITALIAN, + Locale.JAPANESE, + Locale("jv"), + Locale("ka"), + Locale("kk"), + Locale.KOREAN, + Locale("la"), + Locale("lt"), + Locale("mn"), + Locale("ms"), + Locale("my"), + Locale("ne"), + Locale("nl"), + Locale("no"), + Locale("pl"), + Locale("pt"), + Locale("pt", "br"), + Locale("ro"), + Locale("ru"), + Locale("sk"), + Locale("sl"), + Locale("sq"), + Locale("sr"), + Locale("sv"), + Locale("ta"), + Locale("te"), + Locale("th"), + Locale("tr"), + Locale("uk"), + Locale("ur"), + Locale("uz"), + Locale("vi"), + Locale.SIMPLIFIED_CHINESE, + Locale.TRADITIONAL_CHINESE, + ) + + private fun JSONObject.optGenreId(): String = optString("genre_id").ifBlank { optString("id") } + + private fun JSONObject.optGenreName(): String = + optString("genre_name") + .ifBlank { optString("genreName") } + .ifBlank { optString("name") } + .ifBlank { optString("title") } + } } From 9f6d852a8503722271f891080a51531cd364bfe5 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 14 Aug 2026 23:53:05 +0200 Subject: [PATCH 315/434] Delete src/main/kotlin/tsuki/site/id/KomikCast.kt dead --- src/main/kotlin/tsuki/site/id/KomikCast.kt | 342 --------------------- 1 file changed, 342 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/id/KomikCast.kt diff --git a/src/main/kotlin/tsuki/site/id/KomikCast.kt b/src/main/kotlin/tsuki/site/id/KomikCast.kt deleted file mode 100644 index 72c6d264..00000000 --- a/src/main/kotlin/tsuki/site/id/KomikCast.kt +++ /dev/null @@ -1,342 +0,0 @@ -package tsuki.site.id - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.ContentType -import tsuki.model.SortOrder - -import tsuki.util.generateUid -import tsuki.util.oneOrThrowIfMany - -import org.json.JSONObject -import java.text.DecimalFormat -import java.text.DecimalFormatSymbols -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale -import kotlin.collections.toSet - -@MangaSourceParser("KOMIKCAST", "KomikCast", "id") -internal class Komikcast(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.KOMIKCAST, pageSize = 12) { - - override val configKeyDomain = ConfigKey.Domain("v3.komikcast.fit") - - private val apiBase = "https://be.komikcast.cc" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("Accept", "application/json") - .add("Accept-language", "en-US,en;q=0.9,id;q=0.8") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = false, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val genreMap = fetchGenreMap() - return MangaListFilterOptions( - availableTags = genreMap.values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA - ), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append(apiBase) - append("/series?includeMeta=true") - append("&take=").append(pageSize) - append("&page=").append(page) - - append("&sort=") - append( - when (order) { - SortOrder.POPULARITY -> "popularity" - SortOrder.UPDATED -> "latest" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RATING -> "rating" - else -> "latest" - } - ) - append("&sortOrder=desc") - - if (!filter.query.isNullOrEmpty()) { - val escaped = filter.query!!.replace("\"", "\\\"") - append("&filter=title=like=\"$escaped\",nativeTitle=like=\"$escaped\"") - } - - filter.states.oneOrThrowIfMany()?.let { state -> - append("&status=") - append( - when (state) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - else -> "" - } - ) - } - - filter.types.oneOrThrowIfMany()?.let { type -> - append("&format=") - append( - when (type) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> "" - } - ) - } - - if (filter.tags.isNotEmpty()) { - append("&genreIds=") - append(filter.tags.joinToString(",") { it.key }) - } - } - - val json = webClient.httpGet(url).body?.string().orEmpty() - return parseSeriesList(json) - } - - private fun parseSeriesList(json: String): List<Manga> { - val result = mutableListOf<Manga>() - try { - val root = JSONObject(json) - val dataArray = root.getJSONArray("data") - for (i in 0 until dataArray.length()) { - val item = dataArray.getJSONObject(i) - val data = item.getJSONObject("data") - val slug = data.getString("slug") - val relativeUrl = "/series/$slug" - - result.add( - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - publicUrl = "https://$domain$relativeUrl", - title = data.getString("title"), - altTitles = emptySet(), - rating = data.optDouble("rating", -1.0).let { - if (it >= 0.0) (it / 10.0).toFloat() else RATING_UNKNOWN - }, - contentRating = null, - coverUrl = data.optString("coverImage", null), - tags = emptySet(), - state = when (data.optString("status").lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - else -> null - }, - authors = setOfNotNull(data.optString("author").takeIf { it.isNotEmpty() }), - source = source, - ) - ) - } - } catch (_: Exception) { } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.removeSuffix("/").substringAfterLast("/") - val url = "$apiBase/series/$slug" - val json = webClient.httpGet(url).body?.string().orEmpty() - val seriesData = parseSeriesDetail(json) - - val chapters = fetchChapters(slug) - - return manga.copy( - title = seriesData.title, - description = seriesData.synopsis, - state = when (seriesData.status.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - else -> null - }, - authors = setOfNotNull(seriesData.author), - tags = seriesData.genres.map { - MangaTag(title = it.name, key = it.id.toString(), source = source) - }.toSet(), - coverUrl = seriesData.coverImage, - rating = if (seriesData.rating > 0) seriesData.rating / 10f else RATING_UNKNOWN, - chapters = chapters, - ) - } - - private fun parseSeriesDetail(json: String): SeriesData { - val root = JSONObject(json) - val data = root.getJSONObject("data").getJSONObject("data") - - val genres = mutableListOf<GenreData>() - val genresArray = data.optJSONArray("genres") - if (genresArray != null) { - for (i in 0 until genresArray.length()) { - val g = genresArray.getJSONObject(i) - genres.add( - GenreData( - id = g.getInt("id"), - name = g.getJSONObject("data").getString("name") - ) - ) - } - } - - return SeriesData( - title = data.getString("title"), - synopsis = data.optString("synopsis", ""), - status = data.getString("status"), - author = data.optString("author").takeIf { it.isNotEmpty() }, - coverImage = data.optString("coverImage", ""), - rating = data.optDouble("rating", 0.0).toFloat(), - genres = genres, - ) - } - - private suspend fun fetchChapters(slug: String): List<MangaChapter> { - val url = "$apiBase/series/$slug/chapters" - val json = webClient.httpGet(url).body?.string().orEmpty() - return parseChaptersJson(json, slug) - } - - private fun parseChaptersJson(json: String, slug: String): List<MangaChapter> { - val result = mutableListOf<MangaChapter>() - try { - val root = JSONObject(json) - val dataArray = root.getJSONArray("data") - for (i in 0 until dataArray.length()) { - val item = dataArray.getJSONObject(i) - val data = item.getJSONObject("data") - val index = data.getDouble("index") - val formattedIndex = chapterNumberFormatter.format(index) - - val chapterUrl = "/series/$slug/chapter/$formattedIndex" - val title = data.optString("title").takeIf { it.isNotEmpty() && it != "null" } - ?: "Chapter $formattedIndex" - - result.add( - MangaChapter( - id = generateUid(chapterUrl), - title = title, - url = chapterUrl, - number = index.toFloat(), - volume = 0, - scanlator = null, - uploadDate = parseChapterDate(item.optString("createdAt", "")), - branch = null, - source = source, - ) - ) - } - } catch (_: Exception) { } - return result.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - // chapter.url = /series/{slug}/chapter/{formattedIndex} - val parts = chapter.url.split("/") - val slug = parts[2] - val chapterIndex = parts[4] - val url = "$apiBase/series/$slug/chapters/$chapterIndex" - - val json = webClient.httpGet(url).body?.string().orEmpty() - try { - val root = JSONObject(json) - val data = root.getJSONObject("data").getJSONObject("data") - val imagesArray = data.getJSONArray("images") - return (0 until imagesArray.length()).mapNotNull { i -> - val imgUrl = imagesArray.getString(i) - if (imgUrl.isBlank()) return@mapNotNull null - MangaPage( - id = generateUid(imgUrl), - url = imgUrl, - preview = null, - source = source, - ) - } - } catch (_: Exception) { - return emptyList() - } - } - - @Volatile - private var genreCache: Map<String, MangaTag>? = null - - private suspend fun fetchGenreMap(): Map<String, MangaTag> { - genreCache?.let { return it } - val url = "$apiBase/genres" - val jsonStr = webClient.httpGet(url).body?.string().orEmpty() - val map = mutableMapOf<String, MangaTag>() - try { - val dataArray = JSONObject(jsonStr).getJSONArray("data") - for (i in 0 until dataArray.length()) { - val genreObj = dataArray.getJSONObject(i) - val name = genreObj.getJSONObject("data").getString("name") - val id = genreObj.getInt("id") - map[name] = MangaTag(title = name, key = id.toString(), source = source) - } - } catch (_: Exception) { } - genreCache = map - return map - } - - private fun parseChapterDate(dateStr: String): Long { - if (dateStr.isBlank()) return 0L - return try { - isoDateFormat.parse(dateStr)?.time ?: 0L - } catch (_: Exception) { 0L } - } - - private data class SeriesData( - val title: String, - val synopsis: String, - val status: String, - val author: String?, - val coverImage: String, - val rating: Float, - val genres: List<GenreData>, - ) - - private data class GenreData(val id: Int, val name: String) - - companion object { - private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT) - private val chapterNumberFormatter = DecimalFormat( - "#.##", - DecimalFormatSymbols.getInstance(Locale.US) - ) - } -} From 6e779396fa38484e54cfd422eb9b0664027640e9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 00:29:23 +0200 Subject: [PATCH 316/434] Create VoraToon.kt --- src/main/kotlin/tsuki/site/id/VoraToon.kt | 406 ++++++++++++++++++++++ 1 file changed, 406 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/VoraToon.kt diff --git a/src/main/kotlin/tsuki/site/id/VoraToon.kt b/src/main/kotlin/tsuki/site/id/VoraToon.kt new file mode 100644 index 00000000..7f74f630 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/VoraToon.kt @@ -0,0 +1,406 @@ +package tsuki.site.id + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONArray +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany + +import org.json.JSONObject +import java.net.URLEncoder +import java.text.DecimalFormat +import java.text.DecimalFormatSymbols +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("VORATOON", "VoraToon", "id") +internal class VoraToon(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VORATOON, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("v1.voratoon.com") + + private val apiBase = "https://api.voratoon.com" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept", "application/json") + .add("Accept-language", "en-US,en;q=0.9,id;q=0.8") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.RATING, + SortOrder.RATING_ASC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genreMap = fetchGenreMap() + return MangaListFilterOptions( + availableTags = genreMap.values.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.POPULARITY -> "popularity" to "desc" + SortOrder.POPULARITY_ASC -> "popularity" to "asc" + SortOrder.UPDATED -> "latest" to "desc" + SortOrder.UPDATED_ASC -> "updated" to "asc" + SortOrder.ALPHABETICAL -> "alphabetical" to "asc" + SortOrder.ALPHABETICAL_DESC -> "alphabetical" to "desc" + SortOrder.RATING -> "rating" to "desc" + SortOrder.RATING_ASC -> "rating" to "asc" + else -> null + } + + val url = buildString { + append(apiBase) + append("/series?includeMeta=true") + append("&take=").append(pageSize) + append("&page=").append(page) + + sortParam?.let { (field, dir) -> + append("&sort=").append(field) + append("&sortOrder=").append(dir) + } + + if (!filter.query.isNullOrEmpty()) { + val q = URLEncoder.encode(filter.query, "UTF-8") + append("&filter=title=like=\"$q\",nativeTitle=like=\"$q\"") + } + + filter.states.oneOrThrowIfMany()?.let { state -> + append("&status=") + append( + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "canceled" + else -> "" + }, + ) + } + + filter.types.oneOrThrowIfMany()?.let { type -> + append("&format=") + append( + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + }, + ) + } + + if (filter.tags.isNotEmpty()) { + append("&genreIds=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: String): List<Manga> { + val result = mutableListOf<Manga>() + try { + val root = JSONObject(json) + val dataArray = root.getJSONArray("data") + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val data = item.getJSONObject("data") + val slug = data.getString("slug") + val relativeUrl = "/series/$slug" + + result.add( + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + title = data.getString("title"), + altTitles = emptySet(), + rating = data.toRating(), + contentRating = null, + coverUrl = data.nonNullString("coverImage"), + tags = emptySet(), + state = data.toMangaState(), + authors = setOfNotNull(data.nonNullString("author")), + source = source, + ), + ) + } + } catch (_: Exception) { + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removeSuffix("/").substringAfterLast("/") + val url = "$apiBase/series?includeMeta=true&take=1&page=1&takeChapter=9999&filter=slug%3D%3D$slug" + val json = webClient.httpGet(url).body?.string().orEmpty() + + return try { + val root = JSONObject(json) + val dataArray = root.getJSONArray("data") + if (dataArray.length() == 0) return manga + + val seriesItem = dataArray.getJSONObject(0) + val data = seriesItem.getJSONObject("data") + + val title = data.getString("title") + val nativeTitle = data.nonNullString("nativeTitle") + val synopsis = data.optString("synopsis", "") + val author = data.nonNullString("author") + val coverImage = data.optString("coverImage", "") + val backgroundImage = data.nonNullString("backgroundImage") + val format = data.nonNullString("format")?.lowercase(Locale.ROOT) + + val genres = parseGenres(data) + val chapters = parseChapters(seriesItem, slug) + + val releaseYear = data.nonNullString("releaseDate") + + val extraTags = buildSet { + releaseYear?.let { add(MangaTag(it, it, source)) } + format?.let { + add(MangaTag(it.replaceFirstChar { c -> c.uppercase() }, it, source)) + } + } + + manga.copy( + title = title, + altTitles = setOfNotNull(nativeTitle), + description = synopsis, + state = data.toMangaState(), + authors = setOfNotNull(author), + tags = genres + extraTags, + coverUrl = coverImage, + largeCoverUrl = backgroundImage ?: coverImage, + rating = data.toRating(), + chapters = chapters, + ) + } catch (_: Exception) { + manga + } + } + + private fun parseGenres(data: JSONObject): Set<MangaTag> { + val genres = mutableSetOf<MangaTag>() + data.optJSONArray("genres")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.toGenreTagOrNull()?.let(genres::add) + } + } + return genres + } + + private fun parseChapters(seriesItem: JSONObject, slug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + val chaptersArray = seriesItem.optJSONArray("chapters") ?: JSONArray() + + for (i in 0 until chaptersArray.length()) { + val ch = chaptersArray.getJSONObject(i) + val chapterIndex = ch.getInt("chapterIndex") + val formattedIndex = chapterNumberFormatter.format(chapterIndex.toDouble()) + val chapterUrl = "/series/$slug/chapter/$formattedIndex" + val titleText = ch.optJSONObject("data")?.nonNullString("title") + ?: "Chapter $formattedIndex" + val uploadDate = parseChapterDate(ch.optString("createdAt", "")) + + chapters.add( + MangaChapter( + id = generateUid(chapterUrl), + title = titleText, + url = chapterUrl, + number = chapterIndex.toFloat(), + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ), + ) + } + + chapters.sortBy { it.number } + return chapters + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("/") + if (parts.size < 5) return emptyList() + val slug = parts[2] + val chapterIndex = parts[4] + val url = "$apiBase/series/$slug/chapters/$chapterIndex" + + val json = webClient.httpGet(url).body?.string().orEmpty() + try { + val root = JSONObject(json) + val data = root.optJSONObject("data") ?: return emptyList() + + val imagesArray = data.optJSONObject("data")?.optJSONArray("images") + if (imagesArray != null) { + return (0 until imagesArray.length()).mapNotNull { i -> + val imgUrl = imagesArray.getString(i) + if (imgUrl.isBlank()) { + null + } else { + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + } + + val dataImages = data.optJSONObject("dataImages") + if (dataImages != null) { + val pages = mutableListOf<MangaPage>() + val sortedKeys = dataImages.keys().asSequence().toList() + .sortedBy { it.toIntOrNull() ?: Int.MAX_VALUE } + + for (key in sortedKeys) { + val imgUrl = dataImages.getString(key) + if (imgUrl.isNotBlank()) { + pages.add( + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ), + ) + } + } + return pages + } + + return emptyList() + } catch (_: Exception) { + return emptyList() + } + } + + @Volatile + private var genreCache: Map<String, MangaTag>? = null + private val genreMutex = Mutex() + + private suspend fun fetchGenreMap(): Map<String, MangaTag> { + return genreMutex.withLock { + genreCache ?: fetchGenreMapInternal().also { genreCache = it } + } + } + + private suspend fun fetchGenreMapInternal(): Map<String, MangaTag> { + val url = "$apiBase/genres" + val jsonStr = webClient.httpGet(url).body?.string().orEmpty() + val map = mutableMapOf<String, MangaTag>() + + try { + val dataArray = JSONObject(jsonStr).getJSONArray("data") + for (i in 0 until dataArray.length()) { + dataArray.getJSONObject(i).toGenreTagOrNull()?.let { + map[it.title] = it + } + } + } catch (_: Exception) { + } + return map + } + + private fun parseChapterDate(dateStr: String): Long { + if (dateStr.isBlank()) return 0L + return try { + isoDateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + + private fun JSONObject.nonNullString(key: String): String? { + return optString(key).takeIf { it.isNotBlank() && it != "null" } + } + + private fun JSONObject.toMangaState(): MangaState? { + val status = nonNullString("status")?.lowercase(Locale.ROOT) ?: return null + return when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> null + } + } + + private fun JSONObject.toRating(): Float { + val raw = optDouble("rating", 0.0) + return if (raw > 0.0) (raw / 10.0).toFloat() else RATING_UNKNOWN + } + + private fun JSONObject.toGenreTagOrNull(): MangaTag? { + val name = optJSONObject("data")?.optString("name").orEmpty().trim() + val id = optInt("id", -1) + if (name.isEmpty() || id == -1) return null + return MangaTag(title = name, key = id.toString(), source = source) + } + + companion object { + private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT) + private val chapterNumberFormatter = DecimalFormat( + "#.##", + DecimalFormatSymbols.getInstance(Locale.US), + ) + } +} From 49c7020676e8b6b06e8faf720c56cc156347b37d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:54:40 +0200 Subject: [PATCH 317/434] Update MangaK.kt --- src/main/kotlin/tsuki/site/en/MangaK.kt | 29 ++++++++++++++++++++----- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt index 41ad3c0d..74c6b9ce 100644 --- a/src/main/kotlin/tsuki/site/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -33,6 +33,8 @@ import tsuki.util.json.mapJSON import tsuki.util.json.mapJSONNotNullToSet import tsuki.util.json.mapJSONToSet +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import org.json.JSONObject import java.text.SimpleDateFormat import java.util.EnumSet @@ -46,7 +48,7 @@ internal class MangaK(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("mangak.io") - private val apiUrl = "https://api.mangak.io" + private val apiUrl = "https://api.$domain" private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) @@ -127,7 +129,7 @@ internal class MangaK(context: MangaLoaderContext) : append(page) append("&limit=") append(pageSize) - + // To have "Best match" results if (filter.query.isNullOrBlank()) { append("&sort=") @@ -256,18 +258,32 @@ internal class MangaK(context: MangaLoaderContext) : val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) }.orEmpty() - - override suspend fun getDetails(manga: Manga): Manga { + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() .getJSONObject("data").getJSONObject("title") + val cv = json.optLong("cv") + val chaptersDeferred = async { + fetchChapters(manga.url, cv) + } val authors = json.optJSONArray("authors")?.mapJSONNotNullToSet { it.optString("name").nullIfEmpty() }.orEmpty() val artists = json.optJSONArray("artists")?.mapJSONNotNullToSet { it.optString("name").nullIfEmpty() }.orEmpty() - return manga.copy( + + val contentRating = json.optString("content_rating").nullIfEmpty()?.let { + when (it) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + } + + manga.copy( title = json.optString("name").nullIfEmpty() ?: manga.title, altTitles = json.parseAltTitles().ifEmpty { manga.altTitles }, description = json.optString("summary").nullIfEmpty() ?: manga.description, @@ -275,7 +291,8 @@ internal class MangaK(context: MangaLoaderContext) : authors = authors + artists, state = json.optString("status").toMangaState() ?: manga.state, rating = json.optDouble("rating", 0.0).toRating(), - chapters = fetchChapters(manga.url, cv), + contentRating = contentRating ?: manga.contentRating, + chapters = chaptersDeferred.await(), ) } From 33a3859ab327813df8fc16220a9e992dbb62cf68 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 11:55:07 +0200 Subject: [PATCH 318/434] Update VoraToon.kt --- src/main/kotlin/tsuki/site/id/VoraToon.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/VoraToon.kt b/src/main/kotlin/tsuki/site/id/VoraToon.kt index 7f74f630..abc1b18d 100644 --- a/src/main/kotlin/tsuki/site/id/VoraToon.kt +++ b/src/main/kotlin/tsuki/site/id/VoraToon.kt @@ -1,7 +1,5 @@ package tsuki.site.id -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import org.json.JSONArray import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -24,6 +22,8 @@ import tsuki.model.SortOrder import tsuki.util.generateUid import tsuki.util.oneOrThrowIfMany +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import org.json.JSONObject import java.net.URLEncoder import java.text.DecimalFormat @@ -368,7 +368,7 @@ internal class VoraToon(context: MangaLoaderContext) : 0L } } - + private fun JSONObject.nonNullString(key: String): String? { return optString(key).takeIf { it.isNotBlank() && it != "null" } } From 4e6eab4f5398927c2b39765f6ec88306c9da7417 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:18:53 +0200 Subject: [PATCH 319/434] Create LumosKomik.kt --- src/main/kotlin/tsuki/site/id/LumosKomik.kt | 282 ++++++++++++++++++++ 1 file changed, 282 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/LumosKomik.kt diff --git a/src/main/kotlin/tsuki/site/id/LumosKomik.kt b/src/main/kotlin/tsuki/site/id/LumosKomik.kt new file mode 100644 index 00000000..c00373f4 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/LumosKomik.kt @@ -0,0 +1,282 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import java.time.Instant +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("LUMOSKOMIK", "LumosKomik", "id") +internal class LumosKomik(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.LUMOSKOMIK, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("03.lumosgg.com") + + private val baseUrl = "https://$domain" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + return MangaListFilterOptions( + availableTags = genres, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + private suspend fun fetchGenres(): Set<MangaTag> { + val doc = webClient.httpGet("$baseUrl/browse").parseHtml() + return doc.select("label[data-bf-genre-name] input.bf-genre-cb[value]") + .mapNotNull { input -> + val value = input.attr("value").trim() + val name = input.parent()?.selectFirst("span.truncate")?.text()?.trim() + if (value.isNotEmpty() && !name.isNullOrEmpty()) { + MangaTag(name, value, source) + } else null + } + .toSet() + } + + /** + * When scraping they have a hidden honeypot in the listing crazy + */ + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "rating" + SortOrder.ALPHABETICAL -> "az" + else -> "latest" + } + + val url = "$baseUrl/browse".toHttpUrl().newBuilder().apply { + if (!filter.query.isNullOrBlank()) addQueryParameter("q", filter.query) + if (sort.isNotEmpty()) addQueryParameter("sort", sort) + if (filter.states.isNotEmpty()) addQueryParameter("status", mapState(filter.states.first())) + if (filter.types.isNotEmpty()) addQueryParameter("type", mapType(filter.types.first())) + if (filter.tags.isNotEmpty()) addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) + addQueryParameter("page", page.toString()) + }.build() + + val doc = webClient.httpGet(url).parseHtml() + val list = parseMangaList(doc) + return list.filterNot { it.url.contains("honeypot") } + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a[href^=\"/comic/\"]:has(img:not([src*=\"flagcdn\"]))") + .mapNotNull { element -> + val img = element.selectFirst("img") ?: return@mapNotNull null + val title = img.attr("alt").takeIf { it.isNotBlank() } ?: element.text().trim() + if (title.isBlank()) return@mapNotNull null + + val href = element.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = title, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = img.attr("abs:src"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + .distinctBy { it.url } + } + + private fun mapState(state: MangaState?): String = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> "" + } + + private fun mapType(type: ContentType?): String = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("h1")?.text()?.trim() ?: manga.title + val cover = doc.selectFirst("img[src*=\"/cover\"]")?.attr("abs:src") + ?: doc.selectFirst("aside img")?.attr("abs:src") + ?: manga.coverUrl + val description = run { + val encoded = doc.selectFirst("div[data-sr]")?.attr("data-sr")?.takeIf { it.isNotBlank() } + val decoded = encoded?.let { decodeBase64(it) }?.trim()?.takeIf { it.isNotBlank() } + decoded ?: doc.selectFirst("p.text-sm")?.text()?.trim()?.takeIf { it.isNotBlank() } + } + + val genres = doc.select("a[href*=\"genre=\"]") + .asSequence() + .map { it.text().trim() } + .filter { it.isNotBlank() } + .distinct() + .toSet() + .map { name -> + MangaTag( + title = name, + key = name.lowercase(Locale.ROOT).replace(" ", "-"), + source = source, + ) + } + .toSet() + + val statusText = doc.selectFirst("main")?.text() ?: doc.text() + val state = when { + statusText.contains("completed", ignoreCase = true) || + statusText.contains("tamat", ignoreCase = true) -> MangaState.FINISHED + statusText.contains("ongoing", ignoreCase = true) || + statusText.contains("berjalan", ignoreCase = true) -> MangaState.ONGOING + statusText.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED + else -> null + } + + val chapters = parseChapterList(doc) + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + tags = genres, + state = state, + chapters = chapters, + ) + } + + private fun parseChapterList(doc: Document): List<MangaChapter> { + return doc.select("a[href*=\"/read/\"][data-chapter]").mapNotNull { element -> + val href = element.attrAsRelativeUrl("href") + val name = element.selectFirst("span.text-sm, span[class*=\"font-semibold\"]")?.text()?.trim() + ?: element.text().trim() + val number = element.attr("data-chapter").trim().toFloatOrNull() + ?: name.extractChapterNumber() + val dateStr = element.selectFirst("span.tabular-nums, span[class*=\"tabular-nums\"]")?.text()?.trim() + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = parseRelativeDate(dateStr), + source = source, + scanlator = null, + branch = null, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("img[src*=\"file/comic\"], img[src*=\"imgsvr\"]") + .mapNotNull { img -> + val src = img.attr("abs:src") + if (src.isBlank() || src.contains("/api/image/p/")) { + null + } else { + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + } + + private fun decodeBase64(encoded: String): String? = try { + val padded = encoded.trim().let { s -> + val rem = s.length % 4 + if (rem != 0) s + "=".repeat(4 - rem) else s + } + String(Base64.getDecoder().decode(padded), Charsets.UTF_8).trim().takeIf { it.isNotBlank() } + } catch (_: Exception) { + null + } + + private fun parseRelativeDate(dateStr: String?): Long { + if (dateStr.isNullOrBlank()) return 0L + val trimmed = dateStr.lowercase(Locale.ROOT).trim() + if (trimmed == "baru saja") return System.currentTimeMillis() + + val number = Regex("\\d+").find(trimmed)?.value?.toLongOrNull() ?: return 0L + val nowInstant = Instant.now() + val nowZoned = ZonedDateTime.now(ZoneId.systemDefault()) + + return when { + "detik" in trimmed -> nowInstant.minus(number, ChronoUnit.SECONDS).toEpochMilli() + "menit" in trimmed -> nowInstant.minus(number, ChronoUnit.MINUTES).toEpochMilli() + "jam" in trimmed -> nowInstant.minus(number, ChronoUnit.HOURS).toEpochMilli() + "hari" in trimmed -> nowInstant.minus(number, ChronoUnit.DAYS).toEpochMilli() + "minggu" in trimmed -> nowInstant.minus(number * 7, ChronoUnit.DAYS).toEpochMilli() + "bulan" in trimmed -> nowZoned.minusMonths(number).toInstant().toEpochMilli() + "tahun" in trimmed -> nowZoned.minusYears(number).toInstant().toEpochMilli() + else -> 0L + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} From d3142871e575388c4363252eb967d5d11abca24f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 12:24:57 +0200 Subject: [PATCH 320/434] remove novels --- src/main/kotlin/tsuki/site/en/Atsumaru.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt index f3372a98..5d0f74be 100644 --- a/src/main/kotlin/tsuki/site/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -196,6 +196,7 @@ internal class Atsumaru(context: MangaLoaderContext) : val showAdult = config[showAdultKey] return (0 until items.length()).mapNotNull { i -> val obj = items.getJSONObject(i) + if (obj.optString("medium") == "Novel") return@mapNotNull null val isAdult = obj.optBoolean("isAdult", false) if (!showAdult && isAdult) return@mapNotNull null val id = obj.getString("id") @@ -235,7 +236,7 @@ internal class Atsumaru(context: MangaLoaderContext) : addQueryParameter("page", page.toString()) addQueryParameter("per_page", pageSize.toString()) - val filterParts = mutableListOf("hidden:!=true", "views:>0") + val filterParts = mutableListOf("hidden:!=true", "views:>0", "medium:!=[`Novel`]") val showAdult = config[showAdultKey] From 515541c1b2e0c38f1594abe5f9b5a16dfa429967 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:19:39 +0200 Subject: [PATCH 321/434] Update LumosKomik.kt --- src/main/kotlin/tsuki/site/id/LumosKomik.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/LumosKomik.kt b/src/main/kotlin/tsuki/site/id/LumosKomik.kt index c00373f4..65659456 100644 --- a/src/main/kotlin/tsuki/site/id/LumosKomik.kt +++ b/src/main/kotlin/tsuki/site/id/LumosKomik.kt @@ -226,7 +226,7 @@ internal class LumosKomik(context: MangaLoaderContext) : scanlator = null, branch = null, ) - } + }.reversed() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { From 13120eb04cb7724ec8a0abd0e584f6bab8c53524 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 15 Aug 2026 13:20:35 +0200 Subject: [PATCH 322/434] Create StellarSaber.kt --- src/main/kotlin/tsuki/site/ar/StellarSaber.kt | 364 ++++++++++++++++++ 1 file changed, 364 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/StellarSaber.kt diff --git a/src/main/kotlin/tsuki/site/ar/StellarSaber.kt b/src/main/kotlin/tsuki/site/ar/StellarSaber.kt new file mode 100644 index 00000000..07ded6fc --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/StellarSaber.kt @@ -0,0 +1,364 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.await +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl + +import okhttp3.FormBody +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.MultipartBody +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.asResponseBody +import okio.buffer +import okio.cipherSource +import org.jsoup.nodes.Document +import java.util.EnumSet +import java.util.Base64 as JavaBase64 +import java.util.Locale +import javax.crypto.Cipher +import javax.crypto.spec.GCMParameterSpec +import javax.crypto.spec.SecretKeySpec +import kotlin.time.Duration +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes + +@MangaSourceParser("STELLARSABER", "StellarSaber", "ar") +internal class StellarSaber(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.STELLARSABER, pageSize = 24), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("stellarsaber.pro") + private val baseUrl = "https://$domain" + + private var nonce: String? = null + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = false, + isMultipleTagsSupported = false, + ) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.isNotEmpty()) { + return searchManga(page, query) + } + val url = when (order) { + SortOrder.POPULARITY -> "$baseUrl/manga/${page.pageNumber()}?sort=rating" + else -> "$baseUrl/manga/${page.pageNumber()}?sort=latest" + } + val doc = webClient.httpGet(url).parseHtml() + fetchNonce(doc) + return parseMangaList(doc) + } + + private suspend fun searchManga(page: Int, query: String): List<Manga> { + val nonce = fetchNonce() + val body = MultipartBody.Builder().apply { + setType(MultipartBody.FORM) + addFormDataPart("action", "flavor_ajax_filter_content") + addFormDataPart("nonce", nonce) + addFormDataPart("page", page.toString()) + addFormDataPart("keyword", query) + }.build() + + val request = Request.Builder() + .url("$baseUrl/wp-admin/admin-ajax.php") + .post(body) + .headers(getRequestHeaders()) + .build() + + val response = context.httpClient.newCall(request).await() + val json = response.parseJson() + val results = json.optJSONObject("data")?.optJSONArray("results") ?: return emptyList() + + val list = mutableListOf<Manga>() + for (i in 0 until results.length()) { + val item = results.getJSONObject(i) + val type = item.optString("type") + if (type in listOf("novel", "anime")) continue + list.add( + Manga( + id = generateUid(item.getString("url")), + url = item.getString("url"), + publicUrl = item.getString("url").toAbsoluteUrl(domain), + title = item.getString("title"), + coverUrl = item.optString("cover").takeIf { it.isNotBlank() }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + } + return list + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select(".card-grid .card").mapNotNull { element -> + val a = element.select("a").firstOrNull() ?: return@mapNotNull null + val title = element.selectFirst(".card__title")?.text() ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + coverUrl = element.selectFirst("img")?.attr("abs:src"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun fetchNonce(): String { + nonce?.let { return it } + val doc = webClient.httpGet("$baseUrl/manga/").parseHtml() + return fetchNonce(doc) + } + + private fun fetchNonce(doc: Document): String { + nonce?.let { return it } + val script = doc.selectFirst("#flavor-ajax-js-extra")?.data() + ?: throw ParseException("Nonce script not found", "$baseUrl/manga/") + val nonceValue = script.extract("nonce\":\"", '"') + nonce = nonceValue + return nonceValue + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val updated = parseMangaDetails(doc, manga) + val chapters = parseChapterList(doc) + return updated.copy(chapters = chapters) + } + + private fun parseMangaDetails(doc: Document, original: Manga): Manga { + val element = doc.selectFirst(".detail-content") + ?: return original + + val title = element.selectFirst(".detail-info__title")?.text() ?: original.title + val cover = element.selectFirst(".detail-poster img")?.attr("abs:src") + val genres = element.select(".detail-genres a").map { it.text() }.toSet() + val description = element.select("#detail-desc p").joinToString { it.wholeText().trim() } + val otherName = element.select(".detail-info__alt-title").text() + val authors = mutableSetOf<String>() + val artists = mutableSetOf<String>() + var state: MangaState? = null + + element.select(".detail-meta__label").forEach { labelEl -> + val label = labelEl.text() + val value = labelEl.nextElementSibling()?.text() ?: return@forEach + when { + label.contains("المؤلف") -> authors.add(value) + label.contains("الرسّام") -> artists.add(value) + label.contains("الحالة") -> state = when { + "مستمر" in value -> MangaState.ONGOING + "مكتمل" in value -> MangaState.FINISHED + "ملغا" in value -> MangaState.ABANDONED + "متوقف" in value -> MangaState.PAUSED + else -> null + } + } + } + + return original.copy( + title = title, + coverUrl = cover ?: original.coverUrl, + description = buildString { + if (description.isNotBlank()) append(description) + if (otherName.isNotBlank()) { + if (isNotEmpty()) append("\n\n") + append(otherName) + } + }.trim().ifBlank { null }, + authors = authors + artists, + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT).replace(" ", "-"), source) }.toSet(), + state = state, + ) + } + + private fun parseChapterList(doc: Document): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + doc.select(".volume-group").forEach { volume -> + val volumeName = volume.selectFirst(".volume-group__label")?.text() + volume.select(".chapter-item").forEach { element -> + val a = element.selectFirst("a") ?: return@forEach + val href = a.attrAsRelativeUrl("href") + val title = element.selectFirst(".chapter-item__title")?.text() ?: return@forEach + val number = element.selectFirst(".chapter-item__number")?.text().orEmpty() + val name = buildString { + if (!volumeName.isNullOrEmpty()) append("$volumeName - ") + if (number.isNotEmpty() && !title.contains(number)) append("$number: ") + append(title) + } + chapters.add( + MangaChapter( + id = generateUid(href), + title = name, + number = number.toFloatOrNull() ?: -1f, + volume = 0, + url = href, + uploadDate = parseRelativeDate(element.selectFirst(".chapter-item__date")?.text()), + scanlator = element.selectFirst(".chapter-item__team")?.text(), + branch = null, + source = source, + ) + ) + } + } + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val key = fetchCdnKey(doc) + return doc.select("img.reader__page[data-cdn-url]") + .mapIndexed { index, img -> + val url = img.attr("abs:data-cdn-url") + MangaPage( + id = generateUid(url), + url = "$url#$key", + preview = null, + source = source, + ) + } + } + + private suspend fun fetchCdnKey(doc: Document): String { + val script = doc.selectFirst("script:containsData(flavorReaderData)")?.data() + ?: throw ParseException("Script not found", doc.location()) + val nonce = script.extract("cdnNonce: '", '\'') + val chapterId = script.extract("chapterId: ", ',') + + val formBody = FormBody.Builder() + .add("action", "flavor_cdn_get_key") + .add("nonce", nonce) + .add("chapter_id", chapterId) + .build() + + val request = Request.Builder() + .url("$baseUrl/wp-admin/admin-ajax.php") + .post(formBody) + .headers(getRequestHeaders()) + .build() + + val response = context.httpClient.newCall(request).await() + val json = response.parseJson() + return json.getJSONObject("data").getString("key") + } + + private fun parseRelativeDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val trimmed = date.trim() + val now = System.currentTimeMillis() + val number = Regex("""(\d+)""").find(trimmed)?.value?.toIntOrNull() + + val duration: Duration? = if (number == null) { + when { + trimmed.contains("دقيقتين") -> 2.minutes + trimmed.contains("دقيقة") -> 1.minutes + trimmed.contains("ساعتين") -> 2.hours + trimmed.contains("ساعة") -> 1.hours + trimmed.contains("يومين") -> 2.days + trimmed.contains("يوم") -> 1.days + trimmed.contains("أسبوعين") -> 14.days + trimmed.contains("أسبوع") -> 7.days + trimmed.contains("شهرين") -> 60.days + trimmed.contains("شهر") -> 30.days + trimmed.contains("سنتين") -> 730.days + trimmed.contains("سنة") -> 365.days + else -> null + } + } else { + when { + trimmed.contains("دقيقة") || trimmed.contains("دقائق") -> number.minutes + trimmed.contains("ساعة") || trimmed.contains("ساعات") -> number.hours + trimmed.contains("يوم") || trimmed.contains("أيام") -> number.days + trimmed.contains("أسبوع") || trimmed.contains("أسابيع") -> (number * 7).days + trimmed.contains("شهر") || trimmed.contains("أشهر") -> (number * 30).days + trimmed.contains("سنة") || trimmed.contains("سنوات") -> (number * 365).days + else -> null + } + } + return duration?.let { now - it.inWholeMilliseconds } ?: 0L + } + + private fun String.extract(startKey: String, endChar: Char): String { + val startIndex = indexOf(startKey) + require(startIndex != -1) { "$startKey not found" } + val valueStart = startIndex + startKey.length + val valueEnd = indexOf(endChar, valueStart) + require(valueEnd != -1) { "End char not found for $startKey" } + return substring(valueStart, valueEnd) + } + + private fun Int.pageNumber() = if (this > 1) "page/$this/" else "" + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val fragment = request.url.fragment ?: return response + + val keyBytes = JavaBase64.getDecoder().decode(fragment) + val source = response.body?.source() ?: return response + val iv = source.readByteArray(12) + + val cipher = Cipher.getInstance("AES/GCM/NoPadding").apply { + init( + Cipher.DECRYPT_MODE, + SecretKeySpec(keyBytes, "AES"), + GCMParameterSpec(128, iv), + ) + } + + val cipherSource = source.cipherSource(cipher) + return response.newBuilder() + .body(cipherSource.buffer().asResponseBody("image/jpeg".toMediaType(), -1)) + .build() + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} From 1efeefdf9cdded1fabe99c653b885bbe67a3f9a8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:15:49 +0200 Subject: [PATCH 323/434] Update AnimeSama.kt --- src/main/kotlin/tsuki/site/fr/AnimeSama.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt index b8cf751b..d90afe9e 100644 --- a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt +++ b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt @@ -17,6 +17,8 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import tsuki.util.generateUid import tsuki.util.parseHtml @@ -47,6 +49,15 @@ internal class AnimeSama(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("anime-sama.to") private val baseUrl = "https://$domain" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://cdn.jsdelivr.net/gh/Anime-Sama/IMG@img/autres/logo.png", 2600, null), + ), + domain, + ) + } + override val webClient by lazy { OkHttpWebClient(context.httpClient.newBuilder().build(), source) } From 0d9b8f74fbdd327c4857b98cd986302af8921e7f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 11:16:02 +0200 Subject: [PATCH 324/434] Update Westmanga.kt --- src/main/kotlin/tsuki/site/id/Westmanga.kt | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/id/Westmanga.kt b/src/main/kotlin/tsuki/site/id/Westmanga.kt index 3339c00e..26e0faa2 100644 --- a/src/main/kotlin/tsuki/site/id/Westmanga.kt +++ b/src/main/kotlin/tsuki/site/id/Westmanga.kt @@ -17,6 +17,8 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import tsuki.util.generateUid import tsuki.util.oneOrThrowIfMany @@ -40,11 +42,20 @@ import javax.crypto.spec.SecretKeySpec internal class WestmangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.WESTMANGA, pageSize = 20) { - override val configKeyDomain = ConfigKey.Domain("v1.westmanga.cc") + override val configKeyDomain = ConfigKey.Domain("westmanga.co") private val apiDomain: String get() = "data.mantweh.online" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/favicon.ico", 48, null), + ), + domain, + ) + } + private val accessKey = "WM_WEB_FRONT_END" private val secretKey = "xxxoidj" From 5455f7aeda869750748dde327b03d85f8bec5115 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:04:03 +0200 Subject: [PATCH 325/434] Update MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 88 ++++++++++++--------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 55014a8c..15ce62db 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -6,6 +6,7 @@ import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.network.OkHttpWebClient import tsuki.MangaParserAuthProvider +import tsuki.exception.ParseException import tsuki.model.ContentRating import tsuki.model.ContentType @@ -24,6 +25,7 @@ import tsuki.model.SortOrder import tsuki.util.generateUid import tsuki.util.parseJson +import tsuki.util.nullIfEmpty import okhttp3.Interceptor import org.jsoup.Jsoup @@ -37,6 +39,7 @@ import java.util.concurrent.ScheduledThreadPoolExecutor import java.util.concurrent.Semaphore import okio.ByteString import okio.ByteString.Companion.decodeBase64 +import okhttp3.HttpUrl.Builder internal class VrfSigner { fun interceptor() = Interceptor { chain -> @@ -266,12 +269,8 @@ internal abstract class MangaFireParser( availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> = try { - val urlBuilder = okhttp3.HttpUrl.Builder() + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val urlBuilder = Builder() .scheme("https") .host(domain) .addPathSegments("api/titles") @@ -349,43 +348,54 @@ internal abstract class MangaFireParser( } val url = urlBuilder.build().toString() - val response = apiClient.httpGet(url).parseJson() - val items = response.optJSONArray("items") ?: return emptyList() - val mangas = mutableListOf<Manga>() - for (i in 0 until items.length()) { - try { - val obj = items.getJSONObject(i) - val hid = obj.getString("hid") - val title = obj.getString("title") - val slug = obj.optString("slug", null) - val poster = obj.optJSONObject("poster") - val cover = poster?.optString("large") ?: poster?.optString("medium") - ?: poster?.optString("small") ?: "" - val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" - mangas.add( - Manga( - id = generateUid(urlPath), - url = urlPath, - publicUrl = "https://$domain$urlPath", - title = title, - coverUrl = cover, - source = source, - altTitles = emptySet(), - largeCoverUrl = null, - authors = emptySet(), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - tags = emptySet(), + try { + val response = apiClient.httpGet(url).parseJson() + val items = response.optJSONArray("items") + ?: throw ParseException("Missing 'items' array in API response", url) + + val mangas = mutableListOf<Manga>() + for (i in 0 until items.length()) { + try { + val obj = items.getJSONObject(i) + val hid = obj.getString("hid") + val title = obj.getString("title") + val slug = obj.optString("slug").nullIfEmpty() + val poster = obj.optJSONObject("poster") + val cover = poster?.optString("large") + ?: poster?.optString("medium") + ?: poster?.optString("small") + ?: "" + + val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" + + mangas.add( + Manga( + id = generateUid(urlPath), + url = urlPath, + publicUrl = "https://$domain$urlPath", + title = title, + coverUrl = cover, + source = source, + altTitles = emptySet(), + largeCoverUrl = null, + authors = emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + tags = emptySet(), + ) ) - ) - } catch (_: Exception) { + } catch (e: Exception) { + throw ParseException("Failed to parse manga item at index $i: ${e.message}", url, e) + } } + return mangas + } catch (e: ParseException) { + throw e + } catch (e: Exception) { + throw ParseException("Failed to load list page: ${e.message}\nOpen Webview and then reload Page", url, e) } - mangas - } catch (_: Exception) { - emptyList() } private val detailsCacheLock = Any() From 2f6097e4d186e4d4c174190b1d7d1f05e7050202 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 12:04:17 +0200 Subject: [PATCH 326/434] Fix domain in ScytheScans class --- src/main/kotlin/tsuki/site/en/ScytheScans.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/ScytheScans.kt b/src/main/kotlin/tsuki/site/en/ScytheScans.kt index 7b9f7cb7..42d14f42 100644 --- a/src/main/kotlin/tsuki/site/en/ScytheScans.kt +++ b/src/main/kotlin/tsuki/site/en/ScytheScans.kt @@ -16,7 +16,7 @@ import java.util.Base64 @MangaSourceParser("SYCTHESCANS", "Scythe Scans", "en") internal class ScytheScans(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.SYCTHESCANS, "erosscans.com", pageSize = 20) { + MangaThemesia(context, MangaParserSource.SYCTHESCANS, "scythescans.com", pageSize = 20) { override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val chapterUrl = chapter.url.toAbsoluteUrl(domain) From 46275afb0aede5ae0e27c683748ef85c96e68636 Mon Sep 17 00:00:00 2001 From: biglag <dmitriy.gudkov@dotcross.ai> Date: Mon, 17 Aug 2026 00:15:54 +0800 Subject: [PATCH 327/434] Add and repair Russian manga sources - add 25 missing Russian source adapters - expand the Russian catalog to 27 sources - repair six previously broken reading flows - add deterministic parser contracts and live diagnostics - update generated source metadata and documentation --- .github/summary.yaml | 2 +- .gitignore | 7 +- build.gradle.kts | 16 + .../site/kotatsu/all/NineMangaParser.kt | 274 +++++++++ .../parsers/site/kotatsu/ru/AComics.kt | 187 ++++++ .../parsers/site/kotatsu/ru/ComXParser.kt | 251 ++++++++ .../parsers/site/kotatsu/ru/DesuMeParser.kt | 283 +++++++++ .../parsers/site/kotatsu/ru/MangaWtfParser.kt | 263 ++++++++ .../parsers/site/kotatsu/ru/NudeMoonParser.kt | 198 ++++++ .../parsers/site/kotatsu/ru/RemangaParser.kt | 378 ++++++++++++ .../parsers/site/kotatsu/ru/WaMangaParser.kt | 182 ++++++ .../parsers/site/kotatsu/ru/ZenMangaParser.kt | 482 +++++++++++++++ .../site/kotatsu/ru/chan/ChanParser.kt | 271 ++++++++ .../site/kotatsu/ru/chan/HenChanParser.kt | 89 +++ .../site/kotatsu/ru/chan/MangaChanParser.kt | 11 + .../site/kotatsu/ru/chan/YaoiChanParser.kt | 51 ++ .../kotatsu/ru/grouple/AllHentaiParser.kt | 45 ++ .../site/kotatsu/ru/grouple/GroupleParser.kt | 582 ++++++++++++++++++ .../kotatsu/ru/grouple/MintMangaParser.kt | 28 + .../kotatsu/ru/grouple/ReadmangaParser.kt | 26 + .../site/kotatsu/ru/grouple/SeiMangaParser.kt | 26 + .../kotatsu/ru/grouple/SelfMangaParser.kt | 26 + .../site/kotatsu/ru/grouple/UsagiParser.kt | 23 + .../kotatsu/ru/libsocial/HentaiLibParser.kt | 25 + .../kotatsu/ru/libsocial/LibSocialParser.kt | 473 ++++++++++++++ .../kotatsu/ru/libsocial/MangaLibParser.kt | 26 + .../kotatsu/ru/libsocial/SlashLibParser.kt | 13 + .../site/kotatsu/ru/madara/BestManga.kt | 15 + .../site/kotatsu/ru/madara/MangaMammy.kt | 13 + .../site/kotatsu/ru/madara/MangaoneLove.kt | 13 + .../site/kotatsu/ru/madara/Mangazavr.kt | 16 + .../kotatsu/parsers/AComicsContractTest.kt | 31 + .../kotatsu/parsers/AllHentaiContractTest.kt | 39 ++ .../kotatsu/parsers/DesuContractTest.kt | 30 + .../parsers/FixtureMangaLoaderContext.kt | 136 ++++ .../parsers/LibSocialJsonContractTest.kt | 45 ++ .../kotatsu/parsers/LiveCatalogAuditTest.kt | 285 +++++++++ .../kotatsu/parsers/LiveMangaLoaderContext.kt | 126 ++++ .../kotatsu/parsers/LiveSourceSmokeTest.kt | 80 +++ .../kotatsu/parsers/MangaMammyContractTest.kt | 36 ++ .../kotatsu/parsers/MangaWtfContractTest.kt | 36 ++ .../kotatsu/parsers/NineMangaContractTest.kt | 31 + .../kotatsu/parsers/ReadMangaContractTest.kt | 53 ++ .../parsers/RussianSourceCatalogTest.kt | 160 +++++ .../kotatsu/parsers/SourceReadingContract.kt | 43 ++ .../kotatsu/parsers/WaMangaContractTest.kt | 53 ++ .../kotatsu/parsers/ZenMangaContractTest.kt | 39 ++ .../resources/fixtures/acomics/catalog.html | 7 + .../resources/fixtures/acomics/details.html | 6 + src/test/resources/fixtures/acomics/page.html | 4 + src/test/resources/fixtures/acomics/tags.html | 1 + .../resources/fixtures/allhentai/details.html | 7 + .../resources/fixtures/allhentai/pages.html | 4 + .../resources/fixtures/allhentai/search.html | 4 + src/test/resources/fixtures/desu/catalog.html | 12 + src/test/resources/fixtures/desu/details.html | 15 + src/test/resources/fixtures/desu/pages.html | 3 + src/test/resources/fixtures/desu/pages.json | 13 + .../fixtures/mangamammy/details.html | 20 + .../resources/fixtures/mangamammy/list.html | 16 + .../resources/fixtures/mangamammy/pages.html | 10 + .../resources/fixtures/mangawtf/list.json | 14 + .../resources/fixtures/ninemanga/catalog.html | 10 + .../fixtures/ninemanga/chapters.html | 5 + .../resources/fixtures/ninemanga/details.html | 10 + .../resources/fixtures/ninemanga/page.html | 1 + .../resources/fixtures/ninemanga/pages.html | 1 + .../resources/fixtures/ninemanga/tags.html | 1 + .../resources/fixtures/readmanga/details.html | 37 ++ .../resources/fixtures/readmanga/pages.html | 9 + .../resources/fixtures/readmanga/search.html | 21 + .../resources/fixtures/wamanga/catalog.json | 26 + .../resources/fixtures/wamanga/chapters.json | 15 + .../resources/fixtures/wamanga/details.json | 25 + .../resources/fixtures/wamanga/pages.json | 25 + .../resources/fixtures/zenmanga/catalog.json | 11 + .../resources/fixtures/zenmanga/details.html | 10 + .../resources/fixtures/zenmanga/pages.html | 7 + 78 files changed, 5866 insertions(+), 2 deletions(-) create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/NineMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/AComics.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ComXParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/DesuMeParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/MangaWtfParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/NudeMoonParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/RemangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/WaMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ZenMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/ChanParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/HenChanParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/MangaChanParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/YaoiChanParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/AllHentaiParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/GroupleParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/MintMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/ReadmangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SeiMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SelfMangaParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/UsagiParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/HentaiLibParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/LibSocialParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/MangaLibParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/SlashLibParser.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/BestManga.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaMammy.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaoneLove.kt create mode 100644 src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/Mangazavr.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/AComicsContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/AllHentaiContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/DesuContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/FixtureMangaLoaderContext.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/LibSocialJsonContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/LiveCatalogAuditTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/LiveMangaLoaderContext.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/LiveSourceSmokeTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/MangaMammyContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/MangaWtfContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/NineMangaContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/ReadMangaContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/RussianSourceCatalogTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/SourceReadingContract.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/WaMangaContractTest.kt create mode 100644 src/test/kotlin/org/koitharu/kotatsu/parsers/ZenMangaContractTest.kt create mode 100644 src/test/resources/fixtures/acomics/catalog.html create mode 100644 src/test/resources/fixtures/acomics/details.html create mode 100644 src/test/resources/fixtures/acomics/page.html create mode 100644 src/test/resources/fixtures/acomics/tags.html create mode 100644 src/test/resources/fixtures/allhentai/details.html create mode 100644 src/test/resources/fixtures/allhentai/pages.html create mode 100644 src/test/resources/fixtures/allhentai/search.html create mode 100644 src/test/resources/fixtures/desu/catalog.html create mode 100644 src/test/resources/fixtures/desu/details.html create mode 100644 src/test/resources/fixtures/desu/pages.html create mode 100644 src/test/resources/fixtures/desu/pages.json create mode 100644 src/test/resources/fixtures/mangamammy/details.html create mode 100644 src/test/resources/fixtures/mangamammy/list.html create mode 100644 src/test/resources/fixtures/mangamammy/pages.html create mode 100644 src/test/resources/fixtures/mangawtf/list.json create mode 100644 src/test/resources/fixtures/ninemanga/catalog.html create mode 100644 src/test/resources/fixtures/ninemanga/chapters.html create mode 100644 src/test/resources/fixtures/ninemanga/details.html create mode 100644 src/test/resources/fixtures/ninemanga/page.html create mode 100644 src/test/resources/fixtures/ninemanga/pages.html create mode 100644 src/test/resources/fixtures/ninemanga/tags.html create mode 100644 src/test/resources/fixtures/readmanga/details.html create mode 100644 src/test/resources/fixtures/readmanga/pages.html create mode 100644 src/test/resources/fixtures/readmanga/search.html create mode 100644 src/test/resources/fixtures/wamanga/catalog.json create mode 100644 src/test/resources/fixtures/wamanga/chapters.json create mode 100644 src/test/resources/fixtures/wamanga/details.json create mode 100644 src/test/resources/fixtures/wamanga/pages.json create mode 100644 src/test/resources/fixtures/zenmanga/catalog.json create mode 100644 src/test/resources/fixtures/zenmanga/details.html create mode 100644 src/test/resources/fixtures/zenmanga/pages.html diff --git a/.github/summary.yaml b/.github/summary.yaml index f0d18c2f..d0fe967c 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 76 \ No newline at end of file +total: 172 \ No newline at end of file diff --git a/.gitignore b/.gitignore index a75af06e..183715a2 100644 --- a/.gitignore +++ b/.gitignore @@ -100,4 +100,9 @@ src/test/resources/cookies.txt .vscode/ # Config file -/local.properties \ No newline at end of file +/local.properties + +# Local agent, specification, and worktree artifacts +/.codex/ +/openspec/ +/.worktrees/ diff --git a/build.gradle.kts b/build.gradle.kts index 81ed09e1..d56df61a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,6 +14,22 @@ version = "1.0.1" tasks.test { useJUnitPlatform() + listOf( + "LIVE_SOURCE", + "LIVE_STAGE", + "LIVE_SORT", + "LIVE_QUERY", + "LIVE_EXPECT_TITLE", + "LIVE_AUDIT", + "LIVE_AUDIT_STAGE", + "LIVE_AUDIT_LOCALES", + "LIVE_AUDIT_INCLUDE_BROKEN", + "LIVE_AUDIT_CONCURRENCY", + "LIVE_AUDIT_TIMEOUT_SECONDS", + "LIVE_AUDIT_FAIL_ON_ERROR", + ).forEach { variable -> + inputs.property(variable, providers.environmentVariable(variable).orElse("")) + } } ksp { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/NineMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/NineMangaParser.kt new file mode 100644 index 00000000..20d00e72 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/NineMangaParser.kt @@ -0,0 +1,274 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.all + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class NineMangaParser( + context: MangaLoaderContext, + source: MangaParserSource, + defaultDomain: String, +) : PagedMangaParser(context, source, pageSize = 26), Interceptor { + + override val configKeyDomain = ConfigKey.Domain(defaultDomain) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + init { + context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Accept-Language", "en-US;q=0.7,en;q=0.3") + .build() + + override val availableSortOrders: Set<SortOrder> = Collections.singleton( + SortOrder.POPULARITY, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchWithFiltersSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrCreateTagMap().values.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = if (request.url.host == domain) { + request.newBuilder().removeHeader("Referer").build() + } else { + request + } + return chain.proceed(newRequest) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + + if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.states.isNotEmpty() || !filter.query.isNullOrEmpty()) { + append("/search/") + append("?page=") + append(page.toString()) + + filter.query?.let { + append("&name_sel=contain&name=") + append(it.urlEncoded()) + } + + append("&category_id=") + append(filter.tags.joinToString(separator = ",") { it.key }) + + append("&out_category_id=") + append(filter.tagsExclude.joinToString(separator = ",") { it.key }) + + filter.states.oneOrThrowIfMany()?.let { + append("&completed_series=") + when (it) { + MangaState.ONGOING -> append("NO") + MangaState.FINISHED -> append("YES") + else -> Unit + } + } + + } else { + append("/category/index_") + append(page.toString()) + } + } + val doc = webClient.httpGet(url).parseHtml() + val root = doc.body().selectFirstOrThrow("div.manga-list") + val baseHost = root.baseUri().toHttpUrl().host + return root.select("div.manga-item").map { node -> + val a = node.selectFirstOrThrow("a:has(div.manga-name)") + val href = a.attrAsAbsoluteUrl("href") + val relUrl = href.toRelativeUrl(baseHost) + Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = a.selectFirstOrThrow("div.manga-name").text(), + altTitles = emptySet(), + coverUrl = node.selectFirst("div.manga-img img")?.src(), + rating = RATING_UNKNOWN, + authors = emptySet(), + contentRating = null, + tags = emptySet(), + state = null, + source = source, + description = node.selectFirst("div.manga-intro")?.html(), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val detailsUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(detailsUrl).parseHtml() + val root = doc.body().selectFirstOrThrow("div.book-info-parts") + val tagMap = getOrCreateTagMap() + val tags = root.select("[itemprop=genre]").mapNotNullToSet { tagMap[it.text()] } + val authors = root.select("[itemprop=author] [itemprop=name]").mapToSet { it.text() } + val chaptersUrl = detailsUrl.removeSuffix(".html") + "/chapters.html" + val chaptersRoot = webClient.httpGet(chaptersUrl).parseHtml().body() + return manga.copy( + title = doc.selectFirst("h1.book-headline-name[itemprop=name]")?.textOrNull() + ?: manga.title, + tags = tags, + authors = authors, + state = parseStatus(doc.selectFirst("span.book-status")?.text().orEmpty()), + description = root.select("section.detail-synopsis").last()?.html(), + chapters = chaptersRoot.select("ul.chapter-list > a:has(li.chp-item)") + .mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href").replace("%20", " ") + val li = a.selectFirstOrThrow("li.chp-item") + MangaChapter( + id = generateUid(href), + title = li.selectFirst("span.chp-title")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = parseChapterDateByLang(li.selectFirst("span.chp-time")?.text().orEmpty()), + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().select("div.chp-page-trigger[option_val]").map { option -> + val url = option.attr("option_val") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().selectFirstOrThrow("section.mangaread-img img.manga_pic") + .attrAsAbsoluteUrl("src") + } + + private var tagCache: ArrayMap<String, MangaTag>? = null + private val mutex = Mutex() + + private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap<String, MangaTag>() + val tagElements = webClient.httpGet("https://${domain}/search/?type=high").parseHtml().select("li.genre-item") + for (el in tagElements) { + if (el.text().isEmpty()) continue + val a = el.selectFirstOrThrow("a") + val cateId = a.attr("onclick").substringAfter("clickGenre(this, '").substringBefore("'") + tagMap[el.text()] = MangaTag( + title = a.text().toTitleCase(sourceLocale), + key = cateId, + source = source, + ) + } + tagCache = tagMap + return@withLock tagMap + } + + private fun parseStatus(status: String) = when { + // en + status.contains("Ongoing") -> MangaState.ONGOING + status.contains("Completed") -> MangaState.FINISHED + //es + status.contains("En curso") -> MangaState.ONGOING + status.contains("Completado") -> MangaState.FINISHED + //ru + status.contains("постоянный") -> MangaState.ONGOING + status.contains("завершенный") -> MangaState.FINISHED + //de + status.contains("Laufende") -> MangaState.ONGOING + status.contains("Abgeschlossen") -> MangaState.FINISHED + //pt + status.contains("Completo") -> MangaState.ONGOING + status.contains("Em tradução") -> MangaState.FINISHED + //it + status.contains("In corso") -> MangaState.ONGOING + status.contains("Completato") -> MangaState.FINISHED + //fr + status.contains("En cours") -> MangaState.ONGOING + status.contains("Complété") -> MangaState.FINISHED + else -> null + } + + private fun parseChapterDateByLang(date: String): Long { + val dateWords = date.split(" ") + + if (dateWords.size == 3) { + if (dateWords[1].contains(",")) { + return SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parseSafe(date) + } else { + val timeAgo = Integer.parseInt(dateWords[0]) + return Calendar.getInstance().apply { + when (dateWords[1]) { + "minutes" -> Calendar.MINUTE // EN-FR + "hours" -> Calendar.HOUR // EN + + "minutos" -> Calendar.MINUTE // ES + "horas" -> Calendar.HOUR + + // "minutos" -> Calendar.MINUTE // BR + "hora" -> Calendar.HOUR + + "минут" -> Calendar.MINUTE // RU + "часа" -> Calendar.HOUR + + "Stunden" -> Calendar.HOUR // DE + + "minuti" -> Calendar.MINUTE // IT + "ore" -> Calendar.HOUR + + "heures" -> Calendar.HOUR // FR ("minutes" also French word) + else -> null + }?.let { + add(it, -timeAgo) + } + }.timeInMillis + } + } + return 0L + } + + @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") + class Russian(context: MangaLoaderContext) : NineMangaParser( + context, + MangaParserSource.NINEMANGA_RU, + "ru.niadd.com", + ) + +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/AComics.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/AComics.kt new file mode 100644 index 00000000..c107ca0a --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/AComics.kt @@ -0,0 +1,187 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("ACOMICS", "AComics", "ru", ContentType.COMICS) +internal class AComics(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ACOMICS, pageSize = 10) { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) + + override val configKeyDomain = ConfigKey.Domain("acomics.ru") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + context.cookieJar.insertCookies(domain, "ageRestrict=18") + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrCreateTagMap().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + if (page > 0) { + return emptyList() + } + append("/search?keyword=") + append(filter.query) + } + + else -> { + append("/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=") + append(page * 10) + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "last_update" + SortOrder.ALPHABETICAL -> "serial_name" + SortOrder.POPULARITY -> "subscr_count" + else -> "last_update" + }, + ) + + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { + append("&categories[]=") + append(it.key) + } + } + + if (filter.states.isNotEmpty()) { + append("&updatable=") + append( + filter.states.oneOrThrowIfMany().let { + when (it) { + MangaState.ONGOING -> "yes" + MangaState.FINISHED -> "no" + else -> "0" + } + }, + ) + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List<Manga> { + return docs.select("section.serial-card").map { + val a = it.selectFirstOrThrow("h2.title a") + val publicUrl = a.attrAsAbsoluteUrl("href") + val url = "$publicUrl/about" + Manga( + id = generateUid(url), + url = url, + title = it.selectFirstOrThrow(".title").text(), + altTitles = emptySet(), + publicUrl = publicUrl, + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = it.selectFirstOrThrow("a.cover img").attrAsAbsoluteUrl("data-real-src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + description = it.selectFirst("p.about")?.html(), + ) + } + } + + private var tagCache: ArrayMap<String, MangaTag>? = null + private val mutex = Mutex() + + private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap<String, MangaTag>() + val tagElements = webClient.httpGet("https://$domain/comics").parseHtml() + .select("form.catalog-filters-form fieldset.categories label") + for (el in tagElements) { + val name = el.text().trim() + if (name.isEmpty()) continue + tagMap[name] = MangaTag( + title = name, + key = el.selectFirstOrThrow("input[name=categories[]]").attr("value"), + source = source, + ) + } + tagCache = tagMap + return@withLock tagMap + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val tagMap = getOrCreateTagMap() + val tags = doc.select("p.serial-about-badges .category").mapNotNullToSet { tagMap[it.text()] } + val author = doc.selectFirst("p.serial-about-authors a")?.textOrNull() + return manga.copy( + tags = tags, + description = doc.selectFirst("section.serial-about-text")?.html(), + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 1f, + volume = 0, + url = manga.url.replace("/about", "/"), + scanlator = null, + uploadDate = 0, + branch = null, + source = source, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url + "1").parseHtml() + val totalPages = doc.selectFirstOrThrow("nav.reader-navigator").attr("data-issue-count").toInt() + return (1..totalPages).map { + val url = chapter.url + it + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.selectFirstOrThrow("section.reader-issue img.issue").requireSrc() + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ComXParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ComXParser.kt new file mode 100644 index 00000000..3266b14b --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ComXParser.kt @@ -0,0 +1,251 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import org.json.JSONObject +import okhttp3.Interceptor +import okhttp3.Response +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getFloatOrDefault +import org.koitharu.kotatsu.parsers.util.json.getIntOrDefault +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.suspendlazy.getOrNull +import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("COMX", "Com-X", "ru", ContentType.COMICS) +internal class ComXParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.COMX, 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("com-x.life", "comx.life") + + private val availableTags = suspendLazy(initializer = ::fetchTags) + private val cdnImageUrl = "img.com-x.life/comix/" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + init { + context.cookieJar.insertCookies(domain, "adt-accepted", "1") + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + if (response.request.url.pathSegments.firstOrNull() != "_c") { + return response + } + response.close() + return context.requestBrowserAction(this, request.url.toString()) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags.get(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val urlBuilder = StringBuilder() + val query = filter.query + when { + !query.isNullOrEmpty() -> { + val encodedQuery = query.splitByWhitespace().joinToString(separator = "%20") { part -> + part.urlEncoded() + } + urlBuilder.append("/search/") + urlBuilder.append(encodedQuery) + if (page > 1) { + urlBuilder.append("/page/$page/") + } + } + + else -> { + urlBuilder.append("/ComicList") + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.append("/y[from]=${filter.yearFrom}") + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.append("/y[to]=${filter.yearTo}") + } + if (filter.tags.isNotEmpty()) { + urlBuilder.append("/g=") + urlBuilder.append(filter.tags.joinToString(",") { it.key }) + } + urlBuilder.append("/sort") + if (page > 1) { + urlBuilder.append("/page/$page/") + } + } + } + + val fullUrl = urlBuilder.toString().toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.readed.d-flex.short").map { item -> + val a = item.selectFirstOrThrow("a.readed__img.img-fit-cover.anim") + val img = item.selectFirst("img[data-src]") + val href = a.attrAsRelativeUrl("href") + val titleElement = item.selectFirstOrThrow("h3.readed__title a") + val (mainTitle, altTitle) = titleElement.text() + .split("\\s*/\\s*".toRegex()) + .map { it.trim() } + .let { parts -> + when { + parts.size >= 2 -> parts[1] to parts[0] + parts.isNotEmpty() -> parts[0] to "" + else -> "" to "" + } + } + + Manga( + id = generateUid(href), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + title = mainTitle, + altTitles = if (altTitle.isNotEmpty()) setOf(altTitle) else emptySet(), + authors = emptySet(), + description = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + coverUrl = img?.attrAsAbsoluteUrlOrNull("data-src"), + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + + val scriptData = doc.selectFirst("script:containsData(__DATA__)")?.data() + ?.substringAfter("window.__DATA__ = ") + ?.substringBefore(";</script>") + ?.trim() + ?: throw ParseException("Script data not found", manga.url) + + val jsonData = JSONObject(scriptData) + val chaptersJson = jsonData.getJSONArray("chapters") + val newsId = jsonData.getLong("news_id") + + val chapters = List(chaptersJson.length()) { i -> + val chapter = chaptersJson.getJSONObject(i) + val chapterId = chapter.getLong("id") + + MangaChapter( + id = generateUid("$newsId/$chapterId"), + url = "/reader/$newsId/$chapterId", + number = chapter.getFloatOrDefault("posi", 0f), + title = decodeText(chapter.getStringOrNull("title")), + uploadDate = dateFormat.parseSafe(chapter.getStringOrNull("date")), + source = source, + scanlator = null, + branch = null, + volume = chapter.getIntOrDefault("volume", 0), + ) + }.reversed() + + val author = doc.selectFirst("li:contains(Publisher:)") + ?.textOrNull() + ?.substringAfter("Publisher:") + ?.trim() + ?.nullIfEmpty() + val state = when ( + doc.selectFirst("li:contains(Release type:)")?.text()?.substringAfter("Release type:")?.trim() + ) { + "Ongoing" -> MangaState.ONGOING + else -> MangaState.FINISHED + } + + val tagLinks = doc.getElementsByAttributeValueContaining("href", "/genres/") + val tags = if (tagLinks.isNotEmpty()) { + availableTags.getOrNull()?.let { allTags -> + tagLinks.mapNotNullToSet { a -> + val tagName = a.text() + allTags.find { it.title.equals(tagName, ignoreCase = true) } + } + } + } else { + null + } + + return manga.copy( + authors = setOfNotNull(author), + state = state, + chapters = chapters, + description = doc.select("div.page__text.full-text.clearfix").textOrNull(), + tags = tags ?: manga.tags, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val newsId = chapter.url.substringAfter("/reader/").substringBefore("/") + context.cookieJar.insertCookies(domain, "adult=$newsId") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val data = doc.selectFirst("script:containsData(__DATA__)")?.data() + ?.substringAfter("=") + ?.trim() + ?.removeSuffix(";") + ?.substringAfter("\"images\":[") + ?.substringBefore("]") + ?.split(",") + ?.map { it.trim().removeSurrounding("\"").replace("\\", "") } + ?: throw ParseException("Image data not found", chapter.url) + + return data.map { imageUrl -> + val finalUrl = "https://$cdnImageUrl$imageUrl" + MangaPage( + id = generateUid(imageUrl), + url = finalUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/comix/").parseHtml() + val scriptData = doc.selectFirstOrThrow("script:containsData(__XFILTER__)").data() + + val genresJson = scriptData + .substringAfter("\"g\":{") + .substringBefore("}}}") + "}" + + val genresObj = JSONObject("{$genresJson}") + val valuesArray = genresObj.getJSONArray("values") + + return Set(valuesArray.length()) { i -> + val genre = valuesArray.getJSONObject(i) + MangaTag( + key = genre.getInt("id").toString(), + title = genre.getString("value").toTitleCase(sourceLocale), + source = source, + ) + } + } + + private fun decodeText(text: String?): String? { + if (text == null) return null + return text.replace("\\u([0-9a-fA-F]{4})".toRegex()) { matchResult -> + val codePoint = matchResult.groupValues[1].toInt(16) + codePoint.toChar().toString() + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/DesuMeParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/DesuMeParser.kt new file mode 100644 index 00000000..bfd6adf6 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/DesuMeParser.kt @@ -0,0 +1,283 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.model.ContentRating +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities +import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.util.LinkResolver +import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.parseHtml +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow +import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl +import org.koitharu.kotatsu.parsers.util.toTitleCase +import java.util.EnumSet + +@MangaSourceParser("DESUME", "Desu", "ru") +internal class DesuMeParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DESUME, 20) { + + override val configKeyDomain = ConfigKey.Domain("desu.uno") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalogDocument().parseDesuTags(), + ) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("User-Agent", UserAgents.KOTATSU) + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + if (query != null) { + if (page != searchPaginator.firstPage) return emptyList() + val document = webClient.httpPost( + "https://$domain/manga/search/", + mapOf("q" to query), + ).parseHtml() + return document.parseDesuSearchResults().map(::parseSearchManga) + } + + val url = buildCatalogUrl(page, order, filter) + return webClient.httpGet(url) + .parseHtml() + .parseDesuCatalogItems() + .map(::parseCatalogManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + .parseHtml() + .selectFirstOrThrow("#animeView") + val title = root.selectFirstOrThrow("meta[itemprop=headline]").attr("content") + val alternativeTitles = buildSet { + root.selectFirst("meta[itemprop=alternativeHeadline]") + ?.attr("content") + ?.takeIf { it.isNotBlank() } + ?.let(::add) + root.selectFirst(".alternativeHeadline") + ?.text() + ?.split(',') + ?.map(String::trim) + ?.filterTo(this) { it.isNotEmpty() } + } + val chapters = root.select("ul.chlist > li").map { item -> + val link = item.selectFirstOrThrow("h4 > a") + val chapterUrl = link.attrAsRelativeUrl("href") + val match = DESU_CHAPTER_PATH.find(chapterUrl) + ?: error("Unsupported Desu chapter URL: $chapterUrl") + MangaChapter( + id = generateUid(chapterUrl), + url = chapterUrl, + source = source, + number = match.groupValues[2].toFloat(), + volume = match.groupValues[1].toInt(), + title = link.selectFirst(".title")?.text()?.removePrefix("- ")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = 0L, + branch = null, + ) + } + + return manga.copy( + title = title, + altTitles = alternativeTitles, + publicUrl = root.selectFirstOrThrow("link[itemprop=url]").attr("href"), + largeCoverUrl = root.selectFirstOrThrow("img[itemprop=image]").attr("src"), + description = root.selectFirst("#description .russian")?.html(), + tags = root.select("a[itemprop=genre]").mapTo(linkedSetOf()) { link -> + MangaTag( + key = link.attr("href").substringAfter("genres="), + title = link.text().removePrefix("# ").toTitleCase(), + source = source, + ) + }, + state = root.selectFirst(".b-anime_status_tag")?.classNames()?.toDesuState(), + rating = root.selectFirst("meta[itemprop=ratingValue]") + ?.attr("content") + ?.toFloatOrNull() + ?.div(10f) + ?: RATING_UNKNOWN, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val document = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val scriptData = document.selectFirst("script:containsData(window.MangaReader)")?.data() + ?.substringAfter("window.MangaReader =") + ?.substringBeforeLast(';') + ?.trim() + ?.takeIf { it.isNotEmpty() } + ?: error("Cannot find Desu reader state") + val readerState = JSONObject(scriptData) + val chapterId = readerState.getJSONObject("chapter").getLong("id") + val apiBaseUrl = readerState.getString("apiBaseUrl") + val apiUrl = "$apiBaseUrl/chapters/$chapterId".toAbsoluteUrl(domain) + val images = webClient.httpGet(apiUrl).parseJson() + .getJSONObject("chapter") + .getJSONArray("pages") + + return (0 until images.length()).map { index -> + val imageUrl = images.getJSONObject(index).getString("url") + MangaPage( + id = generateUid(imageUrl), + preview = null, + source = chapter.source, + url = imageUrl, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val document = webClient.httpGet(link).parseHtml() + val root = document.selectFirst("#animeView") ?: return null + val title = root.selectFirst("meta[itemprop=headline]")?.attr("content") ?: return null + val url = root.selectFirst("link[itemprop=url]")?.attr("href") ?: return null + return resolver.resolveManga(this, id = generateUid(url), url = url, title = title) + } + + private suspend fun fetchCatalogDocument(): Document = webClient + .httpGet("https://$domain/manga/") + .parseHtml() + + private fun buildCatalogUrl(page: Int, order: SortOrder, filter: MangaListFilter): String = buildString { + append("https://") + append(domain) + append("/manga/?page=") + append(page) + val sortKey = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "popular" + SortOrder.NEWEST -> "id" + SortOrder.ALPHABETICAL -> "name" + else -> error("Unsupported Desu sort order: $order") + } + if (sortKey != "updated") { + append("&order_by=") + append(sortKey) + } + if (filter.tags.isNotEmpty()) { + append("&genres=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + private fun parseCatalogManga(item: Element): Manga { + val link = item.selectFirstOrThrow("a.animeTitle") + val url = link.attrAsRelativeUrl("href") + val cover = item.selectFirstOrThrow("a.avatar .img") + .attr("style") + .let { style -> DESU_BACKGROUND_IMAGE.find(style)?.groupValues?.get(1) } + ?: error("Cannot find Desu catalog cover") + val russianTitle = item.selectFirstOrThrow(".dimmed.oTitle [itemprop=title]").text() + val originalTitle = link.text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = cover, + largeCoverUrl = null, + state = null, + rating = item.parseDesuCatalogRating(), + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } + + private fun parseSearchManga(item: Element): Manga { + val url = item.attrAsRelativeUrl("href") + val russianTitle = item.selectFirstOrThrow(".itemSubTitle").text() + val originalTitle = item.selectFirstOrThrow(".itemTitle").text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = item.selectFirstOrThrow("img").attr("src"), + largeCoverUrl = null, + state = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } +} + +internal fun Document.parseDesuCatalogItems(): List<Element> = + select("ol.memberList > li.memberListItem") + +internal fun Document.parseDesuSearchResults(): List<Element> = + select("ol.section > li > a[href^=manga/]") + +internal fun Document.parseDesuTags(): Set<MangaTag> = + select("#animeFilter input[data-genre-id][data-genre-slug][data-genre-name]").mapTo(linkedSetOf()) { input -> + MangaTag( + key = "${input.attr("data-genre-id")}-${input.attr("data-genre-slug")}", + title = input.attr("data-genre-name").toTitleCase(), + source = MangaParserSource.DESUME, + ) + } + +private fun Element.parseDesuCatalogRating(): Float { + val rating = select(".animeInfo dl").asSequence() + .flatMap { it.children().asSequence() } + .windowed(2) + .firstOrNull { it[0].tagName() == "dt" && it[0].text() == "Рейтинг:" && it[1].tagName() == "dd" } + ?.get(1) + ?.text() + ?.toFloatOrNull() + return rating?.div(10f) ?: RATING_UNKNOWN +} + +private fun Set<String>.toDesuState(): MangaState? = when { + "ongoing" in this -> MangaState.ONGOING + "released" in this -> MangaState.FINISHED + "anons" in this -> MangaState.UPCOMING + isEmpty() -> null + else -> error("Unsupported Desu status classes: $this") +} + +private val DESU_BACKGROUND_IMAGE = Regex("url\\(['\"]?([^'\")]+)") +private val DESU_CHAPTER_PATH = Regex("/vol(\\d+)/ch([0-9.]+)/") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/MangaWtfParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/MangaWtfParser.kt new file mode 100644 index 00000000..67be7023 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/MangaWtfParser.kt @@ -0,0 +1,263 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import androidx.collection.ArrayMap +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.InternalParsersApi +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("MANGA_WTF", "MangaWtf", "ru") +internal class MangaWtfParser( + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaParserSource.MANGA_WTF, pageSize = 20) { + + override val availableSortOrders: Set<SortOrder> = + EnumSet.of( + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) + + @InternalParsersApi + override val configKeyDomain = ConfigKey.Domain("inkstory.net") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.UPCOMING, + MangaState.PAUSED, + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentRating = EnumSet.allOf(ContentRating::class.java), + ) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("type", "COMIC") + when { + filter.query.isNullOrEmpty() -> { + url.addQueryParameter( + "sort", + when (order) { + SortOrder.UPDATED -> "updatedAt,desc" + SortOrder.POPULARITY -> "viewsCount,desc" + SortOrder.RATING -> "likesCount,desc" + SortOrder.NEWEST -> "createdAt,desc" + else -> throw IllegalArgumentException("Unsupported $order") + }, + ) + if (filter.tags.isNotEmpty()) { + url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) + } + if (filter.tagsExclude.isNotEmpty()) { + url.addQueryParameter("labelsExclude", filter.tagsExclude.joinToString(",") { it.key }) + } + if (filter.states.isNotEmpty()) { + url.addQueryParameter( + "status", + filter.states.joinToString(",") { + when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "DONE" + MangaState.ABANDONED -> "" + MangaState.PAUSED -> "FROZEN" + MangaState.UPCOMING -> "ANNOUNCE" + else -> throw IllegalArgumentException("$it not supported") + } + }, + ) + } + if (filter.contentRating.isNotEmpty()) { + url.addQueryParameter( + "contentStatus", + filter.contentRating.joinToString(",") { + when (it) { + ContentRating.SAFE -> "SAFE" + ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" + ContentRating.ADULT -> "PORNOGRAPHIC" + } + }, + ) + } + } + + else -> { + url.addQueryParameter("search", filter.query) + } + } + val ja = webClient.httpGet(url.build()).parseJsonArray() + return ja.mapJSON { jo -> jo.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga = + coroutineScope { + val chaptersDeferred = async { getChapters(manga.url) } + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addPathSegment(manga.url) + val jo = webClient.httpGet(url.build()).parseJson() + val isNsfwSource = jo.getStringOrNull("contentStatus").isNsfw() + Manga( + id = generateUid(jo.getString("id")), + title = jo.getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(jo.getJSONObject("name").getStringOrNull("en")), + url = jo.getString("id"), + publicUrl = "https://$domain/manga/${jo.getString("slug")}", + rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = jo.getString("poster"), + tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, + state = jo.getStringOrNull("status")?.toMangaState(), + authors = jo.getJSONArray("relations").asTypedList<JSONObject>().mapNotNullToSet { + if (it.getStringOrNull("type") == "AUTHOR") { + it.getJSONObject("publisher").getStringOrNull("name") + } else { + null + } + }, + source = source, + largeCoverUrl = null, + description = jo.getString("description").nl2br(), + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addPathSegment(chapter.url) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("pages").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getString("id")), + url = jo.getString("image"), + preview = null, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val url = urlBuilder("api").addPathSegment("v2").addPathSegment("labels") + val json = webClient.httpGet(url.build()).parseJsonArray() + return json.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getChapters(mangaId: String): List<MangaChapter> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addQueryParameter("bookId", mangaId) + val ja = webClient.httpGet(url.build()).parseJsonArray() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) + val branches = ArrayMap<String, String>() + return ja + .mapJSON { jo -> + val number = jo.getFloatOrDefault("number", 0f) + val volume = jo.getIntOrDefault("volume", 0) + val branchId = jo.getString("branchId") + MangaChapter( + id = generateUid(jo.getString("id")), + title = jo.getStringOrNull("name"), + number = number, + volume = volume, + url = jo.getString("id"), + scanlator = null, + uploadDate = dateFormat.parseSafe(jo.getString("createdAt")), + branch = branches.getOrPut(branchId) { getBranchName(branchId) }, + source = source, + ) + }.reversed() + } + + private suspend fun getBranchName(id: String): String? { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("branches") + .addPathSegment(id) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() + } + + private fun String.toMangaState() = + when (this.uppercase(Locale.ROOT)) { + "DONE" -> MangaState.FINISHED + "ONGOING" -> MangaState.ONGOING + "FROZEN" -> MangaState.PAUSED + "ANNOUNCE" -> MangaState.UPCOMING + else -> null + } + + private fun String?.isNsfw() = + this.equals("EROTIC", ignoreCase = true) || + this.equals("PORNOGRAPHIC", ignoreCase = true) + + private fun JSONObject.toMangaTag() = + MangaTag( + title = getString("name").toTitleCase(sourceLocale), + key = getString("slug"), + source = source, + ) + + private fun JSONObject.toManga(): Manga { + val isNsfwSource = getStringOrNull("contentStatus").isNsfw() + return Manga( + id = generateUid(getString("id")), + title = getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(getJSONObject("name").getStringOrNull("en")), + url = getString("id"), + publicUrl = "https://$domain/manga/${getString("slug")}", + rating = getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = getString("poster"), + tags = setOf(), + state = getStringOrNull("status")?.toMangaState(), + authors = emptySet(), + source = source, + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/NudeMoonParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/NudeMoonParser.kt new file mode 100644 index 00000000..eea2f0fb --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/NudeMoonParser.kt @@ -0,0 +1,198 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.AbstractMangaParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI) +internal class NudeMoonParser( + context: MangaLoaderContext, +) : AbstractMangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain( + "g.nude-moon.fun", + "b.nude-moon.fun", + "nude-moon.org", + "nude-moon.net", + ) + override val authUrl: String + get() = "https://${domain}/index.php" + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name == "fusion_user" + } + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + init { + context.cookieJar.insertCookies( + domain, + "NMfYa=1;", + "nm_mobile=1;", + ) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val domain = domain + val query = filter.query + + val url = when { + !query.isNullOrEmpty() -> { + if (!isAuthorized()) { + throw AuthRequiredException(source) + } + "https://$domain/search?stext=${query.urlEncoded()}&rowstart=$offset" + } + + else -> { + if (filter.tags.isNotEmpty()) { + filter.tags.joinToString( + separator = "_", + prefix = "https://$domain/tags/", + postfix = "&rowstart=$offset", + transform = { it.key.urlEncoded() }, + ) + } else { + val order = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "date" + SortOrder.RATING -> "like" + else -> "like" + } + "https://$domain/all_manga?$order&rowstart=$offset" + } + } + + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.body().select("table.news_pic2").mapNotNull { row -> + val a = row.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = a.attr("title") + val author = row.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.textOrNull() + Manga( + id = generateUid(href), + url = href, + title = title.substringAfter(" / "), + altTitles = setOfNotNull(title.substringBefore(" / ", "").takeUnless { it.isBlank() }), + authors = setOfNotNull(author), + coverUrl = row.selectFirst("img")?.absUrl("src").orEmpty(), + tags = row.selectFirst(".tag-links")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + source = source, + publicUrl = a.absUrl("href"), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + description = row.selectFirst("div.description")?.html(), + state = null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() + val root = body.selectFirstOrThrow("table.news_pic2") + val dateFormat = SimpleDateFormat("dd MMMM yyyy", sourceLocale) + val author = root.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.text() + return manga.copy( + largeCoverUrl = body.selectFirstOrThrow("img[data-src]").attrAsAbsoluteUrl("data-src"), + description = root.selectFirst(".description")?.html() ?: manga.description, + tags = root.getElementsByAttributeValueContaining("href", "/tag/").mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + } + manga.tags, + authors = author?.let { setOf(it) } ?: manga.authors, + chapters = listOf( + MangaChapter( + id = manga.id, + url = manga.url, + source = source, + number = 0f, + volume = 0, + title = manga.title, + scanlator = root.getElementsByAttributeValueContaining("href", "/perevod/").firstOrNull() + ?.textOrNull(), + uploadDate = dateFormat.parseSafe( + root.getElementsByAttributeValueEnding("src", "ico/time.png").firstOrNull() + ?.nextElementSibling()?.text(), + ), + branch = null, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val pages = doc.select("img[data-src]") + return pages.map { img -> + val url = img.attrAsRelativeUrl("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + }.toList() + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val domain = domain + val doc = webClient.httpGet("https://$domain/tags").parseHtml() + val root = doc.body().getElementsByAttributeValue("name", "multitags").first() + ?: doc.parseFailed("Tags form not found") + return root.select("input").mapToSet { + val value = it.attr("value").trim() + MangaTag( + title = value.toTitleCase(sourceLocale), + key = value.replace(' ', '_'), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://${domain}/").parseHtml().body() + return body.getElementsContainingOwnText("Профиль").firstOrNull()?.attr("href")?.substringAfterLast('/') + ?: run { + throw if (body.selectFirst("form[name=\"loginform\"]") != null) { + AuthRequiredException(source) + } else { + body.parseFailed("Cannot find username") + } + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/RemangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/RemangaParser.kt new file mode 100644 index 00000000..2182d1f0 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/RemangaParser.kt @@ -0,0 +1,378 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.exception.ContentUnavailableException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.* + +private const val PAGE_SIZE = 20 +private const val TOO_MANY_REQUESTS = 429 + +@MangaSourceParser("REMANGA", "Реманга", "ru") +internal class RemangaParser( + context: MangaLoaderContext, +) : PagedMangaParser(context, MangaParserSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { + + private val baseHeaders: Headers + get() = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .build() + + override fun getRequestHeaders() = getApiHeaders() + + override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг") + + override val authUrl: String + get() = "https://${domain}" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } + + private val regexLastUrlPath = Regex("/[^/]+/?$") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isYearRangeSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.allOf(MangaState::class.java), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.code == TOO_MANY_REQUESTS) { + response.closeQuietly() + Thread.sleep(1000) + return chain.proceed(chain.request().newBuilder().build()) + } + return response + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + copyCookies() + val domain = domain + val urlBuilder = urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("search") + .addQueryParameter("page", page.toString()) + .addQueryParameter("count", PAGE_SIZE.toString()) + .addQueryParameter("ordering", getSortKey(order)) + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("query", filter.query) + } else { + urlBuilder.addPathSegment("catalog") + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("genres", tag.key) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("exclude_genres", tag.key) + } + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_gte", filter.yearFrom.toString()) + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_lte", filter.yearFrom.toString()) + } + for (age in filter.contentRating) { + when (age) { + ContentRating.SAFE -> urlBuilder.addQueryParameter("age_limit", "0") + ContentRating.SUGGESTIVE -> { + urlBuilder.addQueryParameter("age_limit", "1") + urlBuilder.addQueryParameter("age_limit", "2") + } + + else -> Unit + } + } + for (state in filter.states) { + urlBuilder.addQueryParameter( + "status", + when (state) { + MangaState.ONGOING -> "2" + MangaState.FINISHED -> "1" + MangaState.ABANDONED -> "4" + MangaState.PAUSED -> "3" + MangaState.UPCOMING -> "5" + MangaState.RESTRICTED -> "6" + }, + ) + } + val content = webClient.httpGet(urlBuilder.build()).parseJson() + .getJSONArray("results") + return content.mapJSON { jo -> + parseManga(jo) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + return getDetails(slug, manga.publicUrl) + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val slug = link.pathSegments.getOrNull(1) ?: return super.resolveLink(resolver, link) + return getDetails(slug, link.toString()) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + copyCookies() + val slug = seed.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${seed.url}", seed.publicUrl) + val json = webClient.httpGet( + // https://api.remanga.org/api/v2/titles/the_beginning_after_the_end/relations/ + urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("titles") + .addPathSegment(slug) + .addPathSegment("relations") + .build(), + ).parseJson() + .optJSONArray("titles") ?: return emptyList() + return json.mapJSON { jo -> parseManga(jo) } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain(this, "api"))) + .parseJson() + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).parseSafe(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList<MangaPage>(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item) + is JSONArray -> item.mapJSONTo(result) { parsePage(it) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).parseJson() + return jo.getJSONObject("content").getString("username") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private suspend fun getDetails(slug: String, publicUrl: String): Manga { + copyCookies() + val jo = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/$slug/", + ).parseJson() + jo.optJSONObject("detail")?.getStringOrNull("message")?.let { msg -> + throw ParseException(msg, publicUrl) + } + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + val branches = jo.getJSONArray("branches").mapJSONToSet { + it.getLong("id") to it.optJSONObject("publishers")?.getStringOrNull("name") + } + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = jo.getStringOrNull("another_name")?.split(" / ") + ?.toSet().orEmpty() + setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = when (jo.optJSONObject("age_limit")?.getIntOrDefault("id", -1)) { + 0 -> ContentRating.SAFE + 1, 2 -> ContentRating.SUGGESTIVE + else -> null + }, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + description = jo.getStringOrNull("description"), + chapters = branches.flatMap { (branchId, branchName) -> + grabChapters(branchId).mapChapters(reversed = true) { _, cjo -> + if ( + cjo.getBooleanOrDefault("is_paid", false) && + !cjo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = cjo.getLong("id") + val name = cjo.getStringOrNull("name")?.toTitleCase(Locale.ROOT) + val publishers = cjo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/v2/titles/chapters/$id/", + number = cjo.getFloatOrDefault("chapter", 0f), + volume = cjo.getIntOrDefault("tome", 0), + title = name, + uploadDate = dateFormat.parseSafe(cjo.getStringOrNull("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = source, + branch = branchName, + ) + } + }, + source = source, + ) + } + + private fun parseManga(jo: JSONObject): Manga { + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = null, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + source = source, + ) + } + + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(userCookie.value.urlDecode()) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } + + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } + + private fun parsePage(jo: JSONObject) = MangaPage( + // id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone + id = generateUid(jo.getString("link")), + url = jo.getString("link"), + preview = null, + source = source, + ) + + private suspend fun grabChapters(branchId: Long): List<JSONObject> { + val result = ArrayList<JSONObject>(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/chapters/?branch_id=$branchId&page=$page&count=500", + ).parseJson().getJSONArray("results") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/WaMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/WaMangaParser.kt new file mode 100644 index 00000000..ba394294 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/WaMangaParser.kt @@ -0,0 +1,182 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.SinglePageMangaParser +import org.koitharu.kotatsu.parsers.model.ContentRating +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities +import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaState +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.json.asTypedList +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.generateUid +import org.koitharu.kotatsu.parsers.util.parseJson +import org.koitharu.kotatsu.parsers.util.parseJsonArray +import org.koitharu.kotatsu.parsers.util.parseSafe +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("WAMANGA", "WaManga", "ru", type = ContentType.MANGA) +internal class WaMangaParser( + context: MangaLoaderContext, +) : SinglePageMangaParser(context, MangaParserSource.WAMANGA) { + + override val configKeyDomain = ConfigKey.Domain("wamanga.ru") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalog() + .flatMapTo(linkedSetOf()) { it.parseWaMangaTags() }, + ) + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + val requiredGenres = filter.tags.mapTo(mutableSetOf()) { it.key } + + return fetchCatalog() + .asSequence() + .filter { item -> + query == null || item.parseWaMangaSearchText().any { value -> + value.contains(query, ignoreCase = true) + } + } + .filter { item -> + requiredGenres.isEmpty() || item.parseWaMangaGenres().containsAll(requiredGenres) + } + .map(::parseManga) + .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.title }) + .toList() + } + + override suspend fun getDetails(manga: Manga): Manga { + val details = webClient.httpGet(apiUrl("manga/${manga.url}")).parseJson() + val chapters = webClient.httpGet(apiUrl("manga/${manga.url}/chapters")) + .parseJsonArray() + .mapJSON(::parseChapter) + .sortedByDescending { it.number } + + return parseManga(details).copy( + id = manga.id, + url = manga.url, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + return webClient.httpGet(apiUrl("chapters/${chapter.url}")) + .parseJson() + .getJSONArray("files") + .asTypedList<JSONObject>() + .sortedBy { it.getString("position").toDouble() } + .map { file -> + val imageUrl = file.getString("diskFile").toWaMangaAbsoluteUrl(domain) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchCatalog(): List<JSONObject> = webClient + .httpGet(apiUrl("manga")) + .parseJsonArray() + .asTypedList() + + private fun parseManga(item: JSONObject): Manga { + val id = item.getString("id") + val slug = item.getString("slug") + return Manga( + id = generateUid(id), + url = id, + title = item.getString("title"), + altTitles = item.parseWaMangaAlternativeTitles(), + publicUrl = "https://$domain/manga/$slug", + rating = RATING_UNKNOWN, + coverUrl = item.getString("coverUrl").toWaMangaAbsoluteUrl(domain), + largeCoverUrl = item.getString("imageUrl").toWaMangaAbsoluteUrl(domain), + tags = item.parseWaMangaTags(), + state = item.getString("statusTitle").toWaMangaState(), + authors = item.getJSONArray("authors").asTypedList<String>().toSet(), + source = source, + contentRating = if (item.getBoolean("isAdult")) ContentRating.ADULT else ContentRating.SAFE, + description = item.getStringOrNull("description"), + ) + } + + private fun parseChapter(item: JSONObject): MangaChapter { + val id = item.getString("id") + return MangaChapter( + id = generateUid(id), + url = id, + source = source, + number = item.getDouble("position").toFloat(), + volume = 0, + title = item.getStringOrNull("title")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = WA_MANGA_DATE_FORMAT.get().parseSafe(item.getString("createdAt")), + branch = null, + ) + } + + private fun apiUrl(path: String): String = "https://$domain/api/v1/$path" +} + +internal fun JSONObject.parseWaMangaAlternativeTitles(): Set<String> = buildSet { + getStringOrNull("titleEnglish")?.takeIf { it.isNotBlank() }?.let(::add) + getJSONArray("alternateTitles").asTypedList<String>().filterTo(this) { it.isNotBlank() } +} + +internal fun JSONObject.parseWaMangaGenres(): Set<String> = + getJSONArray("genres").asTypedList<String>().toSet() + +internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { + add(getString("title")) + addAll(parseWaMangaAlternativeTitles()) +} + +private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> + MangaTag( + title = genre, + key = genre, + source = MangaParserSource.WAMANGA, + ) +} + +private fun String.toWaMangaState(): MangaState = when (this) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "abandoned" -> MangaState.ABANDONED + else -> error("Unsupported WaManga status: $this") +} + +private fun String.toWaMangaAbsoluteUrl(domain: String): String = when { + startsWith("https://") || startsWith("http://") -> this + startsWith('/') -> "https://$domain$this" + else -> error("Unsupported WaManga asset URL: $this") +} + +private val WA_MANGA_DATE_FORMAT = ThreadLocal.withInitial { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ZenMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ZenMangaParser.kt new file mode 100644 index 00000000..ad6daa09 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/ZenMangaParser.kt @@ -0,0 +1,482 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.json.JSONArray +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.CloudFlareHelper +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import java.text.SimpleDateFormat +import java.util.* + +@MangaSourceParser("ZENMANGA", "ZenManga", "ru") +internal class ZenMangaParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ZENMANGA, 30), + MangaParserAuthProvider { + + private val astroJsonParser = AstroJsonParser() + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + + init { + setFirstPage(0) + } + + override val configKeyDomain = ConfigKey.Domain("inkstory.net") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC + ) + + override val filterCapabilities: MangaListFilterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isYearRangeSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = true, + ) + + override val authUrl: String + get() = "https://sso.inuko.me/account/sign-in" + + private val apiDomain = if (domain.startsWith("v1.")) domain.replace("v1.", "api.") else "api.$domain" + + private fun checkAuth(): Boolean { + val authCookieName = "__otaku_session" + return context.cookieJar.getCookies(domain).any { it.name == authCookieName } + } + + override suspend fun isAuthorized(): Boolean { + return checkAuth() + } + + override suspend fun getUsername(): String { + val libraryUrl = "/library" + val data = fetchAstroData(libraryUrl) + ?: throw ParseException("Cannot obtain Astro state for the current user", libraryUrl) + + val session = data["session"] as? Map<*, *> + ?: throw ParseException("Cannot find the 'session' key", libraryUrl) + + val currentUser = session["currentUser"] as? Map<*, *> + ?: throw ParseException("Cannot find the 'currentUser' key", libraryUrl) + + return currentUser["username"] as? String + ?: throw ParseException("Cannot find the 'username' key", libraryUrl) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val author = filter.author + if (!author.isNullOrBlank()) { + return getListPageByAuthor(author, page) + } + + val urlBuilder = HttpUrl.Builder() + .scheme("https") + .host(apiDomain) + .addPathSegment("v2") + .addPathSegment("books") + + urlBuilder.addQueryParameter("page", page.toString()) + urlBuilder.addQueryParameter("size", pageSize.toString()) + + urlBuilder.addQueryParameter("sort", getSortParameter(order)) + + if (!filter.query.isNullOrBlank()) { + urlBuilder.addQueryParameter("search", filter.query) + } + + filter.tags.forEach { tag -> + urlBuilder.addQueryParameter("labelsInclude", tag.key) + } + + filter.tagsExclude.forEach { tag -> + urlBuilder.addQueryParameter("labelsExclude", tag.key) + } + + filter.states.forEach { state -> + urlBuilder.addQueryParameter("status", when(state) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "DONE" + MangaState.PAUSED -> "FROZEN" + MangaState.UPCOMING -> "ANNOUNCE" + else -> "" + }) + } + + filter.contentRating.forEach { rating -> + urlBuilder.addQueryParameter("contentStatus", when(rating) { + ContentRating.SAFE -> "SAFE" + ContentRating.SUGGESTIVE -> "UNSAFE" + ContentRating.ADULT -> "EROTIC" + }) + } + + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("yearMin", filter.yearFrom.toString()) + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("yearMax", filter.yearTo.toString()) + } + + val requestUrl = urlBuilder.build() + val response = webClient.httpGet(requestUrl).parseJsonArray() + + return response.mapJSON { parseMangaFromJson(it) } + } + + private suspend fun getListPageByAuthor(authorQuery: String, page: Int): List<Manga> { + val authorSearchUrl = HttpUrl.Builder() + .scheme("https") + .host(apiDomain) + .addPathSegment("v2") + .addPathSegment("publishers") + .addQueryParameter("search", authorQuery) + .build() + + val publishersResponse = webClient.httpGet(authorSearchUrl).parseJsonArray() + + var authorId: String? = null + for (i in 0 until publishersResponse.length()) { + val publisher = publishersResponse.getJSONObject(i) + if (publisher.getStringOrNull("kind") == "AUTHOR") { + authorId = publisher.getStringOrNull("id") + break + } + } + + if (authorId == null) { + return emptyList() + } + + val booksByAuthorUrl = HttpUrl.Builder() + .scheme("https") + .host(apiDomain) + .addPathSegment("v2") + .addPathSegment("books") + .addQueryParameter("publisherId", authorId) + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("sort", "createdAt,desc") + .build() + + val booksResponse = webClient.httpGet(booksByAuthorUrl).parseJsonArray() + + return booksResponse.mapJSON { parseMangaFromJson(it) } + } + + private fun parseMangaFromJson(json: JSONObject): Manga { + val slug = json.getString("slug") + val nameObj = json.getJSONObject("name") + val title = nameObj.getStringOrNull("ru") ?: nameObj.getString("en") + + val publicUrl = "https://$domain/content/$slug" + + val altNames = json.getJSONArray("altNames") + .mapJSON { it.getString("name") } + .toSet() + + return Manga( + id = generateUid(publicUrl), + url = "/content/$slug", + publicUrl = "https://$domain/content/$slug", + title = title, + altTitles = altNames, + coverUrl = json.getStringOrNull("poster"), + source = source, + rating = json.getDouble("averageRating").toFloat() / 10f, + state = when (json.getStringOrNull("status")) { + "ONGOING" -> MangaState.ONGOING + "DONE" -> MangaState.FINISHED + "FROZEN" -> MangaState.PAUSED + "ANNOUNCE" -> MangaState.UPCOMING + else -> null + }, + contentRating = when (json.getStringOrNull("contentStatus")) { + "SAFE" -> ContentRating.SAFE + "UNSAFE" -> ContentRating.SUGGESTIVE + "EROTIC" -> ContentRating.ADULT + else -> null + }, + tags = emptySet(), + authors = emptySet() + ) + } + + private fun getSortParameter(order: SortOrder): String { + val field = when (order) { + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "viewsCount" + SortOrder.RATING, SortOrder.RATING_ASC -> "averageRating" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "createdAt" + else -> "viewsCount" + } + val direction = when (order) { + SortOrder.POPULARITY_ASC, SortOrder.RATING_ASC, SortOrder.NEWEST_ASC -> "asc" + else -> "desc" + } + return "$field,$direction" + } + + override suspend fun getDetails(manga: Manga): Manga { + val data = fetchAstroData(manga.url) + ?: throw ParseException("Cannot obtain Astro state for manga details", manga.publicUrl) + + val bookData = data["current-book"] as? Map<*, *> ?: return manga + val branchesData = data["current-book-branches"] as? List<Map<*, *>> ?: emptyList() + val chaptersData = data["current-book-chapters"] as? List<Map<*, *>> ?: emptyList() + + val description = bookData["description"] as? String + + val tags = (bookData["labels"] as? List<Map<*, *>>)?.mapNotNullTo(HashSet()) { + val tagName = it["name"] as? String + val tagKey = it["slug"] as? String + if (tagName != null && tagKey != null) { + MangaTag(key = tagKey, title = tagName.replaceFirstChar { c -> c.uppercase() }, source = source) + } else null + } ?: emptySet() + + val authors = (bookData["relations"] as? List<Map<*, *>>)?.mapNotNullTo(HashSet()) { + val type = it["type"] as? String + if (type == "AUTHOR" || type == "ARTIST") { + (it["publisher"] as? Map<*, *>)?.get("name") as? String + } else null + } ?: emptySet() + + val branchIdToNameMap = branchesData.associate { branchMap -> + val branchId = branchMap["id"] as? String + + val scanlatorNames = (branchMap["publishers"] as? List<Map<*, *>>) + ?.mapNotNull { publisherMap -> publisherMap["name"] as? String } + ?.joinToString(" & ") + + branchId to scanlatorNames + } + + val slug = manga.url.substringAfterLast('/') + + val chapters = chaptersData.mapNotNull { chapterMap -> + val id = chapterMap["id"] as? String ?: return@mapNotNull null + val branchId = chapterMap["branchId"] as? String + val scanlator = branchIdToNameMap[branchId] + + MangaChapter( + id = generateUid(id), + url = "/content/$slug/$id", + title = chapterMap["name"] as? String, + number = chapterMap["number"].toSafeFloat(), + volume = chapterMap["volume"].toSafeInt(), + uploadDate = dateFormat.parseSafe(chapterMap["createdAt"] as? String), + scanlator = scanlator, + branch = scanlator, + source = source + ) + }.reversed() + + return manga.copy( + description = description, + tags = manga.tags + tags, + authors = authors, + chapters = chapters + ) + } + + override fun getRequestHeaders() = Headers.Builder() + .add("User-Agent", UserAgents.CHROME_DESKTOP) + .build() + + private fun Any?.toSafeInt(): Int { + return when(this) { + is Number -> this.toInt() + is String -> this.toIntOrNull() ?: 0 + else -> 0 + } + } + + private fun Any?.toSafeFloat(): Float { + return when(this) { + is Number -> this.toFloat() + is String -> this.toFloatOrNull() ?: 0f + else -> 0f + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val data = fetchAstroData(chapter.url) + ?: throw ParseException("Cannot obtain Astro state for chapter pages", chapter.url) + + val chapterData = data["reader-current-chapter"] as? Map<*, *> + ?: throw ParseException("Cannot find the 'reader-current-chapter' key", chapter.url) + + val pagesList = chapterData["pages"] as? List<Map<*, *>> + ?: throw ParseException("Cannot find the 'pages' list", chapter.url) + + return pagesList + .sortedBy { it["index"].toSafeInt() } + .mapNotNull { pageMap -> + val id = pageMap["id"] as? String + val imageUrl = pageMap["image"] as? String + if (id == null || imageUrl == null) return@mapNotNull null + + MangaPage( + id = generateUid(id), + url = "$imageUrl&width=1600", + preview = null, + source = source + ) + } + } + + private suspend fun fetchAstroData(relativeUrl: String): Map<*, *>? { + val fullUrl = relativeUrl.toAbsoluteUrl(domain) + + val response = webClient.httpGet(fullUrl) + + val protection = CloudFlareHelper.checkResponseForProtection(response.copy()) + if (protection != CloudFlareHelper.PROTECTION_NOT_DETECTED) { + response.close() + context.requestBrowserAction(this, fullUrl) + return null + } + + val responseHtml = response.parseHtml() + val scriptElement = responseHtml.getElementById("it-astro-state") + ?: throw ParseException("Cannot find <script id='it-astro-state'> at $fullUrl", fullUrl) + + return astroJsonParser.parse(scriptElement.data()) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = allGenres.toSet(), + availableStates = allStates, + availableContentRating = allContentRatings + ) + } + + private val allGenres: List<MangaTag> = listOf( + MangaTag(key = "art", title = "Арт", source = source), + MangaTag(key = "martial_arts", title = "Боевые искусства", source = source), + MangaTag(key = "vampires", title = "Вампиры", source = source), + MangaTag(key = "harem", title = "Гарем", source = source), + MangaTag(key = "gender_intriga", title = "Гендерная интрига", source = source), + MangaTag(key = "detective", title = "Детектив", source = source), + MangaTag(key = "josei", title = "Дзёсэй", source = source), + MangaTag(key = "game", title = "Игра", source = source), + MangaTag(key = "cyberpunk", title = "Киберпанк", source = source), + MangaTag(key = "maho_shoujo", title = "Махо-сёдзё", source = source), + MangaTag(key = "mecha", title = "Меха", source = source), + MangaTag(key = "mystery", title = "Мистика", source = source), + MangaTag(key = "sci_fi", title = "Научная фантастика", source = source), + MangaTag(key = "natural", title = "Повседневность", source = source), + MangaTag(key = "postapocalypse", title = "Постапокалипсис", source = source), + MangaTag(key = "adventure", title = "Приключения", source = source), + MangaTag(key = "psychological", title = "Психология", source = source), + MangaTag(key = "samurai", title = "Самураи", source = source), + MangaTag(key = "supernatural", title = "Сверхъестественное", source = source), + MangaTag(key = "sports", title = "Спорт", source = source), + MangaTag(key = "seinen", title = "Сэйнэн", source = source), + MangaTag(key = "thriller", title = "Триллер", source = source), + MangaTag(key = "horror", title = "Ужасы", source = source), + MangaTag(key = "fantastic", title = "Фантастика", source = source), + MangaTag(key = "fantasy", title = "Фэнтези", source = source), + MangaTag(key = "school", title = "Школа", source = source), + MangaTag(key = "erotica", title = "Эротика", source = source), + MangaTag(key = "ecchi", title = "Этти", source = source), + MangaTag(key = "codomo", title = "Кодомо", source = source), + MangaTag(key = "isekai", title = "Исекай", source = source), + MangaTag(key = "omegavers", title = "Омегаверс", source = source), + MangaTag(key = "comedy", title = "Комедия", source = source), + MangaTag(key = "shounen", title = "Сёнэн", source = source), + MangaTag(key = "romance", title = "Романтика", source = source), + MangaTag(key = "drama", title = "Драма", source = source), + MangaTag(key = "shoujo", title = "Сёдзё", source = source), + MangaTag(key = "historical", title = "История", source = source), + MangaTag(key = "tragedy", title = "Трагедия", source = source), + MangaTag(key = "action", title = "Боевик", source = source) + ).sortedBy { it.title } + + private val allStates: Set<MangaState> = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.UPCOMING + ) + + private val allContentRatings: Set<ContentRating> = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT + ) + + private class AstroJsonParser { + fun parse(compressedJson: String): Map<*, *> { + val rootArray = JSONArray(compressedJson) + if (rootArray.length() == 0) return emptyMap<Any, Any>() + + val cache = mutableMapOf<Int, Any?>() + val overdueMap = decompress(rootArray.get(0), rootArray, cache) as? Map<*, *> ?: emptyMap<Any, Any>() + + return overdueMap["@inox-tools/request-nanostores"] as? Map<*, *> ?: emptyMap<Any, Any>() + } + + private fun decompress(value: Any?, rootArray: JSONArray, cache: MutableMap<Int, Any?>): Any? { + if (value is Int) { + val ref = value + if (cache.containsKey(ref)) return cache[ref] + if (ref < 0 || ref >= rootArray.length()) return ref + + cache[ref] = null + val referencedItem = rootArray.get(ref) + val result = processItem(referencedItem, rootArray, cache) + cache[ref] = result + return result + } + return processItem(value, rootArray, cache) + } + + private fun processItem(item: Any?, rootArray: JSONArray, cache: MutableMap<Int, Any?>): Any? { + return when (item) { + is JSONObject -> { + val map = mutableMapOf<String, Any?>() + for (key in item.keys()) { + map[key] = decompress(item.get(key), rootArray, cache) + } + map + } + is JSONArray -> { + if (item.length() > 0 && item.get(0) is String) { + when (item.getString(0)) { + "Map" -> { + val map = mutableMapOf<Any?, Any?>() + for (i in 1 until item.length() step 2) { + val key = decompress(item.get(i), rootArray, cache) + val value = decompress(item.get(i + 1), rootArray, cache) + if (key != null) map[key] = value + } + return map + } + "URL" -> return if (item.length() > 1) decompress(item.get(1), rootArray, cache) else null + } + } + (0 until item.length()).map { i -> decompress(item.get(i), rootArray, cache) } + } + else -> item + } + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/ChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/ChanParser.kt new file mode 100644 index 00000000..66243ff6 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/ChanParser.kt @@ -0,0 +1,271 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.chan + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.core.AbstractMangaParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class ChanParser( + context: MangaLoaderContext, + source: MangaParserSource, +) : AbstractMangaParser(context, source), MangaParserAuthProvider { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val authUrl: String + get() = "https://${domain}" + + override suspend fun isAuthorized(): Boolean = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val domain = domain + val doc = webClient.httpGet(buildUrl(offset, order, filter)).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + }.orEmpty(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> + val a = tr.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + title = tr.selectFirst("a")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.parseSafe(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val doc = webClient.httpGet(seed.url.replace("/manga/", "/related/").toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("right") + return root.select("div.related").mapNotNull { div -> + val info = div.selectFirst(".related_info") ?: return@mapNotNull null + val a = info.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = info.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = div.selectFirst("img")?.absUrl("src").orEmpty(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + protected open fun buildUrl( + offset: Int, + order: SortOrder, + filter: MangaListFilter, + ): HttpUrl { + val builder = urlBuilder() + builder.addQueryParameter("offset", offset.toString()) + when { + !filter.query.isNullOrEmpty() -> { + builder.addQueryParameter("do", "search") + builder.addQueryParameter("subaction", "search") + builder.addQueryParameter("search_start", ((offset / 40) + 1).toString()) + builder.addQueryParameter("full_search", "0") + builder.addQueryParameter("result_from", (offset + 1).toString()) + builder.addQueryParameter("result_num", "40") + builder.addQueryParameter("story", filter.query) + builder.addQueryParameter("need_sort_date", "false") + } + + else -> { + if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { + builder.addPathSegment("tags") + val joiner = StringUtil.StringJoiner("+") + filter.tags.forEach { joiner.add(it.key) } + filter.tagsExclude.forEach { joiner.add("-"); joiner.append(it.key) } + builder.addPathSegment(joiner.complete()) + builder.addQueryParameter( + "n", + when (order) { + SortOrder.RATING, + SortOrder.POPULARITY, + -> "favdesc" + + SortOrder.ALPHABETICAL -> "abcasc" + else -> "" // SortOrder.NEWEST + }, + ) + } else { + when (order) { + SortOrder.POPULARITY -> builder.addPathSegment("mostviews") + SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog") + SortOrder.RATING -> builder.addPathSegment("mostfavorites") + else -> { // SortOrder.NEWEST + builder.addPathSegment("manga") + builder.addPathSegment("new") + } + } + } + } + } + return builder.build() + } + + private fun String.toTagName() = replace('_', ' ').toTitleCase() + + private fun String.parseTitle(): Pair<String, String?> { + var depth = 0 + for (i in indices.reversed()) { + val c = this[i] + if (c == '(') { + depth-- + if (depth == 0 && (i + 2) < lastIndex && i > 0) { + return substring(i + 1, lastIndex).trim() to substring(0, i).trim().nullIfEmpty() + } + } else if (c == ')') { + depth++ + } + } + return this to null + } + + private fun Element.attrAsRelativeUrlAnyHost(attributeKey: String): String { + val attr = attr(attributeKey) + if (attr.isEmpty() || attr.startsWith("data:")) { + throw IllegalArgumentException("Wrong url $attr") + } + if (attr.startsWith('/')) { + return attr + } + val host = attr.toHttpUrl().host + return attr.substringAfter(host) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/HenChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/HenChanParser.kt new file mode 100644 index 00000000..8220c5ff --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/HenChanParser.kt @@ -0,0 +1,89 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.chan + +import okhttp3.HttpUrl +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import java.util.* + +@MangaSourceParser("HENCHAN", "Хентай-тян", "ru", type = ContentType.HENTAI) +internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.HENCHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "x9.h-chan.me", + "x5.h-chan.me", + "xxxx.henchan.pro", + "xxl.hentaichan.live", + "xxx.henchan.pro", + "y.hentaichan.live", + "xx.hentaichan.live", + "x.henchan.pro", + "hentaichan.live", + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val readLink = manga.url.replace("manga", "online") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { + val a = it.children().last() ?: doc.parseFailed("Invalid tag") + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } ?: manga.tags, + chapters = listOf( + MangaChapter( + id = generateUid(readLink), + url = readLink, + source = source, + number = 0f, + volume = 0, + uploadDate = 0L, + title = null, + scanlator = null, + branch = null, + ), + ), + ) + } + + override fun buildUrl(offset: Int, order: SortOrder, filter: MangaListFilter): HttpUrl = when { + filter.query.isNullOrEmpty() && filter.tags.isEmpty() && filter.tagsExclude.isEmpty() -> { + val builder = urlBuilder().addQueryParameter("offset", offset.toString()) + when (order) { + SortOrder.POPULARITY -> { + builder.addPathSegment("mostviews") + builder.addQueryParameter("sort", "manga") + } + + SortOrder.RATING -> { + builder.addPathSegment("mostfavorites") + builder.addQueryParameter("sort", "manga") + } + + else -> { // SortOrder.NEWEST + builder.addPathSegment("manga") + builder.addPathSegment("newest") + } + } + builder.build() + } + + else -> { + super.buildUrl(offset, order, filter) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/MangaChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/MangaChanParser.kt new file mode 100644 index 00000000..435c91c8 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/MangaChanParser.kt @@ -0,0 +1,11 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.chan + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("MANGACHAN", "Манга-тян", "ru") +internal class MangaChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.MANGACHAN) { + override val configKeyDomain = ConfigKey.Domain("manga-chan.me") +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/YaoiChanParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/YaoiChanParser.kt new file mode 100644 index 00000000..f52c9786 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/chan/YaoiChanParser.kt @@ -0,0 +1,51 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.chan + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.Manga +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.util.* + +@MangaSourceParser("YAOICHAN", "Яой-тян", "ru") +internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "v18.yaoi-chan.me", + "v9.yaoi-chan.me", + "v10.yaoi-chan.me", + "v3.yaoi-chan.me", + "v2.yaoi-chan.me", + "v1.yaoi-chan.me", + "yaoi-chan.me", + ) + + override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + chapters = root.select("table.table_cha").flatMap { table -> + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = a.text(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/AllHentaiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/AllHentaiParser.kt new file mode 100644 index 00000000..4c891b52 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/AllHentaiParser.kt @@ -0,0 +1,45 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.util.* + +@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", type = ContentType.HENTAI) +internal class AllHentaiParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.ALLHENTAI, 1) { + + override val detailsTagsSelector = "div.cr-tags" + + override val configKeyDomain = ConfigKey.Domain( + "x.ahen.me", + "20.allhen.online", + "z.ahen.me", + "24.allhen.online", + "2023.allhen.online", + ) + + override suspend fun isAuthorized(): Boolean = + super.isAuthorized() || context.cookieJar.getCookies(domain).any { it.name == "remember_me" } + + override val authUrl: String + get() { + val targetUri = "https://$domain/".urlEncoded() + return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://qawa.org/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else { + res + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/GroupleParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/GroupleParser.kt new file mode 100644 index 00000000..0a6dfca8 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/GroupleParser.kt @@ -0,0 +1,582 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import androidx.collection.MutableScatterMap +import androidx.collection.ScatterMap +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.launch +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import okio.IOException +import org.json.JSONArray +import org.jsoup.nodes.Element +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.AbstractMangaParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.ParseException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.koitharu.kotatsu.parsers.util.json.mapJSON +import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy +import java.net.HttpURLConnection +import java.text.SimpleDateFormat +import java.util.* + +private const val PAGE_SIZE = 70 +private const val NSFW_ALERT = "сексуальные сцены" +private const val NOTHING_FOUND = "Ничего не найдено" +private const val MIN_IMAGE_SIZE = 1024L +private const val HEADER_ACCEPT = "Accept" +private const val RELATED_TITLE = "Связанные произведения" +private const val COPYRIGHT_ALERT = "Запрещена публикация произведения" +private const val NO_CHAPTERS = "В этой манге еще нет ни одной главы" + +internal abstract class GroupleParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val siteId: Int, +) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor { + + protected open val detailsTagsSelector = "div.creation-element-tags" + + @Volatile + private var cachedPagesServer: String? = null + + override val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) + private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .add("Accept-Language", "ru,en-US;q=0.7,en;q=0.3") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.ADDED, + ) + + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://3.grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } + + override suspend fun isAuthorized(): Boolean = hasAuthCookie() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val domain = domain + val root = if (filter.isEmpty()) { + webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(order) + }&offset=${offset upBy PAGE_SIZE}", + ).parseHtml().body().let { doc -> (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) } + } else { + advancedSearch(offset, order, filter).parseHtml() + } + checkNotNull(root) { "Root not found" } + val tiles = root.selectFirst("div.tiles.row") + if (tiles == null) { + if (root.getElementsContainingOwnText(NOTHING_FOUND).isNotEmpty()) { + return emptyList() + } + root.parseFailed("No tiles found") + } + return tiles.select("div.tile").mapNotNull(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + val doc = response.parseHtml() + val root = doc.body().requireElementById("mangaBox") + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("img.cr-hero-poster__img") + val translations = if (config[splitTranslationsKey]) { + root.selectFirst("div.translator-selection") + ?.select(".translator-selection-item") + ?.associate { + it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull() + } + } else { + null + } + val newSource = getSource(response.request.url) + val chaptersList = root.getElementById("chapters-list") + var isRestricted = false + if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { + if (root.getElementsContainingOwnText(COPYRIGHT_ALERT).isNotEmpty()) { + isRestricted = true + } else { + root.parseFailed("No chapters found") + } + } + val hashRegex = Regex("window.user_hash\\s*=\\s*\'([^\']+)\'") + val userHash = doc.select("script").firstNotNullOfOrNull { it.html().findGroupValue(hashRegex) } + val hasNsfwAlert = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) } + return manga.copy( + source = newSource, + title = doc.metaValue("name") ?: manga.title, + altTitles = root.select(".cr-hero-names__alt > span > span") + .filterNot { it.hasClass("cr-hero-names__alt-separator") } + .flatMapTo(linkedSetOf()) { element -> + element.text().split('/').mapNotNull { it.trim().nullIfEmpty() } + } + manga.altTitles, + publicUrl = response.request.url.toString(), + description = root.selectFirst("div.cr-description__content")?.html(), + largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("src"), + coverUrl = manga.coverUrl + ?: coverImg?.attrAsAbsoluteUrlOrNull("src"), + tags = root.selectFirstOrThrow(detailsTagsSelector) + .getElementsByAttributeValueContaining("href", "/list/genre/").mapTo(manga.tags.toMutableSet()) { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) + }, + state = if (isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = manga.authors, + contentRating = (if (hasNsfwAlert) ContentRating.SUGGESTIVE else ContentRating.SAFE) + .coerceAtLeast(manga.contentRating ?: ContentRating.SAFE), + chapters = chaptersList?.select("a.chapter-link") + ?.flatMapChapters(reversed = true) { a -> + val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList() + val href = a.attrAsRelativeUrl("href") + val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f + val volume = tr.attr("data-vol").toIntOrNull() ?: 0 + if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) { + var translators = "" + val translatorElement = a.attr("title") + if (translatorElement.isNotBlank()) { + translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)") + } + listOf( + MangaChapter( + id = generateUid(href), + title = a.text().removePrefix(manga.title).trim().nullIfEmpty(), + number = number, + volume = volume, + url = href.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = newSource, + branch = null, + ), + ) + } else { + val translationData = JSONArray(a.attr("data-translations")) + translationData.mapJSON { jo -> + val personId = jo.getLong("personId") + val link = href.withQueryParam("tran", personId.toString()) + MangaChapter( + id = generateUid(link), + title = a.text().removePrefix(manga.title).trim(), + number = number, + volume = volume, + url = link.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(jo.getStringOrNull("dateCreated")), + scanlator = null, + source = newSource, + branch = translations[personId], + ) + } + } + }.orEmpty(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + if (chapter.source != source && chapter.source is MangaParserSource) { // handle redirects between websites + return context.newParserInstance(chapter.source).getPages(chapter) + } + val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + var pos = data.indexOf("rm_h.readerDoInit(") + if (pos != -1) { + parsePagesV2(data, pos)?.let { return it } + } + pos = data.indexOf("rm_h.readerInit( 0,") + if (pos != -1) { + parsePagesV1(data, pos)?.let { return it } + } + pos = data.indexOf(".readerInit(") + if (pos != -1) { + parsePagesV3(data, pos).let { return it } + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } + + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + if (parts.size < 2) { + return page.url + } + val path = parts.last() + // fast path + cachedPagesServer?.let { host -> + val url = concatUrl("https://$host/", path) + if (tryHead(url)) { + return url + } else { + cachedPagesServer = null + } + } + // slow path + val candidates = HashSet<String>((parts.size - 1) * 2) + for (i in 0 until parts.size - 1) { + val server = parts[i].trim().ifEmpty { "https://$domain/" } + candidates.add(concatUrl(server, path)) + candidates.add(concatUrl(server, path.substringBeforeLast('?'))) + } + val resolvedUrl = channelFlow { + for (url in candidates) { + launch { + if (tryHead(url)) { + send(url) + } + } + } + }.firstOrNull() ?: throw ParseException("No reachable page URL", page.url) + cachedPagesServer = resolvedUrl.toHttpUrlOrNull()?.host + return resolvedUrl + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")?.selectFirst("table.table") + ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { + return chain.proceed(request).checkIfAuthRequired() + } + val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) + return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { + chain.proceed( + request.newBuilder().header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8").build(), + ) + } else { + chain.proceed(request).checkIfAuthRequired() + } + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(splitTranslationsKey) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("mangaBox").select("h4").first { it.ownText() == RELATED_TITLE } + .nextElementSibling() ?: doc.parseFailed("Cannot find root") + return root.select("div.tile").mapNotNull(::parseManga) + } + + protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { + in SeiMangaParser.domains -> MangaParserSource.SEIMANGA + in MintMangaParser.domains -> MangaParserSource.MINTMANGA + in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU + in SelfMangaParser.domains -> MangaParserSource.SELFMANGA + in UsagiParser.domains -> MangaParserSource.USAGI + else -> source + } + + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.ADDED, + SortOrder.NEWEST, + -> "created" + + SortOrder.RATING -> "votes" + else -> "rate" + } + + private suspend fun advancedSearch(offset: Int, order: SortOrder, filter: MangaListFilter): Response { + val tagsMap = if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { + tagsIndex.get() + } else { + null + } + val url = urlBuilder() + .addPathSegment("search") + .addPathSegment("advancedResults") + url.addQueryParameter("q", filter.query) + url.addQueryParameter("offset", offset.toString()) + filter.tags.forEach { tag -> + val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "in") + } + filter.tagsExclude.forEach { tag -> + val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "ex") + } + url.addQueryParameter( + "years", + buildString { + append(filter.yearFrom.ifZero { YEAR_MIN }) + append(',') + append(filter.yearTo.ifZero { YEAR_MAX }) + }, + ) + url.addQueryParameter( + "sortType", + when (order) { + SortOrder.RATING -> "USER_RATING" + SortOrder.ALPHABETICAL -> "NAME" + SortOrder.ADDED -> "YEAR" + SortOrder.POPULARITY -> "POPULARITY" + SortOrder.NEWEST -> "DATE_CREATE" + SortOrder.UPDATED -> "DATE_UPDATE" + else -> "RATING" + }, + ) + filter.states.forEach { state -> + when (state) { + MangaState.FINISHED -> "s_completed" + MangaState.ABANDONED -> "s_abandoned_popular" + MangaState.UPCOMING -> "s_wait_upload" + else -> null + }?.let { + url.addQueryParameter(it, "in") + } + } + + return webClient.httpGet(url.build()) + } + + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).use { response -> + response.isSuccessful && !response.isPumpkin() && response.headersContentLength() >= MIN_IMAGE_SIZE + } + }.getOrDefault(false) + + private fun Response.isPumpkin(): Boolean = request.url.host == "upload.wikimedia.org" + + private fun parseManga(node: Element): Manga? { + val imgDiv = node.selectFirst("div.img") ?: return null + val descDiv = node.selectFirst("div.desc") ?: return null + if (descDiv.selectFirst("i.fa-user") != null || descDiv.selectFirst("i.fa-external-link") != null) { + return null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") ?: return null + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() ?: return null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(domain) + if (relUrl.contains("://")) { + return null + } + val author = tileInfo?.selectFirst("a.person-link")?.text() + return Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitles = setOfNotNull(descDiv.selectFirst("h5")?.textOrNull()), + coverUrl = imgDiv.selectFirst("img.lazy")?.attrAsAbsoluteUrlOrNull("data-original")?.replace("_p.", "."), + rating = node.selectFirst(".compact-rate")?.attr("title")?.toFloatOrNull()?.div(5f) + ?: RATING_UNKNOWN, + authors = setOfNotNull(author), + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + tags = tileInfo?.select("a.element-link")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + state = when { + node.selectFirst("div.tags")?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + + else -> null + }, + source = source, + ) + } + + private fun parsePagesV1(data: String, pos: Int): List<MangaPage>? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV2(data: String, pos: Int): List<MangaPage>? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(0) + val servers = ja.getJSONArray(2).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV3(data: String, pos: Int): List<MangaPage> { + val json = JSONArray( + data.substring(pos) + .substringBetween("(", ")") + .removePrefix("chapterInfo,") + .substringBeforeLast(','), + ) + return (0 until json.length()).map { i -> + val ja = json.getJSONArray(i) + val server = ja.getString(0).ifEmpty { "https://$domain" } + val url = ja.getString(2) + val fullUrl = concatUrl(server, url) + MangaPage( + id = generateUid(url), + url = if (fullUrl.contains("one-way.work")) { + // domain that does not need a token + fullUrl.substringBefore("?") + } else { + fullUrl + }, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTagsMap(): ScatterMap<String, String> { + val url = "https://$domain/search/advanced" + val properties = + webClient.httpGet(url).parseHtml().body().selectFirst("form.search-form")?.select("div.form-group") + ?.find { it.selectFirst("li.property") != null } + ?.select("li.property") + ?: throw ParseException("Genres filter element not found", url) + val result = MutableScatterMap<String, String>(properties.size) + properties.forEach { li -> + val name = li.text().lowercase() + val id = li.selectFirstOrThrow("input").id() + result[name] = id + } + return result + } + + private fun String.withQueryParam(name: String, value: String?): String { + if (value == null) return this + return toAbsoluteUrl(domain) + .toHttpUrl() + .newBuilder() + .setQueryParameter(name, value) + .build() + .toString() + .toRelativeUrl(domain) + } + + @Throws(IOException::class) + private fun Response.checkIfAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() + if (lastPathSegment == "login") { + closeQuietly() + throw AuthRequiredException(source) + } + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + if (!hasAuthCookie()) { + closeQuietly() + throw AuthRequiredException(source) + } else { + return newBuilder().code(HttpURLConnection.HTTP_OK).build() + } + } + return this + } + + private fun hasAuthCookie() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/MintMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/MintMangaParser.kt new file mode 100644 index 00000000..a307378f --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/MintMangaParser.kt @@ -0,0 +1,28 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("MINTMANGA", "MintManga", "ru") +internal class MintMangaParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.MINTMANGA, 2) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .build() + + companion object { + + val domains = arrayOf( + "2.mintmanga.one", + "24.mintmanga.one", + "mintmanga.live", + "mintmanga.com", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/ReadmangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/ReadmangaParser.kt new file mode 100644 index 00000000..43a34d75 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/ReadmangaParser.kt @@ -0,0 +1,26 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("READMANGA_RU", "ReadManga", "ru") +internal class ReadmangaParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.READMANGA_RU, 1) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .build() + + companion object { + + val domains = arrayOf( + "a.zazaza.me", + "3.readmanga.ru", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SeiMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SeiMangaParser.kt new file mode 100644 index 00000000..68ee9bb3 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SeiMangaParser.kt @@ -0,0 +1,26 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("SEIMANGA", "SeiManga", "ru") +internal class SeiMangaParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.SEIMANGA, 21) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .build() + + companion object { + + val domains = arrayOf( + "1.seimanga.me", + "seimanga.me", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SelfMangaParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SelfMangaParser.kt new file mode 100644 index 00000000..b551c23d --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/SelfMangaParser.kt @@ -0,0 +1,26 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("SELFMANGA", "SelfManga", "ru", type = ContentType.OTHER) +internal class SelfMangaParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.SELFMANGA, 3) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .build() + + companion object { + + val domains = arrayOf( + "1.selfmanga.live", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/UsagiParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/UsagiParser.kt new file mode 100644 index 00000000..54ea79ec --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/grouple/UsagiParser.kt @@ -0,0 +1,23 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.grouple + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("USAGI", "Usagi", "ru") +internal class UsagiParser( + context: MangaLoaderContext, +) : GroupleParser(context, MangaParserSource.USAGI, 1) { + + override val configKeyDomain = ConfigKey.Domain(*domains) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .build() + + companion object { + + val domains = arrayOf("web.usagi.one") + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/HentaiLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/HentaiLibParser.kt new file mode 100644 index 00000000..9679726f --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/HentaiLibParser.kt @@ -0,0 +1,25 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.libsocial + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI) +internal class HentaiLibParser(context: MangaLoaderContext) : LibSocialParser( + context = context, + source = MangaParserSource.HENTAILIB, + siteId = 4, + siteDomains = arrayOf("v1.hentailib.org", "hentailib.me"), +) { + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { + super.getPages(chapter) + } catch (e: NotFoundException) { + throw AuthRequiredException(source, e) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/LibSocialParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/LibSocialParser.kt new file mode 100644 index 00000000..4bb4091e --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/LibSocialParser.kt @@ -0,0 +1,473 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.libsocial + +import androidx.collection.* +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.runBlocking +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaParserAuthProvider +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.core.PagedMangaParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.model.* +import org.koitharu.kotatsu.parsers.network.UserAgents +import org.koitharu.kotatsu.parsers.util.* +import org.koitharu.kotatsu.parsers.util.json.* +import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy +import java.text.SimpleDateFormat +import java.util.* + +internal abstract class LibSocialParser( + context: MangaLoaderContext, + source: MangaParserSource, + protected val siteId: Int, + siteDomains: Array<String>, +) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { + + protected val apiHost = "api.cdnlibs.org" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE) + + override val authUrl: String + get() = "https://$domain/ru/front/auth" + + override suspend fun isAuthorized(): Boolean { + val token = getAuthData()?.optJSONObject("token")?.getStringOrNull("access_token") + return !token.isNullOrEmpty() + } + + override suspend fun getUsername(): String = getAuthData() + ?.getJSONObject("auth") + ?.getString("username") + ?: throw AuthRequiredException(source) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + final override val configKeyDomain = ConfigKey.Domain(*siteDomains) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + MangaState.UPCOMING, + ), + ) + + final override fun intercept(chain: Interceptor.Chain): Response { + val token = runBlocking { getAuthData() }?.optJSONObject("token")?.getStringOrNull("access_token") + val requestBuilder = chain.request().newBuilder() + if (!token.isNullOrEmpty()) { + requestBuilder.header("Authorization", "Bearer $token") + } + requestBuilder.header("Site-Id", siteId.toString()) + return chain.proceed(requestBuilder.build()) + } + + private val statesMap = intObjectMapOf( + 1, MangaState.ONGOING, + 2, MangaState.FINISHED, + 3, MangaState.UPCOMING, + 4, MangaState.PAUSED, + 5, MangaState.ABANDONED, + ) + private val imageServers = suspendLazy(initializer = ::fetchServers) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + null to null, + SERVER_MAIN to "Первый", + SERVER_SECONDARY to "Второй", + SERVER_COMPRESS to "Сжатия", + SERVER_DOWNLOAD to "Загрузки", + SERVER_CROP to "Обрезки", + ), + defaultValue = null, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val urlBuilder = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addQueryParameter("site_id[]", siteId.toString()) + .addQueryParameter("fields[]", "rate") + .addQueryParameter("fields[]", "rate_avg") + .addQueryParameter("page", page.toString()) + for (state in filter.states) { + urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString()) + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1)) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1)) + } + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("q", filter.query) + } + urlBuilder.addQueryParameter( + "sort_by", + when (order) { + SortOrder.UPDATED -> "last_chapter_at" + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "rate_avg" + SortOrder.NEWEST -> "created_at" + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + -> "rus_name" + + else -> null + }, + ) + urlBuilder.addQueryParameter( + "sort_type", + when (order) { + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL_DESC, + -> "desc" + + SortOrder.ALPHABETICAL -> "asc" + else -> null + }, + ) + val json = webClient.httpGet(urlBuilder.build()).parseJson() + val data = json.getJSONArray("data") + return data.mapJSON(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { fetchChapters(manga) } + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addQueryParameter("fields[]", "summary") + .addQueryParameter("fields[]", "genres") + .addQueryParameter("fields[]", "tags") + .addQueryParameter("fields[]", "authors") + .addQueryParameter("fields[]", "close_view") + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + val genres = json.getJSONArray("genres").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source) + } + val tags = json.getJSONArray("tags").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source) + } + val authors = json.getJSONArray("authors").mapJSONNotNullToSet { + it.getStringOrNull("name") + } + val chapters = chaptersDeferred.await() + val isRestricted = json.getIntOrDefault("close_view", 0) > 0 + manga.copy( + title = json.getStringOrNull("rus_name") ?: manga.title, + altTitles = setOfNotNull(json.getStringOrNull("name")), + tags = tagsSetOf(tags, genres), + state = if (chapters.isEmpty() && isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = authors, + contentRating = json.optJSONObject("ageRestriction")?.let { + val id = it.getIntOrDefault("id", -1) + if (id >= 4) ContentRating.SUGGESTIVE else sourceContentRating + } ?: manga.contentRating, + description = json.parseLibSocialSummary()?.nl2br(), + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope { + val pages = async { + webClient.httpGet( + concatUrl("https://$apiHost/api/manga/", chapter.url), + ).parseJson().getJSONObject("data") + } + val servers = imageServers.get() + val json = pages.await() + val primaryServer = getPrimaryImageServer(servers) + json.getJSONArray("pages").mapJSON { jo -> + val url = jo.getString("url") + MangaPage( + id = generateUid(jo.getLong("id")), + url = concatUrl(primaryServer, url), + preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) }, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { + val tags = async { fetchTags("tags") } + val genres = async { fetchTags("genres") } + tagsSetOf(tags.await(), genres.await()) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(seed.url) + .addPathSegment("similar") + .build(), + ).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + parseManga(jo.getJSONObject("media")) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + return resolver.resolveManga(this, link.pathSegments.lastOrNull() ?: return null) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(splitTranslationsKey) + keys.add(preferredServerKey) + } + + private fun parseManga(jo: JSONObject): Manga { + val cover = jo.getJSONObject("cover") + val isNsfwSource = jo.getJSONObject("ageRestriction").getIntOrDefault("id", 0) >= 3 + return Manga( + id = generateUid(jo.getLong("id")), + title = jo.getString("rus_name").ifEmpty { jo.getString("name") }, + altTitles = setOfNotNull(jo.getString("name")), + url = jo.getString("slug_url"), + publicUrl = "https://$domain/ru/manga/" + jo.getString("slug_url"), + rating = jo.optJSONObject("rating") + ?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN, + contentRating = sourceContentRating, + coverUrl = cover.getString("thumbnail"), + tags = setOf(), + state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1], + authors = emptySet(), + largeCoverUrl = cover.getString("default"), + source = source, + ) + } + + private fun getPrimaryImageServer(servers: ScatterMap<String, String>): String { + val preferred = config[preferredServerKey] + if (preferred != null) { + servers[preferred]?.let { return it } + } + return checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) { + "No available images servers" + } + } + + private suspend fun fetchChapters(manga: Manga): List<MangaChapter> { + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addPathSegment("chapters") + .build() + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + val builder = ChaptersListBuilder(json.length()) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + val useBranching = config[splitTranslationsKey] + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val volume = jo.getIntOrDefault("volume", 0) + val number = jo.getFloatOrDefault("number", 0f) + val numberString = number.formatSimple() + val name = jo.getStringOrNull("name") + val branches = jo.getJSONArray("branches") + for (j in 0 until branches.length()) { + val bjo = branches.getJSONObject(j) + val isRestricted = bjo.optJSONObject("restricted_view")?.let { + !it.getBooleanOrDefault("is_open", true) + } ?: false + if (isRestricted) { + continue + } + val id = bjo.getLong("id") + val branchId = bjo.getLongOrDefault("branch_id", 0L) + val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name") + builder += MangaChapter( + id = generateUid(id), + title = name, + number = number, + volume = volume, + url = buildString { + append(manga.url) + append("/chapter?number=") + append(numberString) + append("&volume=") + append(volume) + if (branchId != 0L) { + append("&branch_id=") + append(branchId) + } + }, + scanlator = team, + uploadDate = dateFormat.parseSafe(bjo.getStringOrNull("created_at")), + branch = if (useBranching) team else null, + source = source, + ) + } + } + return builder.toList() + } + + private suspend fun fetchTags(type: String): List<MangaTag> { + val data = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api").addPathSegment(type).build(), + ).parseJson().getJSONArray("data") + val prefix = type.first().toString() + return data.mapJSONNotNull { jo -> + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + return@mapJSONNotNull null + } + MangaTag( + title = jo.getString("name"), + key = prefix + jo.getInt("id"), + source = source, + ) + } + } + + private suspend fun fetchServers(): ScatterMap<String, String> { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("constants") + .addQueryParameter("fields[]", "imageServers") + .build(), + ).parseJson().getJSONObject("data").getJSONArray("imageServers") + val result = MutableScatterMap<String, String>() + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + continue + } + result[jo.getString("id")] = jo.getString("url") + } + return result + } + + private fun <V> IntObjectMap<V>.keyOf(value: V): Int { + forEach { k, v -> + if (v == value) { + return k + } + } + throw NoSuchElementException("No key associated with value $value") + } + + private fun JSONArray.toIntSet(): IntSet { + val result = MutableIntSet(length()) + for (i in 0 until length()) { + result.add(getInt(i)) + } + return result + } + + private fun MangaTag.typeKey() = when (key.firstOrNull()) { + 'g' -> "genres" + 't' -> "tags" + else -> throw IllegalArgumentException("Tag $key($title) is of unknown type") + } + + private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> { + val result = ArraySet<MangaTag>(tags.size + genres.size) + val names = HashSet<String>(tags.size + genres.size) + genres.forEach { x -> if (names.add(x.title)) result.add(x) } + tags.forEach { x -> if (names.add(x.title)) result.add(x) } + return result + } + + private suspend fun getAuthData(): JSONObject? { + val raw = WebViewHelper(context).getLocalStorageValue(domain, "auth") ?: return null + return JSONObject(raw.unescapeJson().removeSurrounding('"')) + } + + protected companion object { + + const val SERVER_MAIN = "main" + const val SERVER_SECONDARY = "secondary" + const val SERVER_COMPRESS = "compress" + const val SERVER_DOWNLOAD = "download" + const val SERVER_CROP = "crop" + } +} + +internal fun JSONObject.parseLibSocialSummary(): String? = when (val value = opt("summary")) { + null, + JSONObject.NULL, + -> null + + is String -> value + is JSONObject -> value.parseLibSocialRichText() + else -> throw JSONException("Unsupported summary value: ${value.javaClass.name}") +} + +private fun JSONObject.parseLibSocialRichText(): String { + val type = getString("type") + return when (type) { + "text" -> getString("text") + "hardBreak" -> "\n" + "paragraph", + "heading", + -> parseLibSocialChildren(separator = "") + + "doc", + "blockquote", + "bulletList", + "orderedList", + "listItem", + -> parseLibSocialChildren(separator = "\n") + + else -> throw JSONException("Unsupported summary node type: $type") + } +} + +private fun JSONObject.parseLibSocialChildren(separator: String): String { + val children = optJSONArray("content") ?: return "" + return children.mapJSON { child -> child.parseLibSocialRichText() }.joinToString(separator) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/MangaLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/MangaLibParser.kt new file mode 100644 index 00000000..59e17e94 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/MangaLibParser.kt @@ -0,0 +1,26 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.libsocial + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.exception.AuthRequiredException +import org.koitharu.kotatsu.parsers.exception.NotFoundException +import org.koitharu.kotatsu.parsers.model.MangaChapter +import org.koitharu.kotatsu.parsers.model.MangaPage +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("MANGALIB", "MangaLib", "ru") +internal class MangaLibParser( + context: MangaLoaderContext, +) : LibSocialParser( + context = context, + source = MangaParserSource.MANGALIB, + siteId = 1, + siteDomains = arrayOf("mangalib.org", "mangalib.me"), +) { + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { + super.getPages(chapter) + } catch (e: NotFoundException) { + throw AuthRequiredException(source, e) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/SlashLibParser.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/SlashLibParser.kt new file mode 100644 index 00000000..55c3fc06 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/libsocial/SlashLibParser.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.libsocial + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +@MangaSourceParser("YAOILIB", "SlashLib", "ru") +internal class SlashLibParser(context: MangaLoaderContext) : LibSocialParser( + context = context, + source = MangaParserSource.YAOILIB, + siteId = 2, + siteDomains = arrayOf("v2.slashlib.me"), +) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/BestManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/BestManga.kt new file mode 100644 index 00000000..9bd141af --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/BestManga.kt @@ -0,0 +1,15 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.madara + +import org.koitharu.kotatsu.parsers.Broken +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.parsers.MadaraParser + +@Broken +@MangaSourceParser("BEST_MANGA", "BestManga", "ru") +internal class BestManga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.BEST_MANGA, "bestmanga.club") { + override val datePattern = "dd.MM.yyyy" + override val postReq = true +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaMammy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaMammy.kt new file mode 100644 index 00000000..4cd307db --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaMammy.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.madara + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.parsers.MadaraParser + +@MangaSourceParser("MANGAMAMMY", "Nimanga", "ru") +internal class MangaMammy(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAMAMMY, "p.nimanga.com") { + override val datePattern = "dd.MM.yyyy" + override val postReq = true +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaoneLove.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaoneLove.kt new file mode 100644 index 00000000..25b2b82d --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/MangaoneLove.kt @@ -0,0 +1,13 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.madara + +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.parsers.MadaraParser + +@MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") +internal class MangaoneLove(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.website", 10) { + override val datePattern = "dd.MM.yyyy" + override val postReq = true +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/Mangazavr.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/Mangazavr.kt new file mode 100644 index 00000000..47bfc416 --- /dev/null +++ b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ru/madara/Mangazavr.kt @@ -0,0 +1,16 @@ +package org.koitharu.kotatsu.parsers.site.kotatsu.ru.madara + +import org.koitharu.kotatsu.parsers.Broken +import org.koitharu.kotatsu.parsers.MangaLoaderContext +import org.koitharu.kotatsu.parsers.MangaSourceParser +import org.koitharu.kotatsu.parsers.model.ContentType +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.parsers.MadaraParser + +@Broken +@MangaSourceParser("MANGAZAVR", "Mangazavr", "ru", ContentType.HENTAI) +internal class Mangazavr(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAZAVR, "mangazavr.ru") { + override val listUrl = "/?s=&post_type=wp-manga" + override val datePattern = "dd.MM.yyyy" +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AComicsContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AComicsContractTest.kt new file mode 100644 index 00000000..5610e85a --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AComicsContractTest.kt @@ -0,0 +1,31 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class AComicsContractTest { + + @Test + fun `reads catalog details and pages from current AComics markup`() = runBlocking { + val catalogUrl = "https://acomics.ru/comics?ratings[]=1&ratings[]=2&ratings[]=3" + + "&ratings[]=4&ratings[]=5&ratings[]=6&skip=0&sort=last_update" + val context = FixtureMangaLoaderContext.builder() + .get(catalogUrl, "/fixtures/acomics/catalog.html") + .get("https://acomics.ru/comics", "/fixtures/acomics/tags.html") + .get("https://acomics.ru/~example/about", "/fixtures/acomics/details.html") + .get("https://acomics.ru/~example/1", "/fixtures/acomics/page.html") + .build() + val parser = MangaParserSource.ACOMICS.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.ACOMICS, + expectedTitle = "Example Comic", + sortOrder = SortOrder.UPDATED, + expectedPageUrl = "https://acomics.ru/upload/example-001.jpg", + ) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/AllHentaiContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/AllHentaiContractTest.kt new file mode 100644 index 00000000..f3a7d68b --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/AllHentaiContractTest.kt @@ -0,0 +1,39 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class AllHentaiContractTest { + + @Test + fun `source supports current tags details and pages markup`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://x.ahen.me/search/advancedResults?q=Club&offset=0&years=1900%2C2099&sortType=DATE_CREATE", + "/fixtures/allhentai/search.html", + ) + .get("https://x.ahen.me/example", "/fixtures/allhentai/details.html") + .get("https://x.ahen.me/example/vol1/1?mtr=1", "/fixtures/allhentai/pages.html") + .build() + val parser = MangaParserSource.ALLHENTAI.newParser(context) + + val manga = parser.getList( + offset = 0, + order = SortOrder.NEWEST, + filter = MangaListFilter.EMPTY.copy(query = "Club"), + ).single() + val details = parser.getDetails(manga) + + assertEquals(setOf("Drama"), details.tags.mapTo(linkedSetOf()) { it.title }) + val chapter = details.chapters.orEmpty().single() + val pages = parser.getPages(chapter) + assertFalse(pages.isEmpty()) + assertEquals("https://images.allhentai.test/page-01.jpg", parser.getPageUrl(pages.single())) + context.verifyAllFixturesUsed() + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/DesuContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/DesuContractTest.kt new file mode 100644 index 00000000..51db9227 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/DesuContractTest.kt @@ -0,0 +1,30 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class DesuContractTest { + + @Test + fun `source supports current html reader from catalog through pages`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .get("https://desu.uno/manga/?page=1", "/fixtures/desu/catalog.html") + .get("https://desu.uno/manga/example.42/", "/fixtures/desu/details.html") + .get("https://desu.uno/manga/example.42/vol2/ch3/rus", "/fixtures/desu/pages.html") + .get("https://desu.uno/api/manga/42/chapters/1003", "/fixtures/desu/pages.json") + .build() + val parser = MangaParserSource.DESUME.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.DESUME, + expectedTitle = "Пример манги", + sortOrder = SortOrder.UPDATED, + expectedPageUrl = "https://img3.desu.uno/manga/rus/example/001.webp?1003", + ) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/FixtureMangaLoaderContext.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/FixtureMangaLoaderContext.kt new file mode 100644 index 00000000..7260bb8c --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/FixtureMangaLoaderContext.kt @@ -0,0 +1,136 @@ +package org.koitharu.kotatsu.parsers + +import okhttp3.Cookie +import okhttp3.CookieJar +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.koitharu.kotatsu.parsers.bitmap.Bitmap +import org.koitharu.kotatsu.parsers.bitmap.Rect +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.config.MangaSourceConfig +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaSource +import java.util.concurrent.ConcurrentHashMap + +@Suppress("OVERRIDE_DEPRECATION") +internal class FixtureMangaLoaderContext private constructor( + routes: Map<FixtureRequest, String>, +) : MangaLoaderContext() { + + private val fixtureInterceptor = FixtureHttpInterceptor(routes) + + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .addInterceptor(fixtureInterceptor) + .build() + + override val cookieJar: CookieJar = object : CookieJar { + override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = Unit + + override fun loadForRequest(url: HttpUrl): List<Cookie> = emptyList() + } + + override fun newParserInstance(source: MangaSource): MangaParser { + require(source is MangaParserSource) { "Unsupported fixture source: $source" } + return source.newParser(this) + } + + override suspend fun evaluateJs(script: String): String? { + error("JavaScript evaluation is not registered for fixture tests") + } + + override suspend fun evaluateJs(baseUrl: String, script: String): String? { + error("JavaScript evaluation is not registered for fixture tests: $baseUrl") + } + + override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { + override fun <T> get(key: ConfigKey<T>): T = key.defaultValue + } + + override fun getDefaultUserAgent(): String = "MangaRepoContractTest/1.0" + + override fun redrawImageResponse( + response: Response, + redraw: (image: Bitmap) -> Bitmap, + ): Response = error("Image redraw is not registered for fixture tests") + + override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { + override val width: Int = width + override val height: Int = height + + override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { + error("Bitmap drawing is not registered for fixture tests") + } + } + + fun verifyAllFixturesUsed() { + fixtureInterceptor.verifyAllFixturesUsed() + } + + internal class Builder { + private val routes = linkedMapOf<FixtureRequest, String>() + + fun get(url: String, resourcePath: String): Builder = route("GET", url, resourcePath) + + fun post(url: String, resourcePath: String): Builder = route("POST", url, resourcePath) + + private fun route(method: String, url: String, resourcePath: String): Builder = apply { + val key = FixtureRequest(method, url) + require(routes.put(key, resourcePath) == null) { "Duplicate fixture route: $method $url" } + } + + fun build(): FixtureMangaLoaderContext = FixtureMangaLoaderContext(routes.toMap()) + } + + companion object { + fun builder(): Builder = Builder() + } +} + +internal data class FixtureRequest( + val method: String, + val url: String, +) + +private class FixtureHttpInterceptor( + private val routes: Map<FixtureRequest, String>, +) : Interceptor { + + private val usedRoutes = ConcurrentHashMap.newKeySet<FixtureRequest>() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val key = FixtureRequest(request.method, request.url.toString()) + val resourcePath = routes[key] + ?: error("No fixture registered for ${request.method} ${request.url}") + val body = loadFixture(resourcePath) + usedRoutes.add(key) + return response(request, body) + } + + fun verifyAllFixturesUsed() { + val unusedRoutes = routes.keys - usedRoutes + check(unusedRoutes.isEmpty()) { + "Unused fixture routes: ${unusedRoutes.joinToString { "${it.method} ${it.url}" }}" + } + } + + private fun loadFixture(resourcePath: String): ByteArray { + val stream = FixtureMangaLoaderContext::class.java.getResourceAsStream(resourcePath) + ?: error("Fixture resource not found: $resourcePath") + return stream.use { it.readBytes() } + } + + private fun response(request: Request, body: ByteArray): Response = Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body.toResponseBody("text/html; charset=utf-8".toMediaType())) + .build() +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/LibSocialJsonContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/LibSocialJsonContractTest.kt new file mode 100644 index 00000000..19260081 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/LibSocialJsonContractTest.kt @@ -0,0 +1,45 @@ +package org.koitharu.kotatsu.parsers + +import org.json.JSONObject +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.site.kotatsu.ru.libsocial.parseLibSocialSummary + +class LibSocialJsonContractTest { + + @Test + fun `parses current structured summary`() { + val manga = JSONObject( + """ + { + "summary": { + "type": "doc", + "content": [ + { + "type": "paragraph", + "content": [ + {"type": "text", "text": "First line"}, + {"type": "hardBreak"}, + {"type": "text", "text": "Second line"} + ] + }, + { + "type": "paragraph", + "content": [{"type": "text", "text": "Next paragraph"}] + } + ] + } + } + """.trimIndent(), + ) + + assertEquals("First line\nSecond line\nNext paragraph", manga.parseLibSocialSummary()) + } + + @Test + fun `keeps legacy string summary contract`() { + val manga = JSONObject("""{"summary":"Plain description"}""") + + assertEquals("Plain description", manga.parseLibSocialSummary()) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveCatalogAuditTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveCatalogAuditTest.kt new file mode 100644 index 00000000..b3e752d5 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveCatalogAuditTest.kt @@ -0,0 +1,285 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.TimeoutCancellationException +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withTimeout +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.StandardOpenOption +import kotlin.time.Duration.Companion.seconds + +class LiveCatalogAuditTest { + + @Test + fun `audit configured live catalog`() = runBlocking { + assumeTrue(System.getenv(ENV_ENABLED).toBoolean(), "$ENV_ENABLED is not enabled") + + val targetStage = AuditStage.valueOf( + System.getenv(ENV_STAGE)?.uppercase() ?: AuditStage.DETAILS.name, + ) + val locales = System.getenv(ENV_LOCALES) + ?.split(',') + ?.mapTo(linkedSetOf()) { it.trim().lowercase() } + ?.takeIf { it.isNotEmpty() } + val includeBroken = System.getenv(ENV_INCLUDE_BROKEN)?.toBooleanStrictOrNull() ?: true + val concurrency = System.getenv(ENV_CONCURRENCY)?.toIntOrNull()?.coerceIn(1, 12) ?: 4 + val timeout = (System.getenv(ENV_TIMEOUT_SECONDS)?.toLongOrNull()?.coerceIn(10, 600) ?: 120).seconds + val semaphore = Semaphore(concurrency) + val sources = MangaParserSource.entries.filter { source -> + (includeBroken || !source.isBroken) && (locales == null || source.locale.lowercase() in locales) + } + + val results = coroutineScope { + sources.map { source -> + async { + semaphore.withPermit { + val startedAt = System.nanoTime() + val result = try { + withTimeout(timeout) { + auditSource(source, targetStage) + } + } catch (_: TimeoutCancellationException) { + result( + source = source, + status = AuditStatus.TIMEOUT, + stage = targetStage, + startedAt = startedAt, + details = "Source audit exceeded ${timeout.inWholeSeconds} seconds", + ) + } + println(result.consoleLine()) + result + } + } + }.awaitAll() + }.sortedBy { it.source.name } + + val reportScope = locales?.joinToString("-")?.ifEmpty { "all" } ?: "all" + val reportPath = writeReport(results, targetStage, reportScope) + printSummary(results, reportPath) + + if (System.getenv(ENV_FAIL_ON_ERROR).toBoolean()) { + val failures = results.filter { it.status.isFailure } + check(failures.isEmpty()) { + "${failures.size} live sources failed; see $reportPath" + } + } + } + + private suspend fun auditSource(source: MangaParserSource, targetStage: AuditStage): AuditResult { + val startedAt = System.nanoTime() + var currentStage = AuditStage.LIST + return try { + val context = LiveMangaLoaderContext() + val parser = context.newParserInstance(source) + val order = preferredSortOrder(parser) + val list = parser.getList(0, order, MangaListFilter.EMPTY) + check(list.isNotEmpty()) { "Source returned an empty first page" } + val manga = list.first() + if (targetStage == AuditStage.LIST) { + return passed(source, targetStage, startedAt, "items=${list.size}; title=${manga.title}") + } + + currentStage = AuditStage.DETAILS + var details = parser.getDetails(manga) + if (targetStage == AuditStage.DETAILS) { + return passed( + source, + targetStage, + startedAt, + "items=${list.size}; title=${details.title}; chapters=${details.chapters.orEmpty().size}", + ) + } + + if (details.chapters.isNullOrEmpty()) { + for (candidate in list.drop(1).take(MAX_CHAPTER_CANDIDATES - 1)) { + val candidateDetails = parser.getDetails(candidate) + if (!candidateDetails.chapters.isNullOrEmpty()) { + details = candidateDetails + break + } + } + } + + currentStage = AuditStage.CHAPTERS + val chapter = details.chapters.orEmpty().firstOrNull() + ?: return result( + source, + AuditStatus.NO_CHAPTERS, + currentStage, + startedAt, + "title=${details.title}", + ) + if (targetStage == AuditStage.CHAPTERS) { + return passed(source, targetStage, startedAt, "title=${details.title}; chapter=${chapter.url}") + } + + currentStage = AuditStage.PAGES + val pages = parser.getPages(chapter) + check(pages.isNotEmpty()) { "Chapter returned no pages" } + val pageUrl = parser.getPageUrl(pages.first()) + passed(source, targetStage, startedAt, "pages=${pages.size}; first=$pageUrl") + } catch (error: Throwable) { + result( + source = source, + status = classify(source, error), + stage = currentStage, + startedAt = startedAt, + details = "${error::class.simpleName}: ${error.message.orEmpty()}".singleLine(), + ) + } + } + + private fun preferredSortOrder(parser: MangaParser): SortOrder = listOf( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ).firstOrNull { it in parser.availableSortOrders } + ?: parser.availableSortOrders.first() + + private fun classify(source: MangaParserSource, error: Throwable): AuditStatus { + if (source.isBroken) return AuditStatus.EXPECTED_BROKEN + val message = generateSequence(error) { it.cause } + .joinToString(" | ") { "${it::class.simpleName}: ${it.message.orEmpty()}" } + val normalizedMessage = message.lowercase() + return when { + "requires an Android consumer runtime" in message || + "Browser is not available" in message -> AuditStatus.ANDROID_RUNTIME_REQUIRED + "AuthRequiredException" in message -> AuditStatus.AUTH_REQUIRED + "SocketTimeoutException" in message || "TimeoutCancellationException" in message -> AuditStatus.TIMEOUT + "source returned an empty first page" in normalizedMessage -> AuditStatus.EMPTY_LIST + "ssl" in normalizedMessage || "certificate" in normalizedMessage -> AuditStatus.TLS_ERROR + "HttpStatusException" in message || "NotFoundException" in message || "status=" in normalizedMessage -> + AuditStatus.HTTP_ERROR + "ParseException" in message || "JSONException" in message || "parse" in normalizedMessage -> + AuditStatus.PARSE_ERROR + "HttpException" in message || "UnknownHostException" in message || "StreamResetException" in message -> + AuditStatus.NETWORK_ERROR + else -> AuditStatus.ERROR + } + } + + private fun passed( + source: MangaParserSource, + stage: AuditStage, + startedAt: Long, + details: String, + ): AuditResult = result( + source = source, + status = if (source.isBroken) AuditStatus.BROKEN_MARKER_STALE else AuditStatus.PASSED, + stage = stage, + startedAt = startedAt, + details = details, + ) + + private fun result( + source: MangaParserSource, + status: AuditStatus, + stage: AuditStage, + startedAt: Long, + details: String, + ) = AuditResult( + source = source, + status = status, + stage = stage, + durationMs = (System.nanoTime() - startedAt) / 1_000_000, + details = details.singleLine(), + ) + + private fun writeReport(results: List<AuditResult>, targetStage: AuditStage, reportScope: String): Path { + val report = Path.of( + "build/reports/live-source-audit-${targetStage.name.lowercase()}-$reportScope.tsv", + ) + Files.createDirectories(report.parent) + val rows = buildString { + appendLine("source\ttitle\tlocale\tbroken\ttarget\tstatus\tstage\tduration_ms\tdetails") + results.forEach { result -> + appendLine( + listOf( + result.source.name, + result.source.title, + result.source.locale.ifEmpty { "all" }, + result.source.isBroken, + targetStage, + result.status, + result.stage, + result.durationMs, + result.details, + ).joinToString("\t"), + ) + } + } + Files.writeString( + report, + rows, + StandardOpenOption.CREATE, + StandardOpenOption.TRUNCATE_EXISTING, + StandardOpenOption.WRITE, + ) + return report + } + + private fun printSummary(results: List<AuditResult>, reportPath: Path) { + val counts = results.groupingBy { it.status }.eachCount().toSortedMap(compareBy { it.name }) + println("[LIVE-AUDIT] report=$reportPath") + println("[LIVE-AUDIT] total=${results.size}; ${counts.entries.joinToString { "${it.key}=${it.value}" }}") + } + + private fun String.singleLine(): String = replace('\t', ' ').replace('\r', ' ').replace('\n', ' ') + + private data class AuditResult( + val source: MangaParserSource, + val status: AuditStatus, + val stage: AuditStage, + val durationMs: Long, + val details: String, + ) { + fun consoleLine(): String = + "[LIVE-AUDIT] ${source.name}\t$status\t$stage\t${durationMs}ms\t$details" + } + + private enum class AuditStage { + LIST, + DETAILS, + CHAPTERS, + PAGES, + } + + private enum class AuditStatus(val isFailure: Boolean) { + PASSED(false), + EXPECTED_BROKEN(false), + BROKEN_MARKER_STALE(true), + NO_CHAPTERS(false), + ANDROID_RUNTIME_REQUIRED(false), + AUTH_REQUIRED(false), + TIMEOUT(true), + EMPTY_LIST(true), + HTTP_ERROR(true), + TLS_ERROR(true), + PARSE_ERROR(true), + NETWORK_ERROR(true), + ERROR(true), + } + + private companion object { + const val ENV_ENABLED = "LIVE_AUDIT" + const val ENV_STAGE = "LIVE_AUDIT_STAGE" + const val ENV_LOCALES = "LIVE_AUDIT_LOCALES" + const val ENV_INCLUDE_BROKEN = "LIVE_AUDIT_INCLUDE_BROKEN" + const val ENV_CONCURRENCY = "LIVE_AUDIT_CONCURRENCY" + const val ENV_TIMEOUT_SECONDS = "LIVE_AUDIT_TIMEOUT_SECONDS" + const val ENV_FAIL_ON_ERROR = "LIVE_AUDIT_FAIL_ON_ERROR" + const val MAX_CHAPTER_CANDIDATES = 5 + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveMangaLoaderContext.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveMangaLoaderContext.kt new file mode 100644 index 00000000..cede7046 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveMangaLoaderContext.kt @@ -0,0 +1,126 @@ +package org.koitharu.kotatsu.parsers + +import okhttp3.Cookie +import okhttp3.CookieJar +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.koitharu.kotatsu.parsers.bitmap.Bitmap +import org.koitharu.kotatsu.parsers.bitmap.Rect +import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.koitharu.kotatsu.parsers.config.MangaSourceConfig +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaSource +import java.util.concurrent.CopyOnWriteArrayList +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +@Suppress("OVERRIDE_DEPRECATION") +internal class LiveMangaLoaderContext : MangaLoaderContext() { + + private val storedCookies = CopyOnWriteArrayList<Cookie>() + private val parsers = ConcurrentHashMap<MangaSource, MangaParser>() + + override val cookieJar: CookieJar = object : CookieJar { + override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) { + cookies.forEach { cookie -> + storedCookies.removeIf { + it.name == cookie.name && it.domain == cookie.domain && it.path == cookie.path + } + if (cookie.expiresAt > System.currentTimeMillis()) { + storedCookies.add(cookie) + } + } + } + + override fun loadForRequest(url: HttpUrl): List<Cookie> { + storedCookies.removeIf { it.expiresAt <= System.currentTimeMillis() } + return storedCookies.filter { it.matches(url) } + } + } + + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .cookieJar(cookieJar) + .addInterceptor(::interceptSourceRequest) + .connectTimeout(30, TimeUnit.SECONDS) + .readTimeout(30, TimeUnit.SECONDS) + .writeTimeout(30, TimeUnit.SECONDS) + .callTimeout(60, TimeUnit.SECONDS) + .build() + + override fun newParserInstance(source: MangaSource): MangaParser { + require(source is MangaParserSource) { "Unsupported live source: $source" } + return parsers.computeIfAbsent(source) { source.newParser(this) } + } + + override suspend fun evaluateJs(script: String): String? { + error("JavaScript evaluation requires an Android consumer runtime") + } + + override suspend fun evaluateJs(baseUrl: String, script: String): String? { + if (script.startsWith("window.localStorage.getItem(")) { + return null + } + error("JavaScript evaluation requires an Android consumer runtime: $baseUrl") + } + + override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { + override fun <T> get(key: ConfigKey<T>): T = key.defaultValue + } + + override fun getDefaultUserAgent(): String = + "Mozilla/5.0 (Linux; Android 15; Pixel 8) AppleWebKit/537.36 " + + "(KHTML, like Gecko) Chrome/131.0 Mobile Safari/537.36" + + override fun redrawImageResponse( + response: Response, + redraw: (image: Bitmap) -> Bitmap, + ): Response = error("Image redraw requires an Android consumer runtime") + + override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { + override val width: Int = width + override val height: Int = height + + override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { + error("Bitmap drawing requires an Android consumer runtime") + } + } + + private fun interceptSourceRequest(chain: Interceptor.Chain): Response { + val request = chain.request() + val source = request.tag(MangaSource::class.java) + ?: return chain.proceed(request.withDefaultUserAgent()) + val parser = newParserInstance(source) + val headers = request.headers.newBuilder() + val parserHeaders = parser.getRequestHeaders() + parserHeaders.names().forEach { name -> + if (headers[name] == null) { + parserHeaders.values(name).forEach { value -> headers.add(name, value) } + } + } + if (headers["User-Agent"] == null) { + headers["User-Agent"] = getDefaultUserAgent() + } + if (headers["Referer"] == null) { + headers["Referer"] = "https://${parser.domain}/" + } + val taggedRequest = request.newBuilder().headers(headers.build()).build() + return parser.intercept(SourceRequestChain(chain, taggedRequest)) + } + + private fun Request.withDefaultUserAgent(): Request { + if (header("User-Agent") != null) { + return this + } + return newBuilder().header("User-Agent", getDefaultUserAgent()).build() + } + + private class SourceRequestChain( + private val delegate: Interceptor.Chain, + private val taggedRequest: Request, + ) : Interceptor.Chain by delegate { + override fun request(): Request = taggedRequest + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveSourceSmokeTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveSourceSmokeTest.kt new file mode 100644 index 00000000..e2a08030 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/LiveSourceSmokeTest.kt @@ -0,0 +1,80 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Assumptions.assumeTrue +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class LiveSourceSmokeTest { + + @Test + fun `run configured live source flow`() = runBlocking { + val sourceName = System.getenv(ENV_SOURCE) + assumeTrue(!sourceName.isNullOrBlank(), "$ENV_SOURCE is not configured") + + val source = MangaParserSource.valueOf(sourceName) + val stage = LiveStage.valueOf(System.getenv(ENV_STAGE)?.uppercase() ?: LiveStage.PAGES.name) + val order = SortOrder.valueOf(System.getenv(ENV_SORT)?.uppercase() ?: SortOrder.UPDATED.name) + val query = System.getenv(ENV_QUERY)?.takeIf(String::isNotBlank) + val expectedTitle = System.getenv(ENV_EXPECT_TITLE)?.takeIf(String::isNotBlank) + val context = LiveMangaLoaderContext() + val parser = context.newParserInstance(source) + val filter = MangaListFilter.EMPTY.copy(query = query) + + val list = liveStep(source, "list") { + parser.getList(offset = 0, order = order, filter = filter) + } + assertFalse(list.isEmpty(), "Live list is empty for ${source.name}") + val manga = expectedTitle?.let { title -> + list.firstOrNull { it.title.equals(title, ignoreCase = true) } + ?: error("Cannot find '$title' in ${list.size} live list entries") + } ?: list.first() + println("[LIVE-SOURCE] list ok: count=${list.size}, selected=${manga.title}, url=${manga.url}") + if (stage == LiveStage.LIST) return@runBlocking + + val details = liveStep(source, "details") { parser.getDetails(manga) } + println( + "[LIVE-SOURCE] details ok: title=${details.title}, " + + "chapters=${details.chapters.orEmpty().size}, description=${!details.description.isNullOrBlank()}", + ) + if (stage == LiveStage.DETAILS) return@runBlocking + + val chapter = details.chapters.orEmpty().firstOrNull() + ?: error("Live details contain no chapters for ${source.name}: ${details.title}") + println("[LIVE-SOURCE] chapter selected: title=${chapter.title}, url=${chapter.url}") + if (stage == LiveStage.CHAPTERS) return@runBlocking + + val pages = liveStep(source, "pages") { parser.getPages(chapter) } + assertFalse(pages.isEmpty(), "Live pages are empty for ${source.name}: ${chapter.title}") + val resolvedUrl = liveStep(source, "page-url") { parser.getPageUrl(pages.first()) } + println("[LIVE-SOURCE] pages ok: count=${pages.size}, first=$resolvedUrl") + } + + private suspend fun <T> liveStep( + source: MangaParserSource, + operation: String, + block: suspend () -> T, + ): T = try { + block() + } catch (error: Throwable) { + throw AssertionError("Live $operation failed for ${source.name}: ${error.message}", error) + } + + private enum class LiveStage { + LIST, + DETAILS, + CHAPTERS, + PAGES, + } + + private companion object { + const val ENV_SOURCE = "LIVE_SOURCE" + const val ENV_STAGE = "LIVE_STAGE" + const val ENV_SORT = "LIVE_SORT" + const val ENV_QUERY = "LIVE_QUERY" + const val ENV_EXPECT_TITLE = "LIVE_EXPECT_TITLE" + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaMammyContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaMammyContractTest.kt new file mode 100644 index 00000000..b517c23a --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaMammyContractTest.kt @@ -0,0 +1,36 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class MangaMammyContractTest { + + @Test + fun `source supports reading from list through pages`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .post( + "https://p.nimanga.com/wp-admin/admin-ajax.php", + "/fixtures/mangamammy/list.html", + ) + .get( + "https://p.nimanga.com/manga/example-title", + "/fixtures/mangamammy/details.html", + ) + .get( + "https://p.nimanga.com/chapter/example-title-1?style=list", + "/fixtures/mangamammy/pages.html", + ) + .build() + val parser = MangaParserSource.MANGAMAMMY.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.MANGAMAMMY, + expectedTitle = "Example Title", + sortOrder = SortOrder.UPDATED, + ) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaWtfContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaWtfContractTest.kt new file mode 100644 index 00000000..0bb49299 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/MangaWtfContractTest.kt @@ -0,0 +1,36 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.MangaTag +import org.koitharu.kotatsu.parsers.model.SortOrder + +class MangaWtfContractTest { + + @Test + fun `source sends excluded labels separately from included labels`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://api.inkstory.net/v2/books?page=0&size=20&type=COMIC&sort=updatedAt%2Cdesc&labelsInclude=action&labelsExclude=romance", + "/fixtures/mangawtf/list.json", + ) + .build() + val parser = MangaParserSource.MANGA_WTF.newParser(context) + val source = MangaParserSource.MANGA_WTF + + val manga = parser.getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter.EMPTY.copy( + tags = setOf(MangaTag("Action", "action", source)), + tagsExclude = setOf(MangaTag("Romance", "romance", source)), + ), + ).single() + + assertEquals("Test title", manga.title) + context.verifyAllFixturesUsed() + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/NineMangaContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/NineMangaContractTest.kt new file mode 100644 index 00000000..7677c6fd --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/NineMangaContractTest.kt @@ -0,0 +1,31 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class NineMangaContractTest { + + @Test + fun `reads catalog details chapters and pages from current Niadd markup`() = runBlocking { + val context = FixtureMangaLoaderContext.builder() + .get("https://ru.niadd.com/category/index_1", "/fixtures/ninemanga/catalog.html") + .get("https://ru.niadd.com/manga/example.html", "/fixtures/ninemanga/details.html") + .get("https://ru.niadd.com/manga/example/chapters.html", "/fixtures/ninemanga/chapters.html") + .get("https://ru.niadd.com/search/?type=high", "/fixtures/ninemanga/tags.html") + .get("https://ru.niadd.com/chapter/1_1/123/", "/fixtures/ninemanga/pages.html") + .get("https://ru.niadd.com/chapter/1_1/123-1.html", "/fixtures/ninemanga/page.html") + .build() + val parser = MangaParserSource.NINEMANGA_RU.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.NINEMANGA_RU, + expectedTitle = "Example Manga", + sortOrder = SortOrder.POPULARITY, + expectedPageUrl = "https://cdn.niadd.test/example-001.webp", + ) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/ReadMangaContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/ReadMangaContractTest.kt new file mode 100644 index 00000000..36837c4e --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/ReadMangaContractTest.kt @@ -0,0 +1,53 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class ReadMangaContractTest { + + @Test + fun `source supports title search through current details and pages markup`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://a.zazaza.me/search/advancedResults?q=Artifact&offset=0&years=1900%2C2099&sortType=DATE_CREATE", + "/fixtures/readmanga/search.html", + ) + .get( + "https://a.zazaza.me/igrok__kotoryi_est_artefakty", + "/fixtures/readmanga/details.html", + ) + .get( + "https://a.zazaza.me/igrok__kotoryi_est_artefakty/vol1/45?mtr=1", + "/fixtures/readmanga/pages.html", + ) + .build() + val parser = MangaParserSource.READMANGA_RU.newParser(context) + + val manga = parser.getList( + offset = 0, + order = SortOrder.NEWEST, + filter = MangaListFilter.EMPTY.copy(query = "Artifact"), + ).single() + assertEquals("Игрок, который ест артефакты", manga.title) + + val details = parser.getDetails(manga) + assertEquals("Игрок, который ест артефакты", details.title) + assertEquals("Current ReadManga description.", details.description) + assertEquals(setOf("Боевик", "Драма"), details.tags.mapTo(linkedSetOf()) { it.title }) + assertEquals("https://rm.one-way.work/uploads/pics/example.jpg", details.largeCoverUrl) + + val chapter = details.chapters.orEmpty().single() + assertEquals(45f, chapter.number) + assertEquals(1, chapter.volume) + + val pages = parser.getPages(chapter) + assertFalse(pages.isEmpty()) + assertEquals("https://images.example/page-01.jpg", parser.getPageUrl(pages.single())) + context.verifyAllFixturesUsed() + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/RussianSourceCatalogTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/RussianSourceCatalogTest.kt new file mode 100644 index 00000000..6e35707d --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/RussianSourceCatalogTest.kt @@ -0,0 +1,160 @@ +package org.koitharu.kotatsu.parsers + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource + +class RussianSourceCatalogTest { + + @Test + fun `catalog exposes Russian NineManga successor`() { + assertTrue( + MangaParserSource.entries.any { + it.name == "NINEMANGA_RU" && it.locale == "ru" + }, + "NINEMANGA_RU must be present in the generated Russian catalog", + ) + } + + @Test + fun `NineManga Russian parser uses the live successor domain`() { + val parser = MangaParserSource.NINEMANGA_RU.newParser(FixtureMangaLoaderContext.builder().build()) + + assertEquals("ru.niadd.com", parser.domain) + } + + @Test + fun `catalog exposes Russian Madara sources`() { + val expectedSources = setOf( + "BEST_MANGA", + "MANGAMAMMY", + "MANGAONELOVE", + "MANGAZAVR", + ) + + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertTrue( + actualSources.containsAll(expectedSources), + "Missing Russian Madara sources: ${expectedSources - actualSources}", + ) + } + + @Test + fun `catalog exposes Russian Chan sources`() { + val expectedSources = setOf("HENCHAN", "MANGACHAN", "YAOICHAN") + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertTrue( + actualSources.containsAll(expectedSources), + "Missing Russian Chan sources: ${expectedSources - actualSources}", + ) + } + + @Test + fun `catalog exposes Russian Grouple sources`() { + val expectedSources = setOf( + "ALLHENTAI", + "MINTMANGA", + "READMANGA_RU", + "SEIMANGA", + "SELFMANGA", + "USAGI", + ) + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertTrue( + actualSources.containsAll(expectedSources), + "Missing Russian Grouple sources: ${expectedSources - actualSources}", + ) + } + + @Test + fun `catalog exposes Russian LibSocial sources`() { + val expectedSources = setOf("HENTAILIB", "MANGALIB", "YAOILIB") + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertTrue( + actualSources.containsAll(expectedSources), + "Missing Russian LibSocial sources: ${expectedSources - actualSources}", + ) + } + + @Test + fun `catalog exposes independent Russian sources`() { + val expectedSources = setOf( + "ACOMICS", + "COMX", + "DESUME", + "MANGA_WTF", + "NUDEMOON", + "REMANGA", + "WAMANGA", + "ZENMANGA", + ) + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertTrue( + actualSources.containsAll(expectedSources), + "Missing independent Russian sources: ${expectedSources - actualSources}", + ) + } + + @Test + fun `catalog exposes the complete Russian source set`() { + val expectedSources = setOf( + "ACOMICS", + "ALLHENTAI", + "BEST_MANGA", + "COMX", + "DESUME", + "HENCHAN", + "HENTAILIB", + "MANGABALL_RU", + "MANGACHAN", + "MANGALIB", + "MANGAMAMMY", + "MANGAONELOVE", + "MANGAPLUSPARSER_RU", + "MANGAZAVR", + "MANGA_WTF", + "MINTMANGA", + "NINEMANGA_RU", + "NUDEMOON", + "READMANGA_RU", + "REMANGA", + "SEIMANGA", + "SELFMANGA", + "USAGI", + "WAMANGA", + "YAOICHAN", + "YAOILIB", + "ZENMANGA", + ) + val actualSources = MangaParserSource.entries + .filter { it.locale == "ru" } + .mapTo(mutableSetOf()) { it.name } + + assertEquals(expectedSources, actualSources) + } + + @Test + fun `only currently unavailable Russian sources remain marked broken`() { + val brokenSources = MangaParserSource.entries + .filter { it.locale == "ru" && it.isBroken } + .mapTo(mutableSetOf()) { it.name } + + assertEquals(setOf("BEST_MANGA", "MANGAZAVR"), brokenSources) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/SourceReadingContract.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/SourceReadingContract.kt new file mode 100644 index 00000000..4ff7ebd2 --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/SourceReadingContract.kt @@ -0,0 +1,43 @@ +package org.koitharu.kotatsu.parsers + +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.koitharu.kotatsu.parsers.model.MangaListFilter +import org.koitharu.kotatsu.parsers.model.MangaSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +internal suspend fun verifySourceReadingContract( + parser: MangaParser, + context: FixtureMangaLoaderContext, + expectedSource: MangaSource, + expectedTitle: String, + sortOrder: SortOrder, + expectedPageUrl: String? = null, +) { + val manga = parser.getList( + offset = 0, + order = sortOrder, + filter = MangaListFilter.EMPTY, + ).single() + + assertEquals(expectedSource, manga.source) + assertEquals(expectedTitle, manga.title) + + val details = parser.getDetails(manga) + + assertEquals(manga.id, details.id) + assertEquals(manga.url, details.url) + assertEquals(manga.source, details.source) + + val chapter = details.chapters.orEmpty().single() + assertEquals(details.source, chapter.source) + + val pages = parser.getPages(chapter) + + assertFalse(pages.isEmpty()) + assertEquals(details.source, pages.single().source) + if (expectedPageUrl != null) { + assertEquals(expectedPageUrl, parser.getPageUrl(pages.single())) + } + context.verifyAllFixturesUsed() +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/WaMangaContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/WaMangaContractTest.kt new file mode 100644 index 00000000..1f09914d --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/WaMangaContractTest.kt @@ -0,0 +1,53 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.test.runTest +import org.json.JSONObject +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder +import org.koitharu.kotatsu.parsers.site.kotatsu.ru.parseWaMangaAlternativeTitles +import org.koitharu.kotatsu.parsers.site.kotatsu.ru.parseWaMangaGenres +import org.koitharu.kotatsu.parsers.site.kotatsu.ru.parseWaMangaSearchText + +class WaMangaContractTest { + + @Test + fun `source supports current v1 API from catalog through pages`() = runTest { + val context = FixtureMangaLoaderContext.builder() + .get("https://wamanga.ru/api/v1/manga", "/fixtures/wamanga/catalog.json") + .get("https://wamanga.ru/api/v1/manga/manga-id", "/fixtures/wamanga/details.json") + .get("https://wamanga.ru/api/v1/manga/manga-id/chapters", "/fixtures/wamanga/chapters.json") + .get("https://wamanga.ru/api/v1/chapters/chapter-id", "/fixtures/wamanga/pages.json") + .build() + val parser = MangaParserSource.WAMANGA.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.WAMANGA, + expectedTitle = "Example Manga", + sortOrder = SortOrder.ALPHABETICAL, + expectedPageUrl = "https://wamanga.ru/app/uploads/example/001.webp", + ) + } + + @Test + fun `catalog search contract includes all current title fields`() { + val item = JSONObject( + """{ + "title":"Russian title", + "titleEnglish":"English title", + "alternateTitles":["Original title"], + "genres":["Action","Drama"] + }""", + ) + + assertEquals(setOf("English title", "Original title"), item.parseWaMangaAlternativeTitles()) + assertEquals(setOf("Action", "Drama"), item.parseWaMangaGenres()) + assertEquals( + setOf("Russian title", "English title", "Original title"), + item.parseWaMangaSearchText(), + ) + } +} diff --git a/src/test/kotlin/org/koitharu/kotatsu/parsers/ZenMangaContractTest.kt b/src/test/kotlin/org/koitharu/kotatsu/parsers/ZenMangaContractTest.kt new file mode 100644 index 00000000..e3fb294c --- /dev/null +++ b/src/test/kotlin/org/koitharu/kotatsu/parsers/ZenMangaContractTest.kt @@ -0,0 +1,39 @@ +package org.koitharu.kotatsu.parsers + +import kotlinx.coroutines.runBlocking +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Test +import org.koitharu.kotatsu.parsers.model.MangaParserSource +import org.koitharu.kotatsu.parsers.model.SortOrder + +class ZenMangaContractTest { + + @Test + fun `uses the current InkStory domain`() { + val parser = MangaParserSource.ZENMANGA.newParser(FixtureMangaLoaderContext.builder().build()) + + assertEquals("inkstory.net", parser.domain) + } + + @Test + fun `reads catalog details chapters and pages from InkStory contracts`() = runBlocking { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://api.inkstory.net/v2/books?page=0&size=30&sort=viewsCount%2Cdesc", + "/fixtures/zenmanga/catalog.json", + ) + .get("https://inkstory.net/content/example", "/fixtures/zenmanga/details.html") + .get("https://inkstory.net/content/example/chapter-1", "/fixtures/zenmanga/pages.html") + .build() + val parser = MangaParserSource.ZENMANGA.newParser(context) + + verifySourceReadingContract( + parser = parser, + context = context, + expectedSource = MangaParserSource.ZENMANGA, + expectedTitle = "Example Manga", + sortOrder = SortOrder.POPULARITY, + expectedPageUrl = "https://cdn.inkstory.test/example-001.jpg?format=webp&width=1600", + ) + } +} diff --git a/src/test/resources/fixtures/acomics/catalog.html b/src/test/resources/fixtures/acomics/catalog.html new file mode 100644 index 00000000..4a04bd84 --- /dev/null +++ b/src/test/resources/fixtures/acomics/catalog.html @@ -0,0 +1,7 @@ +<html><body><div class="infinite-scroll-content"> + <section class="serial-card"> + <a href="/~example" class="cover"><img src="/static/loading.svg" data-real-src="/upload/example-cover.jpg"></a> + <h2 class="title"><a href="/~example">Example Comic</a></h2> + <p class="about">Catalog summary</p> + </section> +</div></body></html> diff --git a/src/test/resources/fixtures/acomics/details.html b/src/test/resources/fixtures/acomics/details.html new file mode 100644 index 00000000..8da43cae --- /dev/null +++ b/src/test/resources/fixtures/acomics/details.html @@ -0,0 +1,6 @@ +<html><body> +<h1>Example Comic</h1> +<p class="serial-about-badges"><a class="badge category">Drama</a></p> +<section class="serial-about-text">Details summary</section> +<p class="serial-about-authors"><b>Author:</b> <a>Example Author</a></p> +</body></html> diff --git a/src/test/resources/fixtures/acomics/page.html b/src/test/resources/fixtures/acomics/page.html new file mode 100644 index 00000000..b8af1cf5 --- /dev/null +++ b/src/test/resources/fixtures/acomics/page.html @@ -0,0 +1,4 @@ +<html><body> +<section class="reader-issue"><img class="issue" src="/upload/example-001.jpg"></section> +<nav class="reader-navigator" data-issue-count="1"></nav> +</body></html> diff --git a/src/test/resources/fixtures/acomics/tags.html b/src/test/resources/fixtures/acomics/tags.html new file mode 100644 index 00000000..4f3240b4 --- /dev/null +++ b/src/test/resources/fixtures/acomics/tags.html @@ -0,0 +1 @@ +<html><body><form class="catalog-filters-form"><fieldset class="categories"><label><input type="checkbox" name="categories[]" value="1">Drama</label></fieldset></form></body></html> diff --git a/src/test/resources/fixtures/allhentai/details.html b/src/test/resources/fixtures/allhentai/details.html new file mode 100644 index 00000000..43a53966 --- /dev/null +++ b/src/test/resources/fixtures/allhentai/details.html @@ -0,0 +1,7 @@ +<html><head><meta itemprop="name" content="Example Hentai"></head><body><div id="mangaBox"> + <div class="cr-description"><div class="cr-description__content">Details summary.</div></div> + <div class="cr-tags"><a class="cr-tags__item" href="/list/genre/drama"><span>drama</span></a></div> + <div id="chapters-list"><table><tr data-vol="1" data-num="10"> + <td><a class="chapter-link" href="/example/vol1/1">Chapter 1</a></td><td class="date">16.08.26</td> + </tr></table></div> +</div></body></html> diff --git a/src/test/resources/fixtures/allhentai/pages.html b/src/test/resources/fixtures/allhentai/pages.html new file mode 100644 index 00000000..31d5e5eb --- /dev/null +++ b/src/test/resources/fixtures/allhentai/pages.html @@ -0,0 +1,4 @@ +<html><body><script> +var chapterInfo = {}; +rm_h.readerInit(chapterInfo, [["https://images.allhentai.test/", "", "page-01.jpg", 800, 1200, ""]], false); +</script></body></html> diff --git a/src/test/resources/fixtures/allhentai/search.html b/src/test/resources/fixtures/allhentai/search.html new file mode 100644 index 00000000..e3003ada --- /dev/null +++ b/src/test/resources/fixtures/allhentai/search.html @@ -0,0 +1,4 @@ +<html><body><div class="tiles row"><div class="tile"> + <div class="img"><a href="/example"><img data-original="https://images.allhentai.test/cover.jpg"></a></div> + <div class="desc"><h3><a href="/example">Example Hentai</a></h3><h5>Example Alternative</h5></div> +</div></div></body></html> diff --git a/src/test/resources/fixtures/desu/catalog.html b/src/test/resources/fixtures/desu/catalog.html new file mode 100644 index 00000000..a6aa1f25 --- /dev/null +++ b/src/test/resources/fixtures/desu/catalog.html @@ -0,0 +1,12 @@ +<html><body> +<ol class="section memberList"> + <li class="primaryContent memberListItem"> + <a href="manga/example.42/" class="avatar"><span class="img" style="background-image: url('https://static.desu.uno/example.jpg');"></span></a> + <div> + <h3><a href="manga/example.42/" class="animeTitle oTitle">Example Manga</a></h3> + <div class="dimmed oTitle"><span itemprop="title">Пример манги</span></div> + <div class="animeInfo"><dl><dt>Рейтинг:</dt><dd>8.5</dd></dl></div> + </div> + </li> +</ol> +</body></html> diff --git a/src/test/resources/fixtures/desu/details.html b/src/test/resources/fixtures/desu/details.html new file mode 100644 index 00000000..a2e49392 --- /dev/null +++ b/src/test/resources/fixtures/desu/details.html @@ -0,0 +1,15 @@ +<html><body> +<div id="animeView"> + <link href="https://desu.uno/manga/example.42/" itemprop="url"> + <meta content="Пример манги" itemprop="headline"> + <meta content="Example Manga" itemprop="alternativeHeadline"> + <img src="https://static.desu.uno/example-large.jpg" itemprop="image"> + <span class="b-anime_status_tag ongoing">выходит</span> + <meta content="8.5" itemprop="ratingValue"> + <a itemprop="genre" href="manga/?genres=56-Action">Экшен</a> + <div id="description"><div class="russian">Current description</div></div> + <ul class="chlist"> + <li><h4><a href="manga/example.42/vol2/ch3/rus">Том 2. Глава 3<span class="title"> - Chapter title</span></a></h4></li> + </ul> +</div> +</body></html> diff --git a/src/test/resources/fixtures/desu/pages.html b/src/test/resources/fixtures/desu/pages.html new file mode 100644 index 00000000..434c1a1f --- /dev/null +++ b/src/test/resources/fixtures/desu/pages.html @@ -0,0 +1,3 @@ +<html><body><script> +window.MangaReader = {"manga":{"id":42},"chapter":{"id":1003},"apiBaseUrl":"/api/manga/42"}; +</script></body></html> diff --git a/src/test/resources/fixtures/desu/pages.json b/src/test/resources/fixtures/desu/pages.json new file mode 100644 index 00000000..d3e4fde5 --- /dev/null +++ b/src/test/resources/fixtures/desu/pages.json @@ -0,0 +1,13 @@ +{ + "chapter": { + "id": 1003, + "pages": [ + { + "page": 1, + "url": "https://img3.desu.uno/manga/rus/example/001.webp?1003", + "width": 1000, + "height": 1400 + } + ] + } +} diff --git a/src/test/resources/fixtures/mangamammy/details.html b/src/test/resources/fixtures/mangamammy/details.html new file mode 100644 index 00000000..c9a08bb8 --- /dev/null +++ b/src/test/resources/fixtures/mangamammy/details.html @@ -0,0 +1,20 @@ +<!doctype html> +<html lang="ru"> +<head> + <meta property="og:url" content="https://p.nimanga.com/manga/example-title" /> +</head> +<body> +<h1>Example Title</h1> +<div class="description-summary"><div class="summary__content">Fixture description.</div></div> +<div class="genres-content"><a href="https://p.nimanga.com/manga-genre/fantasy/">Fantasy</a></div> +<div class="post-content_item"><h5>Статус</h5><div class="summary-content">Продолжается</div></div> +<div class="listing-chapters_wrap"> + <ul> + <li class="wp-manga-chapter"> + <a href="https://p.nimanga.com/chapter/example-title-1">Chapter 1</a> + <span class="chapter-release-date"><i>31.07.2026</i></span> + </li> + </ul> +</div> +</body> +</html> diff --git a/src/test/resources/fixtures/mangamammy/list.html b/src/test/resources/fixtures/mangamammy/list.html new file mode 100644 index 00000000..31dd039c --- /dev/null +++ b/src/test/resources/fixtures/mangamammy/list.html @@ -0,0 +1,16 @@ +<!doctype html> +<html lang="ru"> +<body> +<div class="row c-tabs-item__content"> + <a href="https://p.nimanga.com/manga/example-title"> + <img src="https://p.nimanga.com/covers/example-title.jpg" alt="Example Title" /> + </a> + <div class="tab-summary"> + <div class="post-title"><h3>Example Title</h3></div> + <div class="mg_author"><a href="/author/example-author">Example Author</a></div> + <div class="mg_status"><div class="summary-content">Продолжается</div></div> + <div class="mg_genres"><a href="/manga-genre/fantasy/">Fantasy</a></div> + </div> +</div> +</body> +</html> diff --git a/src/test/resources/fixtures/mangamammy/pages.html b/src/test/resources/fixtures/mangamammy/pages.html new file mode 100644 index 00000000..d21791f8 --- /dev/null +++ b/src/test/resources/fixtures/mangamammy/pages.html @@ -0,0 +1,10 @@ +<!doctype html> +<html lang="ru"> +<body> +<div class="main-col-inner"> + <div class="reading-content"> + <div class="page-break"><img src="https://cdn.mangamammy.ru/pages/example-title-1-001.jpg" /></div> + </div> +</div> +</body> +</html> diff --git a/src/test/resources/fixtures/mangawtf/list.json b/src/test/resources/fixtures/mangawtf/list.json new file mode 100644 index 00000000..5a00f9d7 --- /dev/null +++ b/src/test/resources/fixtures/mangawtf/list.json @@ -0,0 +1,14 @@ +[ + { + "id": "book-id", + "slug": "test-title", + "poster": "https://static.inkstory.net/test-title.jpg", + "status": "ONGOING", + "contentStatus": "SAFE", + "name": { + "ru": "Test title", + "en": "Test title" + }, + "averageRating": 8.5 + } +] diff --git a/src/test/resources/fixtures/ninemanga/catalog.html b/src/test/resources/fixtures/ninemanga/catalog.html new file mode 100644 index 00000000..3d055200 --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/catalog.html @@ -0,0 +1,10 @@ +<html><body> +<div class="manga-list"> + <div class="manga-item"> + <a href="https://ru.niadd.com/manga/example.html"><div class="manga-img"><img src="https://cdn.niadd.test/cover.jpg"></div></a> + <a href="https://ru.niadd.com/manga/example.html"><div class="manga-name">Example Manga</div></a> + <div class="manga-intro">Catalog summary</div> + <div class="manga-cate-item"><a href="/category/action.html">Action</a></div> + </div> +</div> +</body></html> diff --git a/src/test/resources/fixtures/ninemanga/chapters.html b/src/test/resources/fixtures/ninemanga/chapters.html new file mode 100644 index 00000000..69559513 --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/chapters.html @@ -0,0 +1,5 @@ +<html><body> +<ul class="chapter-list"> + <a href="https://ru.niadd.com/chapter/1_1/123/"><li class="chp-item"><span class="chp-title">Example Manga Том 1 Глава 1</span><span class="chp-time">Aug 16, 2026</span></li></a> +</ul> +</body></html> diff --git a/src/test/resources/fixtures/ninemanga/details.html b/src/test/resources/fixtures/ninemanga/details.html new file mode 100644 index 00000000..e0bdbd9d --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/details.html @@ -0,0 +1,10 @@ +<html><body> +<div class="book-headline"><h1 itemprop="name" class="book-headline-name">Example Manga</h1><span class="book-status">(постоянный)</span></div> +<div class="book-info-parts"> + <div class="bookside-img" style="background-image: url('https://cdn.niadd.test/cover.jpg')"><img itemprop="image" src="https://cdn.niadd.test/cover.jpg"></div> + <div class="bookside-big-rate">4.5</div> + <div class="bookside-general-cell"><span>Жанры: </span><a href="/category/action.html"><span itemprop="genre">Action</span></a></div> + <div class="bookside-general-cell" itemprop="author"><span>Авторы): </span><a><span itemprop="name">Example Author</span></a></div> + <div class="detail-section-box"><div class="detail-cate-title">конспект</div><section class="detail-section detail-synopsis">Details summary</section></div> +</div> +</body></html> diff --git a/src/test/resources/fixtures/ninemanga/page.html b/src/test/resources/fixtures/ninemanga/page.html new file mode 100644 index 00000000..573ec5dc --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/page.html @@ -0,0 +1 @@ +<html><body><section class="mangaread-img"><img class="manga_pic" src="https://cdn.niadd.test/example-001.webp"></section></body></html> diff --git a/src/test/resources/fixtures/ninemanga/pages.html b/src/test/resources/fixtures/ninemanga/pages.html new file mode 100644 index 00000000..f98ad034 --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/pages.html @@ -0,0 +1 @@ +<html><body><div class="chp-page-trigger" option_val="https://ru.niadd.com/chapter/1_1/123-1.html">1/1</div></body></html> diff --git a/src/test/resources/fixtures/ninemanga/tags.html b/src/test/resources/fixtures/ninemanga/tags.html new file mode 100644 index 00000000..2656d7cf --- /dev/null +++ b/src/test/resources/fixtures/ninemanga/tags.html @@ -0,0 +1 @@ +<html><body><ul><li class="genre-item" rel="Action"><a onclick="return clickGenre(this, '42');">Action</a></li></ul></body></html> diff --git a/src/test/resources/fixtures/readmanga/details.html b/src/test/resources/fixtures/readmanga/details.html new file mode 100644 index 00000000..437067b0 --- /dev/null +++ b/src/test/resources/fixtures/readmanga/details.html @@ -0,0 +1,37 @@ +<!doctype html> +<html> +<head> + <meta itemprop="name" content="Игрок, который ест артефакты"> + <meta itemprop="alternativeHeadline" content="Artifact-Devouring Player"> +</head> +<body> +<div id="mangaBox"> + <div class="cr-hero"> + <img class="cr-hero-poster__img" src="https://rm.one-way.work/uploads/pics/example.jpg" alt="Игрок, который ест артефакты"> + <div class="cr-hero-names"> + <h1 class="cr-hero-names__main">Игрок, который ест артефакты</h1> + <h3 class="cr-hero-names__alt"> + <span><span>Artifact-Devouring Player</span></span> + </h3> + </div> + </div> + <div class="cr-description"> + <div class="cr-description__content">Current ReadManga description.</div> + </div> + <div class="creation-element-tags"> + <a class="creation-element-tags__item" href="/list/genre/action"><span>боевик</span></a> + <a class="creation-element-tags__item" href="/list/genre/drama"><span>драма</span></a> + </div> + <div id="chapters-list"> + <table> + <tr class="item-row" data-vol="1" data-num="450"> + <td class="item-title"> + <a class="chapter-link" href="/igrok__kotoryi_est_artefakty/vol1/45">1 - 45</a> + </td> + <td class="date">08.07.26</td> + </tr> + </table> + </div> +</div> +</body> +</html> diff --git a/src/test/resources/fixtures/readmanga/pages.html b/src/test/resources/fixtures/readmanga/pages.html new file mode 100644 index 00000000..b5b41c48 --- /dev/null +++ b/src/test/resources/fixtures/readmanga/pages.html @@ -0,0 +1,9 @@ +<!doctype html> +<html> +<body> +<script> + var chapterInfo = {}; + rm_h.readerInit(chapterInfo, [["https://images.example/", "", "page-01.jpg", 800, 1200, ""]], false); +</script> +</body> +</html> diff --git a/src/test/resources/fixtures/readmanga/search.html b/src/test/resources/fixtures/readmanga/search.html new file mode 100644 index 00000000..1c812ae6 --- /dev/null +++ b/src/test/resources/fixtures/readmanga/search.html @@ -0,0 +1,21 @@ +<!doctype html> +<html> +<body> +<div class="tiles row"> + <div class="tile"> + <div class="img"> + <a href="/igrok__kotoryi_est_artefakty"> + <img class="lazy" data-original="https://rm.one-way.work/uploads/pics/example_p.jpg" alt="Игрок, который ест артефакты"> + </a> + </div> + <div class="desc"> + <h3><a href="/igrok__kotoryi_est_artefakty">Игрок, который ест артефакты</a></h3> + <h5>Artifact-Devouring Player</h5> + <div class="tile-info"> + <a class="element-link" href="/list/genre/action">боевик</a> + </div> + </div> + </div> +</div> +</body> +</html> diff --git a/src/test/resources/fixtures/wamanga/catalog.json b/src/test/resources/fixtures/wamanga/catalog.json new file mode 100644 index 00000000..8ee0a3bf --- /dev/null +++ b/src/test/resources/fixtures/wamanga/catalog.json @@ -0,0 +1,26 @@ +[ + { + "id": "manga-id", + "slug": "example-manga", + "year": 2026, + "title": "Example Manga", + "titleEnglish": "Example Manga EN", + "alternateTitles": ["Example Manga JP"], + "genres": ["Action", "Drama"], + "description": "Current API description", + "authors": ["Example Author"], + "artists": [], + "publishers": [], + "imageUrl": "/app/uploads/example/banner.webp", + "coverUrl": "/app/uploads/example/cover.webp", + "updatedAt": "2026-07-31T12:00:00.000Z", + "createdAt": "2026-07-30T12:00:00.000Z", + "hidden": false, + "licensed": false, + "isAdult": false, + "type": "manga", + "statusTitle": "ongoing", + "translationStatus": "ongoing", + "pegiRating": "16+" + } +] diff --git a/src/test/resources/fixtures/wamanga/chapters.json b/src/test/resources/fixtures/wamanga/chapters.json new file mode 100644 index 00000000..5cf38900 --- /dev/null +++ b/src/test/resources/fixtures/wamanga/chapters.json @@ -0,0 +1,15 @@ +[ + { + "id": "chapter-id", + "title": "", + "slug": "", + "position": 1, + "isProcessing": false, + "totalFilesExpected": 1, + "processedFilesCount": 1, + "views": 10, + "createdAt": "2026-07-31T12:00:00.000Z", + "updatedAt": "2026-07-31T12:00:00.000Z", + "likesCount": 0 + } +] diff --git a/src/test/resources/fixtures/wamanga/details.json b/src/test/resources/fixtures/wamanga/details.json new file mode 100644 index 00000000..62fb45d2 --- /dev/null +++ b/src/test/resources/fixtures/wamanga/details.json @@ -0,0 +1,25 @@ +{ + "id": "manga-id", + "slug": "example-manga", + "year": 2026, + "title": "Example Manga", + "titleEnglish": "Example Manga EN", + "alternateTitles": ["Example Manga JP"], + "genres": ["Action", "Drama"], + "description": "Current API description", + "authors": ["Example Author"], + "artists": [], + "publishers": [], + "imageUrl": "/app/uploads/example/banner.webp", + "coverUrl": "/app/uploads/example/cover.webp", + "updatedAt": "2026-07-31T12:00:00.000Z", + "createdAt": "2026-07-30T12:00:00.000Z", + "hidden": false, + "licensed": false, + "isAdult": false, + "type": "manga", + "statusTitle": "ongoing", + "translationStatus": "ongoing", + "pegiRating": "16+", + "teams": [{"id": "team-id", "name": "Wa"}] +} diff --git a/src/test/resources/fixtures/wamanga/pages.json b/src/test/resources/fixtures/wamanga/pages.json new file mode 100644 index 00000000..628442d0 --- /dev/null +++ b/src/test/resources/fixtures/wamanga/pages.json @@ -0,0 +1,25 @@ +{ + "id": "chapter-id", + "title": "", + "slug": "", + "position": 1, + "isProcessing": false, + "totalFilesExpected": 1, + "processedFilesCount": 1, + "createdAt": "2026-07-31T12:00:00.000Z", + "updatedAt": "2026-07-31T12:00:00.000Z", + "files": [ + { + "id": "file-id", + "diskFile": "/app/uploads/example/001.webp", + "position": "1.000000000" + } + ], + "manga": { + "id": "manga-id", + "slug": "example-manga", + "title": "Example Manga", + "type": "manga" + }, + "likesCount": 0 +} diff --git a/src/test/resources/fixtures/zenmanga/catalog.json b/src/test/resources/fixtures/zenmanga/catalog.json new file mode 100644 index 00000000..9cb18662 --- /dev/null +++ b/src/test/resources/fixtures/zenmanga/catalog.json @@ -0,0 +1,11 @@ +[ + { + "slug": "example", + "name": {"ru": "Example Manga", "en": "Example Manga"}, + "altNames": [{"name": "Example Alternative"}], + "poster": "https://cdn.inkstory.test/example-cover.jpg", + "averageRating": 9.2, + "status": "ONGOING", + "contentStatus": "SAFE" + } +] diff --git a/src/test/resources/fixtures/zenmanga/details.html b/src/test/resources/fixtures/zenmanga/details.html new file mode 100644 index 00000000..26ff07d6 --- /dev/null +++ b/src/test/resources/fixtures/zenmanga/details.html @@ -0,0 +1,10 @@ +<html><body><script id="it-astro-state" type="application/json">[ + {"@inox-tools/request-nanostores":1}, + {"current-book":2,"current-book-branches":3,"current-book-chapters":4}, + {"description":"Details summary","labels":5,"relations":6}, + [], + [7], + [{"name":"Action","slug":"action"}], + [{"type":"AUTHOR","publisher":{"name":"Example Author"}}], + {"id":"chapter-1","branchId":"branch-1","name":"Chapter 1","number":"1","volume":"1","createdAt":"2026-08-16T00:00:00.000Z"} +]</script></body></html> diff --git a/src/test/resources/fixtures/zenmanga/pages.html b/src/test/resources/fixtures/zenmanga/pages.html new file mode 100644 index 00000000..1858c940 --- /dev/null +++ b/src/test/resources/fixtures/zenmanga/pages.html @@ -0,0 +1,7 @@ +<html><body><script id="it-astro-state" type="application/json">[ + {"@inox-tools/request-nanostores":1}, + {"reader-current-chapter":2}, + {"pages":3}, + [4], + {"id":"page-1","index":"0","image":"https://cdn.inkstory.test/example-001.jpg?format=webp"} +]</script></body></html> From 2e4c5562f8d397c836fde87a6145eb5a9ba2743a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:41:27 +0200 Subject: [PATCH 328/434] fix mangadotnet filter --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 67 ++++++++++++++----- 1 file changed, 49 insertions(+), 18 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 599e69cd..23bea490 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -4,6 +4,7 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException import tsuki.model.ContentType import tsuki.model.Demographic @@ -35,7 +36,7 @@ import java.util.EnumSet import java.util.Locale import java.util.TimeZone -@MangaSourceParser("MANGADOTNET", "Mangadot.net", "en", ContentType.MANGA) +@MangaSourceParser("MANGADOTNET", "Mangadot.net", type = ContentType.MANGA) internal class Mangadotnet(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANGADOTNET, 20) { @@ -82,11 +83,42 @@ internal class Mangadotnet(context: MangaLoaderContext) : } @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscData(url: String, route: String): Map<String, Any?>? { + private suspend fun fetchRscData(url: String, route: String): Map<String, Any?> { val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) ?: return null - val routeValue = (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> ?: return null - return routeValue.asMap("data") + val decoded = decodeRsc(flat) + ?: throw ParseException("Failed to decode RSC data", url) + val routeValue = (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> + ?: throw ParseException("Missing RSC route '$route'", url) + val mangaListKeys = listOf("manga_list", "results", "rows") + for (key in mangaListKeys) { + routeValue[key]?.let { found -> + return mapOf(key to found) + } + } + fun searchRecursively(obj: Any?, depth: Int = 0): Any? { + if (depth > 5) return null + when (obj) { + is Map<*, *> -> { + for (key in mangaListKeys) { + obj[key]?.let { return it } + } + for (value in obj.values) { + val result = searchRecursively(value, depth + 1) + if (result != null) return result + } + } + is List<*> -> { + for (item in obj) { + val result = searchRecursively(item, depth + 1) + if (result != null) return result + } + } + } + return null + } + val found = searchRecursively(routeValue) + ?: throw ParseException("Manga list not found in route '$route'", url) + return mapOf("manga_list" to found) } override val filterCapabilities = MangaListFilterCapabilities( @@ -203,19 +235,19 @@ internal class Mangadotnet(context: MangaLoaderContext) : addQueryParameter("_routes", "pages/SearchPage") }.build().toString() - val dataMap = fetchRscData(url, "pages/SearchPage") ?: return emptyList() - return parseMangaList(dataMap) - } + val dataMap = fetchRscData(url, "pages/SearchPage") - private suspend fun parseViewAllPage(url: String): List<Manga> { - val viewAllData = fetchRscData(url, "pages/ViewAllPage") ?: return emptyList() - val mangaListMap = viewAllData.asMap("data") ?: return emptyList() - return parseMangaList(mangaListMap) + val mangaList = dataMap["manga_list"] as? List<*> + ?: throw ParseException("Missing 'manga_list' in decoded data. Keys: ${dataMap.keys}", url) + + return mangaList.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } } - private fun parseMangaList(data: Map<String, Any?>): List<Manga> { - val list = data["manga_list"] as? List<*> ?: data["results"] as? List<*> ?: return emptyList() - return list.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } + private suspend fun parseViewAllPage(url: String): List<Manga> { + val viewAllData = fetchRscData(url, "pages/ViewAllPage") + val mangaList = viewAllData["manga_list"] as? List<*> + ?: throw ParseException("Missing 'manga_list' in view-all data. Keys: ${viewAllData.keys}", url) + return mangaList.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } } private fun parseMangaFromList(data: Map<String, Any?>): Manga { @@ -253,7 +285,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return@coroutineScope manga + val mangaData = fetchRscData(url, "pages/MangaDetailPage") val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return@coroutineScope manga val title = mangaInfo["title"] as? String ?: manga.title @@ -475,11 +507,10 @@ internal class Mangadotnet(context: MangaLoaderContext) : } } - @Suppress("UNCHECKED_CAST") override suspend fun getRelatedManga(seed: Manga): List<Manga> { val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" - val data = fetchRscData(url, "pages/MangaDetailPage") ?: return emptyList() + val data = fetchRscData(url, "pages/MangaDetailPage") val related = mutableListOf<Manga>() (data["suggestions"] as? List<*>)?.filterIsInstance<Map<String, Any?>>()?.mapTo(related) { parseMangaFromList(it) } From 003ee2d4dd2a8075565463e2c560632097933917 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 16 Aug 2026 23:56:07 +0200 Subject: [PATCH 329/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 93 +++++++++++++------ 1 file changed, 63 insertions(+), 30 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 23bea490..e053e5b9 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -28,6 +28,8 @@ import tsuki.util.parseSafe import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject @@ -41,7 +43,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANGADOTNET, 20) { override val configKeyDomain = ConfigKey.Domain("mangadot.net") - private val baseUrl = "https://mangadot.net" + private val baseUrl = "https://$domain" private val apiBase = "$baseUrl/api" private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") @@ -134,12 +136,12 @@ internal class Mangadotnet(context: MangaLoaderContext) : SortOrder.RATING, // top rated SortOrder.ALPHABETICAL, // a-z SortOrder.RELEVANCE, // most tracked + SortOrder.ADDED // chapters ) override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = GENRES.map { name -> MangaTag(name, name, source) }.toSet() return MangaListFilterOptions( - availableTags = tags, + availableTags = fetchAvailableTags(), availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), availableContentTypes = EnumSet.of( ContentType.MANGA, @@ -166,6 +168,60 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } + @Volatile + private var tagsCache: Set<MangaTag>? = null + private val tagsMutex = Mutex() + + private fun findValueRecursively(obj: Any?, key: String, maxDepth: Int = 6): Any? { + if (maxDepth < 0 || obj == null) return null + + return when (obj) { + is Map<*, *> -> { + obj[key]?.let { return it } + for (value in obj.values) { + findValueRecursively(value, key, maxDepth - 1)?.let { return it } + } + null + } + is List<*> -> { + for (item in obj) { + findValueRecursively(item, key, maxDepth - 1)?.let { return it } + } + null + } + else -> null + } + } + + private suspend fun fetchTags(): Set<MangaTag> = try { + val url = "$baseUrl/search.data?page=1&_routes=pages/SearchPage".toHttpUrl() + + val flat = webClient.httpGet(url).parseJsonArray() + val decoded = decodeRsc(flat) + ?: throw ParseException("Failed to decode RSC data", url.toString()) + + val routeValue = (decoded as? Map<*, *>)?.get("pages/SearchPage") as? Map<*, *> + ?: throw ParseException("Missing RSC route 'pages/SearchPage'", url.toString()) + + val allGenres = findValueRecursively(routeValue, "allGenres") as? List<*> + ?: emptyList<Any?>() + + val ignore = setOf("Josei", "Seinen", "Shoujo", "Shounen", "Manga", "Manhwa", "Manhua", "One Shot") + + allGenres.asSequence().filterIsInstance<String>() + .filter { it !in ignore } + .distinct() + .sortedBy { it.lowercase(Locale.ROOT) } + .map { MangaTag(it, it, source) } + .toSet() + } catch (i: Exception) { + throw Exception("Failed to load: $i") + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> = tagsMutex.withLock { + tagsCache ?: fetchTags().also { tagsCache = it } + } + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { selectedLanguage = filter.locale?.let { localeToLang(it) } ?: "en" if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || @@ -177,10 +233,12 @@ internal class Mangadotnet(context: MangaLoaderContext) : private suspend fun getBrowsePage(page: Int, order: SortOrder): List<Manga> { val sortParam = when (order) { + SortOrder.UPDATED -> "latest" SortOrder.POPULARITY -> "views" SortOrder.RATING -> "rating" SortOrder.ALPHABETICAL -> "alphabetical" SortOrder.RELEVANCE -> "tracked" + SortOrder.ADDED -> "chapters" else -> null } val adultParam = if (config[showAdultKey]) "both" else "0" @@ -206,6 +264,8 @@ internal class Mangadotnet(context: MangaLoaderContext) : SortOrder.ALPHABETICAL -> "alphabetical" to "asc" SortOrder.POPULARITY -> "views" to "desc" SortOrder.RATING -> "rating" to "desc" + SortOrder.RELEVANCE -> "tracked" to "desc" + SortOrder.ADDED -> "chapters" to "desc" else -> "relevance" to "desc" } addQueryParameter("sortBy", sortBy) @@ -531,31 +591,4 @@ internal class Mangadotnet(context: MangaLoaderContext) : @Suppress("UNCHECKED_CAST") private fun Map<String, Any?>.asMap(key: String): Map<String, Any?>? = this[key] as? Map<String, Any?> private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this - - companion object { - private val GENRES = setOf( - "Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Historical", "Horror", "Mecha", "Mystery", "Psychological", - "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", - "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", - "Medical", "Military", "Music", "School Life", "Webtoon", - "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", - "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", - "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", - "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", - "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", - "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", - "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", - "Magical Girls", "Mahou Shoujo", "Mature", "Medieval Area", - "Monster Girls", "Monsters", "Ninja", "Nobility", - "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", - "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", - "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", - "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", - "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", - "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", - "Wuxia", "Yaoi", "Yuri", "Zombies" - ) - - } } From b3643c30b98a87fedd6d2efa8defaac49b6cee38 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 17 Aug 2026 00:13:05 +0200 Subject: [PATCH 330/434] Update Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 125 ++++++++++------------- 1 file changed, 55 insertions(+), 70 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index b70d78d2..52324673 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -5,6 +5,7 @@ import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.exception.ParseException +import tsuki.network.CloudFlareHelper import tsuki.model.ContentRating import tsuki.model.Manga @@ -35,7 +36,6 @@ import okhttp3.Response import kotlinx.coroutines.delay import org.json.JSONArray import org.json.JSONObject -import org.jsoup.HttpStatusException import java.io.IOException import java.net.URI import java.text.SimpleDateFormat @@ -55,7 +55,7 @@ internal class Kagane(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.KAGANE, pageSize = 35) { override val configKeyDomain = ConfigKey.Domain("kagane.to") - private val apiUrl = "https://kagane.to" + private val apiUrl = "https://$domain" private val dataSaverKey = ConfigKey.PreferredImageServer( presetValues = mapOf( @@ -285,27 +285,11 @@ internal class Kagane(context: MangaLoaderContext) : .add("Referer", "https://$domain/") .build() - val responseBody = try { - requestWithCloudflareRetry(url) { - webClient.httpPost(url.toHttpUrl(), jsonBody, headers).parseRaw() - } - } catch (e: HttpStatusException) { - if (e.statusCode == 403 || e.statusCode == 429 || e.statusCode == 503) { - requestCloudflareVerification(url, e) - } else { - throw e - } - } catch (e: ParseException) { - val causeMessage = e.message.orEmpty() + " " + (e.cause?.message.orEmpty()) - if (causeMessage.contains("CloudFlare", ignoreCase = true) || - causeMessage.contains("cf-mitigated", ignoreCase = true) || - causeMessage.contains("cf-error-details", ignoreCase = true) - ) { - requestCloudflareVerification(url, e) - } else { - throw e - } - } + val responseBody = executeWithCloudflareCheck( + url = url, + block = { webClient.httpPost(url.toHttpUrl(), jsonBody, headers) }, + parse = { it.parseRaw() }, + ) if (responseBody.isCloudflareChallenge()) { requestCloudflareVerification(url) @@ -359,9 +343,11 @@ internal class Kagane(context: MangaLoaderContext) : .add("Origin", "https://$domain") .add("Referer", "https://$domain/") .build() - val resp = requestWithCloudflareRetry(url) { - webClient.httpGet(url, headers) - } + val resp = executeWithCloudflareCheck( + url = url, + block = { webClient.httpGet(url, headers) }, + parse = { it }, + ) val respBody = resp.body?.string() ?: "" if (!resp.isSuccessful) throw Exception("Details error ${resp.code}: $respBody") val json = try { @@ -609,45 +595,38 @@ internal class Kagane(context: MangaLoaderContext) : } } - private suspend fun <T> requestWithCloudflareRetry(url: String, block: suspend () -> T): T { - try { - return block() - } catch (e: Exception) { - if (!e.isCloudflareProtectionError()) { - throw e - } - delay(CLOUDFLARE_RETRY_DELAY_MS.milliseconds) - try { - return block() - } catch (retryError: Exception) { - if (e.isCloudflareBlock() || retryError.isCloudflareBlock()) { - requestCloudflareVerification(url, retryError) + private suspend fun <T> executeWithCloudflareCheck( + url: String, + block: suspend () -> Response, + parse: (Response) -> T, + ): T { + CloudFlareHelper.getClearanceCookie(context.cookieJar, "https://$domain") + val response = block() + return when (CloudFlareHelper.checkResponseForProtection(response)) { + CloudFlareHelper.PROTECTION_NOT_DETECTED -> parse(response) + CloudFlareHelper.PROTECTION_CAPTCHA -> { + response.close() + delay(CLOUDFLARE_RETRY_DELAY_MS.milliseconds) + val retryResponse = block() + when (CloudFlareHelper.checkResponseForProtection(retryResponse)) { + CloudFlareHelper.PROTECTION_NOT_DETECTED -> parse(retryResponse) + else -> { + retryResponse.close() + requestCloudflareVerification(url) + } } - throw retryError + } + CloudFlareHelper.PROTECTION_BLOCKED -> { + response.close() + requestCloudflareVerification(url) + } + else -> { + response.close() + requestCloudflareVerification(url) } } } - private fun Throwable.isCloudflareBlock(): Boolean { - val name = javaClass.simpleName - return name.contains("CloudflareBlocked", ignoreCase = true) || - name.contains("CloudFlareBlocked") || - (isCloudflareProtectionError() && message.orEmpty().contains("blocked", ignoreCase = true)) - } - - private fun Throwable.isCloudflareProtectionError(): Boolean { - val name = javaClass.simpleName - if (name.contains("Cloudflare", ignoreCase = true) || name.contains("CloudFlare")) { - return true - } - val message = "${message.orEmpty()} ${cause?.message.orEmpty()}" - return message.contains("cf-mitigated", ignoreCase = true) || - message.contains("Just a moment", ignoreCase = true) || - message.contains("challenges.cloudflare.com", ignoreCase = true) || - message.contains("cf-error-details", ignoreCase = true) || - message.contains("cf-chl-bypass", ignoreCase = true) - } - private fun requestCloudflareVerification(url: String, cause: Throwable? = null): Nothing { try { context.requestBrowserAction(this, "https://$domain/") @@ -717,13 +696,17 @@ internal class Kagane(context: MangaLoaderContext) : .add("Referer", "https://$domain/") .build() - val response = requestWithCloudflareRetry("$apiUrl/api/integrity") { - webClient.httpPost( - urlBuilder().addPathSegments("api/integrity").build(), - JSONObject(), - headers, - ).parseJson() - } + val response = executeWithCloudflareCheck( + url = "$apiUrl/api/integrity", + block = { + webClient.httpPost( + urlBuilder().addPathSegments("api/integrity").build(), + JSONObject(), + headers, + ) + }, + parse = { it.parseJson() }, + ) val token = response.optString("token") if (token.isBlank()) { @@ -742,9 +725,11 @@ internal class Kagane(context: MangaLoaderContext) : .add("x-integrity-token", integrityToken) .build() val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver" - return requestWithCloudflareRetry(challengeUrl) { - webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers).parseJson() - } + return executeWithCloudflareCheck( + url = challengeUrl, + block = { webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers) }, + parse = { it.parseJson() }, + ) } private fun String.toChapterNumberOrNull(): Float? = trim() From 0fa9dfaa041c8b36483f3807a94f0ac775a6e84f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 17 Aug 2026 11:40:11 +0200 Subject: [PATCH 331/434] Update KaliScan.kt --- src/main/kotlin/tsuki/site/en/KaliScan.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/KaliScan.kt b/src/main/kotlin/tsuki/site/en/KaliScan.kt index d656b6e9..98df8601 100644 --- a/src/main/kotlin/tsuki/site/en/KaliScan.kt +++ b/src/main/kotlin/tsuki/site/en/KaliScan.kt @@ -33,7 +33,7 @@ import org.jsoup.nodes.Document import java.text.SimpleDateFormat import java.util.EnumSet -@MangaSourceParser("KALISCAN", "kaliscan.io", "en") +@MangaSourceParser("KALISCAN", "Kaliscan.io", "en") internal class KaliScan(context: MangaLoaderContext) : MadthemeParser(context, MangaParserSource.KALISCAN, "kaliscan.io", pageSize=24) { From 0ce6cd8fb94eae735ca92695eebd4dfab222e964 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Mon, 17 Aug 2026 12:09:03 +0200 Subject: [PATCH 332/434] fix search --- .../kotlin/tsuki/site/all/nsfw/Multporn.kt | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt index cc1f14c4..5f5015d9 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt @@ -26,7 +26,6 @@ import tsuki.util.generateUid import tsuki.util.oneOrThrowIfMany import tsuki.util.parseHtml import tsuki.util.requireSrc -import tsuki.util.selectFirstOrThrow import tsuki.util.splitByWhitespace import tsuki.util.toAbsoluteUrl import tsuki.util.urlEncoded @@ -159,12 +158,35 @@ internal class Multporn(context: MangaLoaderContext) : } val doc = webClient.httpGet(url).parseHtml() - return doc.select(".masonry-item").map { div -> - val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") - val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() + val items = doc.select(".masonry-item") + .ifEmpty { doc.select(".views-row") } + .ifEmpty { doc.select(".node") } + + return items.mapNotNull { div -> + val linkElement = div.selectFirst(".views-field-title a") + ?: div.selectFirst(".views-field-name a") + ?: div.selectFirst(".field-content a") + ?: div.selectFirst("h2 a") + ?: div.selectFirst("h3 a") + ?: return@mapNotNull null + + val href = linkElement.attrAsRelativeUrl("href") + if (href.isBlank()) return@mapNotNull null + val coverElement = div.selectFirst(".views-field-field-files img") + ?: div.selectFirst(".views-field-field-game-preview img") + ?: div.selectFirst(".views-field-field-preview-1 img") + ?: div.selectFirst(".views-field-field-picture-preview img") + ?: div.selectFirst(".views-field img") + ?: div.selectFirst("img") + ?: return@mapNotNull null + + val coverUrl = coverElement.requireSrc() + val title = linkElement.text() + .ifEmpty { div.selectFirst(".views-field-title")?.text() ?: "Unknown" } + Manga( id = generateUid(href), - title = div.select(".views-field-title").text(), + title = title, altTitles = emptySet(), url = href, publicUrl = href.toAbsoluteUrl(domain), From e094a7fb7b3a30392172cc1b1ab73e0d0c13ba17 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 18 Aug 2026 12:51:52 +0200 Subject: [PATCH 333/434] dedup + fix altnames --- .../kotlin/tsuki/parsers/MangaThemesia.kt | 119 +++++++++++------- 1 file changed, 74 insertions(+), 45 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 01081d36..c604e652 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -129,41 +129,50 @@ abstract class MangaThemesia( } filter.states.firstOrNull()?.let { append("&status=") - append(when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "dropped" - else -> "" - }) + append(it.toQueryParam()) } filter.types.firstOrNull()?.let { append("&type=") - append(when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - }) + append(it.toQueryParam()) + } + filter.tags.forEach { + append("&genre[]=") + append(it.key.urlEncoded()) } - filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } append("&order=") - append(when (order) { - SortOrder.UPDATED -> "update" - SortOrder.POPULARITY -> "popular" - SortOrder.ADDED -> "latest" - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - else -> "update" - }) + append(order.toQueryParam() + ) } - val doc = webClient.httpGet(url).parseHtml() return parseMangaList(doc) } + private fun MangaState.toQueryParam(): String = when (this) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "dropped" + else -> "" + } + + private fun ContentType.toQueryParam(): String = when (this) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + } + + private fun SortOrder.toQueryParam(): String = when (this) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ADDED -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "update" + } + protected open fun parseMangaList(doc: Document): List<Manga> { return doc.select(searchSelector).mapNotNull { element -> parseMangaElement(element) @@ -219,25 +228,7 @@ abstract class MangaThemesia( doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() } } - val altnamesFromTable = if (hasTable) { - table.selectFirst("tr:has(td:contains(Alternative)) td:last-child") - ?.text() - ?.trim() - ?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } - ?.let { setOf(it) } - } else emptySet() - - val altnamesFromAlternative = doc.select(".alternative") - .firstOrNull() - ?.text() - ?.trim() - ?.takeIf { it.isNotBlank() } - ?.split(",") - ?.map { it.trim() } - ?.filter { it.isNotBlank() } - ?.toSet() ?: emptySet() - - val altnames = altnamesFromTable?.plus(altnamesFromAlternative) + val altTitles = parseAltTitles(doc, table, hasTable) val state = parseStatus(statusText) @@ -256,10 +247,48 @@ abstract class MangaThemesia( state = state, rating = normalizedRating, chapters = chapters, - altTitles = altnames ?: emptySet() + altTitles = altTitles ) } + protected open fun parseAltTitles(doc: Document, table: Element?, hasTable: Boolean): Set<String> { + return buildSet { + if (hasTable && table != null) { + table.select("tr").forEach { row -> + val label = row.selectFirst("td")?.text()?.trim().orEmpty() + if ( + label.contains("Alternative", ignoreCase = true) || + label.contains("Alt.", ignoreCase = true) || + label.contains("Alt ", ignoreCase = true) + ) { + addAll(splitAltTitles(row.selectFirst("td:last-child")?.text())) + } + } + } else { + val selectors = listOf( + ".tsinfo .imptdt:contains(Alt) i", + ".tsinfo .imptdt:contains(Alternative) i", + ".spe span:contains(Alt) a", + ".spe span:contains(Alternative) a", + ".alternative", + ".alter", + ".post-content_item:contains(Alt) .summary-content", + ) + doc.select(selectors.joinToString(", ")).forEach { element -> + addAll(splitAltTitles(element.text())) + } + } + } + } + + protected fun splitAltTitles(text: String?): Set<String> { + return text.orEmpty() + .split(',', ';', '/', '•', '\n') + .map { it.trim() } + .filter { it.isNotBlank() && !it.equals("n/a", true) } + .toSet() + } + protected open fun parseStatus(text: String?): MangaState? { val status = text?.lowercase() ?: return null return when (status) { From 2a4ec19d9a2a41c3d5e399f9809a7af569f53f51 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 18 Aug 2026 15:14:18 +0200 Subject: [PATCH 334/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 26 +++++++++++++++---- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index e053e5b9..1dd8f4e4 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -123,6 +123,15 @@ internal class Mangadotnet(context: MangaLoaderContext) : return mapOf("manga_list" to found) } + @Suppress("UNCHECKED_CAST") + private suspend fun fetchRscRoute(url: String, route: String): Map<String, Any?> { + val flat = webClient.httpGet(url).parseJsonArray() + val decoded = decodeRsc(flat) + ?: throw ParseException("Failed to decode RSC data", url) + return (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> + ?: throw ParseException("Missing RSC route '$route'", url) + } + override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, isMultipleTagsSupported = true, @@ -345,9 +354,14 @@ internal class Mangadotnet(context: MangaLoaderContext) : override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") - val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return@coroutineScope manga + val dataMap = routeValue.asMap("data") + ?: throw ParseException("Missing 'data' in detail route", url) + + val mangaInfo = dataMap.asMap("mangaData") + ?: throw ParseException("Missing 'mangaData' in detail route", url) + val title = mangaInfo["title"] as? String ?: manga.title val description = mangaInfo["description"] as? String val photo = mangaInfo["photo"] as? String @@ -570,11 +584,13 @@ internal class Mangadotnet(context: MangaLoaderContext) : @Suppress("UNCHECKED_CAST") override suspend fun getRelatedManga(seed: Manga): List<Manga> { val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" - val data = fetchRscData(url, "pages/MangaDetailPage") + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") val related = mutableListOf<Manga>() - (data["suggestions"] as? List<*>)?.filterIsInstance<Map<String, Any?>>()?.mapTo(related) { parseMangaFromList(it) } - val relationsData = data.asMap("relationsData") + (routeValue["suggestions"] as? List<*>)?.filterIsInstance<Map<String, Any?>>()?.mapTo(related) { + parseMangaFromList(it) + } + val relationsData = routeValue.asMap("relationsData") val relations = relationsData?.asMap("relations") as? Map<String, List<Map<String, Any?>>> relations?.values?.forEach { list -> list.mapTo(related) { parseMangaFromList(it) } } From c398500f8d566d3cbc598951e63a78ab05b97151 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:07:38 +0200 Subject: [PATCH 335/434] fix mangadot --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 282 +++++++----------- 1 file changed, 112 insertions(+), 170 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index e053e5b9..9dfae35c 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -85,42 +85,12 @@ internal class Mangadotnet(context: MangaLoaderContext) : } @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscData(url: String, route: String): Map<String, Any?> { + private suspend fun fetchRscRoute(url: String, route: String): Map<String, Any?> { val flat = webClient.httpGet(url).parseJsonArray() val decoded = decodeRsc(flat) ?: throw ParseException("Failed to decode RSC data", url) - val routeValue = (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> + return (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> ?: throw ParseException("Missing RSC route '$route'", url) - val mangaListKeys = listOf("manga_list", "results", "rows") - for (key in mangaListKeys) { - routeValue[key]?.let { found -> - return mapOf(key to found) - } - } - fun searchRecursively(obj: Any?, depth: Int = 0): Any? { - if (depth > 5) return null - when (obj) { - is Map<*, *> -> { - for (key in mangaListKeys) { - obj[key]?.let { return it } - } - for (value in obj.values) { - val result = searchRecursively(value, depth + 1) - if (result != null) return result - } - } - is List<*> -> { - for (item in obj) { - val result = searchRecursively(item, depth + 1) - if (result != null) return result - } - } - } - return null - } - val found = searchRecursively(routeValue) - ?: throw ParseException("Manga list not found in route '$route'", url) - return mapOf("manga_list" to found) } override val filterCapabilities = MangaListFilterCapabilities( @@ -131,29 +101,68 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, // latest - SortOrder.POPULARITY, // most viewed - SortOrder.RATING, // top rated - SortOrder.ALPHABETICAL, // a-z - SortOrder.RELEVANCE, // most tracked - SortOrder.ADDED // chapters + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.RELEVANCE, + SortOrder.ADDED, ) + @Volatile + private var genreNames: Set<String>? = null + @Volatile + private var availableTagsCache: Set<MangaTag>? = null + private val filterMutex = Mutex() + + private suspend fun fetchFilterData() { + if (genreNames != null) return + + val facetsUrl = "$baseUrl/api/search?facets=1".toHttpUrl() + val facetsResponse = runCatching { + webClient.httpGet(facetsUrl).parseJson() + }.getOrNull() + val ignore = setOf("Josei", "Seinen", "Shoujo", "Shounen", "Manga", "Manhwa", "Manhua", "One Shot") + val genres = facetsResponse + ?.optJSONObject("facets") + ?.optJSONArray("genres") + ?.let { arr -> + (0 until arr.length()).mapNotNull { arr.getJSONObject(it).optString("key") } + } + ?.filter { it !in ignore } + ?.toSet() + ?: emptySet() + + genreNames = genres + + availableTagsCache = genres + .map { MangaTag(it, it, source) } + .toSet() + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> = filterMutex.withLock { + if (availableTagsCache == null) { + fetchFilterData() + } + availableTagsCache ?: emptySet() + } + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchAvailableTags() return MangaListFilterOptions( - availableTags = fetchAvailableTags(), + availableTags = genres, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), availableContentTypes = EnumSet.of( ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, - ContentType.OTHER + ContentType.OTHER, ), availableDemographics = EnumSet.of( Demographic.JOSEI, Demographic.SEINEN, Demographic.SHOUJO, - Demographic.SHOUNEN + Demographic.SHOUNEN, ), availableLocales = setOf( Locale.ENGLISH, @@ -168,96 +177,17 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } - @Volatile - private var tagsCache: Set<MangaTag>? = null - private val tagsMutex = Mutex() - - private fun findValueRecursively(obj: Any?, key: String, maxDepth: Int = 6): Any? { - if (maxDepth < 0 || obj == null) return null - - return when (obj) { - is Map<*, *> -> { - obj[key]?.let { return it } - for (value in obj.values) { - findValueRecursively(value, key, maxDepth - 1)?.let { return it } - } - null - } - is List<*> -> { - for (item in obj) { - findValueRecursively(item, key, maxDepth - 1)?.let { return it } - } - null - } - else -> null - } - } - - private suspend fun fetchTags(): Set<MangaTag> = try { - val url = "$baseUrl/search.data?page=1&_routes=pages/SearchPage".toHttpUrl() - - val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) - ?: throw ParseException("Failed to decode RSC data", url.toString()) - - val routeValue = (decoded as? Map<*, *>)?.get("pages/SearchPage") as? Map<*, *> - ?: throw ParseException("Missing RSC route 'pages/SearchPage'", url.toString()) - - val allGenres = findValueRecursively(routeValue, "allGenres") as? List<*> - ?: emptyList<Any?>() - - val ignore = setOf("Josei", "Seinen", "Shoujo", "Shounen", "Manga", "Manhwa", "Manhua", "One Shot") - - allGenres.asSequence().filterIsInstance<String>() - .filter { it !in ignore } - .distinct() - .sortedBy { it.lowercase(Locale.ROOT) } - .map { MangaTag(it, it, source) } - .toSet() - } catch (i: Exception) { - throw Exception("Failed to load: $i") - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> = tagsMutex.withLock { - tagsCache ?: fetchTags().also { tagsCache = it } - } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { selectedLanguage = filter.locale?.let { localeToLang(it) } ?: "en" - if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || - filter.states.isNotEmpty() || filter.types.isNotEmpty() || filter.demographics.isNotEmpty()) { - return getSearchPage(page, order, filter) - } - return getBrowsePage(page, order) - } - - private suspend fun getBrowsePage(page: Int, order: SortOrder): List<Manga> { - val sortParam = when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.POPULARITY -> "views" - SortOrder.RATING -> "rating" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "tracked" - SortOrder.ADDED -> "chapters" - else -> null - } - val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/view-all/latest-updates.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) - if (page > 1) addQueryParameter("page", page.toString()) - sortParam?.let { addQueryParameter("sortBy", it) } - addQueryParameter("_routes", "pages/ViewAllPage") - }.build().toString() - return parseViewAllPage(url) + return getSearchPage(page, order, filter) } private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/search.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) + val url = "$baseUrl/api/search".toHttpUrl().newBuilder().apply { if (!filter.query.isNullOrBlank()) addQueryParameter("search", filter.query!!) addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "56") + addQueryParameter("adult", adultParam) val (sortBy, sortOrder) = when (order) { SortOrder.UPDATED -> "latest" to "desc" @@ -266,7 +196,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : SortOrder.RATING -> "rating" to "desc" SortOrder.RELEVANCE -> "tracked" to "desc" SortOrder.ADDED -> "chapters" to "desc" - else -> "relevance" to "desc" + else -> "latest" to "desc" } addQueryParameter("sortBy", sortBy) addQueryParameter("sortOrder", sortOrder) @@ -286,35 +216,33 @@ internal class Mangadotnet(context: MangaLoaderContext) : } filter.demographics.forEach { demographic -> - addQueryParameter("genre", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) + addQueryParameter("genres", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) } - filter.tags.forEach { tag -> addQueryParameter("genre", tag.key) } - filter.tagsExclude.forEach { tag -> addQueryParameter("genre", "-${tag.key}") } + filter.tags.forEach { tag -> + addQueryParameter("genres", tag.key) + } - addQueryParameter("_routes", "pages/SearchPage") + filter.tagsExclude.forEach { tag -> + addQueryParameter("excludeGenres", tag.key) + } }.build().toString() - val dataMap = fetchRscData(url, "pages/SearchPage") + val json = webClient.httpGet(url).parseJson() + val items = json.optJSONArray("manga_list") + ?: json.optJSONArray("results") + ?: throw ParseException("Missing manga list in search response", url) - val mangaList = dataMap["manga_list"] as? List<*> - ?: throw ParseException("Missing 'manga_list' in decoded data. Keys: ${dataMap.keys}", url) - - return mangaList.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } - } - - private suspend fun parseViewAllPage(url: String): List<Manga> { - val viewAllData = fetchRscData(url, "pages/ViewAllPage") - val mangaList = viewAllData["manga_list"] as? List<*> - ?: throw ParseException("Missing 'manga_list' in view-all data. Keys: ${viewAllData.keys}", url) - return mangaList.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } + return (0 until items.length()).map { i -> + parseMangaFromList(items.getJSONObject(i)) + } } - private fun parseMangaFromList(data: Map<String, Any?>): Manga { - val id = (data["id"] as? Number)?.toInt()?.toString() ?: "" - val title = data["title"] as? String ?: "" - val photo = data["photo"] as? String - val coverUrl = photo?.let { + private fun parseMangaFromList(data: JSONObject): Manga { + val id = (data.optInt("id", data.optInt("manga_id", 0))).takeIf { it > 0 }?.toString() ?: "" + val title = data.optString("title") + val photo = data.optString("photo") + val coverUrl = photo?.takeIf { it.isNotBlank() }?.let { when { it.startsWith("/") -> "$baseUrl$it" it.startsWith("http") -> it @@ -345,9 +273,14 @@ internal class Mangadotnet(context: MangaLoaderContext) : override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") + + val mangaInfo = routeValue + .asMap("data") + ?.asMap("mangaData") + ?.asMap("manga") + ?: throw ParseException("Missing manga data in detail route", url) - val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return@coroutineScope manga val title = mangaInfo["title"] as? String ?: manga.title val description = mangaInfo["description"] as? String val photo = mangaInfo["photo"] as? String @@ -358,6 +291,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : else -> null } } ?: manga.coverUrl + val bannerImage = mangaInfo["banner_image"] as? String val largeCoverUrl = bannerImage?.let { when { @@ -367,9 +301,19 @@ internal class Mangadotnet(context: MangaLoaderContext) : } } - val genres = (mangaInfo["genres"] as? List<*>)?.filterIsInstance<String>()?.mapNotNull { it.trim().ifBlank { null } }.orEmpty() - val altTitles = (mangaInfo["alt_titles"] as? List<*>)?.filterIsInstance<String>()?.mapNotNull { it.trim().ifBlank { null } }?.toSet().orEmpty() + val genres = (mangaInfo["genres"] as? List<*>) + ?.filterIsInstance<String>() + ?.mapNotNull { it.trim().ifBlank { null } } + .orEmpty() + + val altTitles = (mangaInfo["alt_titles"] as? List<*>) + ?.filterIsInstance<String>() + ?.mapNotNull { it.trim().ifBlank { null } } + ?.toSet() + .orEmpty() + val origin = mangaInfo["country_of_origin"] as? String + val authorRaw = mangaInfo["authors"] as? String val author = parseJsonArrayString(authorRaw)?.joinToString(", ") ?: authorRaw @@ -390,33 +334,23 @@ internal class Mangadotnet(context: MangaLoaderContext) : } else -> null } - val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { MangaTag(key = it, title = it, source = source) } + + val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { + MangaTag(key = it, title = it, source = source) + } when (origin) { "JP" -> tagSet.add(MangaTag("Manga", "Manga", source)) "KR" -> tagSet.add(MangaTag("Manhwa", "Manhwa", source)) "CN" -> tagSet.add(MangaTag("Manhua", "Manhua", source)) } - val tagsArray = mangaInfo["tags"] - if (tagsArray is List<*>) { - for (category in tagsArray) { - if (category !is Map<*, *>) continue - val innerTags = category["tags"] - if (innerTags !is List<*>) continue - for (tag in innerTags) { - if (tag !is Map<*, *>) continue - val name = tag["name"] as? String ?: continue - val key = name.lowercase().replace(" ", "-") - tagSet.add(MangaTag(key, name, source)) - } - } - } - val rawRating = (mangaInfo["avg_rating"] as? Number)?.toDouble() ?: -1.0 val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN val chapterLang = selectedLanguage ?: "en" - val chaptersDeferred = async { fetchChaptersAndVolumes(manga.url, chapterLang) } + val chaptersDeferred = async { + fetchChaptersAndVolumes(manga.url, chapterLang) + } manga.copy( title = title, @@ -570,13 +504,21 @@ internal class Mangadotnet(context: MangaLoaderContext) : @Suppress("UNCHECKED_CAST") override suspend fun getRelatedManga(seed: Manga): List<Manga> { val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" - val data = fetchRscData(url, "pages/MangaDetailPage") + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") + + val dataMap = routeValue.asMap("data") + ?: throw ParseException("Missing 'data' in detail route", url) val related = mutableListOf<Manga>() - (data["suggestions"] as? List<*>)?.filterIsInstance<Map<String, Any?>>()?.mapTo(related) { parseMangaFromList(it) } - val relationsData = data.asMap("relationsData") + (dataMap["suggestions"] as? List<*>) + ?.filterIsInstance<Map<String, Any?>>() + ?.mapTo(related) { parseMangaFromList(JSONObject(it)) } + + val relationsData = dataMap.asMap("relationsData") val relations = relationsData?.asMap("relations") as? Map<String, List<Map<String, Any?>>> - relations?.values?.forEach { list -> list.mapTo(related) { parseMangaFromList(it) } } + relations?.values?.forEach { list -> + list.mapTo(related) { parseMangaFromList(JSONObject(it)) } + } return related } From a4ee187affdbb50f10dd9734f6f80fa41b6aa5f9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:17:45 +0200 Subject: [PATCH 336/434] Update print statement from 'Hello' to 'Goodbye' --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 32 +++++++++++++++------ 1 file changed, 24 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 15ce62db..56c8bc6b 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -177,6 +177,22 @@ internal abstract class MangaFireParser( .build() ) + private suspend fun apiGetJsonWithRetry(url: String, maxRetries: Int = 5): org.json.JSONObject { + var lastResponse: org.json.JSONObject? = null + repeat(maxRetries) { + val json = apiClient.httpGet(url).parseJson() + if (json.optString("message").equals("Missing token.", ignoreCase = true)) { + lastResponse = json + return@repeat + } + return json + } + throw ParseException( + "Missing token after $maxRetries retries. Last response: $lastResponse", + url, + ) + } + override val filterCapabilities = MangaListFilterCapabilities( isMultipleTagsSupported = true, isTagsExclusionSupported = true, @@ -350,7 +366,7 @@ internal abstract class MangaFireParser( val url = urlBuilder.build().toString() try { - val response = apiClient.httpGet(url).parseJson() + val response = apiGetJsonWithRetry(url) val items = response.optJSONArray("items") ?: throw ParseException("Missing 'items' array in API response", url) @@ -394,7 +410,7 @@ internal abstract class MangaFireParser( } catch (e: ParseException) { throw e } catch (e: Exception) { - throw ParseException("Failed to load list page: ${e.message}\nOpen Webview and then reload Page", url, e) + throw ParseException("Failed to load list page: ${e.message}\nOpen Webview to solve captcha / Reload Page when missing token.", url, e) } } @@ -410,7 +426,7 @@ internal abstract class MangaFireParser( val result = try { coroutineScope { val hid = extractHid(manga.url) - val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() + val detailsJson = apiGetJsonWithRetry("https://$domain/api/titles/$hid") val data = detailsJson.getJSONObject("data") val hasVolumes = data.optBoolean("hasVolumes", false) @@ -487,7 +503,7 @@ internal abstract class MangaFireParser( try { val firstUrl = "$base/chapters?language=$siteLang&sort=number&order=desc&page=1&limit=200" - val firstJson = apiClient.httpGet(firstUrl).parseJson() + val firstJson = apiGetJsonWithRetry(firstUrl) val firstItems = firstJson.optJSONArray("items") ?: org.json.JSONArray() val meta = firstJson.optJSONObject("meta") val lastPage = meta?.optInt("lastPage", 1) ?: 1 @@ -499,9 +515,9 @@ internal abstract class MangaFireParser( val deferred = (2..lastPage).map { page -> async { try { - val json = apiClient.httpGet( + val json = apiGetJsonWithRetry( "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" - ).parseJson() + ) json.optJSONArray("items") ?: org.json.JSONArray() } catch (_: Exception) { org.json.JSONArray() @@ -518,7 +534,7 @@ internal abstract class MangaFireParser( if (hasVolumes) { try { - val volJson = apiClient.httpGet("$base/volumes?language=$siteLang").parseJson() + val volJson = apiGetJsonWithRetry("$base/volumes?language=$siteLang") val volItems = volJson.optJSONArray("items") ?: org.json.JSONArray() for (i in 0 until volItems.length()) { val vol = volItems.getJSONObject(i) @@ -599,7 +615,7 @@ internal abstract class MangaFireParser( val isVolume = chapter.url.contains("/vol/") val endpoint = if (isVolume) "volumes" else "chapters" - val response = apiClient.httpGet("https://$domain/api/$endpoint/$chapterId").parseJson() + val response = apiGetJsonWithRetry("https://$domain/api/$endpoint/$chapterId") val data = response.optJSONObject("data") ?: return emptyList() val pagesArray = data.optJSONArray("pages") ?: return emptyList() val pages = ArrayList<MangaPage>(pagesArray.length()) From 7768dc378311790c0d32f54dc9821fe72d57fbe1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:28:04 +0200 Subject: [PATCH 337/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 67 ++++++++----------- 1 file changed, 28 insertions(+), 39 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 9dfae35c..716c1809 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -28,8 +28,6 @@ import tsuki.util.parseSafe import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject @@ -109,48 +107,39 @@ internal class Mangadotnet(context: MangaLoaderContext) : SortOrder.ADDED, ) - @Volatile - private var genreNames: Set<String>? = null - @Volatile - private var availableTagsCache: Set<MangaTag>? = null - private val filterMutex = Mutex() - - private suspend fun fetchFilterData() { - if (genreNames != null) return - - val facetsUrl = "$baseUrl/api/search?facets=1".toHttpUrl() - val facetsResponse = runCatching { - webClient.httpGet(facetsUrl).parseJson() - }.getOrNull() - val ignore = setOf("Josei", "Seinen", "Shoujo", "Shounen", "Manga", "Manhwa", "Manhua", "One Shot") - val genres = facetsResponse - ?.optJSONObject("facets") - ?.optJSONArray("genres") - ?.let { arr -> - (0 until arr.length()).mapNotNull { arr.getJSONObject(it).optString("key") } - } - ?.filter { it !in ignore } - ?.toSet() - ?: emptySet() - - genreNames = genres - - availableTagsCache = genres - .map { MangaTag(it, it, source) } - .toSet() + companion object { + private val GENRES = setOf( + "Action", "Adventure", "Comedy", "Drama", "Fantasy", + "Historical", "Horror", "Mecha", "Mystery", "Psychological", + "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", + "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", + "Medical", "Military", "Music", "School Life", "Webtoon", + "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", + "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", + "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", + "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", + "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", + "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", + "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", + "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", + "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", + "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", + "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", + "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", + "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", + "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", + "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", + "Wuxia", "Yaoi", "Yuri", "Zombies" + ) } - private suspend fun fetchAvailableTags(): Set<MangaTag> = filterMutex.withLock { - if (availableTagsCache == null) { - fetchFilterData() - } - availableTagsCache ?: emptySet() - } + private val availableTags = GENRES + .map { MangaTag(it, it, source) } + .toSet() override suspend fun getFilterOptions(): MangaListFilterOptions { - val genres = fetchAvailableTags() return MangaListFilterOptions( - availableTags = genres, + availableTags = availableTags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), availableContentTypes = EnumSet.of( ContentType.MANGA, From b5813df52561ee966f7c3918a66fa82dc808b06c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:30:24 +0200 Subject: [PATCH 338/434] Update print statement from 'Hello' to 'Goodbye' --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 74a16375..716c1809 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -91,15 +91,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : ?: throw ParseException("Missing RSC route '$route'", url) } - @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscRoute(url: String, route: String): Map<String, Any?> { - val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) - ?: throw ParseException("Failed to decode RSC data", url) - return (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> - ?: throw ParseException("Missing RSC route '$route'", url) - } - override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, isMultipleTagsSupported = true, From 27fda23978ec88bc913bcd59e9d701888110c0f4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:31:13 +0200 Subject: [PATCH 339/434] Update Mangadotnet.kt --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 74a16375..716c1809 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -91,15 +91,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : ?: throw ParseException("Missing RSC route '$route'", url) } - @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscRoute(url: String, route: String): Map<String, Any?> { - val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) - ?: throw ParseException("Failed to decode RSC data", url) - return (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> - ?: throw ParseException("Missing RSC route '$route'", url) - } - override val filterCapabilities = MangaListFilterCapabilities( isSearchSupported = true, isMultipleTagsSupported = true, From de31466183190fdb0f5725ed2c1e54b02b56c3cc Mon Sep 17 00:00:00 2001 From: biglag <dmitriy.gudkov@dotcross.ai> Date: Wed, 19 Aug 2026 22:46:16 +0800 Subject: [PATCH 340/434] test(ru): consolidate Russian source contracts --- build.gradle.kts | 16 - .../kotlin/tsuki/FixtureMangaLoaderContext.kt | 138 ------ .../kotlin/tsuki/LibSocialJsonContractTest.kt | 45 -- src/test/kotlin/tsuki/LiveCatalogAuditTest.kt | 273 ----------- .../kotlin/tsuki/LiveMangaLoaderContext.kt | 128 ----- src/test/kotlin/tsuki/LiveSourceSmokeTest.kt | 79 --- .../kotlin/tsuki/MangaMammyContractTest.kt | 36 -- src/test/kotlin/tsuki/MangaWtfContractTest.kt | 36 -- .../kotlin/tsuki/ReadMangaContractTest.kt | 53 -- .../kotlin/tsuki/RussianSourceCatalogTest.kt | 174 ------- .../tsuki/RussianSourceRepairContractTest.kt | 93 ---- .../tsuki/RussianSourcesContractTest.kt | 457 ++++++++++++++++++ .../kotlin/tsuki/SourceReadingContract.kt | 43 -- src/test/kotlin/tsuki/WaMangaContractTest.kt | 53 -- 14 files changed, 457 insertions(+), 1167 deletions(-) delete mode 100644 src/test/kotlin/tsuki/FixtureMangaLoaderContext.kt delete mode 100644 src/test/kotlin/tsuki/LibSocialJsonContractTest.kt delete mode 100644 src/test/kotlin/tsuki/LiveCatalogAuditTest.kt delete mode 100644 src/test/kotlin/tsuki/LiveMangaLoaderContext.kt delete mode 100644 src/test/kotlin/tsuki/LiveSourceSmokeTest.kt delete mode 100644 src/test/kotlin/tsuki/MangaMammyContractTest.kt delete mode 100644 src/test/kotlin/tsuki/MangaWtfContractTest.kt delete mode 100644 src/test/kotlin/tsuki/ReadMangaContractTest.kt delete mode 100644 src/test/kotlin/tsuki/RussianSourceCatalogTest.kt delete mode 100644 src/test/kotlin/tsuki/RussianSourceRepairContractTest.kt create mode 100644 src/test/kotlin/tsuki/RussianSourcesContractTest.kt delete mode 100644 src/test/kotlin/tsuki/SourceReadingContract.kt delete mode 100644 src/test/kotlin/tsuki/WaMangaContractTest.kt diff --git a/build.gradle.kts b/build.gradle.kts index 33205584..cfd7984a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -14,22 +14,6 @@ version = "1.0.0" tasks.test { useJUnitPlatform() - listOf( - "LIVE_SOURCE", - "LIVE_STAGE", - "LIVE_SORT", - "LIVE_QUERY", - "LIVE_EXPECT_TITLE", - "LIVE_AUDIT", - "LIVE_AUDIT_STAGE", - "LIVE_AUDIT_LOCALES", - "LIVE_AUDIT_INCLUDE_BROKEN", - "LIVE_AUDIT_CONCURRENCY", - "LIVE_AUDIT_TIMEOUT_SECONDS", - "LIVE_AUDIT_FAIL_ON_ERROR", - ).forEach { variable -> - inputs.property(variable, providers.environmentVariable(variable).orElse("")) - } } ksp { diff --git a/src/test/kotlin/tsuki/FixtureMangaLoaderContext.kt b/src/test/kotlin/tsuki/FixtureMangaLoaderContext.kt deleted file mode 100644 index f41f0d9c..00000000 --- a/src/test/kotlin/tsuki/FixtureMangaLoaderContext.kt +++ /dev/null @@ -1,138 +0,0 @@ -package tsuki - -import okhttp3.Cookie -import okhttp3.CookieJar -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.MediaType.Companion.toMediaType -import okhttp3.OkHttpClient -import okhttp3.Protocol -import okhttp3.Request -import okhttp3.Response -import okhttp3.ResponseBody.Companion.toResponseBody -import tsuki.bitmap.Bitmap -import tsuki.bitmap.Rect -import tsuki.config.ConfigKey -import tsuki.config.MangaSourceConfig -import tsuki.model.MangaParserSource -import tsuki.model.MangaSource -import java.util.concurrent.ConcurrentHashMap - -@Suppress("OVERRIDE_DEPRECATION") -internal class FixtureMangaLoaderContext private constructor( - routes: Map<FixtureRequest, String>, -) : MangaLoaderContext() { - - private val fixtureInterceptor = FixtureHttpInterceptor(routes) - - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .addInterceptor(fixtureInterceptor) - .build() - - override val cookieJar: CookieJar = object : CookieJar { - override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = Unit - - override fun loadForRequest(url: HttpUrl): List<Cookie> = emptyList() - } - - override fun newParserInstance(source: MangaSource): MangaParser { - require(source is MangaParserSource) { "Unsupported fixture source: $source" } - return source.newParser(this) - } - - override fun getParserSources(): List<MangaSource> = MangaParserSource.entries - - override suspend fun evaluateJs(script: String): String? { - error("JavaScript evaluation is not registered for fixture tests") - } - - override suspend fun evaluateJs(baseUrl: String, script: String): String? { - error("JavaScript evaluation is not registered for fixture tests: $baseUrl") - } - - override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { - override fun <T> get(key: ConfigKey<T>): T = key.defaultValue - } - - override fun getDefaultUserAgent(): String = "MangaRepoContractTest/1.0" - - override fun redrawImageResponse( - response: Response, - redraw: (image: Bitmap) -> Bitmap, - ): Response = error("Image redraw is not registered for fixture tests") - - override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { - override val width: Int = width - override val height: Int = height - - override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { - error("Bitmap drawing is not registered for fixture tests") - } - } - - fun verifyAllFixturesUsed() { - fixtureInterceptor.verifyAllFixturesUsed() - } - - internal class Builder { - private val routes = linkedMapOf<FixtureRequest, String>() - - fun get(url: String, resourcePath: String): Builder = route("GET", url, resourcePath) - - fun post(url: String, resourcePath: String): Builder = route("POST", url, resourcePath) - - private fun route(method: String, url: String, resourcePath: String): Builder = apply { - val key = FixtureRequest(method, url) - require(routes.put(key, resourcePath) == null) { "Duplicate fixture route: $method $url" } - } - - fun build(): FixtureMangaLoaderContext = FixtureMangaLoaderContext(routes.toMap()) - } - - companion object { - fun builder(): Builder = Builder() - } -} - -internal data class FixtureRequest( - val method: String, - val url: String, -) - -private class FixtureHttpInterceptor( - private val routes: Map<FixtureRequest, String>, -) : Interceptor { - - private val usedRoutes = ConcurrentHashMap.newKeySet<FixtureRequest>() - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val key = FixtureRequest(request.method, request.url.toString()) - val resourcePath = routes[key] - ?: error("No fixture registered for ${request.method} ${request.url}") - val body = loadFixture(resourcePath) - usedRoutes.add(key) - return response(request, body) - } - - fun verifyAllFixturesUsed() { - val unusedRoutes = routes.keys - usedRoutes - check(unusedRoutes.isEmpty()) { - "Unused fixture routes: ${unusedRoutes.joinToString { "${it.method} ${it.url}" }}" - } - } - - private fun loadFixture(resourcePath: String): ByteArray { - val stream = FixtureMangaLoaderContext::class.java.getResourceAsStream(resourcePath) - ?: error("Fixture resource not found: $resourcePath") - return stream.use { it.readBytes() } - } - - private fun response(request: Request, body: ByteArray): Response = Response.Builder() - .request(request) - .protocol(Protocol.HTTP_1_1) - .code(200) - .message("OK") - .body(body.toResponseBody("text/html; charset=utf-8".toMediaType())) - .build() -} diff --git a/src/test/kotlin/tsuki/LibSocialJsonContractTest.kt b/src/test/kotlin/tsuki/LibSocialJsonContractTest.kt deleted file mode 100644 index c46d0a32..00000000 --- a/src/test/kotlin/tsuki/LibSocialJsonContractTest.kt +++ /dev/null @@ -1,45 +0,0 @@ -package tsuki - -import org.json.JSONObject -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import tsuki.site.ru.libsocial.parseLibSocialSummary - -class LibSocialJsonContractTest { - - @Test - fun `parses current structured summary`() { - val manga = JSONObject( - """ - { - "summary": { - "type": "doc", - "content": [ - { - "type": "paragraph", - "content": [ - {"type": "text", "text": "First line"}, - {"type": "hardBreak"}, - {"type": "text", "text": "Second line"} - ] - }, - { - "type": "paragraph", - "content": [{"type": "text", "text": "Next paragraph"}] - } - ] - } - } - """.trimIndent(), - ) - - assertEquals("First line\nSecond line\nNext paragraph", manga.parseLibSocialSummary()) - } - - @Test - fun `keeps legacy string summary contract`() { - val manga = JSONObject("""{"summary":"Plain description"}""") - - assertEquals("Plain description", manga.parseLibSocialSummary()) - } -} diff --git a/src/test/kotlin/tsuki/LiveCatalogAuditTest.kt b/src/test/kotlin/tsuki/LiveCatalogAuditTest.kt deleted file mode 100644 index 6f372170..00000000 --- a/src/test/kotlin/tsuki/LiveCatalogAuditTest.kt +++ /dev/null @@ -1,273 +0,0 @@ -package tsuki - -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.TimeoutCancellationException -import kotlinx.coroutines.sync.Semaphore -import kotlinx.coroutines.sync.withPermit -import kotlinx.coroutines.withTimeout -import org.junit.jupiter.api.Assumptions.assumeTrue -import org.junit.jupiter.api.Test -import tsuki.model.MangaListFilter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder -import java.nio.file.Files -import java.nio.file.Path -import java.nio.file.StandardOpenOption -import kotlin.time.Duration.Companion.seconds - -class LiveCatalogAuditTest { - - @Test - fun `audit configured live catalog`() = runBlocking { - assumeTrue(System.getenv(ENV_ENABLED).toBoolean(), "$ENV_ENABLED is not enabled") - - val targetStage = AuditStage.valueOf( - System.getenv(ENV_STAGE)?.uppercase() ?: AuditStage.DETAILS.name, - ) - val locales = System.getenv(ENV_LOCALES) - ?.split(',') - ?.mapTo(linkedSetOf()) { it.trim().lowercase() } - ?.takeIf { it.isNotEmpty() } - val includeBroken = System.getenv(ENV_INCLUDE_BROKEN)?.toBooleanStrictOrNull() ?: true - val concurrency = System.getenv(ENV_CONCURRENCY)?.toIntOrNull()?.coerceIn(1, 12) ?: 4 - val timeout = (System.getenv(ENV_TIMEOUT_SECONDS)?.toLongOrNull()?.coerceIn(10, 600) ?: 120).seconds - val semaphore = Semaphore(concurrency) - val sources = MangaParserSource.entries.filter { source -> - (includeBroken || !source.isBroken) && (locales == null || source.locale.lowercase() in locales) - } - - val results = coroutineScope { - sources.map { source -> - async { - semaphore.withPermit { - val startedAt = System.nanoTime() - val result = try { - withTimeout(timeout) { - auditSource(source, targetStage) - } - } catch (_: TimeoutCancellationException) { - result( - source = source, - status = AuditStatus.TIMEOUT, - stage = targetStage, - startedAt = startedAt, - details = "Source audit exceeded ${timeout.inWholeSeconds} seconds", - ) - } - println(result.consoleLine()) - result - } - } - }.awaitAll() - }.sortedBy { it.source.name } - - val reportScope = locales?.joinToString("-")?.ifEmpty { "all" } ?: "all" - val reportPath = writeReport(results, targetStage, reportScope) - printSummary(results, reportPath) - - if (System.getenv(ENV_FAIL_ON_ERROR).toBoolean()) { - val failures = results.filter { it.status.isFailure } - check(failures.isEmpty()) { - "${failures.size} live sources failed; see $reportPath" - } - } - } - - private suspend fun auditSource(source: MangaParserSource, targetStage: AuditStage): AuditResult { - val startedAt = System.nanoTime() - var currentStage = AuditStage.LIST - return try { - val context = LiveMangaLoaderContext() - val parser = context.newParserInstance(source) - val order = preferredSortOrder(parser) - val list = parser.getList(0, order, MangaListFilter.EMPTY) - check(list.isNotEmpty()) { "Source returned an empty first page" } - val manga = list.first() - if (targetStage == AuditStage.LIST) { - return passed(source, targetStage, startedAt, "items=${list.size}; title=${manga.title}") - } - - currentStage = AuditStage.DETAILS - val details = parser.getDetails(manga) - if (targetStage == AuditStage.DETAILS) { - return passed( - source, - targetStage, - startedAt, - "items=${list.size}; title=${details.title}; chapters=${details.chapters.orEmpty().size}", - ) - } - - currentStage = AuditStage.CHAPTERS - val chapter = details.chapters.orEmpty().firstOrNull() - ?: return result( - source, - AuditStatus.NO_CHAPTERS, - currentStage, - startedAt, - "title=${details.title}", - ) - if (targetStage == AuditStage.CHAPTERS) { - return passed(source, targetStage, startedAt, "title=${details.title}; chapter=${chapter.url}") - } - - currentStage = AuditStage.PAGES - val pages = parser.getPages(chapter) - check(pages.isNotEmpty()) { "Chapter returned no pages" } - val pageUrl = parser.getPageUrl(pages.first()) - passed(source, targetStage, startedAt, "pages=${pages.size}; first=$pageUrl") - } catch (error: Throwable) { - result( - source = source, - status = classify(source, error), - stage = currentStage, - startedAt = startedAt, - details = "${error::class.simpleName}: ${error.message.orEmpty()}".singleLine(), - ) - } - } - - private fun preferredSortOrder(parser: MangaParser): SortOrder = listOf( - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ).firstOrNull { it in parser.availableSortOrders } - ?: parser.availableSortOrders.first() - - private fun classify(source: MangaParserSource, error: Throwable): AuditStatus { - if (source.isBroken) return AuditStatus.EXPECTED_BROKEN - val message = generateSequence(error) { it.cause } - .joinToString(" | ") { "${it::class.simpleName}: ${it.message.orEmpty()}" } - val normalizedMessage = message.lowercase() - return when { - "requires an Android consumer runtime" in message -> AuditStatus.ANDROID_RUNTIME_REQUIRED - "AuthRequiredException" in message -> AuditStatus.AUTH_REQUIRED - "SocketTimeoutException" in message || "TimeoutCancellationException" in message -> AuditStatus.TIMEOUT - "source returned an empty first page" in normalizedMessage -> AuditStatus.EMPTY_LIST - "ssl" in normalizedMessage || "certificate" in normalizedMessage -> AuditStatus.TLS_ERROR - "HttpStatusException" in message || "NotFoundException" in message || "status=" in normalizedMessage -> - AuditStatus.HTTP_ERROR - "ParseException" in message || "JSONException" in message || "parse" in normalizedMessage -> - AuditStatus.PARSE_ERROR - "HttpException" in message || "UnknownHostException" in message || "StreamResetException" in message -> - AuditStatus.NETWORK_ERROR - else -> AuditStatus.ERROR - } - } - - private fun passed( - source: MangaParserSource, - stage: AuditStage, - startedAt: Long, - details: String, - ): AuditResult = result( - source = source, - status = if (source.isBroken) AuditStatus.BROKEN_MARKER_STALE else AuditStatus.PASSED, - stage = stage, - startedAt = startedAt, - details = details, - ) - - private fun result( - source: MangaParserSource, - status: AuditStatus, - stage: AuditStage, - startedAt: Long, - details: String, - ) = AuditResult( - source = source, - status = status, - stage = stage, - durationMs = (System.nanoTime() - startedAt) / 1_000_000, - details = details.singleLine(), - ) - - private fun writeReport(results: List<AuditResult>, targetStage: AuditStage, reportScope: String): Path { - val report = Path.of( - "build/reports/live-source-audit-${targetStage.name.lowercase()}-$reportScope.tsv", - ) - Files.createDirectories(report.parent) - val rows = buildString { - appendLine("source\ttitle\tlocale\tbroken\ttarget\tstatus\tstage\tduration_ms\tdetails") - results.forEach { result -> - appendLine( - listOf( - result.source.name, - result.source.title, - result.source.locale.ifEmpty { "all" }, - result.source.isBroken, - targetStage, - result.status, - result.stage, - result.durationMs, - result.details, - ).joinToString("\t"), - ) - } - } - Files.writeString( - report, - rows, - StandardOpenOption.CREATE, - StandardOpenOption.TRUNCATE_EXISTING, - StandardOpenOption.WRITE, - ) - return report - } - - private fun printSummary(results: List<AuditResult>, reportPath: Path) { - val counts = results.groupingBy { it.status }.eachCount().toSortedMap(compareBy { it.name }) - println("[LIVE-AUDIT] report=$reportPath") - println("[LIVE-AUDIT] total=${results.size}; ${counts.entries.joinToString { "${it.key}=${it.value}" }}") - } - - private fun String.singleLine(): String = replace('\t', ' ').replace('\r', ' ').replace('\n', ' ') - - private data class AuditResult( - val source: MangaParserSource, - val status: AuditStatus, - val stage: AuditStage, - val durationMs: Long, - val details: String, - ) { - fun consoleLine(): String = - "[LIVE-AUDIT] ${source.name}\t$status\t$stage\t${durationMs}ms\t$details" - } - - private enum class AuditStage { - LIST, - DETAILS, - CHAPTERS, - PAGES, - } - - private enum class AuditStatus(val isFailure: Boolean) { - PASSED(false), - EXPECTED_BROKEN(false), - BROKEN_MARKER_STALE(true), - NO_CHAPTERS(false), - ANDROID_RUNTIME_REQUIRED(false), - AUTH_REQUIRED(false), - TIMEOUT(true), - EMPTY_LIST(true), - HTTP_ERROR(true), - TLS_ERROR(true), - PARSE_ERROR(true), - NETWORK_ERROR(true), - ERROR(true), - } - - private companion object { - const val ENV_ENABLED = "LIVE_AUDIT" - const val ENV_STAGE = "LIVE_AUDIT_STAGE" - const val ENV_LOCALES = "LIVE_AUDIT_LOCALES" - const val ENV_INCLUDE_BROKEN = "LIVE_AUDIT_INCLUDE_BROKEN" - const val ENV_CONCURRENCY = "LIVE_AUDIT_CONCURRENCY" - const val ENV_TIMEOUT_SECONDS = "LIVE_AUDIT_TIMEOUT_SECONDS" - const val ENV_FAIL_ON_ERROR = "LIVE_AUDIT_FAIL_ON_ERROR" - } -} diff --git a/src/test/kotlin/tsuki/LiveMangaLoaderContext.kt b/src/test/kotlin/tsuki/LiveMangaLoaderContext.kt deleted file mode 100644 index 072ceef0..00000000 --- a/src/test/kotlin/tsuki/LiveMangaLoaderContext.kt +++ /dev/null @@ -1,128 +0,0 @@ -package tsuki - -import okhttp3.Cookie -import okhttp3.CookieJar -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.Response -import tsuki.bitmap.Bitmap -import tsuki.bitmap.Rect -import tsuki.config.ConfigKey -import tsuki.config.MangaSourceConfig -import tsuki.model.MangaParserSource -import tsuki.model.MangaSource -import java.util.concurrent.CopyOnWriteArrayList -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.TimeUnit - -@Suppress("OVERRIDE_DEPRECATION") -internal class LiveMangaLoaderContext : MangaLoaderContext() { - - private val storedCookies = CopyOnWriteArrayList<Cookie>() - private val parsers = ConcurrentHashMap<MangaSource, MangaParser>() - - override val cookieJar: CookieJar = object : CookieJar { - override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) { - cookies.forEach { cookie -> - storedCookies.removeIf { - it.name == cookie.name && it.domain == cookie.domain && it.path == cookie.path - } - if (cookie.expiresAt > System.currentTimeMillis()) { - storedCookies.add(cookie) - } - } - } - - override fun loadForRequest(url: HttpUrl): List<Cookie> { - storedCookies.removeIf { it.expiresAt <= System.currentTimeMillis() } - return storedCookies.filter { it.matches(url) } - } - } - - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .cookieJar(cookieJar) - .addInterceptor(::interceptSourceRequest) - .connectTimeout(30, TimeUnit.SECONDS) - .readTimeout(30, TimeUnit.SECONDS) - .writeTimeout(30, TimeUnit.SECONDS) - .callTimeout(60, TimeUnit.SECONDS) - .build() - - override fun newParserInstance(source: MangaSource): MangaParser { - require(source is MangaParserSource) { "Unsupported live source: $source" } - return parsers.computeIfAbsent(source) { source.newParser(this) } - } - - override fun getParserSources(): List<MangaSource> = MangaParserSource.entries - - override suspend fun evaluateJs(script: String): String? { - error("JavaScript evaluation requires an Android consumer runtime") - } - - override suspend fun evaluateJs(baseUrl: String, script: String): String? { - if (script.startsWith("window.localStorage.getItem(")) { - return null - } - error("JavaScript evaluation requires an Android consumer runtime: $baseUrl") - } - - override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { - override fun <T> get(key: ConfigKey<T>): T = key.defaultValue - } - - override fun getDefaultUserAgent(): String = - "Mozilla/5.0 (Linux; Android 15; Pixel 8) AppleWebKit/537.36 " + - "(KHTML, like Gecko) Chrome/131.0 Mobile Safari/537.36" - - override fun redrawImageResponse( - response: Response, - redraw: (image: Bitmap) -> Bitmap, - ): Response = error("Image redraw requires an Android consumer runtime") - - override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { - override val width: Int = width - override val height: Int = height - - override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { - error("Bitmap drawing requires an Android consumer runtime") - } - } - - private fun interceptSourceRequest(chain: Interceptor.Chain): Response { - val request = chain.request() - val source = request.tag(MangaSource::class.java) - ?: return chain.proceed(request.withDefaultUserAgent()) - val parser = newParserInstance(source) - val headers = request.headers.newBuilder() - val parserHeaders = parser.getRequestHeaders() - parserHeaders.names().forEach { name -> - if (headers[name] == null) { - parserHeaders.values(name).forEach { value -> headers.add(name, value) } - } - } - if (headers["User-Agent"] == null) { - headers["User-Agent"] = getDefaultUserAgent() - } - if (headers["Referer"] == null) { - headers["Referer"] = "https://${parser.domain}/" - } - val taggedRequest = request.newBuilder().headers(headers.build()).build() - return parser.intercept(SourceRequestChain(chain, taggedRequest)) - } - - private fun Request.withDefaultUserAgent(): Request { - if (header("User-Agent") != null) { - return this - } - return newBuilder().header("User-Agent", getDefaultUserAgent()).build() - } - - private class SourceRequestChain( - private val delegate: Interceptor.Chain, - private val taggedRequest: Request, - ) : Interceptor.Chain by delegate { - override fun request(): Request = taggedRequest - } -} diff --git a/src/test/kotlin/tsuki/LiveSourceSmokeTest.kt b/src/test/kotlin/tsuki/LiveSourceSmokeTest.kt deleted file mode 100644 index cf6bb3cf..00000000 --- a/src/test/kotlin/tsuki/LiveSourceSmokeTest.kt +++ /dev/null @@ -1,79 +0,0 @@ -package tsuki - -import kotlinx.coroutines.runBlocking -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Assumptions.assumeTrue -import org.junit.jupiter.api.Test -import tsuki.model.MangaListFilter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder - -class LiveSourceSmokeTest { - - @Test - fun `run configured live source flow`() = runBlocking { - val sourceName = System.getenv(ENV_SOURCE) - assumeTrue(!sourceName.isNullOrBlank(), "$ENV_SOURCE is not configured") - - val source = MangaParserSource.valueOf(sourceName) - val stage = LiveStage.valueOf(System.getenv(ENV_STAGE)?.uppercase() ?: LiveStage.PAGES.name) - val order = SortOrder.valueOf(System.getenv(ENV_SORT)?.uppercase() ?: SortOrder.UPDATED.name) - val query = System.getenv(ENV_QUERY)?.takeIf(String::isNotBlank) - val expectedTitle = System.getenv(ENV_EXPECT_TITLE)?.takeIf(String::isNotBlank) - val context = LiveMangaLoaderContext() - val parser = context.newParserInstance(source) - val filter = MangaListFilter.EMPTY.copy(query = query) - - val list = liveStep(source, "list") { - parser.getList(offset = 0, order = order, filter = filter) - } - assertFalse(list.isEmpty(), "Live list is empty for ${source.name}") - val manga = expectedTitle?.let { title -> - list.firstOrNull { it.title.equals(title, ignoreCase = true) } - ?: error("Cannot find '$title' in ${list.size} live list entries") - } ?: list.first() - println("[LIVE-SOURCE] list ok: count=${list.size}, selected=${manga.title}, url=${manga.url}") - if (stage == LiveStage.LIST) return@runBlocking - - val details = liveStep(source, "details") { parser.getDetails(manga) } - println( - "[LIVE-SOURCE] details ok: title=${details.title}, " + - "chapters=${details.chapters.orEmpty().size}, description=${!details.description.isNullOrBlank()}", - ) - if (stage == LiveStage.DETAILS) return@runBlocking - - val chapter = details.chapters.orEmpty().firstOrNull() - ?: error("Live details contain no chapters for ${source.name}: ${details.title}") - if (stage == LiveStage.CHAPTERS) return@runBlocking - - val pages = liveStep(source, "pages") { parser.getPages(chapter) } - assertFalse(pages.isEmpty(), "Live pages are empty for ${source.name}: ${chapter.title}") - val resolvedUrl = liveStep(source, "page-url") { parser.getPageUrl(pages.first()) } - println("[LIVE-SOURCE] pages ok: count=${pages.size}, first=$resolvedUrl") - } - - private suspend fun <T> liveStep( - source: MangaParserSource, - operation: String, - block: suspend () -> T, - ): T = try { - block() - } catch (error: Throwable) { - throw AssertionError("Live $operation failed for ${source.name}: ${error.message}", error) - } - - private enum class LiveStage { - LIST, - DETAILS, - CHAPTERS, - PAGES, - } - - private companion object { - const val ENV_SOURCE = "LIVE_SOURCE" - const val ENV_STAGE = "LIVE_STAGE" - const val ENV_SORT = "LIVE_SORT" - const val ENV_QUERY = "LIVE_QUERY" - const val ENV_EXPECT_TITLE = "LIVE_EXPECT_TITLE" - } -} diff --git a/src/test/kotlin/tsuki/MangaMammyContractTest.kt b/src/test/kotlin/tsuki/MangaMammyContractTest.kt deleted file mode 100644 index 9336baef..00000000 --- a/src/test/kotlin/tsuki/MangaMammyContractTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -package tsuki - -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Test -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder - -class MangaMammyContractTest { - - @Test - fun `source supports reading from list through pages`() = runTest { - val context = FixtureMangaLoaderContext.builder() - .post( - "https://p.nimanga.com/wp-admin/admin-ajax.php", - "/fixtures/mangamammy/list.html", - ) - .get( - "https://p.nimanga.com/manga/example-title", - "/fixtures/mangamammy/details.html", - ) - .get( - "https://p.nimanga.com/chapter/example-title-1?style=list", - "/fixtures/mangamammy/pages.html", - ) - .build() - val parser = MangaParserSource.MANGAMAMMY.newParser(context) - - verifySourceReadingContract( - parser = parser, - context = context, - expectedSource = MangaParserSource.MANGAMAMMY, - expectedTitle = "Example Title", - sortOrder = SortOrder.UPDATED, - ) - } -} diff --git a/src/test/kotlin/tsuki/MangaWtfContractTest.kt b/src/test/kotlin/tsuki/MangaWtfContractTest.kt deleted file mode 100644 index a9f26b3e..00000000 --- a/src/test/kotlin/tsuki/MangaWtfContractTest.kt +++ /dev/null @@ -1,36 +0,0 @@ -package tsuki - -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import tsuki.model.MangaListFilter -import tsuki.model.MangaParserSource -import tsuki.model.MangaTag -import tsuki.model.SortOrder - -class MangaWtfContractTest { - - @Test - fun `source sends excluded labels separately from included labels`() = runTest { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://api.inkstory.net/v2/books?page=0&size=20&type=COMIC&sort=updatedAt%2Cdesc&labelsInclude=action&labelsExclude=romance", - "/fixtures/mangawtf/list.json", - ) - .build() - val parser = MangaParserSource.MANGA_WTF.newParser(context) - val source = MangaParserSource.MANGA_WTF - - val manga = parser.getList( - offset = 0, - order = SortOrder.UPDATED, - filter = MangaListFilter.EMPTY.copy( - tags = setOf(MangaTag("Action", "action", source)), - tagsExclude = setOf(MangaTag("Romance", "romance", source)), - ), - ).single() - - assertEquals("Test title", manga.title) - context.verifyAllFixturesUsed() - } -} diff --git a/src/test/kotlin/tsuki/ReadMangaContractTest.kt b/src/test/kotlin/tsuki/ReadMangaContractTest.kt deleted file mode 100644 index f7327efc..00000000 --- a/src/test/kotlin/tsuki/ReadMangaContractTest.kt +++ /dev/null @@ -1,53 +0,0 @@ -package tsuki - -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import org.junit.jupiter.api.Test -import tsuki.model.MangaListFilter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder - -class ReadMangaContractTest { - - @Test - fun `source supports title search through current details and pages markup`() = runTest { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://a.zazaza.me/search/advancedResults?q=Artifact&offset=0&years=1900%2C2099&sortType=DATE_CREATE", - "/fixtures/readmanga/search.html", - ) - .get( - "https://a.zazaza.me/igrok__kotoryi_est_artefakty", - "/fixtures/readmanga/details.html", - ) - .get( - "https://a.zazaza.me/igrok__kotoryi_est_artefakty/vol1/45?mtr=1", - "/fixtures/readmanga/pages.html", - ) - .build() - val parser = MangaParserSource.READMANGA_RU.newParser(context) - - val manga = parser.getList( - offset = 0, - order = SortOrder.NEWEST, - filter = MangaListFilter.EMPTY.copy(query = "Artifact"), - ).single() - assertEquals("Игрок, который ест артефакты", manga.title) - - val details = parser.getDetails(manga) - assertEquals("Игрок, который ест артефакты", details.title) - assertEquals("Current ReadManga description.", details.description) - assertEquals(setOf("Боевик", "Драма"), details.tags.mapTo(linkedSetOf()) { it.title }) - assertEquals("https://rm.one-way.work/uploads/pics/example.jpg", details.largeCoverUrl) - - val chapter = details.chapters.orEmpty().single() - assertEquals(45f, chapter.number) - assertEquals(1, chapter.volume) - - val pages = parser.getPages(chapter) - assertFalse(pages.isEmpty()) - assertEquals("https://images.example/page-01.jpg", parser.getPageUrl(pages.single())) - context.verifyAllFixturesUsed() - } -} diff --git a/src/test/kotlin/tsuki/RussianSourceCatalogTest.kt b/src/test/kotlin/tsuki/RussianSourceCatalogTest.kt deleted file mode 100644 index 71607b30..00000000 --- a/src/test/kotlin/tsuki/RussianSourceCatalogTest.kt +++ /dev/null @@ -1,174 +0,0 @@ -package tsuki - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Test -import tsuki.model.MangaParserSource - -class RussianSourceCatalogTest { - - @Test - fun `catalog exposes Russian NineManga successor`() { - assertTrue( - MangaParserSource.entries.any { - it.name == "NINEMANGA_RU" && it.locale == "ru" - }, - "NINEMANGA_RU must be present in the generated Russian catalog", - ) - } - - @Test - fun `NineManga Russian parser preserves its configured domain`() { - val parser = MangaParserSource.NINEMANGA_RU.newParser(FixtureMangaLoaderContext.builder().build()) - - assertEquals("ru.ninemanga.com", parser.domain) - } - - @Test - fun `catalog exposes Russian Madara sources`() { - val expectedSources = setOf( - "BEST_MANGA", - "MANGAMAMMY", - "MANGAONELOVE", - "MANGAZAVR", - ) - - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertTrue( - actualSources.containsAll(expectedSources), - "Missing Russian Madara sources: ${expectedSources - actualSources}", - ) - } - - @Test - fun `catalog exposes Russian Chan sources`() { - val expectedSources = setOf("HENCHAN", "MANGACHAN", "YAOICHAN") - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertTrue( - actualSources.containsAll(expectedSources), - "Missing Russian Chan sources: ${expectedSources - actualSources}", - ) - } - - @Test - fun `catalog exposes Russian Grouple sources`() { - val expectedSources = setOf( - "ALLHENTAI", - "MINTMANGA", - "READMANGA_RU", - "SEIMANGA", - "SELFMANGA", - "USAGI", - ) - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertTrue( - actualSources.containsAll(expectedSources), - "Missing Russian Grouple sources: ${expectedSources - actualSources}", - ) - } - - @Test - fun `catalog exposes Russian LibSocial sources`() { - val expectedSources = setOf("HENTAILIB", "MANGALIB", "YAOILIB") - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertTrue( - actualSources.containsAll(expectedSources), - "Missing Russian LibSocial sources: ${expectedSources - actualSources}", - ) - } - - @Test - fun `catalog exposes independent Russian sources`() { - val expectedSources = setOf( - "ACOMICS", - "COMX", - "DESUME", - "MANGA_WTF", - "NUDEMOON", - "REMANGA", - "WAMANGA", - "ZENMANGA", - ) - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertTrue( - actualSources.containsAll(expectedSources), - "Missing independent Russian sources: ${expectedSources - actualSources}", - ) - } - - @Test - fun `catalog exposes the complete Russian source set`() { - val expectedSources = setOf( - "ACOMICS", - "ALLHENTAI", - "ASTRAMANGA", - "BEST_MANGA", - "COMX", - "DESUME", - "HENCHAN", - "HENTAILIB", - "MANGABALL_RU", - "MANGACHAN", - "MANGALIB", - "MANGAMAMMY", - "MANGAONELOVE", - "MANGAPLUSPARSER_RU", - "MANGAZAVR", - "MANGA_WTF", - "MINTMANGA", - "NINEMANGA_RU", - "NUDEMOON", - "READMANGA_RU", - "REMANGA", - "SEIMANGA", - "SELFMANGA", - "USAGI", - "WAMANGA", - "YAOICHAN", - "YAOILIB", - "ZENMANGA", - ) - val actualSources = MangaParserSource.entries - .filter { it.locale == "ru" } - .mapTo(mutableSetOf()) { it.name } - - assertEquals(expectedSources, actualSources) - } - - @Test - fun `only currently unavailable Russian sources remain marked broken`() { - val brokenSources = MangaParserSource.entries - .filter { it.locale == "ru" && it.isBroken } - .mapTo(mutableSetOf()) { it.name } - - assertEquals( - setOf( - "ALLHENTAI", - "BEST_MANGA", - "HENCHAN", - "HENTAILIB", - "MANGAZAVR", - "MINTMANGA", - "NINEMANGA_RU", - "NUDEMOON", - "ZENMANGA", - ), - brokenSources, - ) - } -} diff --git a/src/test/kotlin/tsuki/RussianSourceRepairContractTest.kt b/src/test/kotlin/tsuki/RussianSourceRepairContractTest.kt deleted file mode 100644 index d2d593d6..00000000 --- a/src/test/kotlin/tsuki/RussianSourceRepairContractTest.kt +++ /dev/null @@ -1,93 +0,0 @@ -package tsuki - -import kotlinx.coroutines.test.runTest -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import tsuki.model.MangaListFilter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder - -class RussianSourceRepairContractTest { - - @Test - fun `repaired Russian sources satisfy their current reading contracts`() = runTest { - verifyAComics() - verifyDesu() - verifyMangaOneLove() - } - - private suspend fun verifyAComics() { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://acomics.ru/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=0&sort=last_update", - "/fixtures/acomics/catalog.html", - ) - .get("https://acomics.ru/~fixture/about", "/fixtures/acomics/details.html") - .get("https://acomics.ru/comics", "/fixtures/acomics/catalog.html") - .get("https://acomics.ru/~fixture/1", "/fixtures/acomics/page.html") - .build() - val parser = MangaParserSource.ACOMICS.newParser(context) - - val manga = parser.getList( - offset = 0, - order = SortOrder.UPDATED, - filter = MangaListFilter.EMPTY, - ).single() - assertEquals("Fixture Comic", manga.title) - assertEquals("https://acomics.ru/upload/fixture-cover.jpg", manga.coverUrl) - - val details = parser.getDetails(manga) - assertEquals("Current AComics description.", details.description) - assertEquals(setOf("Fantasy"), details.tags.mapTo(linkedSetOf()) { it.title }) - assertEquals(setOf("Fixture Author"), details.authors) - - val pages = parser.getPages(details.chapters.orEmpty().single()) - assertEquals(3, pages.size) - assertEquals("https://acomics.ru/upload/fixture-page.jpg", parser.getPageUrl(pages.first())) - context.verifyAllFixturesUsed() - } - - private suspend fun verifyDesu() { - val context = FixtureMangaLoaderContext.builder() - .get("https://desu.uno/manga/?page=1", "/fixtures/desu/catalog.html") - .get("https://desu.uno/manga/example.42/", "/fixtures/desu/details.html") - .get("https://desu.uno/api/manga/42/chapters/690", "/fixtures/desu/pages.json") - .build() - val parser = MangaParserSource.DESUME.newParser(context) - - verifySourceReadingContract( - parser = parser, - context = context, - expectedSource = MangaParserSource.DESUME, - expectedTitle = "Пример манги", - sortOrder = SortOrder.UPDATED, - expectedPageUrl = "https://img3.desu.uno/manga/rus/example/001.webp", - ) - } - - private suspend fun verifyMangaOneLove() { - val context = FixtureMangaLoaderContext.builder() - .post( - "https://mangaonelove.su/wp-admin/admin-ajax.php", - "/fixtures/mangaonelove/list.html", - ) - .get( - "https://mangaonelove.su/manga/example-title", - "/fixtures/mangaonelove/details.html", - ) - .get( - "https://mangaonelove.su/chapter/example-title-1?style=list", - "/fixtures/mangaonelove/pages.html", - ) - .build() - val parser = MangaParserSource.MANGAONELOVE.newParser(context) - - verifySourceReadingContract( - parser = parser, - context = context, - expectedSource = MangaParserSource.MANGAONELOVE, - expectedTitle = "Example Title", - sortOrder = SortOrder.UPDATED, - ) - } -} diff --git a/src/test/kotlin/tsuki/RussianSourcesContractTest.kt b/src/test/kotlin/tsuki/RussianSourcesContractTest.kt new file mode 100644 index 00000000..83c14350 --- /dev/null +++ b/src/test/kotlin/tsuki/RussianSourcesContractTest.kt @@ -0,0 +1,457 @@ +package tsuki + +import kotlinx.coroutines.test.runTest +import okhttp3.Cookie +import okhttp3.CookieJar +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Protocol +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONObject +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Assertions.assertFalse +import org.junit.jupiter.api.Test +import tsuki.bitmap.Bitmap +import tsuki.bitmap.Rect +import tsuki.config.ConfigKey +import tsuki.config.MangaSourceConfig +import tsuki.model.MangaParserSource +import tsuki.model.MangaListFilter +import tsuki.model.MangaSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.site.ru.libsocial.parseLibSocialSummary +import tsuki.site.ru.parseWaMangaAlternativeTitles +import tsuki.site.ru.parseWaMangaGenres +import tsuki.site.ru.parseWaMangaSearchText +import java.util.concurrent.ConcurrentHashMap + +class RussianSourcesContractTest { + + @Test + fun `Russian catalog and representative reading contracts remain valid`() = runTest { + verifyCatalog() + verifyLibSocialSummary() + verifyAComics() + verifyDesu() + verifyMangaOneLove() + verifyMangaMammy() + verifyMangaWtfFilters() + verifyReadManga() + verifyWaManga() + } + + private fun verifyCatalog() { + val expectedSources = setOf( + "ACOMICS", + "ALLHENTAI", + "ASTRAMANGA", + "BEST_MANGA", + "COMX", + "DESUME", + "HENCHAN", + "HENTAILIB", + "MANGABALL_RU", + "MANGACHAN", + "MANGALIB", + "MANGAMAMMY", + "MANGAONELOVE", + "MANGAPLUSPARSER_RU", + "MANGAZAVR", + "MANGA_WTF", + "MINTMANGA", + "NINEMANGA_RU", + "NUDEMOON", + "READMANGA_RU", + "REMANGA", + "SEIMANGA", + "SELFMANGA", + "USAGI", + "WAMANGA", + "YAOICHAN", + "YAOILIB", + "ZENMANGA", + ) + val expectedBrokenSources = setOf( + "ALLHENTAI", + "BEST_MANGA", + "HENCHAN", + "HENTAILIB", + "MANGAZAVR", + "MINTMANGA", + "NINEMANGA_RU", + "NUDEMOON", + "ZENMANGA", + ) + val sources = MangaParserSource.entries.filter { it.locale == "ru" } + + assertEquals(expectedSources, sources.mapTo(mutableSetOf()) { it.name }) + assertEquals(expectedBrokenSources, sources.filter { it.isBroken }.mapTo(mutableSetOf()) { it.name }) + + val context = FixtureMangaLoaderContext.builder().build() + sources.forEach { source -> + assertEquals(source, source.newParser(context).source) + } + assertEquals("ru.ninemanga.com", MangaParserSource.NINEMANGA_RU.newParser(context).domain) + } + + private fun verifyLibSocialSummary() { + val structured = JSONObject( + """ + { + "summary": { + "type": "doc", + "content": [ + { + "type": "paragraph", + "content": [ + {"type": "text", "text": "First line"}, + {"type": "hardBreak"}, + {"type": "text", "text": "Second line"} + ] + }, + { + "type": "paragraph", + "content": [{"type": "text", "text": "Next paragraph"}] + } + ] + } + } + """.trimIndent(), + ) + + assertEquals("First line\nSecond line\nNext paragraph", structured.parseLibSocialSummary()) + assertEquals( + "Plain description", + JSONObject("""{"summary":"Plain description"}""").parseLibSocialSummary(), + ) + } + + private suspend fun verifyAComics() { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://acomics.ru/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=0&sort=last_update", + "/fixtures/acomics/catalog.html", + ) + .get("https://acomics.ru/~fixture/about", "/fixtures/acomics/details.html") + .get("https://acomics.ru/comics", "/fixtures/acomics/catalog.html") + .get("https://acomics.ru/~fixture/1", "/fixtures/acomics/page.html") + .build() + val parser = MangaParserSource.ACOMICS.newParser(context) + + val manga = parser.getList(0, SortOrder.UPDATED, MangaListFilter.EMPTY).single() + assertEquals("Fixture Comic", manga.title) + assertEquals("https://acomics.ru/upload/fixture-cover.jpg", manga.coverUrl) + + val details = parser.getDetails(manga) + assertEquals("Current AComics description.", details.description) + assertEquals(setOf("Fantasy"), details.tags.mapTo(linkedSetOf()) { it.title }) + assertEquals(setOf("Fixture Author"), details.authors) + + val pages = parser.getPages(details.chapters.orEmpty().single()) + assertEquals(3, pages.size) + assertEquals("https://acomics.ru/upload/fixture-page.jpg", parser.getPageUrl(pages.first())) + context.verifyAllFixturesUsed() + } + + private suspend fun verifyDesu() { + val context = FixtureMangaLoaderContext.builder() + .get("https://desu.uno/manga/?page=1", "/fixtures/desu/catalog.html") + .get("https://desu.uno/manga/example.42/", "/fixtures/desu/details.html") + .get("https://desu.uno/api/manga/42/chapters/690", "/fixtures/desu/pages.json") + .build() + + verifyReadingContract( + source = MangaParserSource.DESUME, + context = context, + expectedTitle = "Пример манги", + sortOrder = SortOrder.UPDATED, + expectedPageUrl = "https://img3.desu.uno/manga/rus/example/001.webp", + ) + } + + private suspend fun verifyMangaOneLove() { + val context = FixtureMangaLoaderContext.builder() + .post( + "https://mangaonelove.su/wp-admin/admin-ajax.php", + "/fixtures/mangaonelove/list.html", + ) + .get("https://mangaonelove.su/manga/example-title", "/fixtures/mangaonelove/details.html") + .get( + "https://mangaonelove.su/chapter/example-title-1?style=list", + "/fixtures/mangaonelove/pages.html", + ) + .build() + + verifyReadingContract( + source = MangaParserSource.MANGAONELOVE, + context = context, + expectedTitle = "Example Title", + sortOrder = SortOrder.UPDATED, + ) + } + + private suspend fun verifyMangaMammy() { + val context = FixtureMangaLoaderContext.builder() + .post( + "https://p.nimanga.com/wp-admin/admin-ajax.php", + "/fixtures/mangamammy/list.html", + ) + .get("https://p.nimanga.com/manga/example-title", "/fixtures/mangamammy/details.html") + .get( + "https://p.nimanga.com/chapter/example-title-1?style=list", + "/fixtures/mangamammy/pages.html", + ) + .build() + + verifyReadingContract( + source = MangaParserSource.MANGAMAMMY, + context = context, + expectedTitle = "Example Title", + sortOrder = SortOrder.UPDATED, + ) + } + + private suspend fun verifyMangaWtfFilters() { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://api.inkstory.net/v2/books?page=0&size=20&type=COMIC&sort=updatedAt%2Cdesc&labelsInclude=action&labelsExclude=romance", + "/fixtures/mangawtf/list.json", + ) + .build() + val source = MangaParserSource.MANGA_WTF + val manga = source.newParser(context).getList( + offset = 0, + order = SortOrder.UPDATED, + filter = MangaListFilter.EMPTY.copy( + tags = setOf(MangaTag("Action", "action", source)), + tagsExclude = setOf(MangaTag("Romance", "romance", source)), + ), + ).single() + + assertEquals("Test title", manga.title) + context.verifyAllFixturesUsed() + } + + private suspend fun verifyReadManga() { + val context = FixtureMangaLoaderContext.builder() + .get( + "https://a.zazaza.me/search/advancedResults?q=Artifact&offset=0&years=1900%2C2099&sortType=DATE_CREATE", + "/fixtures/readmanga/search.html", + ) + .get( + "https://a.zazaza.me/igrok__kotoryi_est_artefakty", + "/fixtures/readmanga/details.html", + ) + .get( + "https://a.zazaza.me/igrok__kotoryi_est_artefakty/vol1/45?mtr=1", + "/fixtures/readmanga/pages.html", + ) + .build() + val parser = MangaParserSource.READMANGA_RU.newParser(context) + val manga = parser.getList( + offset = 0, + order = SortOrder.NEWEST, + filter = MangaListFilter.EMPTY.copy(query = "Artifact"), + ).single() + assertEquals("Игрок, который ест артефакты", manga.title) + + val details = parser.getDetails(manga) + assertEquals("Current ReadManga description.", details.description) + assertEquals(setOf("Боевик", "Драма"), details.tags.mapTo(linkedSetOf()) { it.title }) + assertEquals("https://rm.one-way.work/uploads/pics/example.jpg", details.largeCoverUrl) + + val chapter = details.chapters.orEmpty().single() + assertEquals(45f, chapter.number) + assertEquals(1, chapter.volume) + + val pages = parser.getPages(chapter) + assertFalse(pages.isEmpty()) + assertEquals("https://images.example/page-01.jpg", parser.getPageUrl(pages.single())) + context.verifyAllFixturesUsed() + } + + private suspend fun verifyWaManga() { + val context = FixtureMangaLoaderContext.builder() + .get("https://wamanga.ru/api/v1/manga", "/fixtures/wamanga/catalog.json") + .get("https://wamanga.ru/api/v1/manga/manga-id", "/fixtures/wamanga/details.json") + .get("https://wamanga.ru/api/v1/manga/manga-id/chapters", "/fixtures/wamanga/chapters.json") + .get("https://wamanga.ru/api/v1/chapters/chapter-id", "/fixtures/wamanga/pages.json") + .build() + + verifyReadingContract( + source = MangaParserSource.WAMANGA, + context = context, + expectedTitle = "Example Manga", + sortOrder = SortOrder.ALPHABETICAL, + expectedPageUrl = "https://wamanga.ru/app/uploads/example/001.webp", + ) + + val item = JSONObject( + """{ + "title":"Russian title", + "titleEnglish":"English title", + "alternateTitles":["Original title"], + "genres":["Action","Drama"] + }""", + ) + assertEquals(setOf("English title", "Original title"), item.parseWaMangaAlternativeTitles()) + assertEquals(setOf("Action", "Drama"), item.parseWaMangaGenres()) + assertEquals( + setOf("Russian title", "English title", "Original title"), + item.parseWaMangaSearchText(), + ) + } + + private suspend fun verifyReadingContract( + source: MangaParserSource, + context: FixtureMangaLoaderContext, + expectedTitle: String, + sortOrder: SortOrder, + expectedPageUrl: String? = null, + ) { + val parser = source.newParser(context) + val manga = parser.getList(0, sortOrder, MangaListFilter.EMPTY).single() + assertEquals(source, manga.source) + assertEquals(expectedTitle, manga.title) + + val details = parser.getDetails(manga) + assertEquals(manga.id, details.id) + assertEquals(manga.url, details.url) + assertEquals(source, details.source) + + val chapter = details.chapters.orEmpty().single() + assertEquals(source, chapter.source) + + val pages = parser.getPages(chapter) + assertFalse(pages.isEmpty()) + assertEquals(source, pages.single().source) + if (expectedPageUrl != null) { + assertEquals(expectedPageUrl, parser.getPageUrl(pages.single())) + } + context.verifyAllFixturesUsed() + } +} + +@Suppress("OVERRIDE_DEPRECATION") +internal class FixtureMangaLoaderContext private constructor( + routes: Map<FixtureRequest, String>, +) : MangaLoaderContext() { + + private val fixtureInterceptor = FixtureHttpInterceptor(routes) + + override val httpClient: OkHttpClient = OkHttpClient.Builder() + .addInterceptor(fixtureInterceptor) + .build() + + override val cookieJar: CookieJar = object : CookieJar { + override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = Unit + + override fun loadForRequest(url: HttpUrl): List<Cookie> = emptyList() + } + + override fun newParserInstance(source: MangaSource): MangaParser { + require(source is MangaParserSource) { "Unsupported fixture source: $source" } + return source.newParser(this) + } + + override fun getParserSources(): List<MangaSource> = MangaParserSource.entries + + override suspend fun evaluateJs(script: String): String? { + error("JavaScript evaluation is not registered for fixture tests") + } + + override suspend fun evaluateJs(baseUrl: String, script: String): String? { + error("JavaScript evaluation is not registered for fixture tests: $baseUrl") + } + + override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { + override fun <T> get(key: ConfigKey<T>): T = key.defaultValue + } + + override fun getDefaultUserAgent(): String = "MangaRepoContractTest/1.0" + + override fun redrawImageResponse( + response: Response, + redraw: (image: Bitmap) -> Bitmap, + ): Response = error("Image redraw is not registered for fixture tests") + + override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { + override val width: Int = width + override val height: Int = height + + override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { + error("Bitmap drawing is not registered for fixture tests") + } + } + + fun verifyAllFixturesUsed() { + fixtureInterceptor.verifyAllFixturesUsed() + } + + internal class Builder { + private val routes = linkedMapOf<FixtureRequest, String>() + + fun get(url: String, resourcePath: String): Builder = route("GET", url, resourcePath) + + fun post(url: String, resourcePath: String): Builder = route("POST", url, resourcePath) + + private fun route(method: String, url: String, resourcePath: String): Builder = apply { + val key = FixtureRequest(method, url) + require(routes.put(key, resourcePath) == null) { "Duplicate fixture route: $method $url" } + } + + fun build(): FixtureMangaLoaderContext = FixtureMangaLoaderContext(routes.toMap()) + } + + companion object { + fun builder(): Builder = Builder() + } +} + +internal data class FixtureRequest( + val method: String, + val url: String, +) + +private class FixtureHttpInterceptor( + private val routes: Map<FixtureRequest, String>, +) : Interceptor { + + private val usedRoutes = ConcurrentHashMap.newKeySet<FixtureRequest>() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val key = FixtureRequest(request.method, request.url.toString()) + val resourcePath = routes[key] + ?: error("No fixture registered for ${request.method} ${request.url}") + val body = loadFixture(resourcePath) + usedRoutes.add(key) + return response(request, body) + } + + fun verifyAllFixturesUsed() { + val unusedRoutes = routes.keys - usedRoutes + check(unusedRoutes.isEmpty()) { + "Unused fixture routes: ${unusedRoutes.joinToString { "${it.method} ${it.url}" }}" + } + } + + private fun loadFixture(resourcePath: String): ByteArray { + val stream = FixtureMangaLoaderContext::class.java.getResourceAsStream(resourcePath) + ?: error("Fixture resource not found: $resourcePath") + return stream.use { it.readBytes() } + } + + private fun response(request: Request, body: ByteArray): Response = Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(body.toResponseBody("text/html; charset=utf-8".toMediaType())) + .build() +} diff --git a/src/test/kotlin/tsuki/SourceReadingContract.kt b/src/test/kotlin/tsuki/SourceReadingContract.kt deleted file mode 100644 index dd2f484d..00000000 --- a/src/test/kotlin/tsuki/SourceReadingContract.kt +++ /dev/null @@ -1,43 +0,0 @@ -package tsuki - -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse -import tsuki.model.MangaListFilter -import tsuki.model.MangaSource -import tsuki.model.SortOrder - -internal suspend fun verifySourceReadingContract( - parser: MangaParser, - context: FixtureMangaLoaderContext, - expectedSource: MangaSource, - expectedTitle: String, - sortOrder: SortOrder, - expectedPageUrl: String? = null, -) { - val manga = parser.getList( - offset = 0, - order = sortOrder, - filter = MangaListFilter.EMPTY, - ).single() - - assertEquals(expectedSource, manga.source) - assertEquals(expectedTitle, manga.title) - - val details = parser.getDetails(manga) - - assertEquals(manga.id, details.id) - assertEquals(manga.url, details.url) - assertEquals(manga.source, details.source) - - val chapter = details.chapters.orEmpty().single() - assertEquals(details.source, chapter.source) - - val pages = parser.getPages(chapter) - - assertFalse(pages.isEmpty()) - assertEquals(details.source, pages.single().source) - if (expectedPageUrl != null) { - assertEquals(expectedPageUrl, parser.getPageUrl(pages.single())) - } - context.verifyAllFixturesUsed() -} diff --git a/src/test/kotlin/tsuki/WaMangaContractTest.kt b/src/test/kotlin/tsuki/WaMangaContractTest.kt deleted file mode 100644 index a77e9f6f..00000000 --- a/src/test/kotlin/tsuki/WaMangaContractTest.kt +++ /dev/null @@ -1,53 +0,0 @@ -package tsuki - -import kotlinx.coroutines.test.runTest -import org.json.JSONObject -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder -import tsuki.site.ru.parseWaMangaAlternativeTitles -import tsuki.site.ru.parseWaMangaGenres -import tsuki.site.ru.parseWaMangaSearchText - -class WaMangaContractTest { - - @Test - fun `source supports current v1 API from catalog through pages`() = runTest { - val context = FixtureMangaLoaderContext.builder() - .get("https://wamanga.ru/api/v1/manga", "/fixtures/wamanga/catalog.json") - .get("https://wamanga.ru/api/v1/manga/manga-id", "/fixtures/wamanga/details.json") - .get("https://wamanga.ru/api/v1/manga/manga-id/chapters", "/fixtures/wamanga/chapters.json") - .get("https://wamanga.ru/api/v1/chapters/chapter-id", "/fixtures/wamanga/pages.json") - .build() - val parser = MangaParserSource.WAMANGA.newParser(context) - - verifySourceReadingContract( - parser = parser, - context = context, - expectedSource = MangaParserSource.WAMANGA, - expectedTitle = "Example Manga", - sortOrder = SortOrder.ALPHABETICAL, - expectedPageUrl = "https://wamanga.ru/app/uploads/example/001.webp", - ) - } - - @Test - fun `catalog search contract includes all current title fields`() { - val item = JSONObject( - """{ - "title":"Russian title", - "titleEnglish":"English title", - "alternateTitles":["Original title"], - "genres":["Action","Drama"] - }""", - ) - - assertEquals(setOf("English title", "Original title"), item.parseWaMangaAlternativeTitles()) - assertEquals(setOf("Action", "Drama"), item.parseWaMangaGenres()) - assertEquals( - setOf("Russian title", "English title", "Original title"), - item.parseWaMangaSearchText(), - ) - } -} From d1226e0dd9b9ceb489b380976a9ceb13010170a6 Mon Sep 17 00:00:00 2001 From: biglag <dmitriy.gudkov@dotcross.ai> Date: Wed, 19 Aug 2026 23:33:11 +0800 Subject: [PATCH 341/434] organize --- .github/summary.yaml | 2 +- .../{site/all => parsers}/NineMangaParser.kt | 13 +- src/main/kotlin/tsuki/site/id/Softkomik.kt | 495 ------------------ .../site/id/{ => nsfw}/KomikDewasaART.kt | 0 .../kotlin/tsuki/site/ru/NudeMoonParser.kt | 198 ------- .../kotlin/tsuki/site/ru/ZenMangaParser.kt | 486 ----------------- .../tsuki/site/ru/chan/HenChanParser.kt | 90 ---- .../tsuki/site/ru/grouple/AllHentaiParser.kt | 44 -- .../tsuki/site/ru/grouple/GroupleParser.kt | 1 - .../tsuki/site/ru/grouple/MintMangaParser.kt | 30 -- .../site/ru/libsocial/HentaiLibParser.kt | 16 - .../kotlin/tsuki/site/ru/madara/BestManga.kt | 15 - .../kotlin/tsuki/site/ru/madara/Mangazavr.kt | 16 - .../tsuki/RussianSourcesContractTest.kt | 384 +------------- .../resources/fixtures/acomics/catalog.html | 21 - .../resources/fixtures/acomics/details.html | 12 - src/test/resources/fixtures/acomics/page.html | 11 - src/test/resources/fixtures/desu/catalog.html | 12 - src/test/resources/fixtures/desu/details.html | 18 - src/test/resources/fixtures/desu/pages.json | 20 - .../fixtures/mangamammy/details.html | 20 - .../resources/fixtures/mangamammy/list.html | 16 - .../resources/fixtures/mangamammy/pages.html | 10 - .../fixtures/mangaonelove/details.html | 20 - .../resources/fixtures/mangaonelove/list.html | 16 - .../fixtures/mangaonelove/pages.html | 10 - .../resources/fixtures/mangawtf/list.json | 14 - .../resources/fixtures/readmanga/details.html | 37 -- .../resources/fixtures/readmanga/pages.html | 9 - .../resources/fixtures/readmanga/search.html | 21 - .../resources/fixtures/wamanga/catalog.json | 26 - .../resources/fixtures/wamanga/chapters.json | 15 - .../resources/fixtures/wamanga/details.json | 25 - .../resources/fixtures/wamanga/pages.json | 25 - 34 files changed, 17 insertions(+), 2131 deletions(-) rename src/main/kotlin/tsuki/{site/all => parsers}/NineMangaParser.kt (96%) delete mode 100644 src/main/kotlin/tsuki/site/id/Softkomik.kt rename src/main/kotlin/tsuki/site/id/{ => nsfw}/KomikDewasaART.kt (100%) delete mode 100644 src/main/kotlin/tsuki/site/ru/NudeMoonParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/ZenMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/chan/HenChanParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/AllHentaiParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/MintMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/libsocial/HentaiLibParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/madara/BestManga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/madara/Mangazavr.kt delete mode 100644 src/test/resources/fixtures/acomics/catalog.html delete mode 100644 src/test/resources/fixtures/acomics/details.html delete mode 100644 src/test/resources/fixtures/acomics/page.html delete mode 100644 src/test/resources/fixtures/desu/catalog.html delete mode 100644 src/test/resources/fixtures/desu/details.html delete mode 100644 src/test/resources/fixtures/desu/pages.json delete mode 100644 src/test/resources/fixtures/mangamammy/details.html delete mode 100644 src/test/resources/fixtures/mangamammy/list.html delete mode 100644 src/test/resources/fixtures/mangamammy/pages.html delete mode 100644 src/test/resources/fixtures/mangaonelove/details.html delete mode 100644 src/test/resources/fixtures/mangaonelove/list.html delete mode 100644 src/test/resources/fixtures/mangaonelove/pages.html delete mode 100644 src/test/resources/fixtures/mangawtf/list.json delete mode 100644 src/test/resources/fixtures/readmanga/details.html delete mode 100644 src/test/resources/fixtures/readmanga/pages.html delete mode 100644 src/test/resources/fixtures/readmanga/search.html delete mode 100644 src/test/resources/fixtures/wamanga/catalog.json delete mode 100644 src/test/resources/fixtures/wamanga/chapters.json delete mode 100644 src/test/resources/fixtures/wamanga/details.json delete mode 100644 src/test/resources/fixtures/wamanga/pages.json diff --git a/.github/summary.yaml b/.github/summary.yaml index 61d0f101..b6db931f 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 221 \ No newline at end of file +total: 211 \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/NineMangaParser.kt b/src/main/kotlin/tsuki/parsers/NineMangaParser.kt similarity index 96% rename from src/main/kotlin/tsuki/site/all/NineMangaParser.kt rename to src/main/kotlin/tsuki/parsers/NineMangaParser.kt index 03c2a4c0..fbe40e38 100644 --- a/src/main/kotlin/tsuki/site/all/NineMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/NineMangaParser.kt @@ -1,4 +1,4 @@ -package tsuki.site.all +package tsuki.parsers import androidx.collection.ArrayMap import kotlinx.coroutines.sync.Mutex @@ -6,9 +6,7 @@ import kotlinx.coroutines.sync.withLock import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Interceptor import okhttp3.Response -import tsuki.Broken import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser import tsuki.model.* @@ -266,13 +264,4 @@ internal abstract class NineMangaParser( } return 0L } - - @Broken - @MangaSourceParser("NINEMANGA_RU", "NineManga Русский", "ru") - class Russian(context: MangaLoaderContext) : NineMangaParser( - context, - MangaParserSource.NINEMANGA_RU, - "ru.ninemanga.com", - ) - } diff --git a/src/main/kotlin/tsuki/site/id/Softkomik.kt b/src/main/kotlin/tsuki/site/id/Softkomik.kt deleted file mode 100644 index 8e407288..00000000 --- a/src/main/kotlin/tsuki/site/id/Softkomik.kt +++ /dev/null @@ -1,495 +0,0 @@ -package tsuki.site.id - -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.ContentRating -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder - -import tsuki.util.generateUid -import tsuki.util.getCookies -import tsuki.util.parseHtml -import tsuki.util.parseJson -import tsuki.util.toAbsoluteUrl -import tsuki.util.urlEncoded - -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject -import java.net.URLDecoder -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.TimeUnit - -import tsuki.Broken -@Broken -@MangaSourceParser("SOFTKOMIK", "Softkomik", "id") -internal class Softkomik(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.SOFTKOMIK, pageSize = 24), MangaParserAuthProvider { - - override val authUrl: String get() = "https://$domain/akun/login" - override suspend fun isAuthorized(): Boolean { - val cookies = context.cookieJar.getCookies(domain) - return cookies.any { it.name == "tokkey" } - } - override suspend fun getUsername(): String = "" - - override val configKeyDomain = ConfigKey.Domain("softkomik.co") - - private val apiUrl = "https://v2.softdevices.my.id" - private val coverUrl = "https://cover.softdevices.my.id/softkomik-cover" - - private val cdnUrls = listOf( - "https://psy1.komik.im", - "https://image.komik.im/softkomik", - "https://cdn1.softkomik.online/softkomik", - "https://cd1.softkomik.online/softkomik", - "https://f1.softkomik.com/file/softkomik-image", - "https://img.softdevices.my.id/softkomik-image", - "https://image.softkomik.com/softkomik", - ) - - private val loginRequiredGenres = setOf("ecchi", "mature") - private val requiredLoginSuffix = "login-required" - private val requiredLoginFragment = "#$requiredLoginSuffix" - - override val availableSortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY) - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - ) - - private val customClient: OkHttpClient by lazy { - context.httpClient.newBuilder() - .addInterceptor(::apiAuthInterceptor) - .addInterceptor(::imageInterceptor) - .build() - } - - override val webClient by lazy { - tsuki.network.OkHttpWebClient(customClient, source) - } - - private fun pageHeaders() = getRequestHeaders().newBuilder() - .add("rsc", "1") - .build() - - private fun apiHeaders(session: SessionDto): Headers { - val builder = Headers.Builder() - .add("Accept", "application/json") - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - if (session.sign.isNotEmpty()) { - builder.add("X-Token", session.token.cleanB64()) - builder.add("X-Sign", session.sign.take(64)) - } else { - builder.add("Authorization", "Bearer ${session.token}") - } - return builder.build() - } - - private data class SessionDto(val ex: Long, val token: String, val sign: String) - private val sessionsByKey = ConcurrentHashMap<String, SessionDto>() - private val sessionKeyChapterList = "chapter-list" - private val sessionKeyChapterImage = "chapter-image" - - private suspend fun getSessionAsync(route: SessionRoute): SessionDto { - sessionsByKey[route.key]?.takeIf { it.ex > System.currentTimeMillis() + 30_000L }?.let { return it } - - try { - val apiHeaders = Headers.Builder() - .add("Accept", "application/json") - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("X-Requested-With", "XMLHttpRequest") - .build() - val res = webClient.httpGet(route.sessionApiUrl, apiHeaders).parseJson() - val rawSign = res.optString("sign", "") - val session = SessionDto( - ex = res.optLong("ex", System.currentTimeMillis() + 60_000L), - token = res.optString("token", ""), - sign = rawSign.substringBefore('|'), - ) - sessionsByKey[route.key] = session - return session - } catch (_: Exception) { - val token = getBearerTokenFromCookie() - if (token != null) { - val session = SessionDto( - ex = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(2), - token = token, - sign = "", - ) - sessionsByKey[route.key] = session - return session - } - throw Exception("Gagal mendapatkan session. Coba lagi.") - } - } - - private fun getSessionBlocking(route: SessionRoute): SessionDto = runBlocking { - getSessionAsync(route) - } - - private fun getBearerTokenFromCookie(): String? { - val cookies = context.cookieJar.getCookies(domain) - val tokkey = cookies.firstOrNull { it.name == "tokkey" }?.value ?: return null - val decoded = runCatching { URLDecoder.decode(tokkey, Charsets.UTF_8.name()) }.getOrDefault(tokkey) - return decoded.cleanB64() - } - - private fun String.cleanB64(): String = - substringBefore('=').let { it + "=".repeat((4 - (it.length % 4)) % 4) } - - private fun apiAuthInterceptor(chain: Interceptor.Chain): Response { - val request = chain.request() - if (!request.url.toString().startsWith(apiUrl)) { - return chain.proceed(request) - } - val route = resolveSessionRoute(request.url) - val session = getSessionBlocking(route) - val newRequest = request.newBuilder() - .apply { - if (session.sign.isNotEmpty()) { - header("X-Token", session.token.cleanB64()) - header("X-Sign", session.sign.take(64)) - } else { - header("Authorization", "Bearer ${session.token}") - } - } - .build() - return chain.proceed(newRequest) - } - - private fun imageInterceptor(chain: Interceptor.Chain): Response { - val request = chain.request() - val userAgent = request.header("User-Agent") - val normalizedUserAgent = userAgent?.replace( - Regex("""\s*Mobile Safari/\d+(?:\.\d+)*""", RegexOption.IGNORE_CASE), - "", - )?.trim()?.ifEmpty { null } - - val modRequest = if (normalizedUserAgent != userAgent) { - request.newBuilder().header("User-Agent", normalizedUserAgent ?: "").build() - } else request - - val response: Response? = try { - chain.proceed(modRequest) - } catch (_: java.net.UnknownHostException) { - null - } - - if (response?.isSuccessful == true) return response - - val currentHost = cdnUrls.firstOrNull { request.url.toString().startsWith(it) } - if (currentHost == null) return response ?: throw java.net.UnknownHostException(request.url.host) - - response?.close() - val imagePath = request.url.toString().removePrefix(currentHost).removePrefix("/") - for (newHost in cdnUrls) { - if (newHost == currentHost) continue - val newUrl = "$newHost/$imagePath".toHttpUrl() - try { - val newResp = chain.proceed(request.newBuilder().url(newUrl).build()) - if (newResp.isSuccessful) return newResp - newResp.close() - } catch (_: Exception) {} - } - throw java.net.UnknownHostException("All CDN hosts failed for $imagePath") - } - - private data class SessionRoute( - val key: String, - val sessionApiUrl: String, - val slug: String?, - val isChapterListRequest: Boolean, - val isChapterImageRequest: Boolean, - ) - - private fun resolveSessionRoute(url: okhttp3.HttpUrl): SessionRoute { - val segments = url.pathSegments - val komikIndex = segments.indexOf("komik") - val slug = if (komikIndex != -1) segments.getOrNull(komikIndex + 1) else null - val isChapterListRequest = komikIndex != -1 && segments.getOrNull(komikIndex + 2) == "chapter" - val isChapterImageRequest = isChapterListRequest && (segments.contains("imgs") || segments.contains("img")) - - val key = if (isChapterImageRequest) sessionKeyChapterImage else sessionKeyChapterList - val sessionApiUrl = if (isChapterImageRequest) { - "https://$domain/api/session/chapter/oioa" - } else { - "https://$domain/api/session/iuiuiwqw" - } - return SessionRoute( - key = key, - sessionApiUrl = sessionApiUrl, - slug = slug, - isChapterListRequest = isChapterListRequest, - isChapterImageRequest = isChapterImageRequest, - ) - } - - @Volatile - private var genresCache: Set<MangaTag>? = null - private val genresMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - ) - - private suspend fun getOrFetchGenres(): Set<MangaTag> { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: fetchGenres().also { genresCache = it } - } - } - - private suspend fun fetchGenres(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/komik/library", pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) ?: return emptySet() - val arr = pageProps.optJSONArray("genre") ?: return emptySet() - return LinkedHashSet<MangaTag>(arr.length()).apply { - for (i in 0 until arr.length()) { - val jo = arr.optJSONObject(i) ?: continue - val value = jo.optString("value", "").ifEmpty { jo.optString("label", "") } - val label = jo.optString("label", "").ifEmpty { value } - if (value.isNotEmpty() && label.isNotEmpty()) { - add(MangaTag(title = label, key = value, source = source)) - } - } - } - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim().orEmpty() - if (query.isNotEmpty()) { - return searchByQuery(query, page) - } - val url = buildString { - append("https://$domain/komik/library") - append("?page=").append(page) - append("&sortBy=").append(if (order == SortOrder.POPULARITY) "popular" else "newKomik") - filter.tags.firstOrNull()?.key?.let { - append("&genre=").append(it.urlEncoded()) - } - } - val doc = webClient.httpGet(url, pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) ?: return emptyList() - val libData = pageProps.optJSONObject("libData") ?: pageProps - val dataArray = libData.optJSONArray("data") ?: return emptyList() - return parseListData(dataArray) - } - - private suspend fun searchByQuery(query: String, page: Int): List<Manga> { - val urlPath = "$apiUrl/komik?name=${query.urlEncoded()}&search=true&limit=24&page=$page" - val session = getSessionAsync(resolveSessionRoute(urlPath.toHttpUrl())) - val headers = apiHeaders(session) - val json = webClient.httpGet(urlPath, headers).parseJson() - val dataArray = json.optJSONArray("data") ?: return emptyList() - return parseListData(dataArray) - } - - private fun parseListData(dataArray: JSONArray): List<Manga> { - val result = ArrayList<Manga>(dataArray.length()) - for (i in 0 until dataArray.length()) { - val jo = dataArray.optJSONObject(i) ?: continue - val slug = jo.optString("title_slug", "").ifEmpty { jo.optString("id", "") } - if (slug.isEmpty()) continue - val gambar = jo.optString("gambar", "").removePrefix("/") - result.add( - Manga( - id = generateUid(slug), - title = jo.optString("title", "Untitled"), - altTitles = emptySet(), - url = "/$slug", - publicUrl = "https://$domain/$slug", - rating = RATING_UNKNOWN, - contentRating = ContentRating.SAFE, - coverUrl = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else "", - tags = emptySet(), - state = parseStatus(jo.optString("status")), - authors = emptySet(), - source = source, - ), - ) - } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.publicUrl, pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) - val detail = pageProps?.optJSONObject("data") - - val title = detail?.optString("title")?.takeIf { it.isNotBlank() } ?: manga.title - val gambar = detail?.optString("gambar", "")?.removePrefix("/") ?: "" - val coverImg = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else manga.coverUrl - val description = detail?.optString("sinopsis")?.takeIf { it.isNotBlank() } - val author = detail?.optString("author")?.takeIf { it.isNotBlank() } - val state = parseStatus(detail?.optString("status")) - - val tags = LinkedHashSet<MangaTag>() - detail?.optJSONArray("Genre")?.let { arr -> - for (i in 0 until arr.length()) { - val name = arr.optString(i, "").trim() - if (name.isNotEmpty()) tags.add(MangaTag(name, name.lowercase(), source)) - } - } - - val isLoginRequired = tags.any { it.title.lowercase() in loginRequiredGenres } - val slug = manga.url.trim('/').substringBefore('/') - val chapters = fetchChapterList(slug, isLoginRequired) - - return manga.copy( - title = title, - coverUrl = coverImg, - description = description, - tags = tags, - authors = setOfNotNull(author), - state = state, - chapters = chapters, - ) - } - - private suspend fun fetchChapterList(slug: String, isLoginRequired: Boolean): List<MangaChapter> { - val apiPath = "$apiUrl/komik/$slug/chapter?limit=9999999" - val headers = if (isLoginRequired) { - val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan untuk mengakses chapter") - Headers.Builder() - .add("Authorization", "Bearer $token") - .add("Referer", "https://$domain/$slug") - .add("Origin", "https://$domain") - .build() - } else { - val session = getSessionAsync(resolveSessionRoute(apiPath.toHttpUrl())) - apiHeaders(session) - } - - val json = webClient.httpGet(apiPath, headers).parseJson() - val chapterArray = json.optJSONArray("chapter") ?: return emptyList() - val chapters = ArrayList<MangaChapter>(chapterArray.length()) - for (i in 0 until chapterArray.length()) { - val ch = chapterArray.optJSONObject(i) ?: continue - val chStr = ch.optString("chapter", "") - if (chStr.isEmpty()) continue - val number = chStr.substringBefore(".").toFloatOrNull() ?: continue - var chapterUrl = "/$slug/chapter/$chStr" - if (isLoginRequired) chapterUrl += requiredLoginFragment - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = "Chapter ${formatChapterDisplay(chStr)}", - url = chapterUrl, - number = number, - volume = 0, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ), - ) - } - chapters.sortBy { it.number } - return chapters - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val isLoginRequired = chapter.url.contains(requiredLoginFragment) - val cleanUrl = chapter.url.substringBefore('#') - - val doc = webClient.httpGet(cleanUrl.toAbsoluteUrl(domain), pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) - val data = pageProps?.optJSONObject("data") - val komik = data?.optJSONObject("komik") - val chapterData = data?.optJSONObject("data") - - val slug = cleanUrl.trim('/').substringBefore("/chapter/") - val chNum = cleanUrl.substringAfterLast("/chapter/").trim('/') - - var imageSrc = chapterData?.optJSONArray("imageSrc") ?: JSONArray() - val storageInter2 = chapterData?.optBoolean("storageInter2", false) ?: false - - if (imageSrc.length() == 0) { - val id = chapterData?.optString("_id") ?: komik?.optString("_id") ?: return emptyList() - imageSrc = fetchChapterImages(slug, chNum, id, isLoginRequired) - } - - if (imageSrc.length() == 0) return emptyList() - - val imageBaseUrl = if (storageInter2) cdnUrls[2] else cdnUrls[0] - return (0 until imageSrc.length()).mapNotNull { i -> - val path = imageSrc.optString(i, "").removePrefix("/") - if (path.isEmpty()) return@mapNotNull null - MangaPage( - id = generateUid(path), - url = "$imageBaseUrl/$path", - preview = null, - source = source, - ) - } - } - - private suspend fun fetchChapterImages( - slug: String, - chapter: String, - id: String, - isLoginRequired: Boolean, - ): JSONArray { - val url = "$apiUrl/komik/$slug/chapter/$chapter/imgs/$id" - val headers = if (isLoginRequired) { - val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan") - Headers.Builder() - .add("Authorization", "Bearer $token") - .add("Referer", "https://$domain/$slug/chapter/$chapter") - .add("Origin", "https://$domain") - .build() - } else { - val session = getSessionAsync(resolveSessionRoute(url.toHttpUrl())) - apiHeaders(session) - } - val json = webClient.httpGet(url, headers).parseJson() - return json.optJSONArray("imageSrc") ?: JSONArray() - } - - private fun extractNextJs(doc: org.jsoup.nodes.Document): JSONObject? { - val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() ?: return null - return runCatching { - JSONObject(raw).optJSONObject("props")?.optJSONObject("pageProps") - }.getOrNull() - } - - private fun parseStatus(text: String?): MangaState? = when { - text == null -> null - text.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING - text.contains("tamat", ignoreCase = true) || text.contains("completed", ignoreCase = true) -> MangaState.FINISHED - else -> null - } - - private fun formatChapterDisplay(chStr: String): String { - val parts = chStr.split(".") - val numPart = parts[0].toFloatOrNull() ?: return chStr - val formatted = if (numPart == numPart.toLong().toFloat()) numPart.toLong().toString() - else numPart.toString().trimEnd('0').trimEnd('.') - val suffix = parts.drop(1).joinToString(".") - return if (suffix.isNotEmpty()) "$formatted.$suffix" else formatted - } -} diff --git a/src/main/kotlin/tsuki/site/id/KomikDewasaART.kt b/src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt similarity index 100% rename from src/main/kotlin/tsuki/site/id/KomikDewasaART.kt rename to src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt diff --git a/src/main/kotlin/tsuki/site/ru/NudeMoonParser.kt b/src/main/kotlin/tsuki/site/ru/NudeMoonParser.kt deleted file mode 100644 index f129ac38..00000000 --- a/src/main/kotlin/tsuki/site/ru/NudeMoonParser.kt +++ /dev/null @@ -1,198 +0,0 @@ -package tsuki.site.ru - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.AbstractMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.util.* -import java.text.SimpleDateFormat -import java.util.* - -@Broken -@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI) -internal class NudeMoonParser( - context: MangaLoaderContext, -) : AbstractMangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain( - "b.nude-moon.fun", - "nude-moon.org", - "nude-moon.net", - ) - override val authUrl: String - get() = "https://${domain}/index.php" - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name == "fusion_user" - } - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - init { - context.cookieJar.insertCookies( - domain, - "NMfYa=1;", - "nm_mobile=1;", - ) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val domain = domain - - val url = when { - !filter.query.isNullOrEmpty() -> { - if (!isAuthorized()) { - throw AuthRequiredException(source) - } - "https://$domain/search?stext=${filter.query.urlEncoded()}&rowstart=$offset" - } - - else -> { - if (filter.tags.isNotEmpty()) { - filter.tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - } else { - val order = when (order) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - "https://$domain/all_manga?$order&rowstart=$offset" - } - } - - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.body().select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = a.attr("title") - val author = row.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.textOrNull() - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitles = setOfNotNull(title.substringBefore(" / ", "").takeUnless { it.isBlank() }), - authors = setOfNotNull(author), - coverUrl = row.selectFirst("img")?.absUrl("src").orEmpty(), - tags = row.selectFirst(".tag-links")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - val root = body.selectFirstOrThrow("table.news_pic2") - val dateFormat = SimpleDateFormat("dd MMMM yyyy", sourceLocale) - val author = root.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.text() - return manga.copy( - largeCoverUrl = body.selectFirstOrThrow("img[data-src]").attrAsAbsoluteUrl("data-src"), - description = root.selectFirst(".description")?.html() ?: manga.description, - tags = root.getElementsByAttributeValueContaining("href", "/tag/").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } + manga.tags, - authors = author?.let { setOf(it) } ?: manga.authors, - chapters = listOf( - MangaChapter( - id = manga.id, - url = manga.url, - source = source, - number = 0f, - volume = 0, - title = manga.title, - scanlator = root.getElementsByAttributeValueContaining("href", "/perevod/").firstOrNull() - ?.textOrNull(), - uploadDate = dateFormat.parseSafe( - root.getElementsByAttributeValueEnding("src", "ico/time.png").firstOrNull() - ?.nextElementSibling()?.text(), - ), - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val pages = doc.select("img[data-src]") - return pages.map { img -> - val url = img.attrAsRelativeUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - }.toList() - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val domain = domain - val doc = webClient.httpGet("https://$domain/tags").parseHtml() - val root = doc.body().getElementsByAttributeValue("name", "multitags").first() - ?: doc.parseFailed("Tags form not found") - return root.select("input").mapToSet { - val value = it.attr("value").trim() - MangaTag( - title = value.toTitleCase(sourceLocale), - key = value.replace(' ', '_'), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${domain}/").parseHtml().body() - return body.getElementsContainingOwnText("Профиль").firstOrNull()?.attr("href")?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } -} diff --git a/src/main/kotlin/tsuki/site/ru/ZenMangaParser.kt b/src/main/kotlin/tsuki/site/ru/ZenMangaParser.kt deleted file mode 100644 index c52d86da..00000000 --- a/src/main/kotlin/tsuki/site/ru/ZenMangaParser.kt +++ /dev/null @@ -1,486 +0,0 @@ -package tsuki.site.ru - -import okhttp3.Headers -import okhttp3.HttpUrl -import org.json.JSONArray -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.MangaParserAuthProvider -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.network.CloudFlareHelper -import tsuki.network.UserAgents -import tsuki.util.* -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.Broken -import java.text.SimpleDateFormat -import java.util.* - -@Broken -@MangaSourceParser("ZENMANGA", "ZenManga", "ru") -internal class ZenMangaParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ZENMANGA, 30), - MangaParserAuthProvider { - - private val astroJsonParser = AstroJsonParser() - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) - - init { - setFirstPage(0) - } - - override val configKeyDomain = ConfigKey.Domain("inkstory.me") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.POPULARITY_ASC, - SortOrder.RATING, - SortOrder.RATING_ASC, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC - ) - - override val filterCapabilities: MangaListFilterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isYearRangeSupported = true, - isSearchWithFiltersSupported = true, - isAuthorSearchSupported = true, - ) - - override val authUrl: String - get() = "https://sso.inuko.me/account/sign-in" - - private val apiDomain = if (domain.startsWith("v1.")) domain.replace("v1.", "api.") else "api.$domain" - - private fun checkAuth(): Boolean { - val authCookieName = "__otaku_session" - return context.cookieJar.getCookies("zenmanga.io").any { it.name == authCookieName } || - context.cookieJar.getCookies("v1.zenmanga.one").any { it.name == authCookieName } || - context.cookieJar.getCookies("v1.zenmanga.me").any { it.name == authCookieName } - } - - override suspend fun isAuthorized(): Boolean { - return checkAuth() - } - - override suspend fun getUsername(): String { - val libraryUrl = "/library" - val data = fetchAstroData(libraryUrl) - ?: throw ParseException("Не удалось получить Astro JSON для получения имени пользователя", libraryUrl) - - val session = data["session"] as? Map<*, *> - ?: throw ParseException("Ключ 'session' не найден", libraryUrl) - - val currentUser = session["currentUser"] as? Map<*, *> - ?: throw ParseException("Ключ 'currentUser' не найден", libraryUrl) - - return currentUser["username"] as? String - ?: throw ParseException("Ключ 'username' не найден", libraryUrl) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val author = filter.author - if (!author.isNullOrBlank()) { - return getListPageByAuthor(author, page) - } - - val urlBuilder = HttpUrl.Builder() - .scheme("https") - .host(apiDomain) - .addPathSegment("v2") - .addPathSegment("books") - - urlBuilder.addQueryParameter("page", page.toString()) - urlBuilder.addQueryParameter("size", pageSize.toString()) - - urlBuilder.addQueryParameter("sort", getSortParameter(order)) - - if (!filter.query.isNullOrBlank()) { - urlBuilder.addQueryParameter("search", filter.query) - } - - filter.tags.forEach { tag -> - urlBuilder.addQueryParameter("labelsInclude", tag.key) - } - - filter.tagsExclude.forEach { tag -> - urlBuilder.addQueryParameter("labelsExclude", tag.key) - } - - filter.states.forEach { state -> - urlBuilder.addQueryParameter("status", when(state) { - MangaState.ONGOING -> "ONGOING" - MangaState.FINISHED -> "DONE" - MangaState.PAUSED -> "FROZEN" - MangaState.UPCOMING -> "ANNOUNCE" - else -> "" - }) - } - - filter.contentRating.forEach { rating -> - urlBuilder.addQueryParameter("contentStatus", when(rating) { - ContentRating.SAFE -> "SAFE" - ContentRating.SUGGESTIVE -> "UNSAFE" - ContentRating.ADULT -> "EROTIC" - }) - } - - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("yearMin", filter.yearFrom.toString()) - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("yearMax", filter.yearTo.toString()) - } - - val requestUrl = urlBuilder.build() - val response = webClient.httpGet(requestUrl).parseJsonArray() - - return response.mapJSON { parseMangaFromJson(it) } - } - - private suspend fun getListPageByAuthor(authorQuery: String, page: Int): List<Manga> { - val authorSearchUrl = HttpUrl.Builder() - .scheme("https") - .host(apiDomain) - .addPathSegment("v2") - .addPathSegment("publishers") - .addQueryParameter("search", authorQuery) - .build() - - val publishersResponse = webClient.httpGet(authorSearchUrl).parseJsonArray() - - var authorId: String? = null - for (i in 0 until publishersResponse.length()) { - val publisher = publishersResponse.getJSONObject(i) - if (publisher.getStringOrNull("kind") == "AUTHOR") { - authorId = publisher.getStringOrNull("id") - break - } - } - - if (authorId == null) { - return emptyList() - } - - val booksByAuthorUrl = HttpUrl.Builder() - .scheme("https") - .host(apiDomain) - .addPathSegment("v2") - .addPathSegment("books") - .addQueryParameter("publisherId", authorId) - .addQueryParameter("page", page.toString()) - .addQueryParameter("size", pageSize.toString()) - .addQueryParameter("sort", "createdAt,desc") - .build() - - val booksResponse = webClient.httpGet(booksByAuthorUrl).parseJsonArray() - - return booksResponse.mapJSON { parseMangaFromJson(it) } - } - - private fun parseMangaFromJson(json: JSONObject): Manga { - val slug = json.getString("slug") - val nameObj = json.getJSONObject("name") - val title = nameObj.getStringOrNull("ru") ?: nameObj.getString("en") - - val publicUrl = "https://$domain/content/$slug" - - val altNames = json.getJSONArray("altNames") - .mapJSON { it.getString("name") } - .toSet() - - return Manga( - id = generateUid(publicUrl), - url = "/content/$slug", - publicUrl = "https://$domain/content/$slug", - title = title, - altTitles = altNames, - coverUrl = json.getStringOrNull("poster"), - source = source, - rating = json.getDouble("averageRating").toFloat() / 10f, - state = when (json.getStringOrNull("status")) { - "ONGOING" -> MangaState.ONGOING - "DONE" -> MangaState.FINISHED - "FROZEN" -> MangaState.PAUSED - "ANNOUNCE" -> MangaState.UPCOMING - else -> null - }, - contentRating = when (json.getStringOrNull("contentStatus")) { - "SAFE" -> ContentRating.SAFE - "UNSAFE" -> ContentRating.SUGGESTIVE - "EROTIC" -> ContentRating.ADULT - else -> null - }, - tags = emptySet(), - authors = emptySet() - ) - } - - private fun getSortParameter(order: SortOrder): String { - val field = when (order) { - SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "viewsCount" - SortOrder.RATING, SortOrder.RATING_ASC -> "averageRating" - SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "createdAt" - else -> "viewsCount" - } - val direction = when (order) { - SortOrder.POPULARITY_ASC, SortOrder.RATING_ASC, SortOrder.NEWEST_ASC -> "asc" - else -> "desc" - } - return "$field,$direction" - } - - override suspend fun getDetails(manga: Manga): Manga { - val data = fetchAstroData(manga.url) - ?: throw ParseException("Не удалось получить Astro JSON для деталей манги", manga.publicUrl) - - val bookData = data["current-book"] as? Map<*, *> ?: return manga - val branchesData = data["current-book-branches"] as? List<Map<*, *>> ?: emptyList() - val chaptersData = data["current-book-chapters"] as? List<Map<*, *>> ?: emptyList() - - val description = bookData["description"] as? String - - val tags = (bookData["labels"] as? List<Map<*, *>>)?.mapNotNullTo(HashSet()) { - val tagName = it["name"] as? String - val tagKey = it["slug"] as? String - if (tagName != null && tagKey != null) { - MangaTag(key = tagKey, title = tagName.replaceFirstChar { c -> c.uppercase() }, source = source) - } else null - } ?: emptySet() - - val authors = (bookData["relations"] as? List<Map<*, *>>)?.mapNotNullTo(HashSet()) { - val type = it["type"] as? String - if (type == "AUTHOR" || type == "ARTIST") { - (it["publisher"] as? Map<*, *>)?.get("name") as? String - } else null - } ?: emptySet() - - val branchIdToNameMap = branchesData.associate { branchMap -> - val branchId = branchMap["id"] as? String - - val scanlatorNames = (branchMap["publishers"] as? List<Map<*, *>>) - ?.mapNotNull { publisherMap -> publisherMap["name"] as? String } - ?.joinToString(" & ") - - branchId to scanlatorNames - } - - val slug = manga.url.substringAfterLast('/') - - val chapters = chaptersData.mapNotNull { chapterMap -> - val id = chapterMap["id"] as? String ?: return@mapNotNull null - val branchId = chapterMap["branchId"] as? String - val scanlator = branchIdToNameMap[branchId] - - MangaChapter( - id = generateUid(id), - url = "/content/$slug/$id", - title = chapterMap["name"] as? String, - number = chapterMap["number"].toSafeFloat(), - volume = chapterMap["volume"].toSafeInt(), - uploadDate = dateFormat.parseSafe(chapterMap["createdAt"] as? String), - scanlator = scanlator, - branch = scanlator, - source = source - ) - }.reversed() - - return manga.copy( - description = description, - tags = manga.tags + tags, - authors = authors, - chapters = chapters - ) - } - - override fun getRequestHeaders() = Headers.Builder() - .add("User-Agent", UserAgents.CHROME_DESKTOP) - .build() - - private fun Any?.toSafeInt(): Int { - return when(this) { - is Number -> this.toInt() - is String -> this.toIntOrNull() ?: 0 - else -> 0 - } - } - - private fun Any?.toSafeFloat(): Float { - return when(this) { - is Number -> this.toFloat() - is String -> this.toFloatOrNull() ?: 0f - else -> 0f - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val data = fetchAstroData(chapter.url) - ?: throw ParseException("Не удалось получить Astro JSON для страниц главы", chapter.url) - - val chapterData = data["reader-current-chapter"] as? Map<*, *> - ?: throw ParseException("Ключ 'reader-current-chapter' не найден", chapter.url) - - val pagesList = chapterData["pages"] as? List<Map<*, *>> - ?: throw ParseException("Список страниц 'pages' не найден", chapter.url) - - return pagesList - .sortedBy { it["index"].toSafeInt() } - .mapNotNull { pageMap -> - val id = pageMap["id"] as? String - val imageUrl = pageMap["image"] as? String - if (id == null || imageUrl == null) return@mapNotNull null - - MangaPage( - id = generateUid(id), - url = "$imageUrl&width=1600", - preview = null, - source = source - ) - } - } - - private suspend fun fetchAstroData(relativeUrl: String): Map<*, *>? { - val fullUrl = relativeUrl.toAbsoluteUrl(domain) - - val response = webClient.httpGet(fullUrl) - - val protection = CloudFlareHelper.checkResponseForProtection(response.copy()) - if (protection != CloudFlareHelper.PROTECTION_NOT_DETECTED) { - response.close() - context.requestBrowserAction(this, fullUrl) - return null - } - - val responseHtml = response.parseHtml() - val scriptElement = responseHtml.getElementById("it-astro-state") - ?: throw ParseException("Не удалось найти <script id='it-astro-state'> на странице $fullUrl.", fullUrl) - - return astroJsonParser.parse(scriptElement.data()) - } - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = allGenres.toSet(), - availableStates = allStates, - availableContentRating = allContentRatings - ) - } - - private val allGenres: List<MangaTag> = listOf( - MangaTag(key = "art", title = "Арт", source = source), - MangaTag(key = "martial_arts", title = "Боевые искусства", source = source), - MangaTag(key = "vampires", title = "Вампиры", source = source), - MangaTag(key = "harem", title = "Гарем", source = source), - MangaTag(key = "gender_intriga", title = "Гендерная интрига", source = source), - MangaTag(key = "detective", title = "Детектив", source = source), - MangaTag(key = "josei", title = "Дзёсэй", source = source), - MangaTag(key = "game", title = "Игра", source = source), - MangaTag(key = "cyberpunk", title = "Киберпанк", source = source), - MangaTag(key = "maho_shoujo", title = "Махо-сёдзё", source = source), - MangaTag(key = "mecha", title = "Меха", source = source), - MangaTag(key = "mystery", title = "Мистика", source = source), - MangaTag(key = "sci_fi", title = "Научная фантастика", source = source), - MangaTag(key = "natural", title = "Повседневность", source = source), - MangaTag(key = "postapocalypse", title = "Постапокалипсис", source = source), - MangaTag(key = "adventure", title = "Приключения", source = source), - MangaTag(key = "psychological", title = "Психология", source = source), - MangaTag(key = "samurai", title = "Самураи", source = source), - MangaTag(key = "supernatural", title = "Сверхъестественное", source = source), - MangaTag(key = "sports", title = "Спорт", source = source), - MangaTag(key = "seinen", title = "Сэйнэн", source = source), - MangaTag(key = "thriller", title = "Триллер", source = source), - MangaTag(key = "horror", title = "Ужасы", source = source), - MangaTag(key = "fantastic", title = "Фантастика", source = source), - MangaTag(key = "fantasy", title = "Фэнтези", source = source), - MangaTag(key = "school", title = "Школа", source = source), - MangaTag(key = "erotica", title = "Эротика", source = source), - MangaTag(key = "ecchi", title = "Этти", source = source), - MangaTag(key = "codomo", title = "Кодомо", source = source), - MangaTag(key = "isekai", title = "Исекай", source = source), - MangaTag(key = "omegavers", title = "Омегаверс", source = source), - MangaTag(key = "comedy", title = "Комедия", source = source), - MangaTag(key = "shounen", title = "Сёнэн", source = source), - MangaTag(key = "romance", title = "Романтика", source = source), - MangaTag(key = "drama", title = "Драма", source = source), - MangaTag(key = "shoujo", title = "Сёдзё", source = source), - MangaTag(key = "historical", title = "История", source = source), - MangaTag(key = "tragedy", title = "Трагедия", source = source), - MangaTag(key = "action", title = "Боевик", source = source) - ).sortedBy { it.title } - - private val allStates: Set<MangaState> = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.UPCOMING - ) - - private val allContentRatings: Set<ContentRating> = EnumSet.of( - ContentRating.SAFE, - ContentRating.SUGGESTIVE, - ContentRating.ADULT - ) - - private class AstroJsonParser { - fun parse(compressedJson: String): Map<*, *> { - val rootArray = JSONArray(compressedJson) - if (rootArray.length() == 0) return emptyMap<Any, Any>() - - val cache = mutableMapOf<Int, Any?>() - val overdueMap = decompress(rootArray.get(0), rootArray, cache) as? Map<*, *> ?: emptyMap<Any, Any>() - - return overdueMap["@inox-tools/request-nanostores"] as? Map<*, *> ?: emptyMap<Any, Any>() - } - - private fun decompress(value: Any?, rootArray: JSONArray, cache: MutableMap<Int, Any?>): Any? { - if (value is Int) { - val ref = value - if (cache.containsKey(ref)) return cache[ref] - if (ref < 0 || ref >= rootArray.length()) return ref - - cache[ref] = null - val referencedItem = rootArray.get(ref) - val result = processItem(referencedItem, rootArray, cache) - cache[ref] = result - return result - } - return processItem(value, rootArray, cache) - } - - private fun processItem(item: Any?, rootArray: JSONArray, cache: MutableMap<Int, Any?>): Any? { - return when (item) { - is JSONObject -> { - val map = mutableMapOf<String, Any?>() - for (key in item.keys()) { - map[key] = decompress(item.get(key), rootArray, cache) - } - map - } - is JSONArray -> { - if (item.length() > 0 && item.get(0) is String) { - when (item.getString(0)) { - "Map" -> { - val map = mutableMapOf<Any?, Any?>() - for (i in 1 until item.length() step 2) { - val key = decompress(item.get(i), rootArray, cache) - val value = decompress(item.get(i + 1), rootArray, cache) - if (key != null) map[key] = value - } - return map - } - "URL" -> return if (item.length() > 1) decompress(item.get(1), rootArray, cache) else null - } - } - (0 until item.length()).map { i -> decompress(item.get(i), rootArray, cache) } - } - else -> item - } - } - } -} diff --git a/src/main/kotlin/tsuki/site/ru/chan/HenChanParser.kt b/src/main/kotlin/tsuki/site/ru/chan/HenChanParser.kt deleted file mode 100644 index 2426817e..00000000 --- a/src/main/kotlin/tsuki/site/ru/chan/HenChanParser.kt +++ /dev/null @@ -1,90 +0,0 @@ -package tsuki.site.ru.chan - -import okhttp3.HttpUrl -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.* -import tsuki.util.* -import java.util.* - -@Broken -@MangaSourceParser("HENCHAN", "Хентай-тян", "ru", type = ContentType.HENTAI) -internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.HENCHAN) { - - override val configKeyDomain = ConfigKey.Domain( - "x5.h-chan.me", - "xxxx.henchan.pro", - "xxl.hentaichan.live", - "xxx.henchan.pro", - "y.hentaichan.live", - "xx.hentaichan.live", - "x.henchan.pro", - "hentaichan.live", - ) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - val readLink = manga.url.replace("manga", "online") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { - val a = it.children().last() ?: doc.parseFailed("Invalid tag") - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - } ?: manga.tags, - chapters = listOf( - MangaChapter( - id = generateUid(readLink), - url = readLink, - source = source, - number = 0f, - volume = 0, - uploadDate = 0L, - title = null, - scanlator = null, - branch = null, - ), - ), - ) - } - - override fun buildUrl(offset: Int, order: SortOrder, filter: MangaListFilter): HttpUrl = when { - filter.query.isNullOrEmpty() && filter.tags.isEmpty() && filter.tagsExclude.isEmpty() -> { - val builder = urlBuilder().addQueryParameter("offset", offset.toString()) - when (order) { - SortOrder.POPULARITY -> { - builder.addPathSegment("mostviews") - builder.addQueryParameter("sort", "manga") - } - - SortOrder.RATING -> { - builder.addPathSegment("mostfavorites") - builder.addQueryParameter("sort", "manga") - } - - else -> { // SortOrder.NEWEST - builder.addPathSegment("manga") - builder.addPathSegment("newest") - } - } - builder.build() - } - - else -> { - super.buildUrl(offset, order, filter) - } - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/AllHentaiParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/AllHentaiParser.kt deleted file mode 100644 index 072a5461..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/AllHentaiParser.kt +++ /dev/null @@ -1,44 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.exception.AuthRequiredException -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.util.* - -@Broken -@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", type = ContentType.HENTAI) -internal class AllHentaiParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.ALLHENTAI, 1) { - - override val configKeyDomain = ConfigKey.Domain( - "20.allhen.online", - "z.ahen.me", - "24.allhen.online", - "2023.allhen.online", - ) - - override suspend fun isAuthorized(): Boolean = - super.isAuthorized() || context.cookieJar.getCookies(domain).any { it.name == "remember_me" } - - override val authUrl: String - get() { - val targetUri = "https://$domain/".urlEncoded() - return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" - } - - override suspend fun getUsername(): String { - val root = webClient.httpGet("https://qawa.org/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else { - res - } - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt index a1f9b9e3..4fdf518d 100644 --- a/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt +++ b/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt @@ -328,7 +328,6 @@ internal abstract class GroupleParser( protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { in SeiMangaParser.domains -> MangaParserSource.SEIMANGA - in MintMangaParser.domains -> MangaParserSource.MINTMANGA in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU in SelfMangaParser.domains -> MangaParserSource.SELFMANGA in UsagiParser.domains -> MangaParserSource.USAGI diff --git a/src/main/kotlin/tsuki/site/ru/grouple/MintMangaParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/MintMangaParser.kt deleted file mode 100644 index 1ca7de9f..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/MintMangaParser.kt +++ /dev/null @@ -1,30 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@Broken -@MangaSourceParser("MINTMANGA", "MintManga", "ru") -internal class MintMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.MINTMANGA, 2) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "2.mintmanga.one", - "24.mintmanga.one", - "mintmanga.live", - "mintmanga.com", - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/libsocial/HentaiLibParser.kt b/src/main/kotlin/tsuki/site/ru/libsocial/HentaiLibParser.kt deleted file mode 100644 index f997b18e..00000000 --- a/src/main/kotlin/tsuki/site/ru/libsocial/HentaiLibParser.kt +++ /dev/null @@ -1,16 +0,0 @@ -package tsuki.site.ru.libsocial - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@Broken -@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI) -internal class HentaiLibParser(context: MangaLoaderContext) : LibSocialParser( - context = context, - source = MangaParserSource.HENTAILIB, - siteId = 4, - siteDomains = arrayOf("v1.hentailib.org", "hentailib.me"), -) diff --git a/src/main/kotlin/tsuki/site/ru/madara/BestManga.kt b/src/main/kotlin/tsuki/site/ru/madara/BestManga.kt deleted file mode 100644 index 57327f82..00000000 --- a/src/main/kotlin/tsuki/site/ru/madara/BestManga.kt +++ /dev/null @@ -1,15 +0,0 @@ -package tsuki.site.ru.madara - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -@Broken -@MangaSourceParser("BEST_MANGA", "BestManga", "ru") -internal class BestManga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.BEST_MANGA, "bestmanga.club") { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} diff --git a/src/main/kotlin/tsuki/site/ru/madara/Mangazavr.kt b/src/main/kotlin/tsuki/site/ru/madara/Mangazavr.kt deleted file mode 100644 index bd1d0a5b..00000000 --- a/src/main/kotlin/tsuki/site/ru/madara/Mangazavr.kt +++ /dev/null @@ -1,16 +0,0 @@ -package tsuki.site.ru.madara - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -@Broken -@MangaSourceParser("MANGAZAVR", "Mangazavr", "ru", ContentType.HENTAI) -internal class Mangazavr(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAZAVR, "mangazavr.ru") { - override val listUrl = "/?s=&post_type=wp-manga" - override val datePattern = "dd.MM.yyyy" -} diff --git a/src/test/kotlin/tsuki/RussianSourcesContractTest.kt b/src/test/kotlin/tsuki/RussianSourcesContractTest.kt index 83c14350..ada7eb8b 100644 --- a/src/test/kotlin/tsuki/RussianSourcesContractTest.kt +++ b/src/test/kotlin/tsuki/RussianSourcesContractTest.kt @@ -1,71 +1,35 @@ package tsuki -import kotlinx.coroutines.test.runTest import okhttp3.Cookie import okhttp3.CookieJar import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.MediaType.Companion.toMediaType import okhttp3.OkHttpClient -import okhttp3.Protocol -import okhttp3.Request import okhttp3.Response -import okhttp3.ResponseBody.Companion.toResponseBody -import org.json.JSONObject import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Test import tsuki.bitmap.Bitmap import tsuki.bitmap.Rect import tsuki.config.ConfigKey import tsuki.config.MangaSourceConfig import tsuki.model.MangaParserSource -import tsuki.model.MangaListFilter import tsuki.model.MangaSource -import tsuki.model.MangaTag -import tsuki.model.SortOrder -import tsuki.site.ru.libsocial.parseLibSocialSummary -import tsuki.site.ru.parseWaMangaAlternativeTitles -import tsuki.site.ru.parseWaMangaGenres -import tsuki.site.ru.parseWaMangaSearchText -import java.util.concurrent.ConcurrentHashMap class RussianSourcesContractTest { @Test - fun `Russian catalog and representative reading contracts remain valid`() = runTest { - verifyCatalog() - verifyLibSocialSummary() - verifyAComics() - verifyDesu() - verifyMangaOneLove() - verifyMangaMammy() - verifyMangaWtfFilters() - verifyReadManga() - verifyWaManga() - } - - private fun verifyCatalog() { + fun `Russian catalog metadata and parser construction remain valid`() { val expectedSources = setOf( "ACOMICS", - "ALLHENTAI", "ASTRAMANGA", - "BEST_MANGA", "COMX", "DESUME", - "HENCHAN", - "HENTAILIB", "MANGABALL_RU", "MANGACHAN", "MANGALIB", "MANGAMAMMY", "MANGAONELOVE", "MANGAPLUSPARSER_RU", - "MANGAZAVR", "MANGA_WTF", - "MINTMANGA", - "NINEMANGA_RU", - "NUDEMOON", "READMANGA_RU", "REMANGA", "SEIMANGA", @@ -74,279 +38,27 @@ class RussianSourcesContractTest { "WAMANGA", "YAOICHAN", "YAOILIB", - "ZENMANGA", - ) - val expectedBrokenSources = setOf( - "ALLHENTAI", - "BEST_MANGA", - "HENCHAN", - "HENTAILIB", - "MANGAZAVR", - "MINTMANGA", - "NINEMANGA_RU", - "NUDEMOON", - "ZENMANGA", ) val sources = MangaParserSource.entries.filter { it.locale == "ru" } assertEquals(expectedSources, sources.mapTo(mutableSetOf()) { it.name }) - assertEquals(expectedBrokenSources, sources.filter { it.isBroken }.mapTo(mutableSetOf()) { it.name }) - - val context = FixtureMangaLoaderContext.builder().build() - sources.forEach { source -> - assertEquals(source, source.newParser(context).source) - } - assertEquals("ru.ninemanga.com", MangaParserSource.NINEMANGA_RU.newParser(context).domain) - } - - private fun verifyLibSocialSummary() { - val structured = JSONObject( - """ - { - "summary": { - "type": "doc", - "content": [ - { - "type": "paragraph", - "content": [ - {"type": "text", "text": "First line"}, - {"type": "hardBreak"}, - {"type": "text", "text": "Second line"} - ] - }, - { - "type": "paragraph", - "content": [{"type": "text", "text": "Next paragraph"}] - } - ] - } - } - """.trimIndent(), - ) - - assertEquals("First line\nSecond line\nNext paragraph", structured.parseLibSocialSummary()) - assertEquals( - "Plain description", - JSONObject("""{"summary":"Plain description"}""").parseLibSocialSummary(), - ) - } - - private suspend fun verifyAComics() { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://acomics.ru/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=0&sort=last_update", - "/fixtures/acomics/catalog.html", - ) - .get("https://acomics.ru/~fixture/about", "/fixtures/acomics/details.html") - .get("https://acomics.ru/comics", "/fixtures/acomics/catalog.html") - .get("https://acomics.ru/~fixture/1", "/fixtures/acomics/page.html") - .build() - val parser = MangaParserSource.ACOMICS.newParser(context) - - val manga = parser.getList(0, SortOrder.UPDATED, MangaListFilter.EMPTY).single() - assertEquals("Fixture Comic", manga.title) - assertEquals("https://acomics.ru/upload/fixture-cover.jpg", manga.coverUrl) - - val details = parser.getDetails(manga) - assertEquals("Current AComics description.", details.description) - assertEquals(setOf("Fantasy"), details.tags.mapTo(linkedSetOf()) { it.title }) - assertEquals(setOf("Fixture Author"), details.authors) - - val pages = parser.getPages(details.chapters.orEmpty().single()) - assertEquals(3, pages.size) - assertEquals("https://acomics.ru/upload/fixture-page.jpg", parser.getPageUrl(pages.first())) - context.verifyAllFixturesUsed() - } - - private suspend fun verifyDesu() { - val context = FixtureMangaLoaderContext.builder() - .get("https://desu.uno/manga/?page=1", "/fixtures/desu/catalog.html") - .get("https://desu.uno/manga/example.42/", "/fixtures/desu/details.html") - .get("https://desu.uno/api/manga/42/chapters/690", "/fixtures/desu/pages.json") - .build() - - verifyReadingContract( - source = MangaParserSource.DESUME, - context = context, - expectedTitle = "Пример манги", - sortOrder = SortOrder.UPDATED, - expectedPageUrl = "https://img3.desu.uno/manga/rus/example/001.webp", - ) - } - - private suspend fun verifyMangaOneLove() { - val context = FixtureMangaLoaderContext.builder() - .post( - "https://mangaonelove.su/wp-admin/admin-ajax.php", - "/fixtures/mangaonelove/list.html", - ) - .get("https://mangaonelove.su/manga/example-title", "/fixtures/mangaonelove/details.html") - .get( - "https://mangaonelove.su/chapter/example-title-1?style=list", - "/fixtures/mangaonelove/pages.html", - ) - .build() - - verifyReadingContract( - source = MangaParserSource.MANGAONELOVE, - context = context, - expectedTitle = "Example Title", - sortOrder = SortOrder.UPDATED, - ) - } - - private suspend fun verifyMangaMammy() { - val context = FixtureMangaLoaderContext.builder() - .post( - "https://p.nimanga.com/wp-admin/admin-ajax.php", - "/fixtures/mangamammy/list.html", - ) - .get("https://p.nimanga.com/manga/example-title", "/fixtures/mangamammy/details.html") - .get( - "https://p.nimanga.com/chapter/example-title-1?style=list", - "/fixtures/mangamammy/pages.html", - ) - .build() - - verifyReadingContract( - source = MangaParserSource.MANGAMAMMY, - context = context, - expectedTitle = "Example Title", - sortOrder = SortOrder.UPDATED, - ) - } - - private suspend fun verifyMangaWtfFilters() { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://api.inkstory.net/v2/books?page=0&size=20&type=COMIC&sort=updatedAt%2Cdesc&labelsInclude=action&labelsExclude=romance", - "/fixtures/mangawtf/list.json", - ) - .build() - val source = MangaParserSource.MANGA_WTF - val manga = source.newParser(context).getList( - offset = 0, - order = SortOrder.UPDATED, - filter = MangaListFilter.EMPTY.copy( - tags = setOf(MangaTag("Action", "action", source)), - tagsExclude = setOf(MangaTag("Romance", "romance", source)), - ), - ).single() - - assertEquals("Test title", manga.title) - context.verifyAllFixturesUsed() - } - - private suspend fun verifyReadManga() { - val context = FixtureMangaLoaderContext.builder() - .get( - "https://a.zazaza.me/search/advancedResults?q=Artifact&offset=0&years=1900%2C2099&sortType=DATE_CREATE", - "/fixtures/readmanga/search.html", - ) - .get( - "https://a.zazaza.me/igrok__kotoryi_est_artefakty", - "/fixtures/readmanga/details.html", - ) - .get( - "https://a.zazaza.me/igrok__kotoryi_est_artefakty/vol1/45?mtr=1", - "/fixtures/readmanga/pages.html", - ) - .build() - val parser = MangaParserSource.READMANGA_RU.newParser(context) - val manga = parser.getList( - offset = 0, - order = SortOrder.NEWEST, - filter = MangaListFilter.EMPTY.copy(query = "Artifact"), - ).single() - assertEquals("Игрок, который ест артефакты", manga.title) - - val details = parser.getDetails(manga) - assertEquals("Current ReadManga description.", details.description) - assertEquals(setOf("Боевик", "Драма"), details.tags.mapTo(linkedSetOf()) { it.title }) - assertEquals("https://rm.one-way.work/uploads/pics/example.jpg", details.largeCoverUrl) - - val chapter = details.chapters.orEmpty().single() - assertEquals(45f, chapter.number) - assertEquals(1, chapter.volume) - - val pages = parser.getPages(chapter) - assertFalse(pages.isEmpty()) - assertEquals("https://images.example/page-01.jpg", parser.getPageUrl(pages.single())) - context.verifyAllFixturesUsed() - } - - private suspend fun verifyWaManga() { - val context = FixtureMangaLoaderContext.builder() - .get("https://wamanga.ru/api/v1/manga", "/fixtures/wamanga/catalog.json") - .get("https://wamanga.ru/api/v1/manga/manga-id", "/fixtures/wamanga/details.json") - .get("https://wamanga.ru/api/v1/manga/manga-id/chapters", "/fixtures/wamanga/chapters.json") - .get("https://wamanga.ru/api/v1/chapters/chapter-id", "/fixtures/wamanga/pages.json") - .build() - - verifyReadingContract( - source = MangaParserSource.WAMANGA, - context = context, - expectedTitle = "Example Manga", - sortOrder = SortOrder.ALPHABETICAL, - expectedPageUrl = "https://wamanga.ru/app/uploads/example/001.webp", - ) - - val item = JSONObject( - """{ - "title":"Russian title", - "titleEnglish":"English title", - "alternateTitles":["Original title"], - "genres":["Action","Drama"] - }""", - ) - assertEquals(setOf("English title", "Original title"), item.parseWaMangaAlternativeTitles()) - assertEquals(setOf("Action", "Drama"), item.parseWaMangaGenres()) assertEquals( - setOf("Russian title", "English title", "Original title"), - item.parseWaMangaSearchText(), + emptySet<String>(), + sources.filter { it.isBroken }.mapTo(mutableSetOf()) { it.name }, + "Broken sources must not be shipped", ) - } - - private suspend fun verifyReadingContract( - source: MangaParserSource, - context: FixtureMangaLoaderContext, - expectedTitle: String, - sortOrder: SortOrder, - expectedPageUrl: String? = null, - ) { - val parser = source.newParser(context) - val manga = parser.getList(0, sortOrder, MangaListFilter.EMPTY).single() - assertEquals(source, manga.source) - assertEquals(expectedTitle, manga.title) - - val details = parser.getDetails(manga) - assertEquals(manga.id, details.id) - assertEquals(manga.url, details.url) - assertEquals(source, details.source) - val chapter = details.chapters.orEmpty().single() - assertEquals(source, chapter.source) - - val pages = parser.getPages(chapter) - assertFalse(pages.isEmpty()) - assertEquals(source, pages.single().source) - if (expectedPageUrl != null) { - assertEquals(expectedPageUrl, parser.getPageUrl(pages.single())) + val context = CatalogMangaLoaderContext() + sources.forEach { source -> + assertEquals(source, source.newParser(context).source) } - context.verifyAllFixturesUsed() } } @Suppress("OVERRIDE_DEPRECATION") -internal class FixtureMangaLoaderContext private constructor( - routes: Map<FixtureRequest, String>, -) : MangaLoaderContext() { +private class CatalogMangaLoaderContext : MangaLoaderContext() { - private val fixtureInterceptor = FixtureHttpInterceptor(routes) - - override val httpClient: OkHttpClient = OkHttpClient.Builder() - .addInterceptor(fixtureInterceptor) - .build() + override val httpClient: OkHttpClient = OkHttpClient() override val cookieJar: CookieJar = object : CookieJar { override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = Unit @@ -355,103 +67,37 @@ internal class FixtureMangaLoaderContext private constructor( } override fun newParserInstance(source: MangaSource): MangaParser { - require(source is MangaParserSource) { "Unsupported fixture source: $source" } + require(source is MangaParserSource) { "Unsupported catalog source: $source" } return source.newParser(this) } override fun getParserSources(): List<MangaSource> = MangaParserSource.entries override suspend fun evaluateJs(script: String): String? { - error("JavaScript evaluation is not registered for fixture tests") + error("JavaScript evaluation is not available in catalog tests") } override suspend fun evaluateJs(baseUrl: String, script: String): String? { - error("JavaScript evaluation is not registered for fixture tests: $baseUrl") + error("JavaScript evaluation is not available in catalog tests: $baseUrl") } override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { override fun <T> get(key: ConfigKey<T>): T = key.defaultValue } - override fun getDefaultUserAgent(): String = "MangaRepoContractTest/1.0" + override fun getDefaultUserAgent(): String = "MangaRepoCatalogTest/1.0" override fun redrawImageResponse( response: Response, redraw: (image: Bitmap) -> Bitmap, - ): Response = error("Image redraw is not registered for fixture tests") + ): Response = error("Image redraw is not available in catalog tests") override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { override val width: Int = width override val height: Int = height override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { - error("Bitmap drawing is not registered for fixture tests") + error("Bitmap drawing is not available in catalog tests") } } - - fun verifyAllFixturesUsed() { - fixtureInterceptor.verifyAllFixturesUsed() - } - - internal class Builder { - private val routes = linkedMapOf<FixtureRequest, String>() - - fun get(url: String, resourcePath: String): Builder = route("GET", url, resourcePath) - - fun post(url: String, resourcePath: String): Builder = route("POST", url, resourcePath) - - private fun route(method: String, url: String, resourcePath: String): Builder = apply { - val key = FixtureRequest(method, url) - require(routes.put(key, resourcePath) == null) { "Duplicate fixture route: $method $url" } - } - - fun build(): FixtureMangaLoaderContext = FixtureMangaLoaderContext(routes.toMap()) - } - - companion object { - fun builder(): Builder = Builder() - } -} - -internal data class FixtureRequest( - val method: String, - val url: String, -) - -private class FixtureHttpInterceptor( - private val routes: Map<FixtureRequest, String>, -) : Interceptor { - - private val usedRoutes = ConcurrentHashMap.newKeySet<FixtureRequest>() - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val key = FixtureRequest(request.method, request.url.toString()) - val resourcePath = routes[key] - ?: error("No fixture registered for ${request.method} ${request.url}") - val body = loadFixture(resourcePath) - usedRoutes.add(key) - return response(request, body) - } - - fun verifyAllFixturesUsed() { - val unusedRoutes = routes.keys - usedRoutes - check(unusedRoutes.isEmpty()) { - "Unused fixture routes: ${unusedRoutes.joinToString { "${it.method} ${it.url}" }}" - } - } - - private fun loadFixture(resourcePath: String): ByteArray { - val stream = FixtureMangaLoaderContext::class.java.getResourceAsStream(resourcePath) - ?: error("Fixture resource not found: $resourcePath") - return stream.use { it.readBytes() } - } - - private fun response(request: Request, body: ByteArray): Response = Response.Builder() - .request(request) - .protocol(Protocol.HTTP_1_1) - .code(200) - .message("OK") - .body(body.toResponseBody("text/html; charset=utf-8".toMediaType())) - .build() } diff --git a/src/test/resources/fixtures/acomics/catalog.html b/src/test/resources/fixtures/acomics/catalog.html deleted file mode 100644 index b0620fb5..00000000 --- a/src/test/resources/fixtures/acomics/catalog.html +++ /dev/null @@ -1,21 +0,0 @@ -<!doctype html> -<html> -<body> -<form class="catalog-filters-form"> - <fieldset class="categories"> - <label class="category-fantasy"> - <input type="checkbox" name="categories[]" value="3">Fantasy - </label> - </fieldset> -</form> -<div class="infinite-scroll-content"> - <section class="serial-card"> - <a class="cover" href="/~fixture"> - <img src="/static/img/tail-spin.svg" data-real-src="/upload/fixture-cover.jpg" alt="Fixture Comic"> - </a> - <h2 class="title"><a href="/~fixture">Fixture Comic</a></h2> - <p class="about">Catalog summary.</p> - </section> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/acomics/details.html b/src/test/resources/fixtures/acomics/details.html deleted file mode 100644 index 46b2aaf5..00000000 --- a/src/test/resources/fixtures/acomics/details.html +++ /dev/null @@ -1,12 +0,0 @@ -<!doctype html> -<html> -<body> -<p class="serial-about-badges"> - <a class="badge category category-fantasy" href="/comics/fantasy">Fantasy</a> -</p> -<section class="serial-about-text"> - <p>Current AComics description.</p> -</section> -<p class="serial-about-authors"><b>Author:</b> <a href="/-fixture-author">Fixture Author</a></p> -</body> -</html> diff --git a/src/test/resources/fixtures/acomics/page.html b/src/test/resources/fixtures/acomics/page.html deleted file mode 100644 index 27a8e29b..00000000 --- a/src/test/resources/fixtures/acomics/page.html +++ /dev/null @@ -1,11 +0,0 @@ -<!doctype html> -<html> -<body> -<h1 class="reader-issue-title"> - <span class="number-with-name">Issue 1<span class="number">1/3</span></span> -</h1> -<section class="reader-issue"> - <img class="issue" src="/upload/fixture-page.jpg" alt="Issue 1"> -</section> -</body> -</html> diff --git a/src/test/resources/fixtures/desu/catalog.html b/src/test/resources/fixtures/desu/catalog.html deleted file mode 100644 index a6aa1f25..00000000 --- a/src/test/resources/fixtures/desu/catalog.html +++ /dev/null @@ -1,12 +0,0 @@ -<html><body> -<ol class="section memberList"> - <li class="primaryContent memberListItem"> - <a href="manga/example.42/" class="avatar"><span class="img" style="background-image: url('https://static.desu.uno/example.jpg');"></span></a> - <div> - <h3><a href="manga/example.42/" class="animeTitle oTitle">Example Manga</a></h3> - <div class="dimmed oTitle"><span itemprop="title">Пример манги</span></div> - <div class="animeInfo"><dl><dt>Рейтинг:</dt><dd>8.5</dd></dl></div> - </div> - </li> -</ol> -</body></html> diff --git a/src/test/resources/fixtures/desu/details.html b/src/test/resources/fixtures/desu/details.html deleted file mode 100644 index 8cbd297a..00000000 --- a/src/test/resources/fixtures/desu/details.html +++ /dev/null @@ -1,18 +0,0 @@ -<html><body> -<div id="animeView"> - <link href="https://desu.uno/manga/example.42/" itemprop="url"> - <meta content="Пример манги" itemprop="headline"> - <meta content="Example Manga" itemprop="alternativeHeadline"> - <img src="https://static.desu.uno/example-large.jpg" itemprop="image"> - <span class="b-anime_status_tag ongoing">выходит</span> - <meta content="8.5" itemprop="ratingValue"> - <a itemprop="genre" href="manga/?genres=56-Action">Экшен</a> - <div id="description"><div class="russian">Current description</div></div> - <ul class="chlist" data-manga_id="42"> - <li> - <i data-chapters_id="690"></i> - <h4><a href="manga/example.42/vol2/ch3/rus">Том 2. Глава 3<span class="title"> - Chapter title</span></a></h4> - </li> - </ul> -</div> -</body></html> diff --git a/src/test/resources/fixtures/desu/pages.json b/src/test/resources/fixtures/desu/pages.json deleted file mode 100644 index 3fb3e677..00000000 --- a/src/test/resources/fixtures/desu/pages.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "chapter": { - "id": 690, - "volume": "2", - "number": "3", - "title": "Chapter title", - "status": "saved", - "is_readable": true, - "publish_date": 1786900000, - "view_url": "/manga/example.42/vol2/ch3/rus", - "pages": [ - { - "page": 1, - "url": "https://img3.desu.uno/manga/rus/example/001.webp", - "width": 1000, - "height": 1400 - } - ] - } -} diff --git a/src/test/resources/fixtures/mangamammy/details.html b/src/test/resources/fixtures/mangamammy/details.html deleted file mode 100644 index c9a08bb8..00000000 --- a/src/test/resources/fixtures/mangamammy/details.html +++ /dev/null @@ -1,20 +0,0 @@ -<!doctype html> -<html lang="ru"> -<head> - <meta property="og:url" content="https://p.nimanga.com/manga/example-title" /> -</head> -<body> -<h1>Example Title</h1> -<div class="description-summary"><div class="summary__content">Fixture description.</div></div> -<div class="genres-content"><a href="https://p.nimanga.com/manga-genre/fantasy/">Fantasy</a></div> -<div class="post-content_item"><h5>Статус</h5><div class="summary-content">Продолжается</div></div> -<div class="listing-chapters_wrap"> - <ul> - <li class="wp-manga-chapter"> - <a href="https://p.nimanga.com/chapter/example-title-1">Chapter 1</a> - <span class="chapter-release-date"><i>31.07.2026</i></span> - </li> - </ul> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangamammy/list.html b/src/test/resources/fixtures/mangamammy/list.html deleted file mode 100644 index 31dd039c..00000000 --- a/src/test/resources/fixtures/mangamammy/list.html +++ /dev/null @@ -1,16 +0,0 @@ -<!doctype html> -<html lang="ru"> -<body> -<div class="row c-tabs-item__content"> - <a href="https://p.nimanga.com/manga/example-title"> - <img src="https://p.nimanga.com/covers/example-title.jpg" alt="Example Title" /> - </a> - <div class="tab-summary"> - <div class="post-title"><h3>Example Title</h3></div> - <div class="mg_author"><a href="/author/example-author">Example Author</a></div> - <div class="mg_status"><div class="summary-content">Продолжается</div></div> - <div class="mg_genres"><a href="/manga-genre/fantasy/">Fantasy</a></div> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangamammy/pages.html b/src/test/resources/fixtures/mangamammy/pages.html deleted file mode 100644 index d21791f8..00000000 --- a/src/test/resources/fixtures/mangamammy/pages.html +++ /dev/null @@ -1,10 +0,0 @@ -<!doctype html> -<html lang="ru"> -<body> -<div class="main-col-inner"> - <div class="reading-content"> - <div class="page-break"><img src="https://cdn.mangamammy.ru/pages/example-title-1-001.jpg" /></div> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangaonelove/details.html b/src/test/resources/fixtures/mangaonelove/details.html deleted file mode 100644 index 0d1a9e3e..00000000 --- a/src/test/resources/fixtures/mangaonelove/details.html +++ /dev/null @@ -1,20 +0,0 @@ -<!doctype html> -<html lang="ru"> -<head> - <meta property="og:url" content="https://mangaonelove.su/manga/example-title" /> -</head> -<body> -<h1>Example Title</h1> -<div class="description-summary"><div class="summary__content">Fixture description.</div></div> -<div class="genres-content"><a href="https://mangaonelove.su/manga-genre/fantasy/">Fantasy</a></div> -<div class="post-content_item"><h5>Статус</h5><div class="summary-content">Продолжается</div></div> -<div class="listing-chapters_wrap"> - <ul> - <li class="wp-manga-chapter"> - <a href="https://mangaonelove.su/chapter/example-title-1">Chapter 1</a> - <span class="chapter-release-date"><i>31.07.2026</i></span> - </li> - </ul> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangaonelove/list.html b/src/test/resources/fixtures/mangaonelove/list.html deleted file mode 100644 index 7c19d585..00000000 --- a/src/test/resources/fixtures/mangaonelove/list.html +++ /dev/null @@ -1,16 +0,0 @@ -<!doctype html> -<html lang="ru"> -<body> -<div class="row c-tabs-item__content"> - <a href="https://mangaonelove.su/manga/example-title"> - <img src="https://mangaonelove.su/covers/example-title.jpg" alt="Example Title" /> - </a> - <div class="tab-summary"> - <div class="post-title"><h3>Example Title</h3></div> - <div class="mg_author"><a href="/author/example-author">Example Author</a></div> - <div class="mg_status"><div class="summary-content">Продолжается</div></div> - <div class="mg_genres"><a href="/manga-genre/fantasy/">Fantasy</a></div> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangaonelove/pages.html b/src/test/resources/fixtures/mangaonelove/pages.html deleted file mode 100644 index 2fe7f74c..00000000 --- a/src/test/resources/fixtures/mangaonelove/pages.html +++ /dev/null @@ -1,10 +0,0 @@ -<!doctype html> -<html lang="ru"> -<body> -<div class="main-col-inner"> - <div class="reading-content"> - <div class="page-break"><img src="https://cdn.mangaonelove.su/pages/example-title-1-001.jpg" /></div> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/mangawtf/list.json b/src/test/resources/fixtures/mangawtf/list.json deleted file mode 100644 index 5a00f9d7..00000000 --- a/src/test/resources/fixtures/mangawtf/list.json +++ /dev/null @@ -1,14 +0,0 @@ -[ - { - "id": "book-id", - "slug": "test-title", - "poster": "https://static.inkstory.net/test-title.jpg", - "status": "ONGOING", - "contentStatus": "SAFE", - "name": { - "ru": "Test title", - "en": "Test title" - }, - "averageRating": 8.5 - } -] diff --git a/src/test/resources/fixtures/readmanga/details.html b/src/test/resources/fixtures/readmanga/details.html deleted file mode 100644 index 437067b0..00000000 --- a/src/test/resources/fixtures/readmanga/details.html +++ /dev/null @@ -1,37 +0,0 @@ -<!doctype html> -<html> -<head> - <meta itemprop="name" content="Игрок, который ест артефакты"> - <meta itemprop="alternativeHeadline" content="Artifact-Devouring Player"> -</head> -<body> -<div id="mangaBox"> - <div class="cr-hero"> - <img class="cr-hero-poster__img" src="https://rm.one-way.work/uploads/pics/example.jpg" alt="Игрок, который ест артефакты"> - <div class="cr-hero-names"> - <h1 class="cr-hero-names__main">Игрок, который ест артефакты</h1> - <h3 class="cr-hero-names__alt"> - <span><span>Artifact-Devouring Player</span></span> - </h3> - </div> - </div> - <div class="cr-description"> - <div class="cr-description__content">Current ReadManga description.</div> - </div> - <div class="creation-element-tags"> - <a class="creation-element-tags__item" href="/list/genre/action"><span>боевик</span></a> - <a class="creation-element-tags__item" href="/list/genre/drama"><span>драма</span></a> - </div> - <div id="chapters-list"> - <table> - <tr class="item-row" data-vol="1" data-num="450"> - <td class="item-title"> - <a class="chapter-link" href="/igrok__kotoryi_est_artefakty/vol1/45">1 - 45</a> - </td> - <td class="date">08.07.26</td> - </tr> - </table> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/readmanga/pages.html b/src/test/resources/fixtures/readmanga/pages.html deleted file mode 100644 index b5b41c48..00000000 --- a/src/test/resources/fixtures/readmanga/pages.html +++ /dev/null @@ -1,9 +0,0 @@ -<!doctype html> -<html> -<body> -<script> - var chapterInfo = {}; - rm_h.readerInit(chapterInfo, [["https://images.example/", "", "page-01.jpg", 800, 1200, ""]], false); -</script> -</body> -</html> diff --git a/src/test/resources/fixtures/readmanga/search.html b/src/test/resources/fixtures/readmanga/search.html deleted file mode 100644 index 1c812ae6..00000000 --- a/src/test/resources/fixtures/readmanga/search.html +++ /dev/null @@ -1,21 +0,0 @@ -<!doctype html> -<html> -<body> -<div class="tiles row"> - <div class="tile"> - <div class="img"> - <a href="/igrok__kotoryi_est_artefakty"> - <img class="lazy" data-original="https://rm.one-way.work/uploads/pics/example_p.jpg" alt="Игрок, который ест артефакты"> - </a> - </div> - <div class="desc"> - <h3><a href="/igrok__kotoryi_est_artefakty">Игрок, который ест артефакты</a></h3> - <h5>Artifact-Devouring Player</h5> - <div class="tile-info"> - <a class="element-link" href="/list/genre/action">боевик</a> - </div> - </div> - </div> -</div> -</body> -</html> diff --git a/src/test/resources/fixtures/wamanga/catalog.json b/src/test/resources/fixtures/wamanga/catalog.json deleted file mode 100644 index 8ee0a3bf..00000000 --- a/src/test/resources/fixtures/wamanga/catalog.json +++ /dev/null @@ -1,26 +0,0 @@ -[ - { - "id": "manga-id", - "slug": "example-manga", - "year": 2026, - "title": "Example Manga", - "titleEnglish": "Example Manga EN", - "alternateTitles": ["Example Manga JP"], - "genres": ["Action", "Drama"], - "description": "Current API description", - "authors": ["Example Author"], - "artists": [], - "publishers": [], - "imageUrl": "/app/uploads/example/banner.webp", - "coverUrl": "/app/uploads/example/cover.webp", - "updatedAt": "2026-07-31T12:00:00.000Z", - "createdAt": "2026-07-30T12:00:00.000Z", - "hidden": false, - "licensed": false, - "isAdult": false, - "type": "manga", - "statusTitle": "ongoing", - "translationStatus": "ongoing", - "pegiRating": "16+" - } -] diff --git a/src/test/resources/fixtures/wamanga/chapters.json b/src/test/resources/fixtures/wamanga/chapters.json deleted file mode 100644 index 5cf38900..00000000 --- a/src/test/resources/fixtures/wamanga/chapters.json +++ /dev/null @@ -1,15 +0,0 @@ -[ - { - "id": "chapter-id", - "title": "", - "slug": "", - "position": 1, - "isProcessing": false, - "totalFilesExpected": 1, - "processedFilesCount": 1, - "views": 10, - "createdAt": "2026-07-31T12:00:00.000Z", - "updatedAt": "2026-07-31T12:00:00.000Z", - "likesCount": 0 - } -] diff --git a/src/test/resources/fixtures/wamanga/details.json b/src/test/resources/fixtures/wamanga/details.json deleted file mode 100644 index 62fb45d2..00000000 --- a/src/test/resources/fixtures/wamanga/details.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "id": "manga-id", - "slug": "example-manga", - "year": 2026, - "title": "Example Manga", - "titleEnglish": "Example Manga EN", - "alternateTitles": ["Example Manga JP"], - "genres": ["Action", "Drama"], - "description": "Current API description", - "authors": ["Example Author"], - "artists": [], - "publishers": [], - "imageUrl": "/app/uploads/example/banner.webp", - "coverUrl": "/app/uploads/example/cover.webp", - "updatedAt": "2026-07-31T12:00:00.000Z", - "createdAt": "2026-07-30T12:00:00.000Z", - "hidden": false, - "licensed": false, - "isAdult": false, - "type": "manga", - "statusTitle": "ongoing", - "translationStatus": "ongoing", - "pegiRating": "16+", - "teams": [{"id": "team-id", "name": "Wa"}] -} diff --git a/src/test/resources/fixtures/wamanga/pages.json b/src/test/resources/fixtures/wamanga/pages.json deleted file mode 100644 index 628442d0..00000000 --- a/src/test/resources/fixtures/wamanga/pages.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "id": "chapter-id", - "title": "", - "slug": "", - "position": 1, - "isProcessing": false, - "totalFilesExpected": 1, - "processedFilesCount": 1, - "createdAt": "2026-07-31T12:00:00.000Z", - "updatedAt": "2026-07-31T12:00:00.000Z", - "files": [ - { - "id": "file-id", - "diskFile": "/app/uploads/example/001.webp", - "position": "1.000000000" - } - ], - "manga": { - "id": "manga-id", - "slug": "example-manga", - "title": "Example Manga", - "type": "manga" - }, - "likesCount": 0 -} From 6b9a7b179dac21e14ba86ac50eaa4f809abf532d Mon Sep 17 00:00:00 2001 From: biglag <dmitriy.gudkov@dotcross.ai> Date: Thu, 20 Aug 2026 13:46:23 +0800 Subject: [PATCH 342/434] remove test --- .../tsuki/RussianSourcesContractTest.kt | 103 ------------------ 1 file changed, 103 deletions(-) delete mode 100644 src/test/kotlin/tsuki/RussianSourcesContractTest.kt diff --git a/src/test/kotlin/tsuki/RussianSourcesContractTest.kt b/src/test/kotlin/tsuki/RussianSourcesContractTest.kt deleted file mode 100644 index ada7eb8b..00000000 --- a/src/test/kotlin/tsuki/RussianSourcesContractTest.kt +++ /dev/null @@ -1,103 +0,0 @@ -package tsuki - -import okhttp3.Cookie -import okhttp3.CookieJar -import okhttp3.HttpUrl -import okhttp3.OkHttpClient -import okhttp3.Response -import org.junit.jupiter.api.Assertions.assertEquals -import org.junit.jupiter.api.Test -import tsuki.bitmap.Bitmap -import tsuki.bitmap.Rect -import tsuki.config.ConfigKey -import tsuki.config.MangaSourceConfig -import tsuki.model.MangaParserSource -import tsuki.model.MangaSource - -class RussianSourcesContractTest { - - @Test - fun `Russian catalog metadata and parser construction remain valid`() { - val expectedSources = setOf( - "ACOMICS", - "ASTRAMANGA", - "COMX", - "DESUME", - "MANGABALL_RU", - "MANGACHAN", - "MANGALIB", - "MANGAMAMMY", - "MANGAONELOVE", - "MANGAPLUSPARSER_RU", - "MANGA_WTF", - "READMANGA_RU", - "REMANGA", - "SEIMANGA", - "SELFMANGA", - "USAGI", - "WAMANGA", - "YAOICHAN", - "YAOILIB", - ) - val sources = MangaParserSource.entries.filter { it.locale == "ru" } - - assertEquals(expectedSources, sources.mapTo(mutableSetOf()) { it.name }) - assertEquals( - emptySet<String>(), - sources.filter { it.isBroken }.mapTo(mutableSetOf()) { it.name }, - "Broken sources must not be shipped", - ) - - val context = CatalogMangaLoaderContext() - sources.forEach { source -> - assertEquals(source, source.newParser(context).source) - } - } -} - -@Suppress("OVERRIDE_DEPRECATION") -private class CatalogMangaLoaderContext : MangaLoaderContext() { - - override val httpClient: OkHttpClient = OkHttpClient() - - override val cookieJar: CookieJar = object : CookieJar { - override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) = Unit - - override fun loadForRequest(url: HttpUrl): List<Cookie> = emptyList() - } - - override fun newParserInstance(source: MangaSource): MangaParser { - require(source is MangaParserSource) { "Unsupported catalog source: $source" } - return source.newParser(this) - } - - override fun getParserSources(): List<MangaSource> = MangaParserSource.entries - - override suspend fun evaluateJs(script: String): String? { - error("JavaScript evaluation is not available in catalog tests") - } - - override suspend fun evaluateJs(baseUrl: String, script: String): String? { - error("JavaScript evaluation is not available in catalog tests: $baseUrl") - } - - override fun getConfig(source: MangaSource): MangaSourceConfig = object : MangaSourceConfig { - override fun <T> get(key: ConfigKey<T>): T = key.defaultValue - } - - override fun getDefaultUserAgent(): String = "MangaRepoCatalogTest/1.0" - - override fun redrawImageResponse( - response: Response, - redraw: (image: Bitmap) -> Bitmap, - ): Response = error("Image redraw is not available in catalog tests") - - override fun createBitmap(width: Int, height: Int): Bitmap = object : Bitmap { - override val width: Int = width - override val height: Int = height - - override fun drawBitmap(sourceBitmap: Bitmap, src: Rect, dst: Rect) { - error("Bitmap drawing is not available in catalog tests") - } - } -} From 77e4643fb5c2fa4ce97aab5fb05418c83f3115e4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 21 Aug 2026 12:47:41 +0200 Subject: [PATCH 343/434] Rename EHentai class to EXHentai --- src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt index 2adb3d60..aa26836c 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt @@ -68,8 +68,8 @@ private const val DOMAIN_AUTHORIZED = "exhentai.org" private val TAG_PREFIXES = arrayOf("male:", "female:", "other:") private const val BANNED_RESPONSE_LENGTH = 256L -@MangaSourceParser("EXHENTAI", "EXHentai", type = ContentType.HENTAI) -internal class EHentai(context: MangaLoaderContext) : +@MangaSourceParser("EXHENTAI", "EXHentai / Ehentai", type = ContentType.HENTAI) +internal class EXHentai(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor { override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) From 069c9c4df3bdffa7639fbe379e281231c96705c1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:10:19 +0200 Subject: [PATCH 344/434] Create AnisaScans.kt --- src/main/kotlin/tsuki/site/en/AnisaScans.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/AnisaScans.kt diff --git a/src/main/kotlin/tsuki/site/en/AnisaScans.kt b/src/main/kotlin/tsuki/site/en/AnisaScans.kt new file mode 100644 index 00000000..1fa74cdf --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/AnisaScans.kt @@ -0,0 +1,13 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("ANISASCANS", "AnisaScans", "en") +internal class AnisaScans(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.ANISASCANS, "anisascans.in") { + override val datePattern = "d MMMM, yyyy" +} From 2573928973a0206a811e88aa06f42f8e8439fc80 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 21 Aug 2026 13:10:38 +0200 Subject: [PATCH 345/434] Create Apkomik.kt --- src/main/kotlin/tsuki/site/id/Apkomik.kt | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Apkomik.kt diff --git a/src/main/kotlin/tsuki/site/id/Apkomik.kt b/src/main/kotlin/tsuki/site/id/Apkomik.kt new file mode 100644 index 00000000..ecfeb609 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Apkomik.kt @@ -0,0 +1,24 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("APKOMIK", "Apkomik", "id") +internal class Apkomik(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.APKOMIK, "01.apkomik.com") { + override val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("id")) + + override fun parseAltTitles(doc: Document, table: Element?, hasTable: Boolean): Set<String> { + return doc.select(".wd-full:contains(Alternative) span, .wd-full:contains(Alt) span") + .flatMap { splitAltTitles(it.text()) } + .toSet() + } +} From 5c2a58efb5cc4e76fa789f8b7cd4acf063d1dbfc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:10:46 +0200 Subject: [PATCH 346/434] Add new file KomikDewasaART.kt for NSFW content --- src/main/kotlin/tsuki/site/id/{ => nsfw}/KomikDewasaART.kt | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/main/kotlin/tsuki/site/id/{ => nsfw}/KomikDewasaART.kt (100%) diff --git a/src/main/kotlin/tsuki/site/id/KomikDewasaART.kt b/src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt similarity index 100% rename from src/main/kotlin/tsuki/site/id/KomikDewasaART.kt rename to src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt From b0c23ba03e37ccd217442d92ab9b47d0fedd4c3b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:19:06 +0200 Subject: [PATCH 347/434] fix getpages images --- src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt index 7799755b..c4c4a50e 100644 --- a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -2,7 +2,6 @@ package tsuki.site.fr import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.config.ConfigKey import tsuki.parsers.MadaraParser import tsuki.model.Manga @@ -33,7 +32,7 @@ internal class MangasOriginesFr(context: MangaLoaderContext) : override val datePattern = "dd/MM/yyyy" override val tagPrefix = "manga-genres/" override val listUrl = "oeuvre/" - + override val availableSortOrders: Set<SortOrder> = setOf( SortOrder.UPDATED, SortOrder.POPULARITY, @@ -134,7 +133,10 @@ internal class MangasOriginesFr(context: MangaLoaderContext) : val fullUrl = "https://$domain/oeuvre/${chapter.url}/" val doc = webClient.httpGet(fullUrl).parseHtml() return doc.select("div.reading-content img.wp-manga-chapter-img").mapIndexed { _, img -> - val url = img.attr("src").trim() + val url = when { + img.hasAttr("data-src") -> img.absUrl("data-src").trim() + else -> img.absUrl("src").trim() + } MangaPage( id = generateUid(url), url = url, From cf487315cf1d59a266527196da13fa042c3bb2ca Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:20:13 +0200 Subject: [PATCH 348/434] Update MangaFire.kt --- src/main/kotlin/tsuki/site/all/MangaFire.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt index 56c8bc6b..435311b0 100644 --- a/src/main/kotlin/tsuki/site/all/MangaFire.kt +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -410,7 +410,7 @@ internal abstract class MangaFireParser( } catch (e: ParseException) { throw e } catch (e: Exception) { - throw ParseException("Failed to load list page: ${e.message}\nOpen Webview to solve captcha / Reload Page when missing token.", url, e) + throw ParseException("Failed to load list page: ${e.message}\nSolve captcha in webview and then reload page manually.", url, e) } } From 5f4e40a0bdaa343553ccf9de7afc2571c8626e03 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 23 Aug 2026 12:48:22 +0200 Subject: [PATCH 349/434] Create Bacami.kt --- src/main/kotlin/tsuki/site/id/Bacami.kt | 326 ++++++++++++++++++++++++ 1 file changed, 326 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Bacami.kt diff --git a/src/main/kotlin/tsuki/site/id/Bacami.kt b/src/main/kotlin/tsuki/site/id/Bacami.kt new file mode 100644 index 00000000..40a6d11a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Bacami.kt @@ -0,0 +1,326 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import kotlinx.coroutines.coroutineScope +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("BACAMI", "Bacami", "id") +internal class Bacami(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BACAMI, 20) { + + override val configKeyDomain = ConfigKey.Domain("v1.bacami.site") + + private val dateFormat = SimpleDateFormat("dd MMMM, yyyy", Locale.ENGLISH) + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (key, title) -> + MangaTag(key = key, title = title, source = source) + }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = filter.query + ?.takeIf { it.isNotBlank() } + ?.let { q -> + val cleaned = q.replace("Bahasa Indonesia", "").trim() + "https://$domain/search/${cleaned.urlEncoded()}/page/$page/" + } + ?: buildString { + append("https://$domain/custom-search/") + + if (filter.tags.isNotEmpty()) { + append("genre/") + append(filter.tags.first().key) + append("/") + } + + filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> append("status/hot/") + MangaState.FINISHED -> append("status/tamat/") + else -> {} + } + } + + filter.types.firstOrNull()?.let { type -> + when (type) { + ContentType.MANGA -> append("type/manga/") + ContentType.MANHWA -> append("type/manhwa/") + ContentType.MANHUA -> append("type/manhua/") + else -> {} + } + } + + when (order) { + SortOrder.POPULARITY -> append("orderby/score/") + SortOrder.UPDATED -> append("orderby/latest/") + SortOrder.ALPHABETICAL -> append("orderby/name/") + else -> append("orderby/latest/") + } + + append("page/") + append(page) + append("/") + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("article.genre-card").mapNotNull { element -> + val link = element.selectFirst("div.genre-cover > a") ?: return@mapNotNull null + val title = element.selectFirst("div.genre-info > a")?.text() ?: return@mapNotNull null + val href = link.attr("href") + val cover = element.selectFirst("div.genre-cover > a > img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + }?.toAbsoluteUrl(domain) + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val content = doc.selectFirst("#komik > section.manga-content") + + val title = content?.selectFirst("header > h1")?.text() ?: manga.title + val cover = content?.selectFirst("figure .image-wrap img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + }?.toAbsoluteUrl(domain) ?: manga.coverUrl.orEmpty() + + val description = content?.selectFirst("p.manga-description")?.text().orEmpty() + val altTitle = content?.selectFirst("p.manga-altname")?.text().orEmpty() + val fullDescription = if (altTitle.isNotBlank()) { + "$description\n\nAlternative Title: $altTitle".trim() + } else { + description + } + + val author = content?.selectFirst(".info-item:contains(Author) .info-value")?.text() + ?: content?.selectFirst("div > div > div:nth-child(3) > span.info-value")?.text() + ?: "" + + val tags = content?.select("nav > span > a")?.map { it.text() } + ?.map { MangaTag(key = it.lowercase(), title = it, source = source) } + ?.toSet().orEmpty() + + val state = when { + doc.selectFirst(".hot-tag, .project-tag") != null -> MangaState.ONGOING + doc.selectFirst(".tamat-tag") != null -> MangaState.FINISHED + else -> null + } + + val chapters = doc.select("ol.chapter-list > li").mapNotNull { element -> + val link = element.selectFirst("a.ch-link") ?: return@mapNotNull null + val href = link.attr("href") + val name = link.text().substringAfter("–").trim() + val date = element.selectFirst("span.ch-date")?.text() + + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = parseDate(date), + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + + manga.copy( + title = title, + coverUrl = cover, + description = fullDescription, + authors = setOfNotNull(author.takeIf { it.isNotBlank() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private fun parseDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + return runCatching { dateFormat.parse(date)?.time ?: 0L }.getOrDefault(0L) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val scriptContent = doc.selectFirst("script:containsData(imageUrls)")?.data() + ?: return emptyList() + + val jsonString = scriptContent.substringAfter("imageUrls:").substringBefore("],").plus("]") + val jsonArray = JSONArray(jsonString) + + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + companion object { + private val GENRES = listOf( + "action-2" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "apocalypse" to "Apocalypse", + "comedy" to "Comedy", + "comedy-mystery-romance-slice-of-life-supernatural" to "Comedy Mystery Romance Slice Of Life Supernatural", + "comedy-romance-slice-of-life" to "Comedy Romance Slice Of Life", + "cooking" to "Cooking", + "crime" to "Crime", + "cultivation" to "Cultivation", + "demons" to "Demons", + "doujinshi" to "Doujinshi", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "furry" to "Furry", + "game" to "Game", + "gender-bender" to "Gender Bender", + "genius" to "Genius", + "gore" to "Gore", + "harem" to "Harem", + "hentai" to "Hentai", + "historical" to "Historical", + "horror" to "Horror", + "isekai" to "Isekai", + "josei" to "Josei", + "lolicon" to "Lolicon", + "long-strip" to "Long Strip", + "love-polygon" to "Love Polygon", + "magic" to "Magic", + "magical-girl" to "Magical Girl", + "manhua" to "Manhua", + "manhwa" to "Manhwa", + "martial-art" to "Martial Art", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medical" to "Medical", + "military" to "Military", + "monster" to "Monster", + "monster-girls" to "Monster Girls", + "monsters" to "Monsters", + "music" to "Music", + "mystery" to "Mystery", + "mystery-shounen" to "Mystery Shounen", + "mythology" to "Mythology", + "one-shot" to "One Shot", + "oneshot" to "Oneshot", + "parody" to "Parody", + "philosophical" to "Philosophical", + "police" to "Police", + "post-apocalyptic" to "Post-Apocalyptic", + "psychological" to "Psychological", + "rebirth" to "Rebirth", + "reincarnation" to "Reincarnation", + "romance" to "Romance", + "romantic-subtext" to "Romantic Subtext", + "samurai" to "Samurai", + "school" to "School", + "school-life" to "School Life", + "sci-fi" to "Sci-fi", + "seinen" to "Seinen", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "shounen-ai" to "Shounen Ai", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "space" to "Space", + "sports" to "Sports", + "superhero" to "Superhero", + "supernatural" to "Supernatural", + "super-power" to "Super Power", + "survival" to "Survival", + "suspense" to "Suspense", + "system" to "System", + "team-sports" to "Team Sports", + "thriller" to "Thriller", + "time-travel" to "Time Travel", + "tragedy" to "Tragedy", + "urban" to "Urban", + "urban-fantasy" to "Urban Fantasy", + "vampire" to "Vampire", + "video-game" to "Video Game", + "villainess" to "Villainess", + "visual-arts" to "Visual Arts", + "webtoon" to "Webtoon", + "webtoons" to "Webtoons", + "wuxia" to "Wuxia", + "yaoi" to "Yaoi", + "yuri" to "Yuri", + "zombies" to "Zombies", + ) + } +} From d89beddf076aceb4b63c156fdf177314d13586c4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 25 Aug 2026 13:28:16 +0200 Subject: [PATCH 350/434] fix hitomila --- src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt index 6bbe24e6..071b97eb 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt @@ -608,7 +608,7 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser chapters = listOf( MangaChapter( id = generateUid(manga.url), - url = manga.url, + url = json.getString("galleryurl").toAbsoluteurl(domain), title = json.getStringOrNull("title"), scanlator = json.getString("type").toTitleCase(), number = 1f, @@ -673,7 +673,8 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") + val galleryId = chapter.url.substringAfterLast("/").substringBefore(".") + val json = webClient.httpGet("$ltnBaseUrl/galleries/${galleryId}.js") .parseRaw() .substringAfter("var galleryinfo = ") .let(::JSONObject) From c8dc1a14fff99fe1c3ef676f7c81ff77e10cbb74 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Tue, 25 Aug 2026 15:43:35 +0200 Subject: [PATCH 351/434] deleted manga18 --- src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt deleted file mode 100644 index 3a05bd69..00000000 --- a/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt +++ /dev/null @@ -1,12 +0,0 @@ -package tsuki.site.en.nsfw - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.Manga18Parser - -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) -internal class Manga18(context: MangaLoaderContext) : - Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") \ No newline at end of file From 8a26fcd66a4ad7bd8b0c2cfe31f64257b1268736 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:55:28 +0200 Subject: [PATCH 352/434] Add TheManga source parser implementation Implement TheManga source parser for manga retrieval and details. --- src/main/kotlin/tsuki/site/id/TheManga.kt | 338 ++++++++++++++++++++++ 1 file changed, 338 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/TheManga.kt diff --git a/src/main/kotlin/tsuki/site/id/TheManga.kt b/src/main/kotlin/tsuki/site/id/TheManga.kt new file mode 100644 index 00000000..2de842c2 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/TheManga.kt @@ -0,0 +1,338 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("THEMANGA", "TheManga", "id") +internal class TheManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.THEMANGA, 24) { + + override val configKeyDomain = ConfigKey.Domain("themanga.site") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX", Locale.ROOT) + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + isYearSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (key, title) -> + MangaTag(key = key, title = title, source = source) + }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + val noFilters = filter.query.isNullOrBlank() && + filter.tags.isEmpty() && + filter.states.isEmpty() && + filter.types.isEmpty() && + filter.year == 0 && + filter.author.isNullOrBlank() + + if (noFilters) { + return when (order) { + SortOrder.POPULARITY -> "https://$domain/?q=&sort=popular&page=$page" + SortOrder.UPDATED -> "https://$domain/?q=&sort=latest_update&page=$page" + else -> "https://$domain/explore?q=&page=$page" + } + } + + return buildString { + append("https://$domain/explore?") + val params = mutableListOf<String>() + + filter.query?.takeIf { it.isNotBlank() }?.let { + params.add("q=${it.urlEncoded()}") + } + + params.add("page=$page") + + filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> params.add("status=ongoing") + MangaState.FINISHED -> params.add("status=completed") + else -> {} + } + } + + filter.tags.firstOrNull()?.let { tag -> + params.add("genre=${tag.key.urlEncoded()}") + } + + filter.types.firstOrNull()?.let { type -> + when (type) { + ContentType.MANGA -> params.add("type=manga") + ContentType.MANHWA -> params.add("type=manhwa") + ContentType.MANHUA -> params.add("type=manhua") + else -> {} + } + } + + if (filter.year != 0) { + params.add("year=${filter.year}") + } + + filter.author?.takeIf { it.isNotBlank() }?.let { + params.add("author=${it.urlEncoded()}") + } + + append(params.joinToString("&")) + } + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a.card, a.manga-card").mapNotNull { element -> + val href = element.attr("href") + val title = element.selectFirst(".card-title")?.text() + ?: return@mapNotNull null + val cover = element.selectFirst(".card-cover img, .cover img")?.absUrl("src") + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst(".hero-title")?.text() ?: manga.title + val cover = doc.selectFirst(".hero-cover img")?.absUrl("src") ?: manga.coverUrl.orEmpty() + val description = doc.selectFirst(".synopsis-text")?.text().orEmpty() + + val author = doc.selectFirst(".meta-item-label:matchesOwn(^Author$) + .meta-item-value")?.text() + val artist = doc.selectFirst(".meta-item-label:matchesOwn(^Artist$) + .meta-item-value")?.text() + val authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet() + + val tags = doc.select(".meta-pill-row .meta-pill").map { it.text() } + .map { MangaTag(key = it.lowercase(), title = it, source = source) } + .toSet() + + val typeMeta = doc.selectFirst(".meta-item-label:matchesOwn(^Type$) + .meta-item-value")?.text() + val allTags = tags.toMutableSet().apply { + typeMeta?.takeIf { it.isNotBlank() }?.let { + add(MangaTag(key = it.lowercase(), title = it, source = source)) + } + } + + val state = when (doc.selectFirst(".hero-status-badge")?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val chapterUrl = "$fullUrl?all=1" + val chapterDoc = webClient.httpGet(chapterUrl).parseHtml() + val chapters = chapterDoc.select(".chapter-row").mapNotNull { element -> + val href = element.attr("data-href") + val name = element.selectFirst(".chapter-title")?.text() + ?: return@mapNotNull null + val dateText = element.selectFirst("[data-local-time]")?.attr("data-local-time") + val date = dateText?.let { + runCatching { dateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + } ?: 0L + + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = authors, + tags = allTags, + state = state, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = if (!chapter.url.contains("/chapter/")) { + // Support old Madara URLs + val segments = chapter.url.split("/").filter { it.isNotBlank() } + if (segments.size >= 2) { + val mangaSlug = segments[0] + val number = segments[1].removePrefix("chapter-").replace("-", ".") + val dotIndex = number.indexOf('.') + val formatted = if (dotIndex >= 0) { + number.padEnd(dotIndex + 3, '0') + } else { + "$number.00" + } + "https://$domain/manga/$mangaSlug/chapter/$formatted" + } else { + chapter.url.toAbsoluteUrl(domain) + } + } else { + chapter.url.toAbsoluteUrl(domain) + } + + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("img.page-img").mapIndexed { _, img -> + val url = img.absUrl("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + companion object { + private val GENRES = listOf( + "4-koma" to "4-Koma", + "action" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "aliens" to "Aliens", + "animals" to "Animals", + "anthology" to "Anthology", + "comedy" to "Comedy", + "cooking" to "Cooking", + "crime" to "Crime", + "crossdressing" to "Crossdressing", + "delinquents" to "Delinquents", + "demon" to "Demon", + "demons" to "Demons", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "game" to "Game", + "gender-bender" to "Gender Bender", + "genderswap" to "Genderswap", + "ghosts" to "Ghosts", + "gore" to "Gore", + "gyaru" to "Gyaru", + "harem" to "Harem", + "historical" to "Historical", + "horror" to "Horror", + "incest" to "Incest", + "isekai" to "Isekai", + "josei" to "Josei", + "loli" to "Loli", + "mafia" to "Mafia", + "magic" to "Magic", + "magical-girls" to "Magical Girls", + "martial-art" to "Martial Art", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medical" to "Medical", + "military" to "Military", + "monster" to "Monster", + "monster-girls" to "Monster Girls", + "monsters" to "Monsters", + "music" to "Music", + "mystery" to "Mystery", + "ninja" to "Ninja", + "office-workers" to "Office Workers", + "oneshot" to "Oneshot", + "philosophical" to "Philosophical", + "police" to "Police", + "psychological" to "Psychological", + "regression" to "Regression", + "reincarnation" to "Reincarnation", + "reverse-harem" to "Reverse Harem", + "romance" to "Romance", + "samurai" to "Samurai", + "school" to "School", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "sexual-violence" to "Sexual Violence", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "sports" to "Sports", + "super-power" to "Super Power", + "supernatural" to "Supernatural", + "survival" to "Survival", + "suspense" to "Suspense", + "system" to "System", + "thriller" to "Thriller", + "time-travel" to "Time Travel", + "tragedy" to "Tragedy", + "urban" to "Urban", + "vampire" to "Vampire", + "video-games" to "Video Games", + "villainess" to "Villainess", + "virtual-reality" to "Virtual Reality", + "web-comic" to "Web Comic", + "webtoons" to "Webtoons", + "yuri" to "Yuri", + "zombies" to "Zombies", + ) + } +} From d54cb596680fbee380bbba2998434f8b84d46460 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 15:59:53 +0200 Subject: [PATCH 353/434] Create Manga18.kt --- src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt new file mode 100644 index 00000000..2153e4c9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.Manga18Parser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) +internal class Manga18(context: MangaLoaderContext) : + Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") From 7f0b15f4967568b42a3b8df2ae6951e579d72aee Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:04:17 +0200 Subject: [PATCH 354/434] Update Manga18Parser.kt --- src/main/kotlin/tsuki/parsers/Manga18Parser.kt | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt index 467bfffe..91d6efb2 100644 --- a/src/main/kotlin/tsuki/parsers/Manga18Parser.kt +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -128,34 +128,37 @@ internal abstract class Manga18Parser( override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val tag = filter.tags.firstOrNull() - val url = buildString(120) { + val url = buildString { append("https://") append(domain) append('/') when { - tag != null && filter.query!!.isNotEmpty() -> { + tag != null && !filter.query.isNullOrBlank() -> throw IllegalArgumentException("Search is not supported with tags") - } + tag != null -> { append(tagUrl) append(tag.key) append('/') append(page) } - filter.query!!.isNotEmpty() -> { + + !filter.query.isNullOrBlank() -> { append(listUrl) append(page) append("?search=") - append(filter.query.urlEncoded()) + append(filter.query.orEmpty().urlEncoded()) append("&order_by=latest") } + else -> { append(listUrl) append(page) } } - if (filter.query!!.isEmpty()) { + + if (filter.query.isNullOrBlank()) { append("?order_by=") when (order) { SortOrder.POPULARITY -> append("views") From 78471386413f2f99a76eb75e653f7dcbd36debda Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:11:29 +0200 Subject: [PATCH 355/434] Update HitomiLaParser.kt --- src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt index 071b97eb..6bbe24e6 100644 --- a/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt @@ -608,7 +608,7 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser chapters = listOf( MangaChapter( id = generateUid(manga.url), - url = json.getString("galleryurl").toAbsoluteurl(domain), + url = manga.url, title = json.getStringOrNull("title"), scanlator = json.getString("type").toTitleCase(), number = 1f, @@ -673,8 +673,7 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val galleryId = chapter.url.substringAfterLast("/").substringBefore(".") - val json = webClient.httpGet("$ltnBaseUrl/galleries/${galleryId}.js") + val json = webClient.httpGet("$ltnBaseUrl/galleries/${chapter.url}.js") .parseRaw() .substringAfter("var galleryinfo = ") .let(::JSONObject) From acdb7fd3fd637fcedcc3f37b79396fce96fe52e6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:13:27 +0200 Subject: [PATCH 356/434] Revert "Add and adapt Russian manga sources for the current UMA structure" --- .github/summary.yaml | 2 +- .../kotlin/tsuki/parsers/NineMangaParser.kt | 267 -------- src/main/kotlin/tsuki/site/ru/AComics.kt | 187 ------ src/main/kotlin/tsuki/site/ru/ComXParser.kt | 238 ------- src/main/kotlin/tsuki/site/ru/DesuMeParser.kt | 280 --------- .../kotlin/tsuki/site/ru/MangaWtfParser.kt | 263 -------- .../kotlin/tsuki/site/ru/RemangaParser.kt | 378 ------------ .../kotlin/tsuki/site/ru/WaMangaParser.kt | 182 ------ .../kotlin/tsuki/site/ru/chan/ChanParser.kt | 271 -------- .../tsuki/site/ru/chan/MangaChanParser.kt | 11 - .../tsuki/site/ru/chan/YaoiChanParser.kt | 50 -- .../tsuki/site/ru/grouple/GroupleParser.kt | 579 ------------------ .../tsuki/site/ru/grouple/ReadmangaParser.kt | 26 - .../tsuki/site/ru/grouple/SeiMangaParser.kt | 26 - .../tsuki/site/ru/grouple/SelfMangaParser.kt | 26 - .../tsuki/site/ru/grouple/UsagiParser.kt | 23 - .../site/ru/libsocial/LibSocialParser.kt | 473 -------------- .../tsuki/site/ru/libsocial/MangaLibParser.kt | 26 - .../tsuki/site/ru/libsocial/SlashLibParser.kt | 13 - .../kotlin/tsuki/site/ru/madara/MangaMammy.kt | 13 - .../tsuki/site/ru/madara/MangaoneLove.kt | 13 - 21 files changed, 1 insertion(+), 3346 deletions(-) delete mode 100644 src/main/kotlin/tsuki/parsers/NineMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/AComics.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/ComXParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/DesuMeParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/RemangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/WaMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/chan/ChanParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/chan/MangaChanParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/chan/YaoiChanParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/ReadmangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/SeiMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/SelfMangaParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/grouple/UsagiParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/libsocial/LibSocialParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/libsocial/MangaLibParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/libsocial/SlashLibParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/madara/MangaMammy.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/madara/MangaoneLove.kt diff --git a/.github/summary.yaml b/.github/summary.yaml index ac3c4a8c..f0d18c2f 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 215 \ No newline at end of file +total: 76 \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/NineMangaParser.kt b/src/main/kotlin/tsuki/parsers/NineMangaParser.kt deleted file mode 100644 index fbe40e38..00000000 --- a/src/main/kotlin/tsuki/parsers/NineMangaParser.kt +++ /dev/null @@ -1,267 +0,0 @@ -package tsuki.parsers - -import androidx.collection.ArrayMap -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import tsuki.MangaLoaderContext -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class NineMangaParser( - context: MangaLoaderContext, - source: MangaParserSource, - defaultDomain: String, -) : PagedMangaParser(context, source, pageSize = 26), Interceptor { - - override val configKeyDomain = ConfigKey.Domain(defaultDomain) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - init { - context.cookieJar.insertCookies(domain, "ninemanga_template_desk=yes") - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("Accept-Language", "en-US;q=0.7,en;q=0.3") - .build() - - override val availableSortOrders: Set<SortOrder> = Collections.singleton( - SortOrder.POPULARITY, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchWithFiltersSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrCreateTagMap().values.toSet(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - ), - ) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = if (request.url.host == domain) { - request.newBuilder().removeHeader("Referer").build() - } else { - request - } - return chain.proceed(newRequest) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append("https://") - append(domain) - - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.states.isNotEmpty() || !filter.query.isNullOrEmpty()) { - append("/search/") - append("?page=") - append(page.toString()) - - filter.query?.let { - append("&name_sel=contain&wd=") - append(filter.query.urlEncoded()) - } - - append("&category_id=") - append(filter.tags.joinToString(separator = ",") { it.key }) - - append("&out_category_id=") - append(filter.tagsExclude.joinToString(separator = ",") { it.key }) - - filter.states.oneOrThrowIfMany()?.let { - append("&completed_series=") - when (it) { - MangaState.ONGOING -> append("no") - MangaState.FINISHED -> append("yes") - else -> append("either") - } - } - - } else { - append("/category/index_") - append(page.toString()) - } - } - val doc = webClient.httpGet(url).parseHtml() - val root = doc.body().selectFirstOrThrow("ul.direlist") - val baseHost = root.baseUri().toHttpUrl().host - return root.select("li").map { node -> - val href = node.selectFirstOrThrow("a").attrAsAbsoluteUrl("href") - val relUrl = href.toRelativeUrl(baseHost) - val dd = node.selectFirst("dd") - Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = dd?.selectFirst("a.bookname")?.text()?.toCamelCase().orEmpty(), - altTitles = emptySet(), - coverUrl = node.selectFirst("img")?.src(), - rating = RATING_UNKNOWN, - authors = emptySet(), - contentRating = null, - tags = emptySet(), - state = null, - source = source, - description = dd?.selectFirst("p")?.html(), - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet( - manga.url.toAbsoluteUrl(domain) + "?waring=1", - ).parseHtml() - val root = doc.body().selectFirstOrThrow("div.manga") - val infoRoot = root.selectFirstOrThrow("div.bookintro") - val tagMap = getOrCreateTagMap() - val selectTag = infoRoot.getElementsByAttributeValue("itemprop", "genre").first()?.select("a") - val tags = selectTag?.mapNotNullToSet { tagMap[it.text()] } - val author = infoRoot.getElementsByAttributeValue("itemprop", "author").first()?.textOrNull() - return manga.copy( - title = root.selectFirst("h1[itemprop=name]")?.textOrNull()?.removeSuffix("Manga")?.trimEnd() - ?: manga.title, - tags = tags.orEmpty(), - authors = setOfNotNull(author), - state = parseStatus(infoRoot.select("li a.red").text()), - description = infoRoot.getElementsByAttributeValue("itemprop", "description").first()?.html() - ?.substringAfter("</b>"), - chapters = root.selectFirst("div.chapterbox")?.select("ul.sub_vol_ul > li") - ?.mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a.chapter_list_a") - val href = a.attrAsRelativeUrl("href").replace("%20", " ") - MangaChapter( - id = generateUid(href), - title = a.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = parseChapterDateByLang(li.selectFirst("span")?.text().orEmpty()), - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.body().requireElementById("page").select("option").map { option -> - val url = option.attr("value") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body() - return root.selectFirstOrThrow("a.pic_download").attrAsAbsoluteUrl("href") - } - - private var tagCache: ArrayMap<String, MangaTag>? = null - private val mutex = Mutex() - - private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock { - tagCache?.let { return@withLock it } - val tagMap = ArrayMap<String, MangaTag>() - val tagElements = webClient.httpGet("https://${domain}/search/?type=high").parseHtml().select("li.cate_list") - for (el in tagElements) { - if (el.text().isEmpty()) continue - val cateId = el.attr("cate_id") - val a = el.selectFirstOrThrow("a") - tagMap[el.text()] = MangaTag( - title = a.text().toTitleCase(sourceLocale), - key = cateId, - source = source, - ) - } - tagCache = tagMap - return@withLock tagMap - } - - private fun parseStatus(status: String) = when { - // en - status.contains("Ongoing") -> MangaState.ONGOING - status.contains("Completed") -> MangaState.FINISHED - //es - status.contains("En curso") -> MangaState.ONGOING - status.contains("Completado") -> MangaState.FINISHED - //ru - status.contains("постоянный") -> MangaState.ONGOING - status.contains("завершенный") -> MangaState.FINISHED - //de - status.contains("Laufende") -> MangaState.ONGOING - status.contains("Abgeschlossen") -> MangaState.FINISHED - //pt - status.contains("Completo") -> MangaState.ONGOING - status.contains("Em tradução") -> MangaState.FINISHED - //it - status.contains("In corso") -> MangaState.ONGOING - status.contains("Completato") -> MangaState.FINISHED - //fr - status.contains("En cours") -> MangaState.ONGOING - status.contains("Complété") -> MangaState.FINISHED - else -> null - } - - private fun parseChapterDateByLang(date: String): Long { - val dateWords = date.split(" ") - - if (dateWords.size == 3) { - if (dateWords[1].contains(",")) { - SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH).parseSafe(date) - } else { - val timeAgo = Integer.parseInt(dateWords[0]) - return Calendar.getInstance().apply { - when (dateWords[1]) { - "minutes" -> Calendar.MINUTE // EN-FR - "hours" -> Calendar.HOUR // EN - - "minutos" -> Calendar.MINUTE // ES - "horas" -> Calendar.HOUR - - // "minutos" -> Calendar.MINUTE // BR - "hora" -> Calendar.HOUR - - "минут" -> Calendar.MINUTE // RU - "часа" -> Calendar.HOUR - - "Stunden" -> Calendar.HOUR // DE - - "minuti" -> Calendar.MINUTE // IT - "ore" -> Calendar.HOUR - - "heures" -> Calendar.HOUR // FR ("minutes" also French word) - else -> null - }?.let { - add(it, -timeAgo) - } - }.timeInMillis - } - } - return 0L - } -} diff --git a/src/main/kotlin/tsuki/site/ru/AComics.kt b/src/main/kotlin/tsuki/site/ru/AComics.kt deleted file mode 100644 index b64f02f9..00000000 --- a/src/main/kotlin/tsuki/site/ru/AComics.kt +++ /dev/null @@ -1,187 +0,0 @@ -package tsuki.site.ru - -import androidx.collection.ArrayMap -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.jsoup.nodes.Document -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import java.util.* - -@MangaSourceParser("ACOMICS", "AComics", "ru", ContentType.COMICS) -internal class AComics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ACOMICS, pageSize = 10) { - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - SortOrder.POPULARITY, - ) - - override val configKeyDomain = ConfigKey.Domain("acomics.ru") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - context.cookieJar.insertCookies(domain, "ageRestrict=18") - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrCreateTagMap().values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - val url = buildString { - append("https://") - append(domain) - when { - !filter.query.isNullOrEmpty() -> { - if (page > 0) { - return emptyList() - } - append("/search?keyword=") - append(filter.query) - } - - else -> { - append("/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=") - append(page * 10) - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "last_update" - SortOrder.ALPHABETICAL -> "serial_name" - SortOrder.POPULARITY -> "subscr_count" - else -> "last_update" - }, - ) - - if (filter.tags.isNotEmpty()) { - append("&categories=") - append(filter.tags.joinToString(separator = ",") { it.key }) - } - - if (filter.states.isNotEmpty()) { - append("&updatable=") - append( - filter.states.oneOrThrowIfMany().let { - when (it) { - MangaState.ONGOING -> "yes" - MangaState.FINISHED -> "no" - else -> "0" - } - }, - ) - } - } - } - } - - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - private fun parseMangaList(docs: Document): List<Manga> { - return docs.select("section.serial-card").map { - val a = it.selectFirstOrThrow("a.cover") - val url = a.attrAsAbsoluteUrl("href") + "/about" - Manga( - id = generateUid(url), - url = url, - title = it.selectFirstOrThrow("h2.title a").text(), - altTitles = emptySet(), - publicUrl = url, - rating = RATING_UNKNOWN, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = a.selectFirstOrThrow("img").attrAsAbsoluteUrl("data-real-src"), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - private var tagCache: ArrayMap<String, MangaTag>? = null - private val mutex = Mutex() - - private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock { - tagCache?.let { return@withLock it } - val tagMap = ArrayMap<String, MangaTag>() - val tagElements = webClient.httpGet("https://$domain/comics") - .parseHtml() - .select("form.catalog-filters-form fieldset.categories label") - for (el in tagElements) { - val name = el.ownText().trim() - val key = el.selectFirstOrThrow("input[name='categories[]']").attr("value") - tagMap[name] = MangaTag( - title = name, - key = key, - source = source, - ) - } - tagCache = tagMap - return@withLock tagMap - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val tagMap = getOrCreateTagMap() - val tags = doc.select("p.serial-about-badges .category").mapNotNullToSet { tagMap[it.text()] } - val author = doc.selectFirst("p.serial-about-authors a")?.textOrNull() - return manga.copy( - tags = tags, - description = doc.selectFirst("section.serial-about-text p")?.text(), - authors = setOfNotNull(author), - chapters = listOf( - MangaChapter( - id = manga.id, - title = manga.title, - number = 1f, - volume = 0, - url = manga.url.replace("/about", "/"), - scanlator = null, - uploadDate = 0, - branch = null, - source = source, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url + "1").parseHtml() - val totalPages = doc.selectFirstOrThrow("h1.reader-issue-title span.number") - .text() - .substringAfterLast('/') - .toInt() - return (1..totalPages).map { - val url = chapter.url + it - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - override suspend fun getPageUrl(page: MangaPage): String { - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() - return doc.selectFirstOrThrow("section.reader-issue img.issue").requireSrc() - } -} diff --git a/src/main/kotlin/tsuki/site/ru/ComXParser.kt b/src/main/kotlin/tsuki/site/ru/ComXParser.kt deleted file mode 100644 index 8b2d7001..00000000 --- a/src/main/kotlin/tsuki/site/ru/ComXParser.kt +++ /dev/null @@ -1,238 +0,0 @@ -package tsuki.site.ru - -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.getFloatOrDefault -import tsuki.util.json.getIntOrDefault -import tsuki.util.json.getStringOrNull -import tsuki.util.suspendlazy.getOrNull -import tsuki.util.suspendlazy.suspendLazy -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("COMX", "Com-X", "ru", ContentType.COMICS) -internal class ComXParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.COMX, 20) { - - override val configKeyDomain = ConfigKey.Domain("com-x.life", "comx.life") - - private val availableTags = suspendLazy(initializer = ::fetchTags) - private val cdnImageUrl = "img.com-x.life/comix/" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - init { - context.cookieJar.insertCookies(domain, "adt-accepted", "1") - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isYearRangeSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags.get(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val urlBuilder = StringBuilder() - when { - !filter.query.isNullOrEmpty() -> { - val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "%20") { part -> - part.urlEncoded() - } - urlBuilder.append("/search/") - urlBuilder.append(encodedQuery) - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - - else -> { - urlBuilder.append("/ComicList") - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.append("/y[from]=${filter.yearFrom}") - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.append("/y[to]=${filter.yearTo}") - } - if (filter.tags.isNotEmpty()) { - urlBuilder.append("/g=") - urlBuilder.append(filter.tags.joinToString(",") { it.key }) - } - urlBuilder.append("/sort") - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - } - - val fullUrl = urlBuilder.toString().toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.readed.d-flex.short").map { item -> - val a = item.selectFirstOrThrow("a.readed__img.img-fit-cover.anim") - val img = item.selectFirst("img[data-src]") - val href = a.attrAsRelativeUrl("href") - val titleElement = item.selectFirstOrThrow("h3.readed__title a") - val (mainTitle, altTitle) = titleElement.text() - .split("\\s*/\\s*".toRegex()) - .map { it.trim() } - .let { parts -> - when { - parts.size >= 2 -> parts[1] to parts[0] - parts.isNotEmpty() -> parts[0] to "" - else -> "" to "" - } - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = mainTitle, - altTitles = if (altTitle.isNotEmpty()) setOf(altTitle) else emptySet(), - authors = emptySet(), - description = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - coverUrl = img?.attrAsAbsoluteUrlOrNull("data-src"), - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - - val scriptData = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("window.__DATA__ = ") - ?.substringBefore(";</script>") - ?.trim() - ?: throw ParseException("Script data not found", manga.url) - - val jsonData = JSONObject(scriptData) - val chaptersJson = jsonData.getJSONArray("chapters") - val newsId = jsonData.getLong("news_id") - - val chapters = List(chaptersJson.length()) { i -> - val chapter = chaptersJson.getJSONObject(i) - val chapterId = chapter.getLong("id") - - MangaChapter( - id = generateUid("$newsId/$chapterId"), - url = "/reader/$newsId/$chapterId", - number = chapter.getFloatOrDefault("posi", 0f), - title = decodeText(chapter.getStringOrNull("title")), - uploadDate = dateFormat.parseSafe(chapter.getStringOrNull("date")), - source = source, - scanlator = null, - branch = null, - volume = chapter.getIntOrDefault("volume", 0), - ) - }.reversed() - - val author = doc.selectFirst("li:contains(Publisher:)") - ?.textOrNull() - ?.substringAfter("Publisher:") - ?.trim() - ?.nullIfEmpty() - val state = when ( - doc.selectFirst("li:contains(Release type:)")?.text()?.substringAfter("Release type:")?.trim() - ) { - "Ongoing" -> MangaState.ONGOING - else -> MangaState.FINISHED - } - - val tagLinks = doc.getElementsByAttributeValueContaining("href", "/genres/") - val tags = if (tagLinks.isNotEmpty()) { - availableTags.getOrNull()?.let { allTags -> - tagLinks.mapNotNullToSet { a -> - val tagName = a.text() - allTags.find { it.title.equals(tagName, ignoreCase = true) } - } - } - } else { - null - } - - return manga.copy( - authors = setOfNotNull(author), - state = state, - chapters = chapters, - description = doc.select("div.page__text.full-text.clearfix").textOrNull(), - tags = tags ?: manga.tags, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val newsId = chapter.url.substringAfter("/reader/").substringBefore("/") - context.cookieJar.insertCookies(domain, "adult=$newsId") - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val data = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("=") - ?.trim() - ?.removeSuffix(";") - ?.substringAfter("\"images\":[") - ?.substringBefore("]") - ?.split(",") - ?.map { it.trim().removeSurrounding("\"").replace("\\", "") } - ?: throw ParseException("Image data not found", chapter.url) - - return data.map { imageUrl -> - val finalUrl = "https://$cdnImageUrl$imageUrl" - MangaPage( - id = generateUid(imageUrl), - url = finalUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/comix/").parseHtml() - val scriptData = doc.selectFirstOrThrow("script:containsData(__XFILTER__)").data() - - val genresJson = scriptData - .substringAfter("\"g\":{") - .substringBefore("}}}") + "}" - - val genresObj = JSONObject("{$genresJson}") - val valuesArray = genresObj.getJSONArray("values") - - return Set(valuesArray.length()) { i -> - val genre = valuesArray.getJSONObject(i) - MangaTag( - key = genre.getInt("id").toString(), - title = genre.getString("value").toTitleCase(sourceLocale), - source = source, - ) - } - } - - private fun decodeText(text: String?): String? { - if (text == null) return null - return text.replace("\\u([0-9a-fA-F]{4})".toRegex()) { matchResult -> - val codePoint = matchResult.groupValues[1].toInt(16) - codePoint.toChar().toString() - } - } -} diff --git a/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt b/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt deleted file mode 100644 index d827b163..00000000 --- a/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt +++ /dev/null @@ -1,280 +0,0 @@ -package tsuki.site.ru - -import okhttp3.Headers -import okhttp3.HttpUrl -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.ContentRating -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder -import tsuki.network.UserAgents -import tsuki.util.LinkResolver -import tsuki.util.attrAsRelativeUrl -import tsuki.util.generateUid -import tsuki.util.parseHtml -import tsuki.util.parseJson -import tsuki.util.selectFirstOrThrow -import tsuki.util.toAbsoluteUrl -import tsuki.util.toTitleCase -import java.util.EnumSet - -@MangaSourceParser("DESUME", "Desu", "ru") -internal class DesuMeParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.DESUME, 20) { - - override val configKeyDomain = ConfigKey.Domain("desu.uno") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchCatalogDocument().parseDesuTags(), - ) - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("User-Agent", UserAgents.KOTATSU) - .build() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - if (query != null) { - if (page != searchPaginator.firstPage) return emptyList() - val document = webClient.httpPost( - "https://$domain/manga/search/", - mapOf("q" to query), - ).parseHtml() - return document.parseDesuSearchResults().map(::parseSearchManga) - } - - val url = buildCatalogUrl(page, order, filter) - return webClient.httpGet(url) - .parseHtml() - .parseDesuCatalogItems() - .map(::parseCatalogManga) - } - - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) - .parseHtml() - .selectFirstOrThrow("#animeView") - val title = root.selectFirstOrThrow("meta[itemprop=headline]").attr("content") - val alternativeTitles = buildSet { - root.selectFirst("meta[itemprop=alternativeHeadline]") - ?.attr("content") - ?.takeIf { it.isNotBlank() } - ?.let(::add) - root.selectFirst(".alternativeHeadline") - ?.text() - ?.split(',') - ?.map(String::trim) - ?.filterTo(this) { it.isNotEmpty() } - } - val mangaId = root.selectFirst("ul.chlist[data-manga_id]") - ?.attr("data-manga_id") - ?.toLongOrNull() - ?: error("Cannot find Desu manga id") - val chapters = root.select("ul.chlist > li").map { item -> - val link = item.selectFirstOrThrow("h4 > a") - val chapterUrl = link.attrAsRelativeUrl("href") - val match = DESU_CHAPTER_PATH.find(chapterUrl) - ?: error("Unsupported Desu chapter URL: $chapterUrl") - val chapterId = item.selectFirst("[data-chapters_id]") - ?.attr("data-chapters_id") - ?.toLongOrNull() - ?: error("Cannot find Desu chapter id") - MangaChapter( - id = generateUid(chapterId), - url = "/api/manga/$mangaId/chapters/$chapterId", - source = source, - number = match.groupValues[2].toFloat(), - volume = match.groupValues[1].toInt(), - title = link.selectFirst(".title")?.text()?.removePrefix("- ")?.takeIf { it.isNotBlank() }, - scanlator = null, - uploadDate = 0L, - branch = null, - ) - } - - return manga.copy( - title = title, - altTitles = alternativeTitles, - publicUrl = root.selectFirstOrThrow("link[itemprop=url]").attr("href"), - largeCoverUrl = root.selectFirstOrThrow("img[itemprop=image]").attr("src"), - description = root.selectFirst("#description .russian")?.html(), - tags = root.select("a[itemprop=genre]").mapTo(linkedSetOf()) { link -> - MangaTag( - key = link.attr("href").substringAfter("genres="), - title = link.text().removePrefix("# ").toTitleCase(), - source = source, - ) - }, - state = root.selectFirst(".b-anime_status_tag")?.classNames()?.toDesuState(), - rating = root.selectFirst("meta[itemprop=ratingValue]") - ?.attr("content") - ?.toFloatOrNull() - ?.div(10f) - ?: RATING_UNKNOWN, - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val pages = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)) - .parseJson() - .getJSONObject("chapter") - .getJSONArray("pages") - - return (0 until pages.length()).map { index -> - val imageUrl = pages.getJSONObject(index).getString("url") - MangaPage( - id = generateUid(imageUrl), - preview = null, - source = chapter.source, - url = imageUrl, - ) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val document = webClient.httpGet(link).parseHtml() - val root = document.selectFirst("#animeView") ?: return null - val title = root.selectFirst("meta[itemprop=headline]")?.attr("content") ?: return null - val url = root.selectFirst("link[itemprop=url]")?.attr("href") ?: return null - return resolver.resolveManga(this, id = generateUid(url), url = url, title = title) - } - - private suspend fun fetchCatalogDocument(): Document = webClient - .httpGet("https://$domain/manga/") - .parseHtml() - - private fun buildCatalogUrl(page: Int, order: SortOrder, filter: MangaListFilter): String = buildString { - append("https://") - append(domain) - append("/manga/?page=") - append(page) - val sortKey = when (order) { - SortOrder.UPDATED -> "updated" - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "id" - SortOrder.ALPHABETICAL -> "name" - else -> error("Unsupported Desu sort order: $order") - } - if (sortKey != "updated") { - append("&order_by=") - append(sortKey) - } - if (filter.tags.isNotEmpty()) { - append("&genres=") - append(filter.tags.joinToString(",") { it.key }) - } - } - - private fun parseCatalogManga(item: Element): Manga { - val link = item.selectFirstOrThrow("a.animeTitle") - val url = link.attrAsRelativeUrl("href") - val cover = item.selectFirstOrThrow("a.avatar .img") - .attr("style") - .let { style -> DESU_BACKGROUND_IMAGE.find(style)?.groupValues?.get(1) } - ?: error("Cannot find Desu catalog cover") - val russianTitle = item.selectFirstOrThrow(".dimmed.oTitle [itemprop=title]").text() - val originalTitle = link.text() - return Manga( - id = generateUid(url), - url = url, - publicUrl = url.toAbsoluteUrl(domain), - source = source, - title = russianTitle, - altTitles = setOf(originalTitle), - coverUrl = cover, - largeCoverUrl = null, - state = null, - rating = item.parseDesuCatalogRating(), - contentRating = null, - tags = emptySet(), - authors = emptySet(), - description = null, - ) - } - - private fun parseSearchManga(item: Element): Manga { - val url = item.attrAsRelativeUrl("href") - val russianTitle = item.selectFirstOrThrow(".itemSubTitle").text() - val originalTitle = item.selectFirstOrThrow(".itemTitle").text() - return Manga( - id = generateUid(url), - url = url, - publicUrl = url.toAbsoluteUrl(domain), - source = source, - title = russianTitle, - altTitles = setOf(originalTitle), - coverUrl = item.selectFirstOrThrow("img").attr("src"), - largeCoverUrl = null, - state = null, - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - description = null, - ) - } -} - -internal fun Document.parseDesuCatalogItems(): List<Element> = - select("ol.memberList > li.memberListItem") - -internal fun Document.parseDesuSearchResults(): List<Element> = - select("ol.section > li > a[href^=manga/]") - -internal fun Document.parseDesuTags(): Set<MangaTag> = - select("#animeFilter input[data-genre-id][data-genre-slug][data-genre-name]").mapTo(linkedSetOf()) { input -> - MangaTag( - key = "${input.attr("data-genre-id")}-${input.attr("data-genre-slug")}", - title = input.attr("data-genre-name").toTitleCase(), - source = MangaParserSource.DESUME, - ) - } - -private fun Element.parseDesuCatalogRating(): Float { - val rating = select(".animeInfo dl").asSequence() - .flatMap { it.children().asSequence() } - .windowed(2) - .firstOrNull { it[0].tagName() == "dt" && it[0].text() == "Рейтинг:" && it[1].tagName() == "dd" } - ?.get(1) - ?.text() - ?.toFloatOrNull() - return rating?.div(10f) ?: RATING_UNKNOWN -} - -private fun Set<String>.toDesuState(): MangaState? = when { - "ongoing" in this -> MangaState.ONGOING - "released" in this -> MangaState.FINISHED - "anons" in this -> MangaState.UPCOMING - isEmpty() -> null - else -> error("Unsupported Desu status classes: $this") -} - -private val DESU_BACKGROUND_IMAGE = Regex("url\\(['\"]?([^'\")]+)") -private val DESU_CHAPTER_PATH = Regex("/vol(\\d+)/ch([0-9.]+)/") diff --git a/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt b/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt deleted file mode 100644 index 1e67bc3d..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt +++ /dev/null @@ -1,263 +0,0 @@ -package tsuki.site.ru - -import androidx.collection.ArrayMap -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONObject -import tsuki.InternalParsersApi -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGA_WTF", "MangaWtf", "ru") -internal class MangaWtfParser( - context: MangaLoaderContext, -) : PagedMangaParser(context, MangaParserSource.MANGA_WTF, pageSize = 20) { - - override val availableSortOrders: Set<SortOrder> = - EnumSet.of( - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) - - @InternalParsersApi - override val configKeyDomain = ConfigKey.Domain("inkstory.net") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.UPCOMING, - MangaState.PAUSED, - MangaState.ONGOING, - MangaState.FINISHED, - ), - availableContentRating = EnumSet.allOf(ContentRating::class.java), - ) - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("books") - .addQueryParameter("page", page.toString()) - .addQueryParameter("size", pageSize.toString()) - .addQueryParameter("type", "COMIC") - when { - filter.query.isNullOrEmpty() -> { - url.addQueryParameter( - "sort", - when (order) { - SortOrder.UPDATED -> "updatedAt,desc" - SortOrder.POPULARITY -> "viewsCount,desc" - SortOrder.RATING -> "likesCount,desc" - SortOrder.NEWEST -> "createdAt,desc" - else -> throw IllegalArgumentException("Unsupported $order") - }, - ) - if (filter.tags.isNotEmpty()) { - url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) - } - if (filter.tagsExclude.isNotEmpty()) { - url.addQueryParameter("labelsExclude", filter.tagsExclude.joinToString(",") { it.key }) - } - if (filter.states.isNotEmpty()) { - url.addQueryParameter( - "status", - filter.states.joinToString(",") { - when (it) { - MangaState.ONGOING -> "ONGOING" - MangaState.FINISHED -> "DONE" - MangaState.ABANDONED -> "" - MangaState.PAUSED -> "FROZEN" - MangaState.UPCOMING -> "ANNOUNCE" - else -> throw IllegalArgumentException("$it not supported") - } - }, - ) - } - if (filter.contentRating.isNotEmpty()) { - url.addQueryParameter( - "contentStatus", - filter.contentRating.joinToString(",") { - when (it) { - ContentRating.SAFE -> "SAFE" - ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" - ContentRating.ADULT -> "PORNOGRAPHIC" - } - }, - ) - } - } - - else -> { - url.addQueryParameter("search", filter.query) - } - } - val ja = webClient.httpGet(url.build()).parseJsonArray() - return ja.mapJSON { jo -> jo.toManga() } - } - - override suspend fun getDetails(manga: Manga): Manga = - coroutineScope { - val chaptersDeferred = async { getChapters(manga.url) } - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("books") - .addPathSegment(manga.url) - val jo = webClient.httpGet(url.build()).parseJson() - val isNsfwSource = jo.getStringOrNull("contentStatus").isNsfw() - Manga( - id = generateUid(jo.getString("id")), - title = jo.getJSONObject("name").getString("ru"), - altTitles = setOfNotNull(jo.getJSONObject("name").getStringOrNull("en")), - url = jo.getString("id"), - publicUrl = "https://$domain/manga/${jo.getString("slug")}", - rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = jo.getString("poster"), - tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, - state = jo.getStringOrNull("status")?.toMangaState(), - authors = jo.getJSONArray("relations").asTypedList<JSONObject>().mapNotNullToSet { - if (it.getStringOrNull("type") == "AUTHOR") { - it.getJSONObject("publisher").getStringOrNull("name") - } else { - null - } - }, - source = source, - largeCoverUrl = null, - description = jo.getString("description").nl2br(), - chapters = chaptersDeferred.await(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("chapters") - .addPathSegment(chapter.url) - val json = webClient.httpGet(url.build()).parseJson() - return json.getJSONArray("pages").mapJSON { jo -> - MangaPage( - id = generateUid(jo.getString("id")), - url = jo.getString("image"), - preview = null, - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val url = urlBuilder("api").addPathSegment("v2").addPathSegment("labels") - val json = webClient.httpGet(url.build()).parseJsonArray() - return json.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(sourceLocale), - key = jo.getString("slug"), - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - override suspend fun getPageUrl(page: MangaPage): String = page.url - - private suspend fun getChapters(mangaId: String): List<MangaChapter> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("chapters") - .addQueryParameter("bookId", mangaId) - val ja = webClient.httpGet(url.build()).parseJsonArray() - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) - val branches = ArrayMap<String, String>() - return ja - .mapJSON { jo -> - val number = jo.getFloatOrDefault("number", 0f) - val volume = jo.getIntOrDefault("volume", 0) - val branchId = jo.getString("branchId") - MangaChapter( - id = generateUid(jo.getString("id")), - title = jo.getStringOrNull("name"), - number = number, - volume = volume, - url = jo.getString("id"), - scanlator = null, - uploadDate = dateFormat.parseSafe(jo.getString("createdAt")), - branch = branches.getOrPut(branchId) { getBranchName(branchId) }, - source = source, - ) - }.reversed() - } - - private suspend fun getBranchName(id: String): String? { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("branches") - .addPathSegment(id) - val json = webClient.httpGet(url.build()).parseJson() - return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() - } - - private fun String.toMangaState() = - when (this.uppercase(Locale.ROOT)) { - "DONE" -> MangaState.FINISHED - "ONGOING" -> MangaState.ONGOING - "FROZEN" -> MangaState.PAUSED - "ANNOUNCE" -> MangaState.UPCOMING - else -> null - } - - private fun String?.isNsfw() = - this.equals("EROTIC", ignoreCase = true) || - this.equals("PORNOGRAPHIC", ignoreCase = true) - - private fun JSONObject.toMangaTag() = - MangaTag( - title = getString("name").toTitleCase(sourceLocale), - key = getString("slug"), - source = source, - ) - - private fun JSONObject.toManga(): Manga { - val isNsfwSource = getStringOrNull("contentStatus").isNsfw() - return Manga( - id = generateUid(getString("id")), - title = getJSONObject("name").getString("ru"), - altTitles = setOfNotNull(getJSONObject("name").getStringOrNull("en")), - url = getString("id"), - publicUrl = "https://$domain/manga/${getString("slug")}", - rating = getFloatOrDefault("averageRating", -10f) / 10f, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = getString("poster"), - tags = setOf(), - state = getStringOrNull("status")?.toMangaState(), - authors = emptySet(), - source = source, - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/RemangaParser.kt b/src/main/kotlin/tsuki/site/ru/RemangaParser.kt deleted file mode 100644 index efb82b26..00000000 --- a/src/main/kotlin/tsuki/site/ru/RemangaParser.kt +++ /dev/null @@ -1,378 +0,0 @@ -package tsuki.site.ru - -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import okhttp3.internal.closeQuietly -import org.json.JSONArray -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ContentUnavailableException -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.* -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -private const val PAGE_SIZE = 20 -private const val TOO_MANY_REQUESTS = 429 - -@MangaSourceParser("REMANGA", "Реманга", "ru") -internal class RemangaParser( - context: MangaLoaderContext, -) : PagedMangaParser(context, MangaParserSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { - - private val baseHeaders: Headers - get() = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .build() - - override fun getRequestHeaders() = getApiHeaders() - - override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг") - - override val authUrl: String - get() = "https://${domain}" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name == "user" - } - } - - private val regexLastUrlPath = Regex("/[^/]+/?$") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isYearRangeSupported = true, - isTagsExclusionSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.allOf(MangaState::class.java), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE), - ) - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (response.code == TOO_MANY_REQUESTS) { - response.closeQuietly() - Thread.sleep(1000) - return chain.proceed(chain.request().newBuilder().build()) - } - return response - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - copyCookies() - val domain = domain - val urlBuilder = urlBuilder(subdomain = "api") - .addPathSegment("api") - .addPathSegment("v2") - .addPathSegment("search") - .addQueryParameter("page", page.toString()) - .addQueryParameter("count", PAGE_SIZE.toString()) - .addQueryParameter("ordering", getSortKey(order)) - if (!filter.query.isNullOrEmpty()) { - urlBuilder.addQueryParameter("query", filter.query) - } else { - urlBuilder.addPathSegment("catalog") - } - for (tag in filter.tags) { - urlBuilder.addQueryParameter("genres", tag.key) - } - for (tag in filter.tagsExclude) { - urlBuilder.addQueryParameter("exclude_genres", tag.key) - } - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("issue_year_gte", filter.yearFrom.toString()) - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("issue_year_lte", filter.yearFrom.toString()) - } - for (age in filter.contentRating) { - when (age) { - ContentRating.SAFE -> urlBuilder.addQueryParameter("age_limit", "0") - ContentRating.SUGGESTIVE -> { - urlBuilder.addQueryParameter("age_limit", "1") - urlBuilder.addQueryParameter("age_limit", "2") - } - - else -> Unit - } - } - for (state in filter.states) { - urlBuilder.addQueryParameter( - "status", - when (state) { - MangaState.ONGOING -> "2" - MangaState.FINISHED -> "1" - MangaState.ABANDONED -> "4" - MangaState.PAUSED -> "3" - MangaState.UPCOMING -> "5" - MangaState.RESTRICTED -> "6" - }, - ) - } - val content = webClient.httpGet(urlBuilder.build()).parseJson() - .getJSONArray("results") - return content.mapJSON { jo -> - parseManga(jo) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.find(regexLastUrlPath)?.removePrefix("/") - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - return getDetails(slug, manga.publicUrl) - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val slug = link.pathSegments.getOrNull(1) ?: return super.resolveLink(resolver, link) - return getDetails(slug, link.toString()) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - copyCookies() - val slug = seed.url.find(regexLastUrlPath)?.removePrefix("/") - ?: throw ParseException("Cannot obtain slug from ${seed.url}", seed.publicUrl) - val json = webClient.httpGet( - // https://api.remanga.org/api/v2/titles/the_beginning_after_the_end/relations/ - urlBuilder(subdomain = "api") - .addPathSegment("api") - .addPathSegment("v2") - .addPathSegment("titles") - .addPathSegment(slug) - .addPathSegment("relations") - .build(), - ).parseJson() - .optJSONArray("titles") ?: return emptyList() - return json.mapJSON { jo -> parseManga(jo) } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain(this, "api"))) - .parseJson() - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).parseSafe(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList<MangaPage>(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item) - is JSONArray -> item.mapJSONTo(result) { parsePage(it) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val jo = webClient.httpGet( - url = "https://api.${domain}/api/users/current/", - ).parseJson() - return jo.getJSONObject("content").getString("username") - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - private suspend fun getDetails(slug: String, publicUrl: String): Manga { - copyCookies() - val jo = webClient.httpGet( - url = "https://api.$domain/api/v2/titles/$slug/", - ).parseJson() - jo.optJSONObject("detail")?.getStringOrNull("message")?.let { msg -> - throw ParseException(msg, publicUrl) - } - val url = "/manga/${jo.getString("dir")}" - val cover = jo.getJSONObject("cover") - val branches = jo.getJSONArray("branches").mapJSONToSet { - it.getLong("id") to it.optJSONObject("publishers")?.getStringOrNull("name") - } - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url/main", - title = jo.getString("main_name"), - altTitles = jo.getStringOrNull("another_name")?.split(" / ") - ?.toSet().orEmpty() + setOfNotNull(jo.getStringOrNull("secondary_name")), - rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, - coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), - largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), - authors = emptySet(), - contentRating = when (jo.optJSONObject("age_limit")?.getIntOrDefault("id", -1)) { - 0 -> ContentRating.SAFE - 1, 2 -> ContentRating.SUGGESTIVE - else -> null - }, - state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { - 1 -> MangaState.FINISHED - 2 -> MangaState.ONGOING - 3 -> MangaState.PAUSED - 4 -> MangaState.ABANDONED - 5 -> MangaState.UPCOMING - 6 -> MangaState.RESTRICTED - else -> null - }, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(sourceLocale), - key = g.getInt("id").toString(), - source = source, - ) - }.orEmpty(), - description = jo.getStringOrNull("description"), - chapters = branches.flatMap { (branchId, branchName) -> - grabChapters(branchId).mapChapters(reversed = true) { _, cjo -> - if ( - cjo.getBooleanOrDefault("is_paid", false) && - !cjo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = cjo.getLong("id") - val name = cjo.getStringOrNull("name")?.toTitleCase(Locale.ROOT) - val publishers = cjo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/v2/titles/chapters/$id/", - number = cjo.getFloatOrDefault("chapter", 0f), - volume = cjo.getIntOrDefault("tome", 0), - title = name, - uploadDate = dateFormat.parseSafe(cjo.getStringOrNull("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = source, - branch = branchName, - ) - } - }, - source = source, - ) - } - - private fun parseManga(jo: JSONObject): Manga { - val url = "/manga/${jo.getString("dir")}" - val cover = jo.getJSONObject("cover") - return Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url/main", - title = jo.getString("main_name"), - altTitles = setOfNotNull(jo.getStringOrNull("secondary_name")), - rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, - coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), - largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), - authors = emptySet(), - contentRating = null, - state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { - 1 -> MangaState.FINISHED - 2 -> MangaState.ONGOING - 3 -> MangaState.PAUSED - 4 -> MangaState.ABANDONED - 5 -> MangaState.UPCOMING - 6 -> MangaState.RESTRICTED - else -> null - }, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(sourceLocale), - key = g.getInt("id").toString(), - source = source, - ) - }.orEmpty(), - source = source, - ) - } - - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(domain).find { - it.name == "user" - } ?: return baseHeaders - val jo = JSONObject(userCookie.value.urlDecode()) - val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders - return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() - } - - private fun copyCookies() { - val domain = domain - context.cookieJar.copyCookies(domain, "api.$domain") - } - - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } - - private fun parsePage(jo: JSONObject) = MangaPage( - // id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone - id = generateUid(jo.getString("link")), - url = jo.getString("link"), - preview = null, - source = source, - ) - - private suspend fun grabChapters(branchId: Long): List<JSONObject> { - val result = ArrayList<JSONObject>(100) - var page = 1 - while (true) { - val content = webClient.httpGet( - url = "https://api.$domain/api/v2/titles/chapters/?branch_id=$branchId&page=$page&count=500", - ).parseJson().getJSONArray("results") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } -} diff --git a/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt b/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt deleted file mode 100644 index 4addd136..00000000 --- a/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt +++ /dev/null @@ -1,182 +0,0 @@ -package tsuki.site.ru - -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.SinglePageMangaParser -import tsuki.model.ContentRating -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder -import tsuki.util.json.asTypedList -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.generateUid -import tsuki.util.parseJson -import tsuki.util.parseJsonArray -import tsuki.util.parseSafe -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("WAMANGA", "WaManga", "ru", type = ContentType.MANGA) -internal class WaMangaParser( - context: MangaLoaderContext, -) : SinglePageMangaParser(context, MangaParserSource.WAMANGA) { - - override val configKeyDomain = ConfigKey.Domain("wamanga.ru") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchCatalog() - .flatMapTo(linkedSetOf()) { it.parseWaMangaTags() }, - ) - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - val requiredGenres = filter.tags.mapTo(mutableSetOf()) { it.key } - - return fetchCatalog() - .asSequence() - .filter { item -> - query == null || item.parseWaMangaSearchText().any { value -> - value.contains(query, ignoreCase = true) - } - } - .filter { item -> - requiredGenres.isEmpty() || item.parseWaMangaGenres().containsAll(requiredGenres) - } - .map(::parseManga) - .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.title }) - .toList() - } - - override suspend fun getDetails(manga: Manga): Manga { - val details = webClient.httpGet(apiUrl("manga/${manga.url}")).parseJson() - val chapters = webClient.httpGet(apiUrl("manga/${manga.url}/chapters")) - .parseJsonArray() - .mapJSON(::parseChapter) - .sortedByDescending { it.number } - - return parseManga(details).copy( - id = manga.id, - url = manga.url, - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - return webClient.httpGet(apiUrl("chapters/${chapter.url}")) - .parseJson() - .getJSONArray("files") - .asTypedList<JSONObject>() - .sortedBy { it.getString("position").toDouble() } - .map { file -> - val imageUrl = file.getString("diskFile").toWaMangaAbsoluteUrl(domain) - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchCatalog(): List<JSONObject> = webClient - .httpGet(apiUrl("manga")) - .parseJsonArray() - .asTypedList() - - private fun parseManga(item: JSONObject): Manga { - val id = item.getString("id") - val slug = item.getString("slug") - return Manga( - id = generateUid(id), - url = id, - title = item.getString("title"), - altTitles = item.parseWaMangaAlternativeTitles(), - publicUrl = "https://$domain/manga/$slug", - rating = RATING_UNKNOWN, - coverUrl = item.getString("coverUrl").toWaMangaAbsoluteUrl(domain), - largeCoverUrl = item.getString("imageUrl").toWaMangaAbsoluteUrl(domain), - tags = item.parseWaMangaTags(), - state = item.getString("statusTitle").toWaMangaState(), - authors = item.getJSONArray("authors").asTypedList<String>().toSet(), - source = source, - contentRating = if (item.getBoolean("isAdult")) ContentRating.ADULT else ContentRating.SAFE, - description = item.getStringOrNull("description"), - ) - } - - private fun parseChapter(item: JSONObject): MangaChapter { - val id = item.getString("id") - return MangaChapter( - id = generateUid(id), - url = id, - source = source, - number = item.getDouble("position").toFloat(), - volume = 0, - title = item.getStringOrNull("title")?.takeIf { it.isNotBlank() }, - scanlator = null, - uploadDate = WA_MANGA_DATE_FORMAT.get().parseSafe(item.getString("createdAt")), - branch = null, - ) - } - - private fun apiUrl(path: String): String = "https://$domain/api/v1/$path" -} - -internal fun JSONObject.parseWaMangaAlternativeTitles(): Set<String> = buildSet { - getStringOrNull("titleEnglish")?.takeIf { it.isNotBlank() }?.let(::add) - getJSONArray("alternateTitles").asTypedList<String>().filterTo(this) { it.isNotBlank() } -} - -internal fun JSONObject.parseWaMangaGenres(): Set<String> = - getJSONArray("genres").asTypedList<String>().toSet() - -internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { - add(getString("title")) - addAll(parseWaMangaAlternativeTitles()) -} - -private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> - MangaTag( - title = genre, - key = genre, - source = MangaParserSource.WAMANGA, - ) -} - -private fun String.toWaMangaState(): MangaState = when (this) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "abandoned" -> MangaState.ABANDONED - else -> error("Unsupported WaManga status: $this") -} - -private fun String.toWaMangaAbsoluteUrl(domain: String): String = when { - startsWith("https://") || startsWith("http://") -> this - startsWith('/') -> "https://$domain$this" - else -> error("Unsupported WaManga asset URL: $this") -} - -private val WA_MANGA_DATE_FORMAT = ThreadLocal.withInitial { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US) -} diff --git a/src/main/kotlin/tsuki/site/ru/chan/ChanParser.kt b/src/main/kotlin/tsuki/site/ru/chan/ChanParser.kt deleted file mode 100644 index 1ed0d149..00000000 --- a/src/main/kotlin/tsuki/site/ru/chan/ChanParser.kt +++ /dev/null @@ -1,271 +0,0 @@ -package tsuki.site.ru.chan - -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Element -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.core.AbstractMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.util.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class ChanParser( - context: MangaLoaderContext, - source: MangaParserSource, -) : AbstractMangaParser(context, source), MangaParserAuthProvider { - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - ) - - override val authUrl: String - get() = "https://${domain}" - - override suspend fun isAuthorized(): Boolean = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val domain = domain - val doc = webClient.httpGet(buildUrl(offset, order, filter)).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.text().parseTitle() - val author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text() - Manga( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitles = setOfNotNull(title.second), - title = title.first, - authors = setOfNotNull(author), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - }.orEmpty(), - rating = RATING_UNKNOWN, - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> - val a = tr.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - title = tr.selectFirst("a")?.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.parseSafe(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = domain - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val domain = domain - val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://${domain}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val doc = webClient.httpGet(seed.url.replace("/manga/", "/related/").toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("right") - return root.select("div.related").mapNotNull { div -> - val info = div.selectFirst(".related_info") ?: return@mapNotNull null - val a = info.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.text().parseTitle() - val author = info.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text() - Manga( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitles = setOfNotNull(title.second), - title = title.first, - authors = setOfNotNull(author), - coverUrl = div.selectFirst("img")?.absUrl("src").orEmpty(), - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - protected open fun buildUrl( - offset: Int, - order: SortOrder, - filter: MangaListFilter, - ): HttpUrl { - val builder = urlBuilder() - builder.addQueryParameter("offset", offset.toString()) - when { - !filter.query.isNullOrEmpty() -> { - builder.addQueryParameter("do", "search") - builder.addQueryParameter("subaction", "search") - builder.addQueryParameter("search_start", ((offset / 40) + 1).toString()) - builder.addQueryParameter("full_search", "0") - builder.addQueryParameter("result_from", (offset + 1).toString()) - builder.addQueryParameter("result_num", "40") - builder.addQueryParameter("story", filter.query) - builder.addQueryParameter("need_sort_date", "false") - } - - else -> { - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { - builder.addPathSegment("tags") - val joiner = StringUtil.StringJoiner("+") - filter.tags.forEach { joiner.add(it.key) } - filter.tagsExclude.forEach { joiner.add("-"); joiner.append(it.key) } - builder.addPathSegment(joiner.complete()) - builder.addQueryParameter( - "n", - when (order) { - SortOrder.RATING, - SortOrder.POPULARITY, - -> "favdesc" - - SortOrder.ALPHABETICAL -> "abcasc" - else -> "" // SortOrder.NEWEST - }, - ) - } else { - when (order) { - SortOrder.POPULARITY -> builder.addPathSegment("mostviews") - SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog") - SortOrder.RATING -> builder.addPathSegment("mostfavorites") - else -> { // SortOrder.NEWEST - builder.addPathSegment("manga") - builder.addPathSegment("new") - } - } - } - } - } - return builder.build() - } - - private fun String.toTagName() = replace('_', ' ').toTitleCase() - - private fun String.parseTitle(): Pair<String, String?> { - var depth = 0 - for (i in indices.reversed()) { - val c = this[i] - if (c == '(') { - depth-- - if (depth == 0 && (i + 2) < lastIndex && i > 0) { - return substring(i + 1, lastIndex).trim() to substring(0, i).trim().nullIfEmpty() - } - } else if (c == ')') { - depth++ - } - } - return this to null - } - - private fun Element.attrAsRelativeUrlAnyHost(attributeKey: String): String { - val attr = attr(attributeKey) - if (attr.isEmpty() || attr.startsWith("data:")) { - throw IllegalArgumentException("Wrong url $attr") - } - if (attr.startsWith('/')) { - return attr - } - val host = attr.toHttpUrl().host - return attr.substringAfter(host) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/chan/MangaChanParser.kt b/src/main/kotlin/tsuki/site/ru/chan/MangaChanParser.kt deleted file mode 100644 index 9a76dd47..00000000 --- a/src/main/kotlin/tsuki/site/ru/chan/MangaChanParser.kt +++ /dev/null @@ -1,11 +0,0 @@ -package tsuki.site.ru.chan - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGACHAN", "Манга-тян", "ru") -internal class MangaChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.MANGACHAN) { - override val configKeyDomain = ConfigKey.Domain("manga-chan.me") -} diff --git a/src/main/kotlin/tsuki/site/ru/chan/YaoiChanParser.kt b/src/main/kotlin/tsuki/site/ru/chan/YaoiChanParser.kt deleted file mode 100644 index 810c1bc1..00000000 --- a/src/main/kotlin/tsuki/site/ru/chan/YaoiChanParser.kt +++ /dev/null @@ -1,50 +0,0 @@ -package tsuki.site.ru.chan - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder -import tsuki.util.* - -@MangaSourceParser("YAOICHAN", "Яой-тян", "ru") -internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { - - override val configKeyDomain = ConfigKey.Domain( - "v9.yaoi-chan.me", - "v10.yaoi-chan.me", - "v3.yaoi-chan.me", - "v2.yaoi-chan.me", - "v1.yaoi-chan.me", - "yaoi-chan.me", - ) - - override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - chapters = root.select("table.table_cha").flatMap { table -> - table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - title = a.text(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt deleted file mode 100644 index 4fdf518d..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/GroupleParser.kt +++ /dev/null @@ -1,579 +0,0 @@ -package tsuki.site.ru.grouple - -import androidx.collection.MutableScatterMap -import androidx.collection.ScatterMap -import kotlinx.coroutines.flow.channelFlow -import kotlinx.coroutines.flow.firstOrNull -import kotlinx.coroutines.launch -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import okhttp3.Interceptor -import okhttp3.Response -import okhttp3.internal.closeQuietly -import okio.IOException -import org.json.JSONArray -import org.jsoup.nodes.Element -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.config.ConfigKey -import tsuki.core.AbstractMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.suspendlazy.suspendLazy -import java.net.HttpURLConnection -import java.text.SimpleDateFormat -import java.util.* - -private const val PAGE_SIZE = 70 -private const val NSFW_ALERT = "сексуальные сцены" -private const val NOTHING_FOUND = "Ничего не найдено" -private const val MIN_IMAGE_SIZE = 1024L -private const val HEADER_ACCEPT = "Accept" -private const val RELATED_TITLE = "Связанные произведения" -private const val COPYRIGHT_ALERT = "Запрещена публикация произведения" -private const val NO_CHAPTERS = "В этой манге еще нет ни одной главы" - -internal abstract class GroupleParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val siteId: Int, -) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor { - - @Volatile - private var cachedPagesServer: String? = null - - override val userAgentKey = ConfigKey.UserAgent( - "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", - ) - private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) - private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .add("Accept-Language", "ru,en-US;q=0.7,en;q=0.3") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.ADDED, - ) - - override val authUrl: String - get() { - val targetUri = "https://${domain}/".urlEncoded() - return "https://3.grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" - } - - override suspend fun isAuthorized(): Boolean = hasAuthCookie() - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearRangeSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of(MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), - ) - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val domain = domain - val root = if (filter.isEmpty()) { - webClient.httpGet( - "https://$domain/list?sortType=${ - getSortKey(order) - }&offset=${offset upBy PAGE_SIZE}", - ).parseHtml().body().let { doc -> (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) } - } else { - advancedSearch(offset, order, filter).parseHtml() - } - checkNotNull(root) { "Root not found" } - val tiles = root.selectFirst("div.tiles.row") - if (tiles == null) { - if (root.getElementsContainingOwnText(NOTHING_FOUND).isNotEmpty()) { - return emptyList() - } - root.parseFailed("No tiles found") - } - return tiles.select("div.tile").mapNotNull(::parseManga) - } - - override suspend fun getDetails(manga: Manga): Manga { - val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) - val doc = response.parseHtml() - val root = doc.body().requireElementById("mangaBox") - val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) - val coverImg = root.selectFirst("img.cr-hero-poster__img") - val translations = if (config[splitTranslationsKey]) { - root.selectFirst("div.translator-selection") - ?.select(".translator-selection-item") - ?.associate { - it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull() - } - } else { - null - } - val newSource = getSource(response.request.url) - val chaptersList = root.getElementById("chapters-list") - var isRestricted = false - if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { - if (root.getElementsContainingOwnText(COPYRIGHT_ALERT).isNotEmpty()) { - isRestricted = true - } else { - root.parseFailed("No chapters found") - } - } - val hashRegex = Regex("window.user_hash\\s*=\\s*\'([^\']+)\'") - val userHash = doc.select("script").firstNotNullOfOrNull { it.html().findGroupValue(hashRegex) } - val hasNsfwAlert = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) } - return manga.copy( - source = newSource, - title = doc.metaValue("name") ?: manga.title, - altTitles = root.select(".cr-hero-names__alt > span > span") - .filterNot { it.hasClass("cr-hero-names__alt-separator") } - .flatMapTo(linkedSetOf()) { element -> - element.text().split('/').mapNotNull { it.trim().nullIfEmpty() } - } + manga.altTitles, - publicUrl = response.request.url.toString(), - description = root.selectFirst("div.cr-description__content")?.html(), - largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("src"), - coverUrl = manga.coverUrl - ?: coverImg?.attrAsAbsoluteUrlOrNull("src"), - tags = root.selectFirstOrThrow("div.creation-element-tags") - .getElementsByAttributeValueContaining("href", "/list/genre/").mapTo(manga.tags.toMutableSet()) { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").removeSuffix('/').substringAfterLast('/'), - source = source, - ) - }, - state = if (isRestricted) { - MangaState.RESTRICTED - } else { - manga.state - }, - authors = manga.authors, - contentRating = (if (hasNsfwAlert) ContentRating.SUGGESTIVE else ContentRating.SAFE) - .coerceAtLeast(manga.contentRating ?: ContentRating.SAFE), - chapters = chaptersList?.select("a.chapter-link") - ?.flatMapChapters(reversed = true) { a -> - val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList() - val href = a.attrAsRelativeUrl("href") - val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f - val volume = tr.attr("data-vol").toIntOrNull() ?: 0 - if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) { - var translators = "" - val translatorElement = a.attr("title") - if (translatorElement.isNotBlank()) { - translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)") - } - listOf( - MangaChapter( - id = generateUid(href), - title = a.text().removePrefix(manga.title).trim().nullIfEmpty(), - number = number, - volume = volume, - url = href.withQueryParam("d", userHash), - uploadDate = dateFormat.parseSafe(tr.selectFirst("td.date")?.text()), - scanlator = translators, - source = newSource, - branch = null, - ), - ) - } else { - val translationData = JSONArray(a.attr("data-translations")) - translationData.mapJSON { jo -> - val personId = jo.getLong("personId") - val link = href.withQueryParam("tran", personId.toString()) - MangaChapter( - id = generateUid(link), - title = a.text().removePrefix(manga.title).trim(), - number = number, - volume = volume, - url = link.withQueryParam("d", userHash), - uploadDate = dateFormat.parseSafe(jo.getStringOrNull("dateCreated")), - scanlator = null, - source = newSource, - branch = translations[personId], - ) - } - } - }.orEmpty(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - if (chapter.source != source && chapter.source is MangaParserSource) { // handle redirects between websites - return context.newParserInstance(chapter.source).getPages(chapter) - } - val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() - val doc = webClient.httpGet(url).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - var pos = data.indexOf("rm_h.readerDoInit(") - if (pos != -1) { - parsePagesV2(data, pos)?.let { return it } - } - pos = data.indexOf("rm_h.readerInit( 0,") - if (pos != -1) { - parsePagesV1(data, pos)?.let { return it } - } - pos = data.indexOf(".readerInit(") - if (pos != -1) { - parsePagesV3(data, pos).let { return it } - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } - - override suspend fun getPageUrl(page: MangaPage): String { - val parts = page.url.split('|') - if (parts.size < 2) { - return page.url - } - val path = parts.last() - // fast path - cachedPagesServer?.let { host -> - val url = concatUrl("https://$host/", path) - if (tryHead(url)) { - return url - } else { - cachedPagesServer = null - } - } - // slow path - val candidates = HashSet<String>((parts.size - 1) * 2) - for (i in 0 until parts.size - 1) { - val server = parts[i].trim().ifEmpty { "https://$domain/" } - candidates.add(concatUrl(server, path)) - candidates.add(concatUrl(server, path.substringBeforeLast('?'))) - } - val resolvedUrl = channelFlow { - for (url in candidates) { - launch { - if (tryHead(url)) { - send(url) - } - } - } - }.firstOrNull() ?: throw ParseException("No reachable page URL", page.url) - cachedPagesServer = resolvedUrl.toHttpUrlOrNull()?.host - return resolvedUrl - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() - val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")?.selectFirst("table.table") - ?: doc.parseFailed("Cannot find root") - return root.select("a.element-link").mapToSet { a -> - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/'), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val root = webClient.httpGet("https://grouple.co/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else res - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { - return chain.proceed(request).checkIfAuthRequired() - } - val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) - return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { - chain.proceed( - request.newBuilder().header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8").build(), - ) - } else { - chain.proceed(request).checkIfAuthRequired() - } - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(splitTranslationsKey) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("mangaBox").select("h4").first { it.ownText() == RELATED_TITLE } - .nextElementSibling() ?: doc.parseFailed("Cannot find root") - return root.select("div.tile").mapNotNull(::parseManga) - } - - protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { - in SeiMangaParser.domains -> MangaParserSource.SEIMANGA - in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU - in SelfMangaParser.domains -> MangaParserSource.SELFMANGA - in UsagiParser.domains -> MangaParserSource.USAGI - else -> source - } - - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { - SortOrder.ALPHABETICAL -> "name" - SortOrder.POPULARITY -> "rate" - SortOrder.UPDATED -> "updated" - SortOrder.ADDED, - SortOrder.NEWEST, - -> "created" - - SortOrder.RATING -> "votes" - else -> "rate" - } - - private suspend fun advancedSearch(offset: Int, order: SortOrder, filter: MangaListFilter): Response { - val tagsMap = if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { - tagsIndex.get() - } else { - null - } - val url = urlBuilder() - .addPathSegment("search") - .addPathSegment("advancedResults") - url.addQueryParameter("q", filter.query) - url.addQueryParameter("offset", offset.toString()) - filter.tags.forEach { tag -> - val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } - url.addQueryParameter(tagId, "in") - } - filter.tagsExclude.forEach { tag -> - val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } - url.addQueryParameter(tagId, "ex") - } - url.addQueryParameter( - "years", - buildString { - append(filter.yearFrom.ifZero { YEAR_MIN }) - append(',') - append(filter.yearTo.ifZero { YEAR_MAX }) - }, - ) - url.addQueryParameter( - "sortType", - when (order) { - SortOrder.RATING -> "USER_RATING" - SortOrder.ALPHABETICAL -> "NAME" - SortOrder.ADDED -> "YEAR" - SortOrder.POPULARITY -> "POPULARITY" - SortOrder.NEWEST -> "DATE_CREATE" - SortOrder.UPDATED -> "DATE_UPDATE" - else -> "RATING" - }, - ) - filter.states.forEach { state -> - when (state) { - MangaState.FINISHED -> "s_completed" - MangaState.ABANDONED -> "s_abandoned_popular" - MangaState.UPCOMING -> "s_wait_upload" - else -> null - }?.let { - url.addQueryParameter(it, "in") - } - } - - return webClient.httpGet(url.build()) - } - - private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { - webClient.httpHead(url).use { response -> - response.isSuccessful && !response.isPumpkin() && response.headersContentLength() >= MIN_IMAGE_SIZE - } - }.getOrDefault(false) - - private fun Response.isPumpkin(): Boolean = request.url.host == "upload.wikimedia.org" - - private fun parseManga(node: Element): Manga? { - val imgDiv = node.selectFirst("div.img") ?: return null - val descDiv = node.selectFirst("div.desc") ?: return null - if (descDiv.selectFirst("i.fa-user") != null || descDiv.selectFirst("i.fa-external-link") != null) { - return null // skip author - } - val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") ?: return null - val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() ?: return null - val tileInfo = descDiv.selectFirst("div.tile-info") - val relUrl = href.toRelativeUrl(domain) - if (relUrl.contains("://")) { - return null - } - val author = tileInfo?.selectFirst("a.person-link")?.text() - return Manga( - id = generateUid(relUrl), - url = relUrl, - publicUrl = href, - title = title, - altTitles = setOfNotNull(descDiv.selectFirst("h5")?.textOrNull()), - coverUrl = imgDiv.selectFirst("img.lazy")?.attrAsAbsoluteUrlOrNull("data-original")?.replace("_p.", "."), - rating = node.selectFirst(".compact-rate")?.attr("title")?.toFloatOrNull()?.div(5f) - ?: RATING_UNKNOWN, - authors = setOfNotNull(author), - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - tags = tileInfo?.select("a.element-link")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - state = when { - node.selectFirst("div.tags")?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED - - else -> null - }, - source = source, - ) - } - - private fun parsePagesV1(data: String, pos: Int): List<MangaPage>? { - val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') - if (json.isEmpty()) { - return null - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(1) - val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { - // domain that does not need a token - "$primaryServer|$serversStr|${url}".substringBefore("?") - } else { - "$primaryServer|$serversStr|$url" - } - MangaPage( - id = generateUid(url), - url = fullSrc, - preview = null, - source = source, - ) - } - } - - private fun parsePagesV2(data: String, pos: Int): List<MangaPage>? { - val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') - if (json.isEmpty()) { - return null - } - val ja = JSONArray("[$json]") - val pages = ja.getJSONArray(0) - val servers = ja.getJSONArray(2).mapJSON { it.getString("path") } - val serversStr = servers.joinToString("|") - return (0 until pages.length()).map { i -> - val page = pages.getJSONArray(i) - val primaryServer = page.getString(0) - val url = page.getString(2) - val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { - // domain that does not need a token - "$primaryServer|$serversStr|${url}".substringBefore("?") - } else { - "$primaryServer|$serversStr|$url" - } - MangaPage( - id = generateUid(url), - url = fullSrc, - preview = null, - source = source, - ) - } - } - - private fun parsePagesV3(data: String, pos: Int): List<MangaPage> { - val json = JSONArray( - data.substring(pos) - .substringBetween("(", ")") - .removePrefix("chapterInfo,") - .substringBeforeLast(','), - ) - return (0 until json.length()).map { i -> - val ja = json.getJSONArray(i) - val server = ja.getString(0).ifEmpty { "https://$domain" } - val url = ja.getString(2) - val fullUrl = concatUrl(server, url) - MangaPage( - id = generateUid(url), - url = if (fullUrl.contains("one-way.work")) { - // domain that does not need a token - fullUrl.substringBefore("?") - } else { - fullUrl - }, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchTagsMap(): ScatterMap<String, String> { - val url = "https://$domain/search/advanced" - val properties = - webClient.httpGet(url).parseHtml().body().selectFirst("form.search-form")?.select("div.form-group") - ?.find { it.selectFirst("li.property") != null } - ?.select("li.property") - ?: throw ParseException("Genres filter element not found", url) - val result = MutableScatterMap<String, String>(properties.size) - properties.forEach { li -> - val name = li.text().lowercase() - val id = li.selectFirstOrThrow("input").id() - result[name] = id - } - return result - } - - private fun String.withQueryParam(name: String, value: String?): String { - if (value == null) return this - return toAbsoluteUrl(domain) - .toHttpUrl() - .newBuilder() - .setQueryParameter(name, value) - .build() - .toString() - .toRelativeUrl(domain) - } - - @Throws(IOException::class) - private fun Response.checkIfAuthRequired(): Response { - val lastPathSegment = request.url.pathSegments.lastOrNull() - if (lastPathSegment == "login") { - closeQuietly() - throw AuthRequiredException(source) - } - if (code == HttpURLConnection.HTTP_NOT_FOUND) { - if (!hasAuthCookie()) { - closeQuietly() - throw AuthRequiredException(source) - } else { - return newBuilder().code(HttpURLConnection.HTTP_OK).build() - } - } - return this - } - - private fun hasAuthCookie() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/ReadmangaParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/ReadmangaParser.kt deleted file mode 100644 index de2960da..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/ReadmangaParser.kt +++ /dev/null @@ -1,26 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("READMANGA_RU", "ReadManga", "ru") -internal class ReadmangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.READMANGA_RU, 1) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "a.zazaza.me", - "3.readmanga.ru", - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/SeiMangaParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/SeiMangaParser.kt deleted file mode 100644 index a07beb1d..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/SeiMangaParser.kt +++ /dev/null @@ -1,26 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("SEIMANGA", "SeiManga", "ru") -internal class SeiMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.SEIMANGA, 21) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "1.seimanga.me", - "seimanga.me", - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/SelfMangaParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/SelfMangaParser.kt deleted file mode 100644 index dd0768b3..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/SelfMangaParser.kt +++ /dev/null @@ -1,26 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("SELFMANGA", "SelfManga", "ru", type = ContentType.OTHER) -internal class SelfMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.SELFMANGA, 3) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "1.selfmanga.live", - ) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/grouple/UsagiParser.kt b/src/main/kotlin/tsuki/site/ru/grouple/UsagiParser.kt deleted file mode 100644 index c7aad5c6..00000000 --- a/src/main/kotlin/tsuki/site/ru/grouple/UsagiParser.kt +++ /dev/null @@ -1,23 +0,0 @@ -package tsuki.site.ru.grouple - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("USAGI", "Usagi", "ru") -internal class UsagiParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.USAGI, 1) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf("web.usagi.one") - } -} diff --git a/src/main/kotlin/tsuki/site/ru/libsocial/LibSocialParser.kt b/src/main/kotlin/tsuki/site/ru/libsocial/LibSocialParser.kt deleted file mode 100644 index 6cf2fc76..00000000 --- a/src/main/kotlin/tsuki/site/ru/libsocial/LibSocialParser.kt +++ /dev/null @@ -1,473 +0,0 @@ -package tsuki.site.ru.libsocial - -import androidx.collection.* -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.runBlocking -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONException -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.network.UserAgents -import tsuki.util.* -import tsuki.util.json.* -import tsuki.util.suspendlazy.suspendLazy -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class LibSocialParser( - context: MangaLoaderContext, - source: MangaParserSource, - protected val siteId: Int, - siteDomains: Array<String>, -) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - - protected val apiHost = "api.cdnlibs.org" - - override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE) - - override val authUrl: String - get() = "https://$domain/ru/front/auth" - - override suspend fun isAuthorized(): Boolean { - val token = getAuthData()?.optJSONObject("token")?.getStringOrNull("access_token") - return !token.isNullOrEmpty() - } - - override suspend fun getUsername(): String = getAuthData() - ?.getJSONObject("auth") - ?.getString("username") - ?: throw AuthRequiredException(source) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - ) - - final override val configKeyDomain = ConfigKey.Domain(*siteDomains) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - MangaState.PAUSED, - MangaState.UPCOMING, - ), - ) - - final override fun intercept(chain: Interceptor.Chain): Response { - val token = runBlocking { getAuthData() }?.optJSONObject("token")?.getStringOrNull("access_token") - val requestBuilder = chain.request().newBuilder() - if (!token.isNullOrEmpty()) { - requestBuilder.header("Authorization", "Bearer $token") - } - requestBuilder.header("Site-Id", siteId.toString()) - return chain.proceed(requestBuilder.build()) - } - - private val statesMap = intObjectMapOf( - 1, MangaState.ONGOING, - 2, MangaState.FINISHED, - 3, MangaState.UPCOMING, - 4, MangaState.PAUSED, - 5, MangaState.ABANDONED, - ) - private val imageServers = suspendLazy(initializer = ::fetchServers) - private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - null to null, - SERVER_MAIN to "Первый", - SERVER_SECONDARY to "Второй", - SERVER_COMPRESS to "Сжатия", - SERVER_DOWNLOAD to "Загрузки", - SERVER_CROP to "Обрезки", - ), - defaultValue = null, - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val urlBuilder = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addQueryParameter("site_id[]", siteId.toString()) - .addQueryParameter("fields[]", "rate") - .addQueryParameter("fields[]", "rate_avg") - .addQueryParameter("page", page.toString()) - for (state in filter.states) { - urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString()) - } - for (tag in filter.tags) { - urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1)) - } - for (tag in filter.tagsExclude) { - urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1)) - } - if (!filter.query.isNullOrEmpty()) { - urlBuilder.addQueryParameter("q", filter.query) - } - urlBuilder.addQueryParameter( - "sort_by", - when (order) { - SortOrder.UPDATED -> "last_chapter_at" - SortOrder.POPULARITY -> "views" - SortOrder.RATING -> "rate_avg" - SortOrder.NEWEST -> "created_at" - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - -> "rus_name" - - else -> null - }, - ) - urlBuilder.addQueryParameter( - "sort_type", - when (order) { - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL_DESC, - -> "desc" - - SortOrder.ALPHABETICAL -> "asc" - else -> null - }, - ) - val json = webClient.httpGet(urlBuilder.build()).parseJson() - val data = json.getJSONArray("data") - return data.mapJSON(::parseManga) - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val chaptersDeferred = async { fetchChapters(manga) } - val url = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(manga.url) - .addQueryParameter("fields[]", "summary") - .addQueryParameter("fields[]", "genres") - .addQueryParameter("fields[]", "tags") - .addQueryParameter("fields[]", "authors") - .addQueryParameter("fields[]", "close_view") - .build() - val json = webClient.httpGet(url).parseJson().getJSONObject("data") - val genres = json.getJSONArray("genres").mapJSON { jo -> - MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source) - } - val tags = json.getJSONArray("tags").mapJSON { jo -> - MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source) - } - val authors = json.getJSONArray("authors").mapJSONNotNullToSet { - it.getStringOrNull("name") - } - val chapters = chaptersDeferred.await() - val isRestricted = json.getIntOrDefault("close_view", 0) > 0 - manga.copy( - title = json.getStringOrNull("rus_name") ?: manga.title, - altTitles = setOfNotNull(json.getStringOrNull("name")), - tags = tagsSetOf(tags, genres), - state = if (chapters.isEmpty() && isRestricted) { - MangaState.RESTRICTED - } else { - manga.state - }, - authors = authors, - contentRating = json.optJSONObject("ageRestriction")?.let { - val id = it.getIntOrDefault("id", -1) - if (id >= 4) ContentRating.SUGGESTIVE else sourceContentRating - } ?: manga.contentRating, - description = json.parseLibSocialSummary()?.nl2br(), - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope { - val pages = async { - webClient.httpGet( - concatUrl("https://$apiHost/api/manga/", chapter.url), - ).parseJson().getJSONObject("data") - } - val servers = imageServers.get() - val json = pages.await() - val primaryServer = getPrimaryImageServer(servers) - json.getJSONArray("pages").mapJSON { jo -> - val url = jo.getString("url") - MangaPage( - id = generateUid(jo.getLong("id")), - url = concatUrl(primaryServer, url), - preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) }, - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { - val tags = async { fetchTags("tags") } - val genres = async { fetchTags("genres") } - tagsSetOf(tags.await(), genres.await()) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val json = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(seed.url) - .addPathSegment("similar") - .build(), - ).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - parseManga(jo.getJSONObject("media")) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - return resolver.resolveManga(this, link.pathSegments.lastOrNull() ?: return null) - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(splitTranslationsKey) - keys.add(preferredServerKey) - } - - private fun parseManga(jo: JSONObject): Manga { - val cover = jo.getJSONObject("cover") - val isNsfwSource = jo.getJSONObject("ageRestriction").getIntOrDefault("id", 0) >= 3 - return Manga( - id = generateUid(jo.getLong("id")), - title = jo.getString("rus_name").ifEmpty { jo.getString("name") }, - altTitles = setOfNotNull(jo.getString("name")), - url = jo.getString("slug_url"), - publicUrl = "https://$domain/ru/manga/" + jo.getString("slug_url"), - rating = jo.optJSONObject("rating") - ?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN, - contentRating = sourceContentRating, - coverUrl = cover.getString("thumbnail"), - tags = setOf(), - state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1], - authors = emptySet(), - largeCoverUrl = cover.getString("default"), - source = source, - ) - } - - private fun getPrimaryImageServer(servers: ScatterMap<String, String>): String { - val preferred = config[preferredServerKey] - if (preferred != null) { - servers[preferred]?.let { return it } - } - return checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) { - "No available images servers" - } - } - - private suspend fun fetchChapters(manga: Manga): List<MangaChapter> { - val url = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(manga.url) - .addPathSegment("chapters") - .build() - val json = webClient.httpGet(url).parseJson().getJSONArray("data") - val builder = ChaptersListBuilder(json.length()) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) - val useBranching = config[splitTranslationsKey] - for (i in 0 until json.length()) { - val jo = json.getJSONObject(i) - val volume = jo.getIntOrDefault("volume", 0) - val number = jo.getFloatOrDefault("number", 0f) - val numberString = number.formatSimple() - val name = jo.getStringOrNull("name") - val branches = jo.getJSONArray("branches") - for (j in 0 until branches.length()) { - val bjo = branches.getJSONObject(j) - val isRestricted = bjo.optJSONObject("restricted_view")?.let { - !it.getBooleanOrDefault("is_open", true) - } ?: false - if (isRestricted) { - continue - } - val id = bjo.getLong("id") - val branchId = bjo.getLongOrDefault("branch_id", 0L) - val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name") - builder += MangaChapter( - id = generateUid(id), - title = name, - number = number, - volume = volume, - url = buildString { - append(manga.url) - append("/chapter?number=") - append(numberString) - append("&volume=") - append(volume) - if (branchId != 0L) { - append("&branch_id=") - append(branchId) - } - }, - scanlator = team, - uploadDate = dateFormat.parseSafe(bjo.getStringOrNull("created_at")), - branch = if (useBranching) team else null, - source = source, - ) - } - } - return builder.toList() - } - - private suspend fun fetchTags(type: String): List<MangaTag> { - val data = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api").addPathSegment(type).build(), - ).parseJson().getJSONArray("data") - val prefix = type.first().toString() - return data.mapJSONNotNull { jo -> - val sites = jo.getJSONArray("site_ids").toIntSet() - if (siteId !in sites) { - return@mapJSONNotNull null - } - MangaTag( - title = jo.getString("name"), - key = prefix + jo.getInt("id"), - source = source, - ) - } - } - - private suspend fun fetchServers(): ScatterMap<String, String> { - val json = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("constants") - .addQueryParameter("fields[]", "imageServers") - .build(), - ).parseJson().getJSONObject("data").getJSONArray("imageServers") - val result = MutableScatterMap<String, String>() - for (i in 0 until json.length()) { - val jo = json.getJSONObject(i) - val sites = jo.getJSONArray("site_ids").toIntSet() - if (siteId !in sites) { - continue - } - result[jo.getString("id")] = jo.getString("url") - } - return result - } - - private fun <V> IntObjectMap<V>.keyOf(value: V): Int { - forEach { k, v -> - if (v == value) { - return k - } - } - throw NoSuchElementException("No key associated with value $value") - } - - private fun JSONArray.toIntSet(): IntSet { - val result = MutableIntSet(length()) - for (i in 0 until length()) { - result.add(getInt(i)) - } - return result - } - - private fun MangaTag.typeKey() = when (key.firstOrNull()) { - 'g' -> "genres" - 't' -> "tags" - else -> throw IllegalArgumentException("Tag $key($title) is of unknown type") - } - - private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> { - val result = ArraySet<MangaTag>(tags.size + genres.size) - val names = HashSet<String>(tags.size + genres.size) - genres.forEach { x -> if (names.add(x.title)) result.add(x) } - tags.forEach { x -> if (names.add(x.title)) result.add(x) } - return result - } - - private suspend fun getAuthData(): JSONObject? { - val raw = WebViewHelper(context).getLocalStorageValue(domain, "auth") ?: return null - return JSONObject(raw.unescapeJson().removeSurrounding('"')) - } - - protected companion object { - - const val SERVER_MAIN = "main" - const val SERVER_SECONDARY = "secondary" - const val SERVER_COMPRESS = "compress" - const val SERVER_DOWNLOAD = "download" - const val SERVER_CROP = "crop" - } -} - -internal fun JSONObject.parseLibSocialSummary(): String? = when (val value = opt("summary")) { - null, - JSONObject.NULL, - -> null - - is String -> value - is JSONObject -> value.parseLibSocialRichText() - else -> throw JSONException("Unsupported summary value: ${value.javaClass.name}") -} - -private fun JSONObject.parseLibSocialRichText(): String { - val type = getString("type") - return when (type) { - "text" -> getString("text") - "hardBreak" -> "\n" - "paragraph", - "heading", - -> parseLibSocialChildren(separator = "") - - "doc", - "blockquote", - "bulletList", - "orderedList", - "listItem", - -> parseLibSocialChildren(separator = "\n") - - else -> throw JSONException("Unsupported summary node type: $type") - } -} - -private fun JSONObject.parseLibSocialChildren(separator: String): String { - val children = optJSONArray("content") ?: return "" - return children.mapJSON { child -> child.parseLibSocialRichText() }.joinToString(separator) -} diff --git a/src/main/kotlin/tsuki/site/ru/libsocial/MangaLibParser.kt b/src/main/kotlin/tsuki/site/ru/libsocial/MangaLibParser.kt deleted file mode 100644 index f8cc4c08..00000000 --- a/src/main/kotlin/tsuki/site/ru/libsocial/MangaLibParser.kt +++ /dev/null @@ -1,26 +0,0 @@ -package tsuki.site.ru.libsocial - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.exception.AuthRequiredException -import tsuki.exception.NotFoundException -import tsuki.model.MangaChapter -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGALIB", "MangaLib", "ru") -internal class MangaLibParser( - context: MangaLoaderContext, -) : LibSocialParser( - context = context, - source = MangaParserSource.MANGALIB, - siteId = 1, - siteDomains = arrayOf("mangalib.org", "mangalib.me"), -) { - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { - super.getPages(chapter) - } catch (e: NotFoundException) { - throw AuthRequiredException(source, e) - } -} diff --git a/src/main/kotlin/tsuki/site/ru/libsocial/SlashLibParser.kt b/src/main/kotlin/tsuki/site/ru/libsocial/SlashLibParser.kt deleted file mode 100644 index 560675d7..00000000 --- a/src/main/kotlin/tsuki/site/ru/libsocial/SlashLibParser.kt +++ /dev/null @@ -1,13 +0,0 @@ -package tsuki.site.ru.libsocial - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource - -@MangaSourceParser("YAOILIB", "SlashLib", "ru") -internal class SlashLibParser(context: MangaLoaderContext) : LibSocialParser( - context = context, - source = MangaParserSource.YAOILIB, - siteId = 2, - siteDomains = arrayOf("v2.slashlib.me"), -) diff --git a/src/main/kotlin/tsuki/site/ru/madara/MangaMammy.kt b/src/main/kotlin/tsuki/site/ru/madara/MangaMammy.kt deleted file mode 100644 index 0fc31943..00000000 --- a/src/main/kotlin/tsuki/site/ru/madara/MangaMammy.kt +++ /dev/null @@ -1,13 +0,0 @@ -package tsuki.site.ru.madara - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -@MangaSourceParser("MANGAMAMMY", "Nimanga", "ru") -internal class MangaMammy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAMAMMY, "p.nimanga.com") { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} diff --git a/src/main/kotlin/tsuki/site/ru/madara/MangaoneLove.kt b/src/main/kotlin/tsuki/site/ru/madara/MangaoneLove.kt deleted file mode 100644 index 5eeddf78..00000000 --- a/src/main/kotlin/tsuki/site/ru/madara/MangaoneLove.kt +++ /dev/null @@ -1,13 +0,0 @@ -package tsuki.site.ru.madara - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -@MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") -internal class MangaoneLove(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} From 6cbe2a696f0c5283eea6e5d09ee06e9bc229339f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:26:16 +0200 Subject: [PATCH 357/434] refractor --- .../kotlin/tsuki/parsers/MangaThemesia.kt | 102 +++++++++++------- 1 file changed, 65 insertions(+), 37 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index c604e652..6cda0d85 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -25,6 +25,7 @@ import tsuki.util.attrAsRelativeUrl import tsuki.util.parseHtml import tsuki.util.urlEncoded import tsuki.util.extractChapterNumber +import tsuki.util.mapNotNullToSet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -140,8 +141,7 @@ abstract class MangaThemesia( append(it.key.urlEncoded()) } append("&order=") - append(order.toQueryParam() - ) + append(order.toQueryParam()) } val doc = webClient.httpGet(url).parseHtml() return parseMangaList(doc) @@ -185,8 +185,9 @@ abstract class MangaThemesia( val chaptersDeferred = async { loadChapters(doc, manga.url) } - val title = doc.selectFirst("h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1") - ?.text() ?: manga.title + val title = doc.selectFirst( + "h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1" + )?.text() ?: manga.title val description = doc.selectFirst( ".desc, .entry-content[itemprop=description], .summary__content, .sinopsis, .entry-content p" @@ -203,21 +204,20 @@ abstract class MangaThemesia( val authorCell = table.selectFirst("tr:has(td:contains(Author)) td:last-child") val artistCell = table.selectFirst("tr:has(td:contains(Artist)) td:last-child") listOfNotNull( - authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, - artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } + authorCell?.text().cleanAuthor(), + artistCell?.text().cleanAuthor() ).toSet() } else { doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Author) i, .spe span:contains(Author) a") - .map { it.text().trim() } - .filter { it.isNotBlank() } + .mapNotNull { it.text().cleanAuthor() } .toSet() } val tags = doc.select(".mgen a, .gnr a, .seriestugenre a, .genres-content a") - .mapNotNull { a -> + .mapNotNullToSet { a -> val text = a.text().trim() if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null - }.toSet() + } val statusText = if (hasTable) { table.selectFirst("tr:has(td:contains(Status)) td:last-child")?.text() @@ -261,7 +261,11 @@ abstract class MangaThemesia( label.contains("Alt.", ignoreCase = true) || label.contains("Alt ", ignoreCase = true) ) { - addAll(splitAltTitles(row.selectFirst("td:last-child")?.text())) + val cell = row.select("td").lastOrNull() + val text = cell?.text() + if (!text.isNullOrBlank()) { + addAll(splitAltTitles(text)) + } } } } else { @@ -275,7 +279,10 @@ abstract class MangaThemesia( ".post-content_item:contains(Alt) .summary-content", ) doc.select(selectors.joinToString(", ")).forEach { element -> - addAll(splitAltTitles(element.text())) + val text = element.text() + if (text.isNotBlank()) { + addAll(splitAltTitles(text)) + } } } } @@ -291,11 +298,11 @@ abstract class MangaThemesia( protected open fun parseStatus(text: String?): MangaState? { val status = text?.lowercase() ?: return null - return when (status) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in paused -> MangaState.PAUSED - in abandoned -> MangaState.ABANDONED + return when { + ongoingWordSet.anyWordIn(status) -> MangaState.ONGOING + finishedWordSet.anyWordIn(status) -> MangaState.FINISHED + pausedWordSet.anyWordIn(status) -> MangaState.PAUSED + abandonedWordSet.anyWordIn(status) -> MangaState.ABANDONED else -> null } } @@ -318,6 +325,7 @@ abstract class MangaThemesia( return elements.mapNotNull { element -> val a = element.selectFirst("a") ?: return@mapNotNull null val href = a.attrAsRelativeUrl("href") + if (href.isBlank()) return@mapNotNull null val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() val dateStr = element.selectFirst(".chapterdate")?.text() MangaChapter( @@ -338,11 +346,11 @@ abstract class MangaThemesia( if (date.isNullOrBlank()) return 0L val clean = date.trim() return when { - WordSet(" ago", "atrás", " hace", " назад", " önce", " trước", "مضت", "قبل").endsWith(clean) -> parseRelativeDate(clean) - WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút").startsWith(clean) -> parseRelativeDate(clean) - WordSet("yesterday", "يوم واحد").startsWith(clean) -> yesterday() - WordSet("today").startsWith(clean) -> today() - WordSet("يومين").startsWith(clean) -> dayBeforeYesterday() + agoWordSet.endsWith(clean) -> parseRelativeDate(clean) + fromWordSet.startsWith(clean) -> parseRelativeDate(clean) + yesterdayWordSet.startsWith(clean) -> yesterday() + todayWordSet.startsWith(clean) -> today() + dayBeforeYesterdayWordSet.startsWith(clean) -> dayBeforeYesterday() else -> dateFormat.parseSafe(clean) } } @@ -351,13 +359,13 @@ abstract class MangaThemesia( val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0L val cal = Calendar.getInstance() return when { - WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان").anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут").anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giờ", "ore", "ساعة", "小时").anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois").anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "año", "năm").anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + secondsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + minutesWordSet.anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + hoursWordSet.anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + daysWordSet.anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + weeksWordSet.anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis + monthsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + yearsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis else -> 0L } } @@ -397,6 +405,7 @@ abstract class MangaThemesia( protected open fun parseMangaElement(element: Element): Manga? { val a = element.selectFirst("a") ?: return null val href = a.attrAsRelativeUrl("href") + if (href.isBlank()) return null val title = a.attr("title").ifBlank { a.text() } val coverUrl = element.selectFirst("img")?.imgAttr() return makeManga(href, title, coverUrl) @@ -431,27 +440,46 @@ abstract class MangaThemesia( return "" } + private fun String?.cleanAuthor(): String? = + this?.trim()?.takeIf { it.isNotBlank() && it !in AUTHOR_PLACEHOLDERS } + private fun SimpleDateFormat.parseSafe(date: String): Long = runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) companion object { - val ongoing = setOf( + private val AUTHOR_PLACEHOLDERS = setOf("n/a", "N/A", "Updating") + + private val ongoingWordSet = WordSet( "ongoing", "on going", "publishing", "updating", "en curso", "ativo", "en cours", "đang tiến hành", "em lançamento", "devam ediyor", "in corso", "güncel", "berjalan", - "продолжается", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة", + "продолжается", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة" ) - val finished = setOf( + private val finishedWordSet = WordSet( "completed", "complete", "finished", "finalizado", "terminé", "tamamlandı", "đã hoàn thành", "hoàn thành", "مكتملة", "завершено", "completato", "one-shot", - "bitti", "tamat", "concluído", "已完结", "bitmiş", "achevé", + "bitti", "tamat", "concluído", "已完结", "bitmiş", "achevé" ) - val paused = setOf( + private val pausedWordSet = WordSet( "hiatus", "on hold", "pausado", "en espera", "en pause", "en attente", - "durduruldu", "beklemede", "đang chờ", "متوقف", "заморожено", + "durduruldu", "beklemede", "đang chờ", "متوقف", "заморожено" ) - val abandoned = setOf( + private val abandonedWordSet = WordSet( "canceled", "cancelled", "cancelado", "cancellato", "dropped", "discontinued", - "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено", + "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено" ) + + private val agoWordSet = WordSet(" ago", "atrás", " hace", " назад", " önce", " trước", "مضت", "قبل") + private val fromWordSet = WordSet("há ", "منذ", "il y a", "hace", "giờ", "phút") + private val yesterdayWordSet = WordSet("yesterday", "يوم واحد") + private val todayWordSet = WordSet("today") + private val dayBeforeYesterdayWordSet = WordSet("يومين") + + private val secondsWordSet = WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان") + private val minutesWordSet = WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут") + private val hoursWordSet = WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giờ", "ore", "ساعة", "小时") + private val daysWordSet = WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день") + private val weeksWordSet = WordSet("week", "semana", "tuần", "أسابيع", "أسبوع") + private val monthsWordSet = WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois") + private val yearsWordSet = WordSet("year", "año", "năm") } } From b2604ab9c4885433dc0c76e31a1be33ed67a183a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:32:18 +0200 Subject: [PATCH 358/434] Update gradle.properties --- gradle.properties | 3 +++ 1 file changed, 3 insertions(+) diff --git a/gradle.properties b/gradle.properties index df06b5a1..83f3a9a7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,3 +22,6 @@ org.gradle.vfs.watch=true org.gradle.workers.max=4 systemProp.org.gradle.unsafe.configuration-cache=false Xannotation-default-target=param-property + +# Enabled parallel sync for Gradle 9.4+ +org.gradle.tooling.parallel=true From 6f06f075c290367cf220320fae7ff3cfb7eca17d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:34:53 +0200 Subject: [PATCH 359/434] Update MGKomik.kt --- src/main/kotlin/tsuki/site/id/MGKomik.kt | 169 ++--------------------- 1 file changed, 8 insertions(+), 161 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/MGKomik.kt b/src/main/kotlin/tsuki/site/id/MGKomik.kt index a4b54960..1c0225c2 100644 --- a/src/main/kotlin/tsuki/site/id/MGKomik.kt +++ b/src/main/kotlin/tsuki/site/id/MGKomik.kt @@ -5,18 +5,10 @@ import tsuki.MangaSourceParser import tsuki.MangaParserAuthProvider import tsuki.parsers.MadaraParser -import tsuki.model.Manga -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterOptions import tsuki.model.MangaParserSource -import tsuki.model.MangaTag -import tsuki.model.SortOrder - -import tsuki.util.parseHtml import okhttp3.Interceptor -import androidx.collection.arraySetOf -import okhttp3.Response +import okhttp3.Headers @MangaSourceParser("MGKOMIK", "MGKomik", "id") internal class MGKomik(context: MangaLoaderContext) : @@ -24,159 +16,14 @@ internal class MGKomik(context: MangaLoaderContext) : override val datePattern = "dd MMM yy" override val tagPrefix = "genres/" + override val listUrl = "komik/" override val selectDesc = "div.description-summary div.summary__content p" - + override val stylePage = "" override val withoutAjax = true + override val postReq = true - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val path = request.url.encodedPath - val isAjax = path.contains("admin-ajax.php") || - path.contains("wp-json") || - path.endsWith("/ajax/chapters") - - val builder = request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - - if (isAjax) { - builder.header("X-Requested-With", "XMLHttpRequest") - .header("Sec-Fetch-Dest", "empty") - .header("Sec-Fetch-Mode", "cors") - .header("Sec-Fetch-Site", "same-origin") - .header("Origin", "https://$domain") - .header("Priority", "u=1, i") - .removeHeader("Sec-Fetch-User") - .removeHeader("Upgrade-Insecure-Requests") - } - - return chain.proceed(builder.build()) - } - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - if (order == SortOrder.POPULARITY) { - val p = page + 1 - val url = buildString { - append("https://") - append(domain) - append("/komik") - if (p > 1) { - append("/page/") - append(p) - } - append("/?m_orderby=trending") - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - return super.getListPage(page, order, filter) - } - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - return super.fetchAvailableTags() - } - - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - - private fun availableTags() = arraySetOf( - MangaTag("2 hours ago", "2-hours-ago", source), - MangaTag("7 hours ago", "7-hours-ago", source), - MangaTag("8 hours ago", "8-hours-ago", source), - MangaTag("11 hours ago", "11-hours-ago", source), - MangaTag("13 hours ago", "13-hours-ago", source), - MangaTag("15 hours ago", "15-hours-ago", source), - MangaTag("1 day ago", "1-day-ago", source), - MangaTag("2 days ago", "2-days-ago", source), - MangaTag("Action", "action", source), - MangaTag("Adaptation", "adaptation", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Animals", "animals", source), - MangaTag("Apocalypse", "apocalypse", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Crime", "crime", source), - MangaTag("Demon", "demon", source), - MangaTag("Demons", "demons", source), - MangaTag("Drama", "drama", source), - MangaTag("Dungeons", "dungeons", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Fight", "fight", source), - MangaTag("Fighting", "fighting", source), - MangaTag("Full Color", "full-color", source), - MangaTag("Game", "game", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Josei(W)", "joseiw", source), - MangaTag("Kids", "kids", source), - MangaTag("Magic", "magic", source), - MangaTag("Manga", "manga", source), - MangaTag("Mangatoon", "mangatoon", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mature", "mature", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Military", "military", source), - MangaTag("Modern", "modern", source), - MangaTag("Monsters", "monsters", source), - MangaTag("Murim", "murim", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Office Workers", "office-workers", source), - MangaTag("One-Shot", "one-shot", source), - MangaTag("Overpowered", "overpowered", source), - MangaTag("Police", "police", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Regression", "regression", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Revenge", "revenge", source), - MangaTag("Reverse Harem", "reverse-harem", source), - MangaTag("Rofan", "rofan", source), - MangaTag("Romance", "romance", source), - MangaTag("School", "school", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujoai", source), - MangaTag("Shoujo AI", "shoujo-ai", source), - MangaTag("Shoujo(G)", "shoujog", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Smut", "smut", source), - MangaTag("Sports", "sports", source), - MangaTag("Super Power", "super-power", source), - MangaTag("Superhero", "superhero", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Supranatural", "supranatural", source), - MangaTag("Survival", "survival", source), - MangaTag("System", "system", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Transmigration", "transmigration", source), - MangaTag("Vampire", "vampire", source), - MangaTag("Villainess", "villainess", source), - MangaTag("Violence", "violence", source), - MangaTag("War", "war", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Webtoons", "webtoons", source), - MangaTag("Wuxia", "wuxia", source), - MangaTag("Yuri", "yuri", source), - ) - + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .add("Referer", "https://$domain/") + .build() } From d98d29fc43908e13120daa3d5e2d7ed3d1f854cd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:39:33 +0200 Subject: [PATCH 360/434] Update MangaJinx.kt --- src/main/kotlin/tsuki/site/en/MangaJinx.kt | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/MangaJinx.kt b/src/main/kotlin/tsuki/site/en/MangaJinx.kt index c4573d7f..c935e554 100644 --- a/src/main/kotlin/tsuki/site/en/MangaJinx.kt +++ b/src/main/kotlin/tsuki/site/en/MangaJinx.kt @@ -15,6 +15,8 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import tsuki.util.attrAsRelativeUrl import tsuki.util.generateUid @@ -33,7 +35,6 @@ import org.jsoup.nodes.Document import java.text.SimpleDateFormat import java.util.EnumSet - @MangaSourceParser("MANGAJINX", "MangaJinx", "en") internal class MangaJinx(context: MangaLoaderContext) : MadthemeParser(context, MangaParserSource.MANGAJINX, "mgjinx.com", pageSize=24) { @@ -45,6 +46,15 @@ internal class MangaJinx(context: MangaLoaderContext) : override val selectDate = ".chapter-update" override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/static/sites/icons/favicon-32x32.png", 32, null), + ), + domain, + ) + } + override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( isAuthorSearchSupported = true, From 92e742ac1a77ccad2173f0db9da5e5bd94faf491 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:39:56 +0200 Subject: [PATCH 361/434] Update RoliaScan.kt --- src/main/kotlin/tsuki/site/en/RoliaScan.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/kotlin/tsuki/site/en/RoliaScan.kt b/src/main/kotlin/tsuki/site/en/RoliaScan.kt index 4efb5a01..f4c399cf 100644 --- a/src/main/kotlin/tsuki/site/en/RoliaScan.kt +++ b/src/main/kotlin/tsuki/site/en/RoliaScan.kt @@ -18,6 +18,8 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import tsuki.util.generateUid import tsuki.util.nullIfEmpty @@ -52,6 +54,15 @@ internal class RoliaScan(context: MangaLoaderContext) : private val relativeDateRegex = Regex("""(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago""") private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/content/themes/mangapeak/assets/img/favicon-32x32.png", 32, null), + ), + domain, + ) + } + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() .set("Referer", "https://$domain/") .build() From daa2ff9f0b22aeeb5fc044e28cfdd68488c00967 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:40:08 +0200 Subject: [PATCH 362/434] Update KaliScan.kt --- src/main/kotlin/tsuki/site/en/KaliScan.kt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/main/kotlin/tsuki/site/en/KaliScan.kt b/src/main/kotlin/tsuki/site/en/KaliScan.kt index 98df8601..6ddeac26 100644 --- a/src/main/kotlin/tsuki/site/en/KaliScan.kt +++ b/src/main/kotlin/tsuki/site/en/KaliScan.kt @@ -15,6 +15,8 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons import tsuki.util.attrAsRelativeUrl import tsuki.util.generateUid @@ -44,6 +46,15 @@ internal class KaliScan(context: MangaLoaderContext) : override val selectDate = ".chapter-update" override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/static/sites/icons/favicon-32x32.png", 32, null), + ), + domain, + ) + } + override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( isAuthorSearchSupported = true, From 22e2f68c5a305bb62f2aa4949880a39151bc2642 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:55:00 +0200 Subject: [PATCH 363/434] Update MadaraParser.kt --- src/main/kotlin/tsuki/parsers/MadaraParser.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt index a7d88535..9b5f2d88 100644 --- a/src/main/kotlin/tsuki/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -636,6 +636,8 @@ internal abstract class MadaraParser( ) } + protected open fun transformChapterName(element: Element, name: String): String = name + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() @@ -688,7 +690,8 @@ internal abstract class MadaraParser( val href = a.attrAsRelativeUrl("href") val link = href + stylePage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() + val baseName = a.selectFirst("p")?.text() ?: a.ownText() + val name = transformChapterName(li, baseName) MangaChapter( id = generateUid(href), title = name, @@ -719,7 +722,8 @@ internal abstract class MadaraParser( val href = a.attrAsRelativeUrl("href") val link = href + stylePage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() + val baseName = a.selectFirst("p")?.text() ?: a.ownText() + val name = transformChapterName(li, baseName) MangaChapter( id = generateUid(href), url = link, From 0ce633f4c185c3ac29382cde1b09f9c1affd6882 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 16:55:23 +0200 Subject: [PATCH 364/434] Create Siikomik.kt --- src/main/kotlin/tsuki/site/id/Siikomik.kt | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Siikomik.kt diff --git a/src/main/kotlin/tsuki/site/id/Siikomik.kt b/src/main/kotlin/tsuki/site/id/Siikomik.kt new file mode 100644 index 00000000..acf8956c --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Siikomik.kt @@ -0,0 +1,24 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +import org.jsoup.nodes.Element + +@MangaSourceParser("SIIKOMIK", "Siikomik", "id") +internal class Siikomik(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.SIIKOMIK, "siikomik.id") { + + override val listUrl = "komik/" + + override fun transformChapterName(element: Element, name: String): String { + return if (element.hasClass("premium") || element.hasClass("premium-block")) { + "🔒 $name" + } else { + name + } + } +} From 13d82e865efb789b3513eca028a3757201b2514c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Fri, 28 Aug 2026 17:25:14 +0200 Subject: [PATCH 365/434] Create KawiiManga.kt --- src/main/kotlin/tsuki/site/ar/KawiiManga.kt | 185 ++++++++++++++++++++ 1 file changed, 185 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ar/KawiiManga.kt diff --git a/src/main/kotlin/tsuki/site/ar/KawiiManga.kt b/src/main/kotlin/tsuki/site/ar/KawiiManga.kt new file mode 100644 index 00000000..32babbe9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/KawiiManga.kt @@ -0,0 +1,185 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded + +import java.time.Instant +import java.util.EnumSet +import kotlinx.coroutines.coroutineScope +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject + +@MangaSourceParser("KAWIIMANGA", "Kawaii Manga", "ar") +internal class KawiiManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KAWIIMANGA, 20) { + + override val configKeyDomain = ConfigKey.Domain("kawaiimanga.org") + + private val apiUrl = "https://manga-api.kawaii-anime.com/api/manga/own" + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("x-app-key", "km_2026_live") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query.orEmpty() + val url = if (query.isNotBlank()) { + "$apiUrl?action=search&q=${query.urlEncoded()}" + } else { + val sort = when (order) { + SortOrder.POPULARITY -> "views" + else -> "latest" + } + "$apiUrl?action=browse&page=$page&sort=$sort" + } + val json = webClient.httpGet(url).parseJsonObject() + return parseMangaList(json) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val slug = manga.url + val json = webClient.httpGet("$apiUrl?action=series&slug=$slug").parseJsonObject() + val detailedManga = parseManga(json) + val chapters = parseChapters(json.optJSONArray("chapters"), slug) + detailedManga.copy(chapters = chapters) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfterLast('#') + val json = webClient.httpGet("$apiUrl?action=pages&chapterId=$chapterId").parseJsonObject() + val pagesArray = json.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun Response.parseJsonObject(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + private fun parseMangaList(json: JSONObject): List<Manga> { + val results = json.optJSONArray("results") ?: return emptyList() + return (0 until results.length()).map { i -> + parseManga(results.getJSONObject(i)) + } + } + + private fun parseManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("title") + val cover = obj.optString("coverUrl").takeIf { it.isNotBlank() } + val author = obj.optString("author").takeIf { it.isNotBlank() && it != "unknown" } + val artist = obj.optString("artist").takeIf { it.isNotBlank() && it != "unknown" } + val description = obj.optString("description").takeIf { it.isNotBlank() } + val type = obj.optString("type") + val status = obj.optString("status") + val genresArray = obj.optJSONArray("genres") + + val tags = buildSet { + when (type) { + "manga" -> add(MangaTag("manga", "Manga", source)) + "manhua" -> add(MangaTag("manhua", "Manhua", source)) + "manhwa" -> add(MangaTag("manhwa", "Manhwa", source)) + } + if (genresArray != null) { + for (i in 0 until genresArray.length()) { + val genre = genresArray.getString(i) + add(MangaTag(genre.lowercase(), genre, source)) + } + } + } + + val state = when (status) { + "ongoing", "coming_soon" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "cancelled", "dropped" -> MangaState.ABANDONED + else -> null + } + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = emptySet(), + authors = setOfNotNull(author, artist), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = tags, + state = state, + description = description, + contentRating = null, + source = source, + ) + } + + private fun parseChapters(jsonArray: JSONArray?, slug: String): List<MangaChapter> { + if (jsonArray == null) return emptyList() + return (0 until jsonArray.length()).map { i -> + val obj = jsonArray.getJSONObject(i) + val id = obj.getString("id") + val title = obj.optString("title") + val number = obj.getInt("number") + val createdAt = obj.optString("createdAt") + val chapterName = if (title.isNotBlank() && title != "$number") { + "الفصل $number - $title" + } else { + "الفصل $number" + } + val date = runCatching { Instant.parse(createdAt).toEpochMilli() }.getOrDefault(0L) + MangaChapter( + id = generateUid("$slug/$number#$id"), + url = "$slug/$number#$id", + title = chapterName, + number = number.toFloat(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } +} From 9b005337997704fd423a7f105d35fea33f7cb7bd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 10:19:30 +0200 Subject: [PATCH 366/434] Create Chikari.kt --- src/main/kotlin/tsuki/site/en/Chikari.kt | 411 +++++++++++++++++++++++ 1 file changed, 411 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/Chikari.kt diff --git a/src/main/kotlin/tsuki/site/en/Chikari.kt b/src/main/kotlin/tsuki/site/en/Chikari.kt new file mode 100644 index 00000000..9016f92d --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Chikari.kt @@ -0,0 +1,411 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.time.Instant +import java.util.EnumSet + +@MangaSourceParser("CHIKARI", "Chikari", "en") +class Chikari(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CHIKARI, pageSize = 36) { + + override val configKeyDomain = ConfigKey.Domain("chikari.moe") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val tags by lazy { + GENRES.entries.map { (slug, title) -> + MangaTag(key = slug, title = title, source = source) + }.toSet() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER, // OEL + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val offset = (page - 1) * pageSize + val url = buildApiUrl(offset, order, filter) + val response = webClient.httpGet(url).parseJson() + return parseMangaList(response) + } + + private fun buildApiUrl(offset: Int, order: SortOrder, filter: MangaListFilter): String { + val sortParam = when (order) { + SortOrder.NEWEST -> "added" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.UPDATED -> "updated" + SortOrder.RATING -> "top_rated" + else -> "popular" + } + + return "https://$domain/api/series".toHttpUrl().newBuilder().apply { + addQueryParameter("sort", sortParam) + addQueryParameter("limit", pageSize.toString()) + addQueryParameter("offset", offset.toString()) + + filter.query?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("q", it) + } + + filter.tags.forEach { tag -> + addQueryParameter("genre", tag.key) + } + + filter.tagsExclude.forEach { tag -> + addQueryParameter("genre_exclude", tag.key) + } + + filter.types.forEach { type -> + addQueryParameter("type", type.toQueryParam()) + } + + filter.states.firstOrNull()?.let { state -> + addQueryParameter("status", state.toQueryParam()) + } + }.build().toString() + } + + private fun parseMangaList(json: JSONObject): List<Manga> { + val items = json.getJSONArray("items") + return (0 until items.length()).mapNotNull { index -> + parseMangaItem(items.getJSONObject(index)) + } + } + + private fun parseStatus(status: String?): MangaState? = when (status?.lowercase()) { + "releasing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + } + + private fun normalizeRating(rating: Double): Float = + if (rating > 0) rating.toFloat() / 10f else RATING_UNKNOWN + + private fun parseContentRating(isNsfw: Boolean): ContentRating = + if (isNsfw) ContentRating.ADULT else ContentRating.SAFE + + private fun parseMangaItem(item: JSONObject): Manga? { + val slug = item.optString("slug").takeIf { it.isNotEmpty() } ?: return null + val title = item.optString("title").takeIf { it.isNotEmpty() } ?: return null + + val url = "/series/$slug".toAbsoluteUrl(domain) + val state = parseStatus(item.optString("status")) + val rating = normalizeRating(item.optDouble("rating", 0.0)) + val contentRating = parseContentRating(item.optBoolean("is_nsfw", false)) + val coverUrl = item.optString("cover_url").takeIf { it.isNotEmpty() } + + return Manga( + id = generateUid(slug), + title = title, + altTitles = emptySet(), + url = url, + publicUrl = url, + rating = rating, + contentRating = contentRating, + coverUrl = coverUrl, + tags = emptySet(), + state = state, + authors = emptySet(), + source = source, + ) + } + + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val result = try { + coroutineScope { + val slug = manga.url.substringAfterLast("/") + val detailsJson = webClient.httpGet("https://$domain/api/series/$slug").parseJson() + val data = detailsJson.optJSONObject("data") ?: detailsJson + + val chaptersDeferred = async { fetchChapters(slug, data) } + + val title = data.optString("title", manga.title) + val cover = data.optString("cover_url").takeIf { it.isNotEmpty() } ?: manga.coverUrl + val description = data.optString("description").takeIf { it.isNotEmpty() } ?: "" + + val state = parseStatus(data.optString("status")) + + val allTags = mutableSetOf<MangaTag>() + + data.optJSONArray("genres")?.let { genresArray -> + for (i in 0 until genresArray.length()) { + val genreObj = genresArray.optJSONObject(i) ?: continue + val genreSlug = genreObj.optString("slug") + val genreName = genreObj.optString("name") + if (genreSlug.isNotBlank() && genreName.isNotBlank()) { + allTags.add(MangaTag(key = genreSlug, title = genreName, source = source)) + } + } + } + + val authors = mutableSetOf<String>() + data.optJSONArray("authors")?.let { authorsArray -> + for (i in 0 until authorsArray.length()) { + val authorObj = authorsArray.optJSONObject(i) ?: continue + val authorName = authorObj.optString("name") + if (authorName.isNotBlank()) { + authors.add(authorName) + } + } + } + + val altTitles = mutableSetOf<String>() + data.optJSONArray("alt_titles")?.let { altArray -> + for (i in 0 until altArray.length()) { + altArray.optString(i)?.takeIf { it.isNotBlank() }?.let { altTitles.add(it) } + } + } + + val rating = normalizeRating(data.optDouble("rating", 0.0)) + val contentRating = parseContentRating(data.optBoolean("is_nsfw", false)) + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = authors, + tags = allTags, + altTitles = altTitles, + state = state, + rating = rating, + contentRating = contentRating, + chapters = chapters, + ) + } + } catch (_: Exception) { + manga + } + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private suspend fun fetchChapters(slug: String, detailsJson: JSONObject? = null): List<MangaChapter> { + try { + val url = "https://$domain/api/series/$slug/chapters".toHttpUrl().newBuilder() + .addQueryParameter("order", "desc") + .addQueryParameter("limit", "9999") + .addQueryParameter("offset", "0") + .build() + .toString() + + val response = webClient.httpGet(url).parseJson() + response.optJSONArray("items")?.let { items -> + return parseChapterArray(items, slug) + } + } catch (_: Exception) { + } + + if (detailsJson != null) { + val allChapters = mutableListOf<MangaChapter>() + detailsJson.optJSONArray("chapters_head")?.let { allChapters.addAll(parseChapterArray(it, slug)) } + detailsJson.optJSONArray("chapters_tail")?.let { allChapters.addAll(parseChapterArray(it, slug)) } + return allChapters.distinctBy { it.number }.sortedBy { it.number } + } + return emptyList() + } + + private fun parseChapterArray(array: JSONArray, slug: String): List<MangaChapter> { + return (0 until array.length()).mapNotNull { index -> + val ch = array.optJSONObject(index) ?: return@mapNotNull null + val number = ch.optDouble("number", 0.0).toFloat() + val title = ch.optString("title").takeIf { it.isNotEmpty() } ?: "Chapter $number" + val createdAt = parseDate(ch.optString("created_at")) + val chapterUrl = "/reader/$slug/$number".toAbsoluteUrl(domain) + + MangaChapter( + id = generateUid("$slug-$number"), + url = chapterUrl, + title = title, + number = number, + volume = 0, + uploadDate = createdAt, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + private fun parseDate(dateStr: String): Long = + try { + Instant.parse(dateStr).toEpochMilli() + } catch (_: Exception) { + 0L + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + try { + val segments = chapter.url.trimEnd('/').split('/') + if (segments.size < 2) return emptyList() + val slug = segments[segments.size - 2] + val number = segments[segments.size - 1] + val response = webClient.httpGet("https://$domain/api/series/$slug/chapters/$number").parseJson() + val pagesArray = response.optJSONArray("pages") ?: return emptyList() + + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } catch (_: Exception) { + return emptyList() + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + try { + val slug = seed.url.substringAfterLast("/") + val response = webClient.httpGet("https://$domain/api/series/$slug/similar") + val array = response.toJSONArray() + return (0 until array.length()).mapNotNull { index -> + parseMangaItem(array.getJSONObject(index)) + } + } catch (_: Exception) { + return emptyList() + } + } + + private fun okhttp3.Response.toJSONArray(): JSONArray { + val body = body?.string() ?: throw tsuki.exception.ParseException("Empty response body", request.url.toString()) + return JSONArray(body) + } + + private fun ContentType.toQueryParam(): String = when (this) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.OTHER -> "oel" + else -> "manga" + } + + private fun MangaState.toQueryParam(): String = when (this) { + MangaState.ONGOING -> "releasing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> "upcoming" + } + + private fun String.toAbsoluteUrl(domain: String): String = + if (startsWith("http")) this else "https://$domain$this" + + companion object { + private val GENRES = mapOf( + "action" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "award-winning" to "Award Winning", + "boys-love" to "Boys Love", + "comedy" to "Comedy", + "doujinshi" to "Doujinshi", + "drama" to "Drama", + "ecchi" to "Ecchi", + "erotica" to "Erotica", + "fantasy" to "Fantasy", + "gender-bender" to "Gender Bender", + "girls-love" to "Girls Love", + "gourmet" to "Gourmet", + "harem" to "Harem", + "historical" to "Historical", + "horror" to "Horror", + "josei" to "Josei", + "mahou-shoujo" to "Mahou Shoujo", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "music" to "Music", + "mystery" to "Mystery", + "psychological" to "Psychological", + "romance" to "Romance", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "shounen-ai" to "Shounen Ai", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "sports" to "Sports", + "supernatural" to "Supernatural", + "suspense" to "Suspense", + "thriller" to "Thriller", + "tragedy" to "Tragedy", + "yaoi" to "Yaoi", + "yuri" to "Yuri", + ) + } +} From afae1925639d8a8d2995682bab8e071fd3a5a13b Mon Sep 17 00:00:00 2001 From: bubz727 <sukaboba791@gmail.com> Date: Sat, 29 Aug 2026 22:07:45 +0700 Subject: [PATCH 367/434] Create Yurilab.kt --- src/main/kotlin/tsuki/site/id/Yurilab.kt | 153 +++++++++++++++++++++++ 1 file changed, 153 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/Yurilab.kt diff --git a/src/main/kotlin/tsuki/site/id/Yurilab.kt b/src/main/kotlin/tsuki/site/id/Yurilab.kt new file mode 100644 index 00000000..55056768 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Yurilab.kt @@ -0,0 +1,153 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.* +import tsuki.util.* + +import kotlinx.coroutines.async +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.* + +@MangaSourceParser("YURILAB", "YuriLab", "id", ContentType.HENTAI) +internal class YuriLab(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.YURILAB, "yurilab.top", pageSize = 30) { + + override val sourceLocale: Locale = Locale.ENGLISH + override val withoutAjax = true + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy(isMultipleTagsSupported = false) + + override fun parseMangaList(doc: Document): List<Manga> { + return super.parseMangaList(doc).map { manga -> + manga.copy(coverUrl = manga.coverUrl?.replace(Regex("""-\d+x\d+(?=\.\w+$)"""), "")) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val author = doc.selectFirst(".author-content a, .manga-author a")?.textOrNull() + return super.getDetails(manga).copy( + authors = setOfNotNull(author).ifEmpty { manga.authors } + ) + } + + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val url = "https://$domain/?s=&post_type=wp-manga" + val docs = webClient.httpGet(url).parseHtml() + val genreLinks = docs.select(".genres-filter .dropdown-menu a[href*='genre=']") + return genreLinks.mapNotNullToSet { el -> + val href = el.attrOrNull("href") ?: return@mapNotNullToSet null + val match = Regex("""genre=([^&]+)""").find(href) + val key = match?.groupValues?.get(1) ?: return@mapNotNullToSet null + val title = el.textOrNull()?.trim()?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null + MangaTag( + title = title, + key = key, + source = source, + ) + } + } + + override val selectGenre = ".genres-content a[href*='genre'], .tags-content a[href*='tag']" + + override suspend fun createMangaTag(a: Element): MangaTag? { + val href = a.attrOrNull("href") ?: return null + val tagKey = extractTagKey(href) ?: return null + val title = a.textOrNull()?.trim() ?: return null + return MangaTag( + title = title, + key = tagKey, + source = source, + ) + } + + private fun extractTagKey(href: String): String? { + val genreMatch = Regex("""genre=([^&/?]+)""").find(href) + if (genreMatch != null) return genreMatch.groupValues[1] + val pattern = Regex("""series-genre/([^/?]+)|series-tag/([^/?]+)""", RegexOption.IGNORE_CASE) + return pattern.find(href)?.groupValues?.getOrNull(1)?.takeIf { it.isNotEmpty() } + ?: pattern.find(href)?.groupValues?.getOrNull(2)?.takeIf { it.isNotEmpty() } + } + + override val selectChapter = "ul.version-chap li.wp-manga-chapter" + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = kotlinx.coroutines.coroutineScope { + val allChapters = mutableListOf<MangaChapter>() + var t = 1 + val batchSize = 5 + + while (true) { + val deferreds = (t until t + batchSize).map { page -> + async { + try { + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/?t=$page" + val ajaxDocs = webClient.httpPost( + ajaxUrl.toHttpUrl(), + emptyMap<String, String>(), + Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build(), + ).parseHtml() + + val lis = ajaxDocs.select(selectChapter) + lis.mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val rawHref = a.attrAsRelativeUrl("href") + + val baseName = a.ownText().ifEmpty { null } ?: a.selectFirst("p")?.textOrNull() + ?: "Chapter" + + val isLocked = rawHref.isBlank() || rawHref == "#" || + li.hasClass("premium") || li.hasClass("premium-block") || + li.selectFirst(".fa-lock, .vip-lock") != null + + val finalHref = if (rawHref.isBlank() || rawHref == "#") mangaUrl else rawHref + val finalName = if (isLocked && !baseName.contains("🔒")) "🔒 ${baseName.trim()}" else baseName.trim() + + MangaChapter( + id = generateUid(finalHref + finalName), + url = finalHref, + title = finalName, + number = 0f, + volume = 0, + branch = null, + uploadDate = 0L, + scanlator = null, + source = source, + ) + } + } catch (e: Exception) { + emptyList<MangaChapter>() + } + } + } + + val batches = deferreds.map { it.await() } + for (pageChapters in batches) { + if (pageChapters.isEmpty()) { + return@coroutineScope allChapters.reversed().mapIndexed { index, chapter -> + val currentTitle = chapter.title + val finalTitle = if (currentTitle == "Chapter" || currentTitle == "🔒 Chapter") { + currentTitle.replace("Chapter", "Chapter ${index + 1}") + } else { + currentTitle + } + chapter.copy( + title = finalTitle, + number = (index + 1).toFloat() + ) + } + } + allChapters.addAll(pageChapters) + } + t += batchSize + } + return@coroutineScope emptyList() + } +} From 6f3d77f87a803be0c63d324ec3f3078788269d34 Mon Sep 17 00:00:00 2001 From: bubz727 <sukaboba791@gmail.com> Date: Sat, 29 Aug 2026 23:50:16 +0700 Subject: [PATCH 368/434] Update Yurilab.kt --- src/main/kotlin/tsuki/site/id/Yurilab.kt | 48 ++++++++++++++++++------ 1 file changed, 36 insertions(+), 12 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/Yurilab.kt b/src/main/kotlin/tsuki/site/id/Yurilab.kt index 55056768..4cf2927d 100644 --- a/src/main/kotlin/tsuki/site/id/Yurilab.kt +++ b/src/main/kotlin/tsuki/site/id/Yurilab.kt @@ -4,15 +4,29 @@ import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.parsers.MadaraParser -import tsuki.model.* -import tsuki.util.* +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase import kotlinx.coroutines.async import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import java.util.* +import java.text.SimpleDateFormat +import java.util.Locale @MangaSourceParser("YURILAB", "YuriLab", "id", ContentType.HENTAI) internal class YuriLab(context: MangaLoaderContext) : @@ -79,16 +93,25 @@ internal class YuriLab(context: MangaLoaderContext) : override val selectChapter = "ul.version-chap li.wp-manga-chapter" + override fun transformChapterName(element: Element, name: String): String { + return if (element.hasClass("premium") || element.hasClass("premium-block")) { + "🔒 ${name.trim()}" + } else { + name.trim() + } + } + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = kotlinx.coroutines.coroutineScope { val allChapters = mutableListOf<MangaChapter>() var t = 1 val batchSize = 5 + val dateFormat = SimpleDateFormat("d MMMM yyyy", sourceLocale) while (true) { val deferreds = (t until t + batchSize).map { page -> async { try { - val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/?t=$page" + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/?t=$page" val ajaxDocs = webClient.httpPost( ajaxUrl.toHttpUrl(), emptyMap<String, String>(), @@ -103,21 +126,22 @@ internal class YuriLab(context: MangaLoaderContext) : val baseName = a.ownText().ifEmpty { null } ?: a.selectFirst("p")?.textOrNull() ?: "Chapter" - val isLocked = rawHref.isBlank() || rawHref == "#" || - li.hasClass("premium") || li.hasClass("premium-block") || - li.selectFirst(".fa-lock, .vip-lock") != null + val finalName = transformChapterName(li, baseName) - val finalHref = if (rawHref.isBlank() || rawHref == "#") mangaUrl else rawHref - val finalName = if (isLocked && !baseName.contains("🔒")) "🔒 ${baseName.trim()}" else baseName.trim() + var dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() + if (dateText != null && !dateText.contains("ago", true) && !dateText.contains(Regex("""\d{4}"""))) { + val year = java.util.Calendar.getInstance().get(java.util.Calendar.YEAR) + dateText = "$dateText $year" + } MangaChapter( - id = generateUid(finalHref + finalName), - url = finalHref, + id = generateUid(rawHref + finalName), + url = rawHref, title = finalName, number = 0f, volume = 0, branch = null, - uploadDate = 0L, + uploadDate = parseChapterDate(dateFormat, dateText), scanlator = null, source = source, ) From 928530ef5aaf75795ae55361c3850c638cd9bca6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:14:42 +0200 Subject: [PATCH 369/434] Update and rename src/main/kotlin/tsuki/site/id/Yurilab.kt to src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt --- .../tsuki/site/id/{ => nsfw}/Yurilab.kt | 69 +++++++++++-------- 1 file changed, 40 insertions(+), 29 deletions(-) rename src/main/kotlin/tsuki/site/id/{ => nsfw}/Yurilab.kt (81%) diff --git a/src/main/kotlin/tsuki/site/id/Yurilab.kt b/src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt similarity index 81% rename from src/main/kotlin/tsuki/site/id/Yurilab.kt rename to src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt index 4cf2927d..31eca85a 100644 --- a/src/main/kotlin/tsuki/site/id/Yurilab.kt +++ b/src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt @@ -1,4 +1,4 @@ -package tsuki.site.id +package tsuki.site.id.nsfw import tsuki.MangaLoaderContext import tsuki.MangaSourceParser @@ -21,11 +21,14 @@ import tsuki.util.toAbsoluteUrl import tsuki.util.toTitleCase import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.jsoup.nodes.Document import org.jsoup.nodes.Element import java.text.SimpleDateFormat +import java.util.Calendar import java.util.Locale @MangaSourceParser("YURILAB", "YuriLab", "id", ContentType.HENTAI) @@ -52,7 +55,6 @@ internal class YuriLab(context: MangaLoaderContext) : ) } - override suspend fun fetchAvailableTags(): Set<MangaTag> { val url = "https://$domain/?s=&post_type=wp-manga" val docs = webClient.httpGet(url).parseHtml() @@ -101,20 +103,29 @@ internal class YuriLab(context: MangaLoaderContext) : } } - override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = kotlinx.coroutines.coroutineScope { + private fun String?.applyChapterNumber(index: Int): String { + val base = this ?: "Chapter" + return if (base == "Chapter" || base == "🔒 Chapter") { + base.replace("Chapter", "Chapter ${index + 1}") + } else { + base + } + } + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = coroutineScope { val allChapters = mutableListOf<MangaChapter>() - var t = 1 + var page = 1 val batchSize = 5 val dateFormat = SimpleDateFormat("d MMMM yyyy", sourceLocale) while (true) { - val deferreds = (t until t + batchSize).map { page -> + val deferreds = (page until page + batchSize).map { currentPage -> async { try { - val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/?t=$page" + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/?t=$currentPage" val ajaxDocs = webClient.httpPost( ajaxUrl.toHttpUrl(), - emptyMap<String, String>(), + emptyMap(), Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build(), ).parseHtml() @@ -122,15 +133,15 @@ internal class YuriLab(context: MangaLoaderContext) : lis.mapNotNull { li -> val a = li.selectFirst("a") ?: return@mapNotNull null val rawHref = a.attrAsRelativeUrl("href") - + val baseName = a.ownText().ifEmpty { null } ?: a.selectFirst("p")?.textOrNull() - ?: "Chapter" - + ?: "Chapter" + val finalName = transformChapterName(li, baseName) - + var dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() if (dateText != null && !dateText.contains("ago", true) && !dateText.contains(Regex("""\d{4}"""))) { - val year = java.util.Calendar.getInstance().get(java.util.Calendar.YEAR) + val year = Calendar.getInstance().get(Calendar.YEAR) dateText = "$dateText $year" } @@ -146,32 +157,32 @@ internal class YuriLab(context: MangaLoaderContext) : source = source, ) } - } catch (e: Exception) { - emptyList<MangaChapter>() + } catch (_: Exception) { + emptyList() } } } - val batches = deferreds.map { it.await() } + val batches = deferreds.awaitAll() + var emptyBatch = false for (pageChapters in batches) { if (pageChapters.isEmpty()) { - return@coroutineScope allChapters.reversed().mapIndexed { index, chapter -> - val currentTitle = chapter.title - val finalTitle = if (currentTitle == "Chapter" || currentTitle == "🔒 Chapter") { - currentTitle.replace("Chapter", "Chapter ${index + 1}") - } else { - currentTitle - } - chapter.copy( - title = finalTitle, - number = (index + 1).toFloat() - ) - } + emptyBatch = true + break } allChapters.addAll(pageChapters) } - t += batchSize + + if (emptyBatch) { + break + } + page += batchSize + } + allChapters.reversed().mapIndexed { index, chapter -> + chapter.copy( + title = chapter.title.applyChapterNumber(index), + number = (index + 1).toFloat() + ) } - return@coroutineScope emptyList() } } From eafd49dba5db7a334306822671800c9e34e439c9 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:26:28 +0200 Subject: [PATCH 370/434] Create MangaScantrad.kt --- src/main/kotlin/tsuki/site/fr/MangaScantrad.kt | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/MangaScantrad.kt diff --git a/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt b/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt new file mode 100644 index 00000000..cbe74576 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt @@ -0,0 +1,13 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGASCANTRAD", "Manga-Scantrad", "fr") +internal class MangaScantrad(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGASCANTRAD, "manga-scantrad.io") { + override val datePattern = "d MMMM yyyy" +} From e996766daf7e1ed38498113ddbb1f3dc6881219c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:34:04 +0200 Subject: [PATCH 371/434] Create SushiScan.kt --- src/main/kotlin/tsuki/site/fr/SushiScan.kt | 114 +++++++++++++++++++++ 1 file changed, 114 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/SushiScan.kt diff --git a/src/main/kotlin/tsuki/site/fr/SushiScan.kt b/src/main/kotlin/tsuki/site/fr/SushiScan.kt new file mode 100644 index 00000000..d0b3f738 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/SushiScan.kt @@ -0,0 +1,114 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SUSHISCAN", "Sushi-Scan", "fr") +internal class SushiScan(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SUSHISCAN, "sushiscan.net") { + + override val mangaDirectory = "catalogue" + override val dateFormat = SimpleDateFormat("dd MMMM yyyy", Locale.FRENCH) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/$mangaDirectory/") + .build() + + override fun parseStatus(text: String?): MangaState? { + val status = text?.lowercase(Locale.FRENCH) ?: return null + return when { + status.contains("en cours") -> MangaState.ONGOING + status.contains("terminé") -> MangaState.FINISHED + status.contains("abandonné") -> MangaState.ABANDONED + status.contains("en pause") -> MangaState.PAUSED + else -> null + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrBlank()) { + val url = "https://$domain/page/$page?s=${filter.query.urlEncoded()}" + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + return super.getListPage(page, order, filter) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val base = super.getDetails(manga) + + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val table = doc.selectFirst("table.infotable") + val hasTable = table != null + val authors = if (hasTable) { + val authorCell = table.selectFirst("tr:has(td:contains(Auteur)) td:last-child") + val artistCell = table.selectFirst("tr:has(td:contains(Artiste)) td:last-child") + listOfNotNull( + authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, + artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } + ).toSet() + } else { + doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Auteur) i, .spe span:contains(Auteur) a") + .mapNotNull { it.text().trim().takeIf { t -> t.isNotBlank() && t != "n/a" && t != "N/A" } } + .toSet() + } + + val statusText = if (hasTable) { + table.selectFirst("tr:has(td:contains(Statut)) td:last-child")?.text() + } else { + doc.select(".imptdt:contains(Statut) i, .tsinfo .imptdt:contains(Statut) a, .fmed b:contains(Statut)+span span") + .text() + .ifEmpty { + doc.selectFirst("div.post-content_item:contains(Statut) div.summary-content")?.text() + } + } + val state = parseStatus(statusText) + + base.copy( + authors = authors, + state = state, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val scriptContent = doc.selectFirst("script:containsData(ts_reader)")?.data() + if (scriptContent != null) { + val jsonString = scriptContent.substringAfter("ts_reader.run(").substringBefore(");") + val json = JSONObject(jsonString) + val sources = json.optJSONArray("sources") + if (sources != null && sources.length() > 0) { + val firstSource = sources.getJSONObject(0) + val images = firstSource.optJSONArray("images") + if (images != null) { + return (0 until images.length()).map { i -> + val url = images.getString(i).replace("http://", "https://") + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + } + } + return super.getPages(chapter) + } +} From 239c483ef48545f009ece299d3757c3233510382 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:40:42 +0200 Subject: [PATCH 372/434] Update MangaDistrict.kt --- .../tsuki/site/en/nsfw/MangaDistrict.kt | 129 ++++-------------- 1 file changed, 23 insertions(+), 106 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt index 6366b0dc..909a9764 100644 --- a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt @@ -12,43 +12,23 @@ import tsuki.model.MangaListFilter import tsuki.model.MangaParserSource import tsuki.model.MangaTag import tsuki.model.SortOrder -import tsuki.model.ContentRating import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaState -import tsuki.util.attrAsRelativeUrl -import tsuki.util.generateUid import tsuki.util.mapNotNullToSet import tsuki.util.parseHtml -import tsuki.util.selectFirstOrThrow import tsuki.util.toTitleCase -import tsuki.util.urlEncoded -import tsuki.util.extractChapterNumber -import tsuki.util.oneOrThrowIfMany import org.jsoup.HttpStatusException import org.jsoup.nodes.Document -import java.net.HttpURLConnection -import java.text.SimpleDateFormat -import java.util.EnumSet @MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) internal class MangaDistrict(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { override val tagPrefix = "publication-genre/" - override val withoutAjax: Boolean = true - override val datePattern: String = "MMMM d, yyyy" - override val stylePage: String = "?style=list" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) + override val withoutAjax = true + override val datePattern = "MMMM d, yyyy" + override val stylePage = "?style=list" override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( @@ -56,8 +36,13 @@ internal class MangaDistrict(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (filter.tags.isEmpty()) { + return super.getListPage(page, order, filter) + } + val pages = page + 1 - fun sortParam(): String = when (order) { + val genreSlug = filter.tags.first().key + val sortParam = when (order) { SortOrder.POPULARITY -> "views" SortOrder.UPDATED -> "latest" SortOrder.NEWEST -> "new-manga" @@ -66,99 +51,31 @@ internal class MangaDistrict(context: MangaLoaderContext) : SortOrder.RELEVANCE -> "" else -> "" } - val url = if (filter.tags.isNotEmpty()) { - val genreSlug = filter.tags.first().key - buildString { - append("https://") - append(domain) - append("/publication-genre/$genreSlug/") - if (pages > 1) { - append("page/") - append(pages) - append("/") - } - append("?m_orderby=") - append(sortParam()) - } - } else { - buildString { - append("https://") - append(domain) - if (pages > 1) { - append("/page/") - append(pages.toString()) - } - append("/?s=") - append(filter.query?.urlEncoded() ?: "") - append("&post_type=wp-manga") - filter.states.forEach { - append("&status[]=") - when (it) { - MangaState.ONGOING -> append("on-going") - MangaState.FINISHED -> append("end") - MangaState.ABANDONED -> append("canceled") - MangaState.PAUSED -> append("on-hold") - MangaState.UPCOMING -> append("upcoming") - else -> throw IllegalArgumentException("$it not supported") - } - } - filter.contentRating.oneOrThrowIfMany()?.let { - append("&adult=") - append( - when (it) { - ContentRating.SAFE -> "0" - ContentRating.ADULT -> "1" - else -> "" - } - ) - } - if (filter.year != 0) { - append("&release=") - append(filter.year.toString()) - } - filter.author?.takeIf { it.isNotEmpty() }?.let { - append("&author=") - append(it.lowercase().replace(" ", "-")) - } - append("&m_orderby=") - append(sortParam()) + + val url = buildString { + append("https://") + append(domain) + append("/$tagPrefix$genreSlug/") + if (pages > 1) { + append("page/") + append(pages) + append("/") } + append("?m_orderby=") + append(sortParam) } + val html = try { webClient.httpGet(url).parseHtml() } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() - else throw ParseException("Can't fetch data from source!", url) + throw ParseException("Failed to load page: ${e.statusCode}", url) } return parseMangaList(html) } override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val chapters = doc.body() - .select("li.wp-manga-chapter") - .mapNotNull { li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - val name = a.selectFirst("p")?.text() - ?: a.ownText() - val dateText = li.selectFirst("a.c-new-tag")?.attr("title") - ?: li.selectFirst("span.chapter-release-date i")?.text() - MangaChapter( - id = generateUid(href), - title = name, - number = name.extractChapterNumber(), - volume = 0, - url = link, - uploadDate = parseChapterDate(dateFormat, dateText), - scanlator = null, - branch = null, - source = source, - ) - } + return super.getChapters(manga, doc) .sortedWith(compareBy<MangaChapter> { it.number }.thenBy { it.title }) - return chapters } override suspend fun fetchAvailableTags(): Set<MangaTag> { From 419bce72f7dcf1f2dc900e0e593b1033fe0cbdc1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:43:35 +0200 Subject: [PATCH 373/434] Update AnisaScans.kt --- src/main/kotlin/tsuki/site/en/AnisaScans.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/main/kotlin/tsuki/site/en/AnisaScans.kt b/src/main/kotlin/tsuki/site/en/AnisaScans.kt index 1fa74cdf..2b10b2e2 100644 --- a/src/main/kotlin/tsuki/site/en/AnisaScans.kt +++ b/src/main/kotlin/tsuki/site/en/AnisaScans.kt @@ -2,7 +2,9 @@ package tsuki.site.en import tsuki.MangaLoaderContext import tsuki.MangaSourceParser +import tsuki.config.ConfigKey import tsuki.parsers.MadaraParser +import tsuki.network.UserAgents import tsuki.model.MangaParserSource @@ -10,4 +12,6 @@ import tsuki.model.MangaParserSource internal class AnisaScans(context: MangaLoaderContext): MadaraParser(context, MangaParserSource.ANISASCANS, "anisascans.in") { override val datePattern = "d MMMM, yyyy" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) } From cdce0851ff84e0414d4bc6ead2d3542847048dae Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:52:31 +0200 Subject: [PATCH 374/434] Update MGKomik.kt --- src/main/kotlin/tsuki/site/id/MGKomik.kt | 163 +++++++++++++++++++++-- 1 file changed, 155 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/MGKomik.kt b/src/main/kotlin/tsuki/site/id/MGKomik.kt index 1c0225c2..77cf70f8 100644 --- a/src/main/kotlin/tsuki/site/id/MGKomik.kt +++ b/src/main/kotlin/tsuki/site/id/MGKomik.kt @@ -5,10 +5,18 @@ import tsuki.MangaSourceParser import tsuki.MangaParserAuthProvider import tsuki.parsers.MadaraParser +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterOptions import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.parseHtml import okhttp3.Interceptor -import okhttp3.Headers +import androidx.collection.arraySetOf +import okhttp3.Response @MangaSourceParser("MGKOMIK", "MGKomik", "id") internal class MGKomik(context: MangaLoaderContext) : @@ -16,14 +24,153 @@ internal class MGKomik(context: MangaLoaderContext) : override val datePattern = "dd MMM yy" override val tagPrefix = "genres/" - override val listUrl = "komik/" override val selectDesc = "div.description-summary div.summary__content p" - override val stylePage = "" + override val withoutAjax = true - override val postReq = true - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") - .add("Referer", "https://$domain/") - .build() + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val path = request.url.encodedPath + val isAjax = path.contains("admin-ajax.php") || + path.contains("wp-json") || + path.endsWith("/ajax/chapters") + + val builder = request.newBuilder() + .header("Sec-CH-UA-Model", "\"\"") + + if (isAjax) { + builder.header("X-Requested-With", "XMLHttpRequest") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Origin", "https://$domain") + .header("Priority", "u=1, i") + .removeHeader("Sec-Fetch-User") + .removeHeader("Upgrade-Insecure-Requests") + } + + return chain.proceed(builder.build()) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (order == SortOrder.POPULARITY) { + val p = page + 1 + val url = buildString { + append("https://") + append(domain) + append("/komik") + if (p > 1) { + append("/page/") + append(p) + } + append("/?m_orderby=trending") + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + return super.getListPage(page, order, filter) + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + return super.fetchAvailableTags() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + private fun availableTags() = arraySetOf( + MangaTag("2 hours ago", "2-hours-ago", source), + MangaTag("7 hours ago", "7-hours-ago", source), + MangaTag("8 hours ago", "8-hours-ago", source), + MangaTag("11 hours ago", "11-hours-ago", source), + MangaTag("13 hours ago", "13-hours-ago", source), + MangaTag("15 hours ago", "15-hours-ago", source), + MangaTag("1 day ago", "1-day-ago", source), + MangaTag("2 days ago", "2-days-ago", source), + MangaTag("Action", "action", source), + MangaTag("Adaptation", "adaptation", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Animals", "animals", source), + MangaTag("Apocalypse", "apocalypse", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Crime", "crime", source), + MangaTag("Demon", "demon", source), + MangaTag("Demons", "demons", source), + MangaTag("Drama", "drama", source), + MangaTag("Dungeons", "dungeons", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Fight", "fight", source), + MangaTag("Fighting", "fighting", source), + MangaTag("Full Color", "full-color", source), + MangaTag("Game", "game", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Josei(W)", "joseiw", source), + MangaTag("Kids", "kids", source), + MangaTag("Magic", "magic", source), + MangaTag("Manga", "manga", source), + MangaTag("Mangatoon", "mangatoon", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mature", "mature", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Military", "military", source), + MangaTag("Modern", "modern", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Murim", "murim", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Office Workers", "office-workers", source), + MangaTag("One-Shot", "one-shot", source), + MangaTag("Overpowered", "overpowered", source), + MangaTag("Police", "police", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Regression", "regression", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Revenge", "revenge", source), + MangaTag("Reverse Harem", "reverse-harem", source), + MangaTag("Rofan", "rofan", source), + MangaTag("Romance", "romance", source), + MangaTag("School", "school", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujoai", source), + MangaTag("Shoujo AI", "shoujo-ai", source), + MangaTag("Shoujo(G)", "shoujog", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Smut", "smut", source), + MangaTag("Sports", "sports", source), + MangaTag("Super Power", "super-power", source), + MangaTag("Superhero", "superhero", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Supranatural", "supranatural", source), + MangaTag("Survival", "survival", source), + MangaTag("System", "system", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Transmigration", "transmigration", source), + MangaTag("Vampire", "vampire", source), + MangaTag("Villainess", "villainess", source), + MangaTag("Violence", "violence", source), + MangaTag("War", "war", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Webtoons", "webtoons", source), + MangaTag("Wuxia", "wuxia", source), + MangaTag("Yuri", "yuri", source), + ) + } From 6684cf48433b703e54b98aaa5e64f79fac1e9cc7 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:56:11 +0200 Subject: [PATCH 375/434] Update AnisaScans.kt --- src/main/kotlin/tsuki/site/en/AnisaScans.kt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/en/AnisaScans.kt b/src/main/kotlin/tsuki/site/en/AnisaScans.kt index 2b10b2e2..0f8d79f7 100644 --- a/src/main/kotlin/tsuki/site/en/AnisaScans.kt +++ b/src/main/kotlin/tsuki/site/en/AnisaScans.kt @@ -12,6 +12,10 @@ import tsuki.model.MangaParserSource internal class AnisaScans(context: MangaLoaderContext): MadaraParser(context, MangaParserSource.ANISASCANS, "anisascans.in") { override val datePattern = "d MMMM, yyyy" - override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } } From 90d450442af19fe73142c65342eab6060c7a2296 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 19:57:30 +0200 Subject: [PATCH 376/434] take code from manga-repo --- .../kotlin/tsuki/site/vi/CuuTruyenParser.kt | 507 ++++++++++++++++++ 1 file changed, 507 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt diff --git a/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt b/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt new file mode 100644 index 00000000..d500d822 --- /dev/null +++ b/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt @@ -0,0 +1,507 @@ +package tsuki.site.vi + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.bitmap.Bitmap +import tsuki.bitmap.Rect +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.network.CloudFlareHelper +import tsuki.network.CommonHeaders +import tsuki.network.UserAgents + +import tsuki.util.copy +import tsuki.util.generateUid +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapChapters +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import androidx.collection.arraySetOf +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okio.IOException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("CUUTRUYEN", "Cứu Truyện", "vi") +internal class CuuTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CUUTRUYEN, 24) { + + private val apiSuffix = "/api/v2" + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override val configKeyDomain = ConfigKey.Domain("cuutruyen.net", "hetcuutruyen.net") + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + DESKTOP_COVER to "Ảnh bìa chất lượng cao (Desktop)", + MOBILE_COVER to "Ảnh bìa chất lượng thấp (Mobile)", + ), + defaultValue = DESKTOP_COVER, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(preferredServerKey) + keys.remove(userAgentKey) + } + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add(CommonHeaders.ORIGIN, "https://$domain") + .add(CommonHeaders.REFERER, "https://$domain") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = availableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + // Move to urlBuilder + val url = buildString { + if (!filter.query.isNullOrEmpty() || filter.tags.isNotEmpty() || filter.states.isNotEmpty()) { + append("/mangas/search?q=") + if (!filter.query.isNullOrEmpty()) { + append(filter.query.urlEncoded()) + } + + // Bug from API: Select both ONGOING + FINISHED will return empty list for all case + val state = listOf(MangaState.ONGOING, MangaState.FINISHED) + if (filter.states.containsAll(state)) { + // oneOrThrowIfMany + throw IllegalArgumentException( + ErrorMessages.FILTER_MULTIPLE_STATES_NOT_SUPPORTED + ) + } + + append("&tags=") + val tags = buildList { + addAll(filter.tags.map { "\"${space2plus(it.title.lowercase())}\"" }) + addAll( + filter.states.map { + when (it) { + MangaState.ONGOING -> "\"đang+tiến+hành\"" + MangaState.FINISHED -> "\"đã+hoàn+thành\"" + else -> "" + } + } + ) + }.filter { it.isNotEmpty() } + append(tags.joinToString(separator = "+AND+")) + + append("&page=") + append(page.toString()) + } else { + append("/mangas") + when (order) { + SortOrder.UPDATED -> append("/recently_updated") + SortOrder.POPULARITY -> append("/top?duration=all") + SortOrder.POPULARITY_WEEK -> append("/top?duration=week") + SortOrder.POPULARITY_MONTH -> append("/top?duration=month") + SortOrder.NEWEST -> { + // clear old buildString + clear() + append("/home_a") + } + else -> append("/recently_updated") + } + + when (order) { + SortOrder.POPULARITY, SortOrder.POPULARITY_WEEK, SortOrder.POPULARITY_MONTH -> { + append("&page=") + append(page.toString()) + } + else -> { + append("?page=") + append(page.toString()) + } + } + } + + append("&per_page=") + append(pageSize) + } + + val fullUrl = "https://$domain$apiSuffix$url" + val response = webClient.httpGet(fullUrl) + + val protection = CloudFlareHelper.checkResponseForProtection(response.copy()) + if (protection != CloudFlareHelper.PROTECTION_NOT_DETECTED) { + response.close() + context.requestBrowserAction(this, fullUrl) + } + + val json = response.parseJson() + val data = json.optJSONArray("data") + ?: json.getJSONObject("data").getJSONArray("new_chapter_mangas") + ?: json.getJSONObject("data").getJSONArray("mangas") + ?: return emptyList() + + return data.mapJSON { jo -> + val author = jo.getStringOrNull("author_name") + val server = config[preferredServerKey] ?: DESKTOP_COVER + val coverUrl = if (server == MOBILE_COVER) jo.getString(MOBILE_COVER) else + jo.getString(DESKTOP_COVER) + Manga( + id = generateUid(jo.getLong("id")), + url = "$apiSuffix/mangas/${jo.getLong("id")}", + publicUrl = "https://$domain/mangas/${jo.getLong("id")}", + title = jo.getString("name"), + altTitles = emptySet(), + coverUrl = coverUrl, + largeCoverUrl = null, + authors = setOfNotNull(author), + tags = emptySet(), + state = null, + description = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + rating = RATING_UNKNOWN, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val server = config[preferredServerKey] ?: DESKTOP_COVER + val url = "https://" + domain + manga.url + val chapters = async { + webClient.httpGet("$url/chapters").parseJson().getJSONArray("data") + } + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("GMT+7") + } + val tags = json.optJSONArray("tags")?.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + }.orEmpty() + + val state = when { + tags.any { it.key == "da-hoan-thanh" } -> MangaState.FINISHED + else -> MangaState.ONGOING // Mostly ONGOING but not marked by site owner + } + + val newTags = tags.filter { it.key != "da-hoan-thanh" && it.key != "dang-tien-hanh" }.toSet() + val author = json.optJSONObject("author")?.getStringOrNull("name")?.substringBefore(',')?.nullIfEmpty() + val title = json.getStringOrNull("name") ?: manga.title + val team = json.optJSONObject("team")?.getStringOrNull("name") + + manga.copy( + title = title, + altTitles = json.optJSONArray("titles")?.mapJSONToSet { it.getString("name") }?.minus(title).orEmpty(), + contentRating = if (json.getBooleanOrDefault("is_nsfw", false)) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + authors = setOfNotNull(author), + description = json.getStringOrNull("full_description"), + tags = newTags, + state = state, + chapters = chapters.await().mapChapters(reversed = true) { _, jo -> + val chapterId = jo.getLong("id") + val number = jo.getFloatOrDefault("number", 0f) + val createdAt = jo.getStringOrNull("created_at") + ?.replace(Regex("([+-]\\d{2}):(\\d{2})$"), "$1$2") + MangaChapter( + id = generateUid(chapterId), + title = jo.getStringOrNull("name"), + number = number, + volume = 0, + url = "$apiSuffix/chapters/$chapterId", + scanlator = team, + uploadDate = chapterDateFormat.parseSafe(createdAt), + branch = null, + source = source, + ) + }, + largeCoverUrl = if (server == MOBILE_COVER) json.getString(MOBILE_PANORAMA) else + json.getString(DESKTOP_PANORAMA), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = "https://$domain${chapter.url}" + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + + return json.getJSONArray("pages").mapJSON { jo -> + val imageUrl = jo.getString("image_url").toHttpUrl().newBuilder() + val id = jo.getLong("id") + val drm = jo.getStringOrNull("drm_data") + if (!drm.isNullOrEmpty()) { + imageUrl.fragment(DRM_DATA_KEY + drm) + } + MangaPage( + id = generateUid(id), + url = imageUrl.build().toString(), + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + var request = chain.request() + STORAGE_HOST_FALLBACK[request.url.host]?.let { host -> + request = request.newBuilder() + .url(request.url.newBuilder().host(host).build()) + .build() + } + + if (request.url.encodedPath.contains("/api/")) { + request = request.newBuilder() + .header(CLIENT_HEADER, CLIENT_VALUE) + .build() + return chain.proceed(request) + } + + val response = chain.proceed(request) + val fragment = response.request.url.fragment + + if (fragment == null || !fragment.contains(DRM_DATA_KEY)) { + return response + } + + return context.redrawImageResponse(response) { bitmap -> + val drmData = fragment.substringAfter(DRM_DATA_KEY) + unscrambleImage(bitmap, drmData) + } + } + + private fun unscrambleImage(bitmap: Bitmap, drmData: String): Bitmap { + val data = context.decodeBase64(drmData) + .decodeXorCipher(DECRYPTION_KEY) + .toString(Charsets.UTF_8) + + if (!data.startsWith("#v4|")) { + throw IOException("Invalid DRM data (does not start with expected magic bytes): $data") + } + + val result = context.createBitmap(bitmap.width, bitmap.height) + var sy = 0 + for (t in data.split('|').drop(1)) { + val (dy, height) = t.split('-').map(String::toInt) + val srcRect = Rect(0, sy, bitmap.width, sy + height) + val dstRect = Rect(0, dy, bitmap.width, dy + height) + + result.drawBitmap(bitmap, srcRect, dstRect) + sy += height + } + + return result + } + + private fun ByteArray.decodeXorCipher(key: String): ByteArray { + val k = key.toByteArray(Charsets.UTF_8) + + return this.mapIndexed { i, b -> + (b.toInt() xor k[i % k.size].toInt()).toByte() + }.toByteArray() + } + + private fun space2plus(input: String): String = input.replace(' ', '+') + + // keep old tags list, key in MangaTag is unnecessary + private fun availableTags() = arraySetOf( // big thanks to beer-psi + MangaTag("School life", "school-life", source), + MangaTag("Nsfw", "nsfw", source), + MangaTag("Monster girls", "monster-girls", source), + MangaTag("Magic", "magic", source), + MangaTag("tình yêu không được đáp lại", "tinh-yeu-khong-duoc-dap-lai", source), + MangaTag("tình yêu thuần khiết", "tinh-yeu-thuan-khiet", source), + MangaTag("Khỏa thân", "khoa-than", source), + MangaTag("Gyaru", "gyaru", source), + MangaTag("4-Koma", "4-koma", source), + MangaTag("Manga", "manga", source), + MangaTag("Thể thao", "the-thao", source), + MangaTag("Hài hước", "hai-huoc", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Học đường", "hoc-duong", source), + MangaTag("Chất lượng cao", "chat-luong-cao", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Action", "action", source), + MangaTag("Horror", "horror", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Aliens", "aliens", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Military", "military", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Web Comic", "web-comic", source), + MangaTag("Phiêu lưu", "phieu-luu", source), + MangaTag("Hậu tận thế", "hau-tan-the", source), + MangaTag("Hành động", "hanh-dong", source), + MangaTag("Sinh tồn", "sinh-ton", source), + MangaTag("Du hành thời gian", "du-hanh-thoi-gian", source), + MangaTag("Khoa học", "khoa-hoc", source), + MangaTag("Tạm ngưng", "tam-ngung", source), + MangaTag("NSFW", "nsfw", source), + MangaTag("Bạo lực", "bao-luc", source), + MangaTag("Khoả thân", "khoa-than", source), + MangaTag("Bí ẩn", "bi-an", source), + MangaTag("Trinh thám", "trinh-tham", source), + MangaTag("Kinh dị", "kinh-di", source), + MangaTag("Máu me", "mau-me", source), + MangaTag("Tình dục", "tinh-duc", source), + MangaTag("Có màu", "co-mau", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Siêu nhiên", "sieu-nhien", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Võ thuật", "vo-thuat", source), + MangaTag("Drama", "drama", source), + MangaTag("Hệ thống", "he-thong", source), + MangaTag("Lãng mạn", "lang-man", source), + MangaTag("Đời thường", "doi-thuong", source), + MangaTag("Công sở", "cong-so", source), + MangaTag("Sát thủ", "sat-thu", source), + MangaTag("Phép thuật", "phep-thuat", source), + MangaTag("Tội phạm", "toi-pham", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Chuyển sinh", "chuyen-sinh", source), + MangaTag("Harem", "harem", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Trung cổ", "trung-co", source), + MangaTag("LGBT", "lgbt", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Game", "game", source), + MangaTag("Bi kịch", "bi-kich", source), + MangaTag("Động vật", "dong-vat", source), + MangaTag("Tâm lý", "tam-ly", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Nam biến nữ", "nam-bien-nu", source), + MangaTag("Romcom", "romcom", source), + MangaTag("Award Winning", "award-winning", source), + MangaTag("Oneshot", "oneshot", source), + MangaTag("Khoa học viễn tưởng", "khoa-hoc-vien-tuong", source), + MangaTag("Dark Fantasy", "dark-fantasy", source), + MangaTag("Zombie", "zombie", source), + MangaTag("Nam x Nam", "nam-x-nam", source), + MangaTag("Giật gân", "giat-gan", source), + MangaTag("Cảnh sát", "canh-sat", source), + MangaTag("NTR", "ntr", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Ẩm thực", "am-thuc", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Quái vật", "quai-vat", source), + MangaTag("Vampires", "vampires", source), + MangaTag("Nam giả nữ", "nam-gia-nu", source), + MangaTag("Yakuza", "yakuza", source), + MangaTag("Romance", "romance", source), + MangaTag("Sport", "sport", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Ninja", "ninja", source), + MangaTag("Lịch sử", "lich-su", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Databook", "databook", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Y học", "y-hoc", source), + MangaTag("Miễn bản quyền", "mien-ban-quyen", source), + MangaTag("Josei", "josei", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Anime", "anime", source), + MangaTag("Yuri", "yuri", source), + MangaTag("Yonkoma", "yonkoma", source), + MangaTag("Quân đội", "quan-doi", source), + MangaTag("Nữ giả nam", "nu-gia-nam", source), + MangaTag("Chính trị", "chinh-tri", source), + MangaTag("Tuyển tập", "tuyen-tap", source), + MangaTag("Tu tiên", "tu-tien", source), + MangaTag("Vô CP", "vo-cp", source), + MangaTag("Xuyên không", "xuyen-khong", source), + MangaTag("Việt Nam", "viet-nam", source), + MangaTag("Việt nam", "viet-nam", source), + MangaTag("Toán học", "toan-hoc", source), + MangaTag("Thiếu niên", "thieu-nien", source), + MangaTag("Tình yêu", "tinh-yeu", source), + MangaTag("Chính kịch", "chinh-kich", source), + MangaTag("Ngọt ngào", "ngot-ngao", source), + MangaTag("Wholesome", "wholesome", source), + MangaTag("Smut", "smut", source), + MangaTag("Gore", "gore", source), + MangaTag("School Life", "school-life", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Atlus", "atlus", source), + MangaTag("Sega", "sega", source), + MangaTag("RPG", "rpg", source), + MangaTag("Chuyển thể", "chuyen-the", source), + MangaTag("Historical", "historical", source), + MangaTag("Medical", "medical", source), + MangaTag("Ghosts", "ghosts", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Animals", "animals", source), + MangaTag("Survival", "survival", source), + MangaTag("Samurai", "samurai", source), + MangaTag("Virtual Reality", "virtual-reality", source), + MangaTag("Video Games", "video-games", source), + MangaTag("Monster Girls", "monster-girls", source), + MangaTag("Adaption", "adaption", source), + MangaTag("Idol", "idol", source), + ) + + private companion object { + const val DRM_DATA_KEY = "drm_data=" + const val DECRYPTION_KEY = "3141592653589793" + const val MOBILE_COVER = "cover_mobile_url" + const val MOBILE_PANORAMA = "panorama_mobile_url" + const val DESKTOP_COVER = "cover_url" + const val DESKTOP_PANORAMA = "panorama_url" + const val CLIENT_HEADER = "Cuutruyen-Client" + const val CLIENT_VALUE = "OfficialWebApp-20250805" + val STORAGE_HOST_FALLBACK = mapOf( + "storage-ct.lrclib.net" to "storage-bravo.cuutruyen.net", + "storage-ct-riften.site" to "storage-charlie.cuutruyen.net", + ) + } +} From a3974c14b64d079bc43e1cf55d0c25555bb4ec85 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 29 Aug 2026 20:35:35 +0200 Subject: [PATCH 377/434] Create SexKomik2COM.kt --- .../tsuki/site/all/nsfw/SexKomik2COM.kt | 302 ++++++++++++++++++ 1 file changed, 302 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt diff --git a/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt b/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt new file mode 100644 index 00000000..be80883e --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt @@ -0,0 +1,302 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient + +import tsuki.model.ContentType +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("SEXKOMIX2COM", "SexKomik2COM", type = ContentType.HENTAI) +class SexKomik2COM(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SEXKOMIX2COM, 24) { + + override val configKeyDomain = tsuki.config.ConfigKey.Domain("sexkomix2.com") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, // date + SortOrder.POPULARITY, // prosmotr (views) + SortOrder.RATING, // like + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + private val cookieInterceptor = Interceptor { chain -> + val original = chain.request() + val request = original.newBuilder() + .header("Cookie", "confirm=true") + .header("Referer", "https://$domain/") + .build() + chain.proceed(request) + } + + override val webClient by lazy { + val client = context.httpClient.newBuilder() + .addInterceptor(cookieInterceptor) + .build() + OkHttpWebClient(client, source) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = setOf( + MangaTag("3D", "3D", source), + MangaTag("Adventure", "Adventure", source), + MangaTag("AI Generated", "AI Generated", source), + MangaTag("BDSM", "BDSM", source), + MangaTag("Big boobs", "Big boobs", source), + MangaTag("Big dick", "Big dick", source), + MangaTag("Black girls", "Black girls", source), + MangaTag("Blondes", "Blondes", source), + MangaTag("Brunettes", "Brunettes", source), + MangaTag("Cheating", "Cheating", source), + MangaTag("Double penetration", "Double penetration", source), + MangaTag("Fantasy", "Fantasy", source), + MangaTag("Fat girls", "Fat girls", source), + MangaTag("Fitness", "Fitness", source), + MangaTag("Furry", "Furry", source), + MangaTag("Gays Yaoi", "Gays Yaoi", source), + MangaTag("Giants", "Giants", source), + MangaTag("Hentai Manga", "Hentai Manga", source), + MangaTag("Inter Komix", "Inter Komix", source), + MangaTag("Interracial sex", "Interracial sex", source), + MangaTag("Lesbians", "Lesbians", source), + MangaTag("Milf", "Milf", source), + MangaTag("Monsters", "Monsters", source), + MangaTag("Neighbor", "Neighbor", source), + MangaTag("NFT", "NFT", source), + MangaTag("Normal boobs", "Normal boobs", source), + MangaTag("Nurses", "Nurses", source), + MangaTag("Parodies", "Parodies", source), + MangaTag("Redheads", "Redheads", source), + MangaTag("Shemale", "Shemale", source), + MangaTag("Simpsons", "Simpsons", source), + MangaTag("Super heroes", "Super heroes", source), + MangaTag("Teacher", "Teacher", source), + MangaTag("Teen", "Teen", source), + MangaTag("Threesome", "Threesome", source), + MangaTag("Unique VIP", "Unique VIP", source), + MangaTag("With sex toys", "With sex toys", source), + ), + availableStates = emptySet(), + availableContentTypes = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.NEWEST -> "date" + SortOrder.POPULARITY -> "prosmotr" + SortOrder.RATING -> "like" + else -> "date" + } + + val tag = filter.tags.firstOrNull() + val basePath = if (tag != null) "categories" else "search" + + val url = "https://$domain/$basePath/".toHttpUrl().newBuilder().apply { + addQueryParameter("lang", "en") + + if (tag != null) { + addQueryParameter("cat", tag.key) + } + + filter.query?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("q", it) + } + + addQueryParameter("page", page.toString()) + addQueryParameter("sort", sortParam) + }.build().toString() + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + val listContainer = doc.selectFirst("#comix_directory") + ?: return emptyList() + + return listContainer.select("li.comix").mapNotNull { element -> + parseMangaItem(element) + } + } + + private fun parseMangaItem(element: Element): Manga? { + val link = element.selectFirst("a[href]") ?: return null + val href = link.attr("abs:href") + if (href.isBlank()) return null + + val titleElement = element.selectFirst(".comix_title h2 p") + val title = titleElement?.text()?.trim() ?: link.attr("title").ifBlank { link.text() }.trim() + if (title.isBlank()) return null + + val img = element.selectFirst(".comix_img_box img") + val cover = img?.imgAttr() ?: "" + + val tagElements = element.select(".tags_ul li a") + val tags = tagElements.mapNotNull { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + }.toSet() + + return Manga( + id = generateUid(href), + title = title, + altTitles = emptySet(), + url = href, + publicUrl = href, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = tags, + state = null, + authors = emptySet(), + source = source, + ) + } + + private fun Element.imgAttr(): String = when { + hasAttr("data-src") -> attr("abs:data-src") + hasAttr("data-url") -> attr("abs:data-url") + hasAttr("data-zoom-src") -> attr("abs:data-zoom-src") + hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") + hasAttr("data-cfsrc") -> attr("abs:data-cfsrc") + else -> attr("abs:src") + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url).parseHtml() + + val title = doc.selectFirst(".info_box h1 a")?.text()?.trim() + ?: doc.selectFirst("h1")?.text()?.trim() + if (title.isNullOrBlank()) { + throw ParseException( + "Failed to extract title from manga page: ${manga.url}\n" + + "HTML snippet: ${doc.html().take(300)}", + manga.url + ) + } + + val description = doc.selectFirst(".info_box p")?.text()?.trim() ?: "" + + val coverElement = doc.selectFirst("#comix_cover_img") + val cover = coverElement?.imgAttr()?.takeIf { it.isNotBlank() } + if (cover == null) { + throw ParseException( + "Failed to extract cover image from manga page: ${manga.url}\n" + + "HTML snippet: ${doc.html().take(300)}", + manga.url + ) + } + + val categoryElements = doc.select(".info_box:has(.tags_ul) .tags_ul li a") + val tags = categoryElements.mapNotNull { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + }.toSet() + + val studioElement = doc.selectFirst(".studio_translator_box .link_button a") + val authors = studioElement?.text()?.trim()?.let { setOf(it) } ?: emptySet() + + val chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + url = manga.url, + title = title, + number = 1f, + volume = 0, + uploadDate = 0L, + scanlator = null, + branch = null, + source = source, + ) + ) + + return manga.copy( + title = title, + description = description, + coverUrl = cover, + tags = tags, + authors = authors, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url).parseHtml() + + val fancyboxLinks = doc.select("#comix_pages_ul li a.fancybox[href]") + if (fancyboxLinks.isNotEmpty()) { + return fancyboxLinks.mapIndexed { _, element -> + val url = element.attr("abs:href") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + val images = doc.select("#comix_pages_ul li img") + if (images.isNotEmpty()) { + return images.mapIndexed { _, img -> + val url = img.imgAttr() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + val scriptMatch = Regex("\"images\"\\s*:\\s*(\\[.*?])", RegexOption.DOT_MATCHES_ALL) + .find(doc.html()) + if (scriptMatch != null) { + val jsonArray = JSONArray(scriptMatch.groupValues[1]) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return emptyList() + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + return emptyList() + } +} From 910c0958876522bf4d13bca4d688186008af8125 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:27:52 +0200 Subject: [PATCH 378/434] Update RavenScans.kt --- src/main/kotlin/tsuki/site/en/RavenScans.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/RavenScans.kt b/src/main/kotlin/tsuki/site/en/RavenScans.kt index 1671b55b..0f07115f 100644 --- a/src/main/kotlin/tsuki/site/en/RavenScans.kt +++ b/src/main/kotlin/tsuki/site/en/RavenScans.kt @@ -8,6 +8,6 @@ import tsuki.model.MangaParserSource @MangaSourceParser("RAVENSCANS", "RavenScans", "en") internal class RavensScan(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.net") { - override val mangaDirectory = "series" + MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.org") { + override val mangaDirectory = "manga" } From 1b39643914a636926b71230026fcd94890ac9495 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 20:36:04 +0200 Subject: [PATCH 379/434] update mangadotnet langauges --- src/main/kotlin/tsuki/site/all/Mangadotnet.kt | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 716c1809..0a6b4040 100644 --- a/src/main/kotlin/tsuki/site/all/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -158,11 +158,13 @@ internal class Mangadotnet(context: MangaLoaderContext) : Locale.FRENCH, Locale.GERMAN, Locale("id"), - Locale("pt", "br"), + Locale("pt", "BR"), Locale("es"), + Locale("es", "419"), Locale("th"), Locale("vi"), - ), + Locale("zh", "Hant"), + ) ) } @@ -177,7 +179,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : if (!filter.query.isNullOrBlank()) addQueryParameter("search", filter.query!!) addQueryParameter("page", page.toString()) addQueryParameter("adult", adultParam) - + selectedLanguage?.let { addQueryParameter("lang", it) } val (sortBy, sortOrder) = when (order) { SortOrder.UPDATED -> "latest" to "desc" SortOrder.ALPHABETICAL -> "alphabetical" to "asc" @@ -256,8 +258,12 @@ internal class Mangadotnet(context: MangaLoaderContext) : private var selectedLanguage: String? = null private fun localeToLang(locale: Locale): String { - if (locale.language == "pt" && locale.country == "br") return "pt-br" - return locale.language + return when (locale.language) { + "pt" if locale.country == "BR" -> "pt-br" + "es" if locale.country == "419" -> "es-la" + "zh" if locale.country == "Hant" -> "zh-hk" + else -> locale.language + } } override suspend fun getDetails(manga: Manga): Manga = coroutineScope { From 8732d9f3243f8b5826884a8d13b804262fcf6c00 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:10:23 +0200 Subject: [PATCH 380/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 6cda0d85..289ba158 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -3,6 +3,7 @@ package tsuki.parsers import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders import tsuki.model.RATING_UNKNOWN import tsuki.model.ContentRating @@ -55,7 +56,7 @@ abstract class MangaThemesia( } override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://$domain/") + .set(CommonHeaders.REFERER, "https://$domain/") .build() protected open val mangaDirectory = "manga" From b33e00b0e42b96b674728ba34a9131173c1b0518 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:10:38 +0200 Subject: [PATCH 381/434] Update MangaThemesia.kt --- src/main/kotlin/tsuki/parsers/MangaThemesia.kt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 6cda0d85..289ba158 100644 --- a/src/main/kotlin/tsuki/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -3,6 +3,7 @@ package tsuki.parsers import tsuki.MangaLoaderContext import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders import tsuki.model.RATING_UNKNOWN import tsuki.model.ContentRating @@ -55,7 +56,7 @@ abstract class MangaThemesia( } override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://$domain/") + .set(CommonHeaders.REFERER, "https://$domain/") .build() protected open val mangaDirectory = "manga" From 2e5ea9c628735636961fefadcb7ef0b4581818b8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:11:00 +0200 Subject: [PATCH 382/434] Update RavenScans.kt --- src/main/kotlin/tsuki/site/en/RavenScans.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/RavenScans.kt b/src/main/kotlin/tsuki/site/en/RavenScans.kt index 1671b55b..0f07115f 100644 --- a/src/main/kotlin/tsuki/site/en/RavenScans.kt +++ b/src/main/kotlin/tsuki/site/en/RavenScans.kt @@ -8,6 +8,6 @@ import tsuki.model.MangaParserSource @MangaSourceParser("RAVENSCANS", "RavenScans", "en") internal class RavensScan(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.net") { - override val mangaDirectory = "series" + MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.org") { + override val mangaDirectory = "manga" } From 688232a92d5f89d98ee2bc049cf0c289729c0123 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Wed, 2 Sep 2026 21:12:01 +0200 Subject: [PATCH 383/434] Create DojingNet.kt --- src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt diff --git a/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt b/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt new file mode 100644 index 00000000..b9a4ae7e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("DOJINGNET", "Dojing.net", "id", ContentType.HENTAI) +internal class DojingNet(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.DOJINGNET, "dojing.net") From 49f91d304e1420bc82d6e48b7a0775515022b6ab Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:31:52 +0200 Subject: [PATCH 384/434] docs: add comprehensive contributing guide with detailed utilities and examples --- CONTRIBUTING.md | 1087 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1087 insertions(+) create mode 100644 CONTRIBUTING.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..43c2ff07 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,1087 @@ +# Contributing to UMA + +This guide provides comprehensive instructions and best practices for contributing to the UMA (Usagi Manga Archive) plugin repository. Please **read it carefully** if you are a new contributor or lack experience with the required languages and technologies. + +This is a living document that evolves over time. If you find issues or have suggestions, feel free to open an [Issue](https://github.com/InvalidDavid/UMA/issues) or submit a Pull Request. + +## Table of Contents + +- [Contributing to UMA](#contributing-to-uma) + - [Table of Contents](#table-of-contents) + - [Prerequisites](#prerequisites) + - [Required Technologies](#required-technologies) + - [Required Tools](#required-tools) + - [Recommended Tools](#recommended-tools) + - [Project Structure](#project-structure) + - [Setup & Build](#setup--build) + - [Understanding the Architecture](#understanding-the-architecture) + - [Core Parser Types](#core-parser-types) + - [Base Classes](#base-classes) + - [Creating a New Parser](#creating-a-new-parser) + - [Parser File Organization](#parser-file-organization) + - [Annotating Your Parser](#annotating-your-parser) + - [Available Utilities & Helpers](#available-utilities--helpers) + - [HTTP Utilities](#http-utilities) + - [JSON Utilities](#json-utilities) + - [HTML/Parsing Utilities](#htmlparsing-utilities) + - [Date & Time Utilities](#date--time-utilities) + - [String & URL Utilities](#string--url-utilities) + - [Model Utilities](#model-utilities) + - [Data Models](#data-models) + - [Manga Model](#manga-model) + - [MangaChapter Model](#mangachapter-model) + - [MangaPage Model](#mangapage-model) + - [MangaTag Model](#mangatag-model) + - [Implementing Parser Methods](#implementing-parser-methods) + - [Popular/Latest Manga](#popularlatest-manga) + - [Search Functionality](#search-functionality) + - [Manga Details](#manga-details) + - [Chapters List](#chapters-list) + - [Page List (Images)](#page-list-images) + - [Filters & Search](#filters--search) + - [Filter Types](#filter-types) + - [Search Capabilities](#search-capabilities) + - [Advanced Features](#advanced-features) + - [Authentication](#authentication) + - [Pagination](#pagination) + - [Coroutines & Async](#coroutines--async) + - [Custom Interceptors](#custom-interceptors) + - [Configuration Keys](#configuration-keys) + - [Common Patterns & Best Practices](#common-patterns--best-practices) + - [URL Handling](#url-handling) + - [Error Handling](#error-handling) + - [Testing Your Parser](#testing-your-parser) + - [Building & Testing](#building--testing) + - [Submitting a Pull Request](#submitting-a-pull-request) + - [Troubleshooting](#troubleshooting) + +## Prerequisites + +Before you start contributing to UMA, ensure you have the following: + +### Required Technologies + +- **Kotlin** - The primary language for this project (Java 11+ compatible) + - Learn from: [Kotlin Official Documentation](https://kotlinlang.org/docs/home.html) + - Key concepts: coroutines, extension functions, DSLs, sealed classes + +- **Android Development Knowledge** + - Basic understanding of Android components + - Familiarity with OkHttp and network requests + +- **Web Scraping & Parsing** + - **HTML/CSS** - Understanding DOM structure and CSS selectors + - **JSoup** - HTML parsing library ([JSoup Documentation](https://jsoup.org/)) + - **JSON** - Parsing and working with JSON APIs + - **Regex** - Pattern matching for data extraction + +- **API Integration** + - RESTful API concepts + - HTTP methods (GET, POST, PUT) + - Headers and request/response handling + +### Required Tools + +- **Java 11 or higher** - JDK installation required +- **Android Studio** (Community Edition) or **IntelliJ IDEA** (Community Edition) +- **Gradle 8.0+** - Build automation (included in project) +- **Git** - Version control +- **Usagi App** (v0.0.32-beta2 or higher) - For testing locally + +### Recommended Tools + +- **Browser DevTools** (F12) - Inspect website structure and network requests +- **Postman** or **Insomnia** - Test API endpoints +- **Online JSoup Selector Tester** - Debug CSS selectors +- **Regex Tester** - Test and validate regex patterns +- **JSON Formatter** - Validate and format JSON responses + +## Project Structure + +``` +UMA/ +├── src/main/kotlin/org/koitharu/kotatsu/parsers/ +│ ├── parsers/ # Base parser implementations (Madara, Liliana, etc.) +│ ├── site/ +│ │ ├── kotatsu/ # Kotatsu-based parsers +│ │ │ ├── all/ # Multi-language sources (MangaDex, MangaPlus, etc.) +│ │ │ ├── en/ # English sources +│ │ │ ├── es/ # Spanish sources +│ │ │ ├── pt/ # Portuguese sources +│ │ │ └── ... # Other language codes +│ │ └── tachiyomi/ # Tachiyomi-compatible parsers +│ ├── util/ # Utility functions and helpers +│ ├── model/ # Data models (Manga, Chapter, Page, etc.) +│ └── core/ # Core parser base classes +├── plugins-ksp/ # KSP annotation processor (code generation) +├── buildSrc/ # Build configuration and custom tasks +├── gradle/ # Gradle wrapper and configuration +└── README.md, CONTRIBUTING.md + +``` + +## Setup & Build + +### Clone the Repository + +```bash +git clone https://github.com/InvalidDavid/UMA.git +cd UMA +``` + +### Build the Project + +**Linux/macOS:** +```bash +chmod +x gradlew +./gradlew buildJar +``` + +**Windows:** +```bash +gradlew.bat buildJar +``` + +**Using Android Studio:** +1. Open Android Studio +2. File → Open → Select UMA directory +3. Wait for Gradle to sync +4. Run `buildJar` task (Build → Gradle Tasks → buildJar) + +### Output + +- Raw JAR: `build/libs/raw.jar` +- DEXed JAR (for Usagi): `build/libs/uma.jar` + +## Understanding the Architecture + +### Core Parser Types + +UMA uses several parser base classes to handle different website structures: + +#### 1. **FlexibleMangaParser** (API-based sources) +Used for JSON/REST API sources. Example: MangaDex + +```kotlin +@MangaSourceParser("MANGADEXORG", "MangaDex") +internal class MangaDexParser(context: MangaLoaderContext) : + FlexibleMangaParser(context, MangaParserSource.MANGADEXORG) { + // Implementation +} +``` + +#### 2. **PagedMangaParser** (HTML-based sources) +Used for HTML scraping with pagination. Example: Madara CMS sites + +```kotlin +internal abstract class MadaraParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 12, +) : PagedMangaParser(context, source, pageSize) { + // Implementation +} +``` + +#### 3. **SimpleMangaParser** (Simple HTML sources) +Lightweight parser for straightforward HTML structures. + +```kotlin +@MangaSourceParser("MYSOURCE", "My Source") +internal class MyParser(context: MangaLoaderContext) : + SimpleMangaParser(context, MangaParserSource.MYSOURCE) { + // Implementation +} +``` + +### Base Classes + +| Base Class | Use Case | Key Features | +|-----------|----------|--------------| +| `FlexibleMangaParser` | JSON/REST APIs | Advanced filtering, sort orders, async operations | +| `PagedMangaParser` | HTML pagination | Tag-based filtering, search support, standard sorting | +| `SimpleMangaParser` | Simple HTML | Lightweight, minimal configuration | +| `MangaParserAuthProvider` | Authentication | Login support, cookie handling | + +## Creating a New Parser + +### Parser File Organization + +For a new parser, use this directory structure: + +``` +src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mysource/ +├── MySource.kt # Main parser class +├── Dto.kt # (Optional) Data transfer objects for JSON +├── Filters.kt # (Optional) Custom filters +└── Constants.kt # (Optional) Constants and helpers +``` + +### Annotating Your Parser + +Always annotate your parser class with `@MangaSourceParser`: + +```kotlin +@MangaSourceParser("MYSOURCEID", "My Source Display Name") +internal class MySourceParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MYSOURCEID) { + + override val domain = "example.com" + + // Required implementations +} +``` + +**Annotation Parameters:** +- First parameter: Unique source ID (UPPERCASE, no spaces) +- Second parameter: Display name shown in the app + +## Available Utilities & Helpers + +UMA provides extensive utilities in `util/` and extensions throughout the codebase. + +### HTTP Utilities + +#### Basic HTTP Requests + +```kotlin +// GET request +val response = webClient.httpGet(url) + +// POST request with parameters +val response = webClient.httpPost(url, mapOf("key" to "value")) + +// POST with form data +val response = webClient.httpPost(url, "raw=body&data") + +// Custom headers +val headers = mapOf( + "User-Agent" to "Mozilla/5.0...", + "Referer" to domain, + "X-Requested-With" to "XMLHttpRequest" +) +val response = webClient.httpGet(url, headers) +``` + +#### Response Handling + +```kotlin +// Parse as HTML +val doc = response.parseHtml() + +// Parse as JSON +val json = response.parseJson() + +// Get raw body +val body = response.body.string() + +// Close response properly +response.use { + // Process response +} +``` + +### JSON Utilities + +#### Parsing JSON + +```kotlin +import org.json.JSONObject +import org.json.JSONArray + +// Parse response +val json = webClient.httpGet(url).parseJson() + +// Access objects and arrays +val data = json.getJSONArray("data") +val item = data.getJSONObject(0) + +// Safe access +val title = item.optString("title", "Unknown") +val count = item.optInt("count", 0) + +// Get or null +val description = item.getStringOrNull("description") +``` + +#### Working with JSONArray + +```kotlin +val array = json.getJSONArray("items") + +// Iterate +repeat(array.length()) { i -> + val obj = array.getJSONObject(i) + // Process obj +} + +// Map +val list = array.mapJSON { obj -> + obj.getString("title") +} + +// Map to Set +val set = array.mapJSONToSet { obj -> + MangaTag( + key = obj.getString("id"), + title = obj.getString("name"), + source = source, + ) +} + +// Flatten (merge multiple JSON objects) +val flattened = jsonArray.flatten() +``` + +#### Serialization with Kotlinx + +```kotlin +import kotlinx.serialization.Serializable +import kotlinx.serialization.SerialName + +@Serializable +class MangaDto( + val id: Int, + val title: String, + @SerialName("cover_url") + val coverUrl: String, +) + +// Parse +val dto = json.decodeFromString<MangaDto>(jsonString) + +// Serialize +val jsonStr = Json.encodeToString(dto) +``` + +### HTML/Parsing Utilities + +#### JSoup Selection + +```kotlin +// Select first element +val title = doc.selectFirst("h1")?.text() + +// Select all matching +val items = doc.select("div.manga-item") + +// CSS selectors +val chapters = doc.select("div.chapter > a") + +// Complex selectors +val status = doc.selectFirst("span:contains(Status)~div")?.text() + +// Safe selection +val element = doc.selectFirstOrThrow("required.selector") +``` + +#### URL Extraction + +```kotlin +// Absolute URL +val absoluteUrl = element.absUrl("href") + +// Relative URL conversion +val relativeUrl = element.attrAsRelativeUrl("href") + +// Parse and build URLs +val httpUrl = url.toHttpUrl() +val newUrl = httpUrl.newBuilder() + .addQueryParameter("page", "1") + .build() +``` + +#### Text Extraction + +```kotlin +// Text with children +val text = element.text() + +// Own text only (no children) +val ownText = element.ownText() + +// Text or null +val value = element.textOrNull() + +// Title case conversion +val title = text.toTitleCase(locale) +``` + +#### HTML Parsing Patterns + +```kotlin +// Parse HTML fragment from JSON +val html = Jsoup.parseBodyFragment(htmlString, baseUrl) + +// Get parent elements +val parent = element.parents().firstOrNull { it.id() == "content" } + +// Filter elements +val filtered = elements.filter { it.hasAttr("data-id") } + +// Map elements +val mangas = elements.mapNotNull { el -> + val url = el.attr("href") ?: return@mapNotNull null + Manga(url = url, title = el.text()) +} +``` + +### Date & Time Utilities + +#### Simple Date Parsing + +```kotlin +import java.text.SimpleDateFormat +import java.util.* + +private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) + +// Parse date +val date = dateFormat.parse("2024-01-15") +val timestamp = date?.time ?: 0L + +// Safe parsing +val safeDate = dateFormat.parseSafe("invalid date") // Returns 0L + +// With timezone +val formatter = SimpleDateFormat("yyyy-MM-dd HH:mm:ss z", Locale.ENGLISH) +val timestamp = formatter.parseSafe(dateString) +``` + +#### Relative Date Parsing + +```kotlin +// Parse relative dates like "2 days ago" +fun parseRelativeDate(text: String): Long { + val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + + return when { + text.contains("hour") -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + text.contains("day") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + text.contains("week") -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis + text.contains("month") -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + else -> 0 + } +} +``` + +### String & URL Utilities + +#### String Encoding/Decoding + +```kotlin +// URL encode +val encoded = string.urlEncoded() + +// Decode from Base64 +val decoded = context.decodeBase64(base64String) + +// Encode to Base64 +val encoded = context.encodeBase64(byteArray) + +// Hex conversion +fun String.decodeHex(): ByteArray { + check(length % 2 == 0) { "Must have an even length" } + return chunked(2).map { it.toInt(16).toByte() }.toByteArray() +} +``` + +#### URL Manipulation + +```kotlin +// Generate unique ID +val uid = generateUid(url) + +// Build URLs +val url = buildString { + append("https://") + append(domain) + append("/manga/") + append(mangaId) +} + +// URL normalization +val normalized = url.removeSuffix('/').substringAfterLast('/') + +// Relative to absolute +val absoluteUrl = relativeUrl.toAbsoluteUrl(domain) +val relativeUrl = absoluteUrl.toRelativeUrl(domain) +``` + +### Model Utilities + +#### Building Models + +```kotlin +// Create Manga +val manga = Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + title = title, + altTitles = setOf(altTitle1, altTitle2), + coverUrl = coverUrl, + largeCoverUrl = largeCoverUrl, + rating = rating, + tags = tagSet, + authors = authorSet, + state = MangaState.ONGOING, + contentRating = ContentRating.SAFE, + description = description, + chapters = chaptersList, + source = source, +) + +// Create MangaChapter +val chapter = MangaChapter( + id = generateUid(chapterId), + url = chapterUrl, + title = chapterTitle, + number = 1.5f, + volume = 1, + branch = "English", + scanlator = "Scan Team", + uploadDate = timestamp, + source = source, +) + +// Create MangaPage +val page = MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = thumbnailUrl, + source = source, +) + +// Create MangaTag +val tag = MangaTag( + key = "action", + title = "Action", + source = source, +) +``` + +## Data Models + +### Manga Model + +```kotlin +data class Manga( + val id: Long, // Unique identifier + val title: String, // Manga title + val altTitles: Set<String> = emptySet(), // Alternative titles + val url: String, // Relative URL to manga page + val publicUrl: String = "", // Full URL for user access + val rating: Float = RATING_UNKNOWN, // 0-10, RATING_UNKNOWN for unknown + val contentRating: ContentRating? = null, // SAFE, ADULT, SUGGESTIVE + val coverUrl: String? = null, // Cover thumbnail + val largeCoverUrl: String? = null, // Full resolution cover + val description: String? = null, // Manga synopsis + val tags: Set<MangaTag> = emptySet(), // Genre/tag set + val authors: Set<String> = emptySet(), // Author names + val state: MangaState? = null, // ONGOING, FINISHED, ABANDONED, etc. + val chapters: List<MangaChapter>? = null, // Chapter list + val source: MangaParserSource, // Source identifier +) +``` + +### MangaChapter Model + +```kotlin +data class MangaChapter( + val id: Long, // Unique ID + val url: String, // Relative URL to chapter + val title: String? = null, // Chapter title/name + val number: Float = 0f, // Chapter number (can be decimal) + val volume: Int = 0, // Volume number + val branch: String? = null, // Scanlation branch/translation + val scanlator: String? = null, // Scanlation group + val uploadDate: Long = 0L, // Unix timestamp in milliseconds + val source: MangaParserSource, // Source identifier +) +``` + +### MangaPage Model + +```kotlin +data class MangaPage( + val id: Long, // Unique ID + val url: String, // Image URL + val preview: String? = null, // Preview/thumbnail URL + val source: MangaParserSource, // Source identifier +) +``` + +### MangaTag Model + +```kotlin +data class MangaTag( + val key: String, // Unique tag key (used in queries) + val title: String, // Display title + val source: MangaParserSource, // Source identifier +) +``` + +## Implementing Parser Methods + +### Popular/Latest Manga + +```kotlin +override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + append("/manga/?page=") + append(page + 1) + + when (order) { + SortOrder.UPDATED -> append("&sort=latest") + SortOrder.POPULARITY -> append("&sort=popular") + SortOrder.NEWEST -> append("&sort=new") + SortOrder.ALPHABETICAL -> append("&sort=az") + else -> {} + } + } + + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div.manga-item").map { element -> + val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = element.selectFirst("h3")?.text().orEmpty(), + coverUrl = element.selectFirst("img")?.src(), + source = source, + ) + } +} +``` + +### Search Functionality + +```kotlin +override suspend fun getList( + page: Int, + query: String, + filter: MangaListFilter, +): List<Manga> { + val url = buildString { + append("https://") + append(domain) + append("/search/?q=") + append(query.urlEncoded()) + append("&page=") + append(page) + + filter.tags.forEach { tag -> + append("&genre=") + append(tag.key) + } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) +} +``` + +### Manga Details + +```kotlin +override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chapters = async { getChapters(doc) } + + return manga.copy( + title = doc.selectFirst("h1")?.text() ?: manga.title, + description = doc.selectFirst("div.synopsis")?.text(), + coverUrl = doc.selectFirst("img.cover")?.src(), + tags = doc.select("a.tag").mapNotNullToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('/'), + title = a.text().toTitleCase(), + source = source, + ) + }, + state = when (doc.selectFirst(".status")?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + else -> null + }, + chapters = chapters.await(), + ) +} +``` + +### Chapters List + +```kotlin +suspend fun getChapters(doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH) + + return doc.select("tr.chapter").mapChapters(reversed = true) { i, tr -> + val a = tr.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + + MangaChapter( + id = generateUid(href), + url = href, + title = a.text(), + number = i + 1f, + uploadDate = dateFormat.parseSafe( + tr.selectFirst("td.date")?.text() + ), + source = source, + ) + } +} +``` + +### Page List (Images) + +```kotlin +override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select("img.page").mapIndexed { index, img -> + MangaPage( + id = generateUid(img.src()), + url = img.absUrl("src"), + source = source, + ) + } +} +``` + +## Filters & Search + +### Filter Types + +```kotlin +override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + ), + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.ADULT, + ), + ) +} + +suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/genres/").parseHtml() + return doc.select("a.genre-link").mapNotNullToSet { a -> + MangaTag( + key = a.attr("data-id"), + title = a.text().toTitleCase(), + source = source, + ) + } +} +``` + +### Search Capabilities + +```kotlin +override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = true, + isAuthorSearchSupported = false, + ) +``` + +## Advanced Features + +### Authentication + +```kotlin +internal class AuthenticatedParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MYSOURCE), + MangaParserAuthProvider { + + override val authUrl: String + get() = "https://$domain/login" + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name == "auth_token" + } + } + + override suspend fun getUsername(): String { + val body = webClient.httpGet("https://$domain/profile/").parseHtml().body() + return body.selectFirst(".username")?.text() + ?: throw AuthRequiredException(source) + } +} +``` + +### Pagination + +```kotlin +// Set pagination starting page +init { + paginator.firstPage = 0 // Start from page 0 + searchPaginator.firstPage = 1 // Or start from page 1 +} + +override suspend fun getListPage(page: Int, ...): List<Manga> { + val pageNumber = page + 1 // If firstPage = 0, convert to 1-based + // or use page directly if firstPage = 1 +} +``` + +### Coroutines & Async + +```kotlin +override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + + // Run multiple requests concurrently + val docDeferred = async { webClient.httpGet(fullUrl).parseHtml() } + val chaptersDeferred = async { loadChapters(manga.url) } + + val doc = docDeferred.await() + val chapters = chaptersDeferred.await() + + // Process and return + manga.copy( + description = doc.selectFirst(".desc")?.text(), + chapters = chapters, + ) +} +``` + +### Custom Interceptors + +```kotlin +override fun OkHttpClient.Builder.configureClient(): OkHttpClient.Builder { + return addInterceptor { chain -> + val request = chain.request() + val newRequest = request.newBuilder() + .header("User-Agent", "Custom/1.0") + .header("Accept-Language", "en-US") + .build() + chain.proceed(newRequest) + } +} +``` + +### Configuration Keys + +```kotlin +override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + + keys.add(ConfigKey.UserAgent("MySource/1.0")) + keys.add(ConfigKey.Domain(domain)) + + val qualityKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "high" to "High Quality", + "med" to "Medium Quality", + ), + defaultValue = "high", + ) + keys.add(qualityKey) +} +``` + +## Common Patterns & Best Practices + +### URL Handling + +```kotlin +// ✅ CORRECT: Using relative URLs +val manga = Manga( + url = "/manga/slug-123", + publicUrl = "/manga/slug-123".toAbsoluteUrl(domain), +) + +// ❌ AVOID: Hardcoding full URLs +val manga = Manga( + url = "https://example.com/manga/slug-123" +) + +// ✅ CORRECT: Building complex URLs +val url = buildString { + append("https://api.") + append(domain) + append("/search?q=") + append(query.urlEncoded()) + filter.tags.forEach { tag -> + append("&tags[]=") + append(tag.key) + } +} +``` + +### Error Handling + +```kotlin +// Handle HTTP errors +try { + val response = webClient.httpGet(url).parseHtml() + return parseMangaList(response) +} catch (e: HttpStatusException) { + when (e.statusCode) { + 404 -> return emptyList() // No results + 429 -> throw ParseException("Rate limited", url) + else -> throw ParseException("HTTP ${e.statusCode}", url) + } +} + +// Safe parsing +val title = doc.selectFirst("h1")?.text() ?: "Unknown" +val count = element.attr("data-count").toIntOrNull() ?: 0 +``` + +### Testing Your Parser + +```kotlin +// Manual testing with main function +suspend fun main() { + val context = MangaLoaderContext(...) + val parser = MySourceParser(context) + + val mangas = parser.getListPage(0, SortOrder.UPDATED, MangaListFilter()) + println("Found ${mangas.size} manga") + + if (mangas.isNotEmpty()) { + val details = parser.getDetails(mangas[0]) + println("Title: ${details.title}") + println("Chapters: ${details.chapters?.size ?: 0}") + } +} +``` + +## Building & Testing + +### Build Commands + +```bash +# Build JAR +./gradlew buildJar + +# Build without DEX +./gradlew jar + +# Clean build +./gradlew clean buildJar + +# Run tests +./gradlew test + +# Lint check +./gradlew lintRelease +``` + +### Installation for Testing + +1. Build the project: `./gradlew buildJar` +2. Transfer `build/libs/uma.jar` to your device +3. In Usagi App: + - Explore → Manage Sources → Manage Plugins + - Click "+" → Import from local storage + - Select the JAR file +4. Enable the plugin and test + +### Debugging + +- Use `println()` and check logcat: `adb logcat | grep "UMA"` +- Add breakpoints in Android Studio debugger +- Check network requests with OkHttp logging + +## Submitting a Pull Request + +### Before Submitting + +1. **Verify your code compiles** + ```bash + ./gradlew build + ``` + +2. **Test your parser thoroughly** + - Popular/Latest manga loads + - Search functionality works + - Chapter list displays correctly + - Images load without errors + +3. **Follow code style** + - Use 4-space indentation + - Remove unused imports + - Keep lines under 120 characters where possible + +4. **Commit message format** + ``` + feat: add MySource parser + + - Supports manga listing and search + - Handles chapter pagination + - Full image support + ``` + +### PR Checklist + +- [ ] Parser compiles without errors +- [ ] Tested on device/emulator +- [ ] New parser follows existing conventions +- [ ] DTOs use @Serializable correctly +- [ ] No hardcoded test URLs or API keys +- [ ] No unnecessary dependencies added +- [ ] Comments added for complex logic +- [ ] Parser name is accurate and follows capitalization + +## Troubleshooting + +### Common Issues + +**"Cannot resolve symbol" errors** +- Run `./gradlew clean build` +- Invalidate Android Studio caches: File → Invalidate Caches + +**"Gradle build failed"** +- Update Gradle: `./gradlew wrapper --gradle-version 8.1` +- Delete `build/` and try again + +**"Parser not appearing in Usagi"** +- Check if JAR was built: `ls build/libs/uma.jar` +- Verify `@MangaSourceParser` annotation is present +- Check app version (v0.0.32-beta2+) + +**"HTTP 403/Cloudflare errors"** +- Add User-Agent header +- Implement interceptor for headers +- Some sites require extra delay between requests + +**"Parse failed - element not found"** +- Website HTML structure may have changed +- Use browser DevTools to inspect current structure +- Update CSS selectors accordingly + +### Getting Help + +- Check existing parsers for patterns +- Review [JSoup Documentation](https://jsoup.org/) +- Ask in [Discord Server](https://discord.gg/CyJeVDP7Cw) +- Open an issue with error logs + +--- + +**Thank you for contributing to UMA!** Your work makes manga reading better for everyone. 🎉 From e4ec215f33d74db1a4afdfd4a322a95f407d60eb Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 13:36:21 +0200 Subject: [PATCH 385/434] Update CONTRIBUTING.md --- CONTRIBUTING.md | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 43c2ff07..96941724 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -952,25 +952,6 @@ val title = doc.selectFirst("h1")?.text() ?: "Unknown" val count = element.attr("data-count").toIntOrNull() ?: 0 ``` -### Testing Your Parser - -```kotlin -// Manual testing with main function -suspend fun main() { - val context = MangaLoaderContext(...) - val parser = MySourceParser(context) - - val mangas = parser.getListPage(0, SortOrder.UPDATED, MangaListFilter()) - println("Found ${mangas.size} manga") - - if (mangas.isNotEmpty()) { - val details = parser.getDetails(mangas[0]) - println("Title: ${details.title}") - println("Chapters: ${details.chapters?.size ?: 0}") - } -} -``` - ## Building & Testing ### Build Commands @@ -1042,7 +1023,6 @@ suspend fun main() { - [ ] Parser compiles without errors - [ ] Tested on device/emulator - [ ] New parser follows existing conventions -- [ ] DTOs use @Serializable correctly - [ ] No hardcoded test URLs or API keys - [ ] No unnecessary dependencies added - [ ] Comments added for complex logic From 64b070c1d4a691bf74b2b0915473ebe278865d3c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 14:34:26 +0200 Subject: [PATCH 386/434] Create RimuScans.kt --- src/main/kotlin/tsuki/site/fr/RimuScans.kt | 334 +++++++++++++++++++++ 1 file changed, 334 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/RimuScans.kt diff --git a/src/main/kotlin/tsuki/site/fr/RimuScans.kt b/src/main/kotlin/tsuki/site/fr/RimuScans.kt new file mode 100644 index 00000000..583b5792 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/RimuScans.kt @@ -0,0 +1,334 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.json.extractNextJs + +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("RIMUSCANS", "Rimu Scans", "fr") +class RimuScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.RIMUSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("rimuscan.fr") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.FRENCH).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val genresCache = mutableListOf<String>() + private var genresLoaded = false + + private suspend fun loadGenres() { + if (genresLoaded) return + runCatching { + val json = webClient.httpGet("https://$domain/api/admin/genres").parseJson() + val arr = json.optJSONArray("genres") ?: return + genresCache.clear() + for (i in 0 until arr.length()) { + genresCache.add(arr.getString(i)) + } + genresLoaded = true + } + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = run { + loadGenres() + genresCache.map { MangaTag(key = it, title = it, source = source) }.toSet()}, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return "https://$domain/api/series".toHttpUrl().newBuilder().apply { + val query = filter.query + if (!query.isNullOrEmpty()) { + addQueryParameter("search", query) + } else { + when (order) { + SortOrder.POPULARITY -> addQueryParameter("sort", "popular") + SortOrder.RATING -> addQueryParameter("sort", "rating") + SortOrder.ALPHABETICAL -> addQueryParameter("sort", "az") + SortOrder.UPDATED -> addQueryParameter("sort", "updated") + SortOrder.NEWEST -> addQueryParameter("sort", "new") + else -> addQueryParameter("sort", "updated") + } + if (filter.types.isNotEmpty()) { + val typeParam = filter.types.mapNotNull { type -> + when (type) { + ContentType.MANHWA -> "webtoon" + ContentType.MANGA -> "manga" + else -> null + } + }.joinToString(",") + if (typeParam.isNotEmpty()) addQueryParameter("types", typeParam) + } + if (filter.states.isNotEmpty()) { + val statusParam = filter.states.mapNotNull { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> null + } + }.joinToString(",") + if (statusParam.isNotEmpty()) addQueryParameter("status", statusParam) + } + if (filter.tags.isNotEmpty()) { + addQueryParameter("genres", filter.tags.joinToString(",") { it.key }) + } + } + addQueryParameter("page", page.toString()) + }.build().toString() + } + + private fun parseSeriesList(json: JSONObject): List<Manga> { + val seriesArr = json.optJSONArray("series") ?: return emptyList() + return (0 until seriesArr.length()).mapNotNull { i -> + val obj = seriesArr.getJSONObject(i) + val slug = obj.optString("slug").takeIf { it.isNotEmpty() } + val title = obj.optString("title").takeIf { it.isNotEmpty() } + val cover = obj.optString("cover_url").takeIf { it.isNotEmpty() } + if (slug == null || title == null) { + null + } else { + Manga( + id = generateUid(slug), + title = title, + altTitles = emptySet(), + contentRating = null, + url = "/manga/$slug", + publicUrl = "https://$domain/manga/$slug", + coverUrl = cover?.toAbsoluteUrl(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl()).parseHtml() + val details = parseDetailsFromLd(doc, manga) + val chapters = extractChapters(doc, manga.url.substringAfterLast("/")) + details.copy( + chapters = chapters, + ) + } + + private fun parseDetailsFromLd(doc: Document, fallback: Manga): Manga { + val ldScripts = doc.select("script[type=application/ld+json]") + for (script in ldScripts) { + val data = script.data() + if (data.contains("\"ComicSeries\"")) { + val json = JSONObject(data) + val name = json.optString("name").ifEmpty { fallback.title } + val description = json.optString("description") + val image = json.optString("image") + val author = json.optJSONObject("author")?.optString("name") + val illustrator = json.optJSONObject("illustrator")?.optString("name") + val genres = json.optJSONArray("genre")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val altNames = json.optJSONArray("alternateName")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + + val h1 = doc.selectFirst("h1") + val badges = h1?.previousElementSibling()?.select("span")?.map { it.text().trim() } ?: emptyList() + val typeLabel = badges.getOrNull(0) + val statusLabel = badges.getOrNull(1) + val state = when (statusLabel?.lowercase()) { + "en cours", "ongoing" -> MangaState.ONGOING + "terminé", "termine", "completed" -> MangaState.FINISHED + "en pause", "hiatus", "on hiatus" -> MangaState.PAUSED + "annulé", "annule", "abandonné", "abandonne", "cancelled" -> MangaState.ABANDONED + else -> null + } + val contentType = when (typeLabel?.lowercase()) { + "webtoon", "manhwa" -> ContentType.MANHWA + "manhua" -> ContentType.MANHUA + "manga" -> ContentType.MANGA + else -> null + } + val allGenres = buildSet { + if (contentType != null) add(contentType.toString().lowercase()) + genres.forEach { add(it) } + } + val tags = allGenres.map { MangaTag(key = it.lowercase(), title = it, source = source) }.toSet() + val fullDescription = buildString { + if (description.isNotEmpty()) append(description) + if (altNames.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Titres alternatifs : ") + append(altNames.joinToString(", ")) + } + }.ifEmpty { null } + + return fallback.copy( + title = name, + description = fullDescription, + coverUrl = image.toAbsoluteUrl(), + authors = setOfNotNull(author, illustrator).filter { it.isNotEmpty() }.toSet(), + tags = tags, + state = state, + ) + } + } + return fallback + } + + private fun extractChapters(doc: Document, mangaSlug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + val seenNumbers = mutableSetOf<Double>() + + doc.extractNextJs { element -> + if (element is JSONObject && element.has("number") && element.has("type")) { + runCatching { + val number = element.optDouble("number", Double.NaN) + if (!number.isNaN() && seenNumbers.add(number)) { + val type = element.optString("type", "NORMAL") + val title = element.optString("title", "").trim() + val releaseDate = element.optString("releaseDate").ifEmpty { null } + val numberString = number.toString().substringBefore(".0") + val chapterName = buildString { + append("Chapitre $numberString") + if (title.isNotEmpty() && title != "Chapitre $numberString") { + if (title.contains("Chapitre", true) || title.contains("Chapter", true)) { + clear() + append(title) + } else { + append(" : $title") + } + } + }.let { if (type.equals("PREMIUM", true)) "🔒 $it" else it } + val date = releaseDate?.let { dateFormat.parse(it)?.time } ?: 0L + chapters.add( + MangaChapter( + id = generateUid("$mangaSlug-$number"), + url = "/read/$mangaSlug/$numberString", + title = chapterName, + number = number.toFloat(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + ) + } + } + } + false + } + + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl()).parseHtml() + val chapterNumber = chapter.number.toDouble() + val chapterObjects = mutableListOf<JSONObject>() + + doc.extractNextJs { element -> + if (element is JSONObject && element.has("number") && element.has("images")) { + chapterObjects.add(element) + } + false + } + + val chapterObj = chapterObjects.firstOrNull { + it.optDouble("number") == chapterNumber && (it.optJSONArray("images")?.length() ?: 0) > 0 + } ?: chapterObjects.firstOrNull { it.optDouble("number") == chapterNumber } + ?: return emptyList() + + if (chapterObj.optString("type").equals("PREMIUM", true)) { + throw ParseException("Ce chapitre est premium. Lisez-le sur le site.", chapter.url.toAbsoluteUrl()) + } + + val imagesArr = chapterObj.optJSONArray("images") ?: return emptyList() + val images = mutableListOf<Pair<Int, String>>() + for (i in 0 until imagesArr.length()) { + val imgObj = imagesArr.getJSONObject(i) + val order = imgObj.optInt("order") + val url = imgObj.optString("url").takeIf { it.isNotEmpty() } ?: continue + images.add(order to url) + } + return images.sortedBy { it.first }.mapIndexed { _, (_, url) -> + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(), + preview = null, + source = source, + ) + } + } + + private fun okhttp3.Response.parseJson(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + private fun String.toAbsoluteUrl(): String = when { + startsWith("http") -> this + startsWith("/") -> "https://$domain$this" + else -> "https://$domain/$this" + } +} From 062d6fc60b4fdc68da92d9e0c41287ec360b02e8 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:41:39 +0200 Subject: [PATCH 387/434] Create OriginesParser.kt --- .../kotlin/tsuki/parsers/OriginesParser.kt | 329 ++++++++++++++++++ 1 file changed, 329 insertions(+) create mode 100644 src/main/kotlin/tsuki/parsers/OriginesParser.kt diff --git a/src/main/kotlin/tsuki/parsers/OriginesParser.kt b/src/main/kotlin/tsuki/parsers/OriginesParser.kt new file mode 100644 index 00000000..5fbb9171 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/OriginesParser.kt @@ -0,0 +1,329 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONObject +import org.jsoup.Jsoup +import java.time.LocalDate +import java.time.ZoneId +import java.util.EnumSet +import java.util.Locale + +abstract class OriginesParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 30, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + /** Path prefix for series, e.g. "manga" or "oeuvre". */ + protected abstract val mangaPath: String + + /** Legacy paths used by older URLs. */ + protected open val legacyMangaPaths: Set<String> = emptySet() + + /** Genres as label to slug. */ + protected abstract val genres: List<Pair<String, String>> + + /** Origins as label to slug (optional). */ + protected open val origins: List<Pair<String, String>> = emptyList() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val knownPaths: Set<String> by lazy { legacyMangaPaths + mangaPath } + + private fun String.pathSegments(): List<String> { + val path = if (startsWith("http")) { + toHttpUrlOrNull()?.encodedPath ?: this + } else this + return path.substringBefore('?') + .substringBefore('#') + .split('/') + .filter { it.isNotBlank() && it !in knownPaths } + } + + private fun String.toMangaSlug(): String = pathSegments().firstOrNull() ?: this + + private fun String.toChapterSlug(): String = pathSegments().take(2).joinToString("/") + + private suspend fun getCatalogue( + page: Int, + query: String = "", + genres: String = "", + status: String = "tous", + rating: String = "0", + origin: String = "", + sort: String = "recents", + chapterMin: String = "0", + chapterMax: String = "0", + ): List<Manga> { + val form = mapOf( + "action" to "madara_child_catalogue", + "s" to query, + "genres" to genres, + "statut" to status, + "note" to rating, + "origine" to origin, + "tri" to sort, + "chmin" to chapterMin, + "chmax" to chapterMax, + "page" to page.toString(), + "auteur" to "", + "artiste" to "", + "annee" to "", + ) + + val response = webClient.httpPost("https://$domain/wp-admin/admin-ajax.php", form) + val json = response.parseJson() + val html = json.getJSONObject("data").optString("html") + if (html.isEmpty()) return emptyList() + + val doc = Jsoup.parseBodyFragment(html) + return doc.select("a.ori-card:has(span.ori-card-title)").map { element -> + val href = element.attr("href").toMangaSlug() + val title = element.selectFirst("span.ori-card-title")!!.text() + val cover = element.selectFirst("img")?.absUrl("src") + Manga( + id = generateUid(href), + url = href, + publicUrl = "https://$domain/$mangaPath/$href/", + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.POPULARITY -> "populaire" + SortOrder.RATING -> "notes" + SortOrder.ALPHABETICAL -> "az" + else -> "recents" + } + + val genreTags = filter.tags.filter { !it.key.startsWith(ORIGIN_PREFIX) } + val originTags = filter.tags.filter { it.key.startsWith(ORIGIN_PREFIX) } + val genreParam = genreTags.joinToString(",") { it.key } + val originParam = originTags.joinToString(",") { it.key.removePrefix(ORIGIN_PREFIX) } + + val statusParam = filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> "en-cours" + MangaState.FINISHED -> "termine" + else -> "tous" + } + } ?: "tous" + + return getCatalogue( + page = page, + query = filter.query.orEmpty(), + genres = genreParam, + status = statusParam, + origin = originParam, + sort = sort, + ) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = buildSet { + genres.forEach { (title, slug) -> + add(MangaTag(key = slug, title = title, source = source)) + } + }, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + ) + + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val slug = manga.url.toMangaSlug() + val fullUrl = "https://$domain/$mangaPath/$slug/" + val doc = webClient.httpGet(fullUrl).parseHtml() + + val infos = doc.select("div.ori-sr-infos dt").associate { dt -> + dt.text().lowercase(Locale.FRENCH) to dt.nextElementSibling()?.text().orEmpty() + } + + val title = doc.selectFirst("h1.ori-sr-title")?.text() ?: manga.title + val cover = doc.selectFirst("div.ori-sr-cover img")?.absUrl("src") ?: manga.coverUrl.orEmpty() + + val description = buildString { + doc.select("div.ori-sr-syn-texte p").eachText().forEach { appendLine(it) } + infos["nom alternatif"]?.takeIf { it.isNotBlank() }?.let { + appendLine() + append("Nom alternatif: ", it) + } + }.trim() + + val authors = listOfNotNull( + infos["auteur"]?.takeIf { it.isNotBlank() }, + infos["scénario"]?.takeIf { it.isNotBlank() }, + infos["artiste"]?.takeIf { it.isNotBlank() }, + infos["dessin"]?.takeIf { it.isNotBlank() } + ).distinct().toSet() + + val tags = doc.select("div.ori-sr-genres a.ori-sr-genre").asSequence().map { it.text() } + .plus(infos["type"].orEmpty()) + .filter { it.isNotBlank() } + .map { MangaTag(it, it, source) } + .toSet() + + val state = when (infos["statut"]?.lowercase(Locale.FRENCH)) { + "en cours" -> MangaState.ONGOING + "terminé" -> MangaState.FINISHED + "en pause" -> MangaState.PAUSED + "abandonné", "annulé" -> MangaState.ABANDONED + else -> null + } + + val chaptersDeferred = async { fetchChapters(slug) } + + manga.copy( + url = slug, + publicUrl = fullUrl, + title = title, + altTitles = setOfNotNull(infos["nom alternatif"]?.takeIf { it.isNotBlank() }), + rating = doc.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + coverUrl = cover, + description = description, + authors = authors, + tags = tags, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + private suspend fun fetchChapters(slug: String): List<MangaChapter> { + val doc = webClient.httpPost( + "https://$domain/$mangaPath/$slug/ajax/chapters/", + emptyMap() + ).parseHtml() + return doc.select("div.ori-chl-row").map { element -> + val link = element.selectFirst("a.ori-chl-corps")!! + val href = link.attr("href").toChapterSlug() + val name = element.selectFirst("span.ori-chl-nom")?.text() ?: link.text() + val dateText = element.selectFirst("span.ori-chl-date")?.attr("title") + ?: element.selectFirst("span.ori-chl-date")?.text() + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = parseChapterDate(dateText), + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = "https://$domain/$mangaPath/${chapter.url}/" + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.reading-content img.wp-manga-chapter-img").mapIndexed { _, img -> + val url = when { + img.hasAttr("data-src") -> img.absUrl("data-src").trim() + else -> img.absUrl("src").trim() + } + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val regex = Regex("""(\d{1,2})\s+(\p{L}+)\.?(?:\s+(\d{4}))?""") + val match = regex.find(date) ?: return 0L + val (day, month, year) = match.destructured + val monthNumber = monthNumber(month) ?: return 0L + val today = LocalDate.now(ZoneId.of("Europe/Paris")) + return runCatching { + var chapterDate = LocalDate.of( + year.toIntOrNull() ?: today.year, + monthNumber, + day.toInt() + ) + if (year.isEmpty() && chapterDate.isAfter(today)) { + chapterDate = chapterDate.minusYears(1) + } + chapterDate.atStartOfDay(ZoneId.of("Europe/Paris")).toInstant().toEpochMilli() + }.getOrDefault(0L) + } + + private fun monthNumber(month: String): Int? { + val name = month.lowercase(Locale.FRENCH) + return when { + name.startsWith("jan") -> 1 + name.startsWith("fev") || name.startsWith("fév") -> 2 + name.startsWith("mar") -> 3 + name.startsWith("avr") -> 4 + name.startsWith("mai") -> 5 + name.startsWith("juin") -> 6 + name.startsWith("juil") -> 7 + name.startsWith("ao") -> 8 + name.startsWith("sep") -> 9 + name.startsWith("oct") -> 10 + name.startsWith("nov") -> 11 + name.startsWith("dec") || name.startsWith("déc") -> 12 + else -> null + } + } + + private fun okhttp3.Response.parseJson(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + companion object { + private const val ORIGIN_PREFIX = "origin:" + } +} From 06876dc43936dbeae95a0e156944802f78f4e111 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:42:07 +0200 Subject: [PATCH 388/434] Update MangasOriginesFr.kt --- .../kotlin/tsuki/site/fr/MangasOriginesFr.kt | 250 +++++------------- 1 file changed, 71 insertions(+), 179 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt index c4c4a50e..69cfac1d 100644 --- a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -2,187 +2,79 @@ package tsuki.site.fr import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser +import tsuki.parsers.OriginesParser -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.SortOrder -import tsuki.util.extractChapterNumber -import tsuki.util.generateUid -import tsuki.util.parseHtml - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import java.time.LocalDate -import java.time.ZoneId -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("MANGASORIGINESFR", "Mangas-Origines.fr", "fr") -internal class MangasOriginesFr(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGASORIGINESFR, "mangas-origines.fr") { - - override val datePattern = "dd/MM/yyyy" - override val tagPrefix = "manga-genres/" - override val listUrl = "oeuvre/" - - override val availableSortOrders: Set<SortOrder> = setOf( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, +@MangaSourceParser("MANGASORIGINESFR", "Mangas Origines", "fr") +class MangasOriginesFr(context: MangaLoaderContext) : + OriginesParser(context, MangaParserSource.MANGASORIGINESFR, "mangas-origines.fr") { + + override val mangaPath = "oeuvre" + + override val legacyMangaPaths = setOf("catalogues") + + override val genres = listOf( + "Action" to "action", + "Adventure" to "adventure", + "Amitié" to "amitie", + "Amour" to "amour", + "Art Martiaux" to "art-martiaux", + "Aventure" to "aventure", + "BL" to "bl", + "Boys" to "boys", + "Combat" to "combat", + "Comedy" to "comedy", + "Comédie" to "comedie", + "Dark Fantasy" to "dark-fantasy", + "Drama" to "drama", + "Drame" to "drame", + "Dystopie" to "dystopie", + "Démon" to "demon", + "Ecchi" to "ecchi", + "Erotique" to "erotique", + "Fantastique" to "fantastique", + "Fantasy" to "fantasy", + "Guerre" to "guerre", + "Harem" to "harem", + "Historique" to "historique", + "Horreur" to "horreur", + "Isekai" to "isekai", + "Jeu" to "jeu", + "Josei" to "josei", + "Magie" to "magie", + "Malédiction" to "malediction", + "Mature" to "mature", + "Moderne" to "moderne", + "Mort" to "mort", + "Murim" to "murim", + "Musique" to "musique", + "Mystère" to "mystere", + "Novel" to "novel", + "Post-Apo" to "post-apo", + "Prison" to "prison", + "Psychologique" to "psychologique", + "Religion" to "religion", + "Returner" to "returner", + "Romance" to "romance", + "Réincarnation" to "reincarnation", + "Régression" to "regression", + "School life" to "school-life", + "Sci-fi" to "sci-fi", + "Seinen" to "seinen", + "Shojo" to "shojo", + "Shonen" to "shonen", + "Slice of Life" to "slice-of-life", + "Société" to "societe", + "Sorcellerie" to "sorcellerie", + "Sport" to "sport", + "Steampunk" to "steampunk", + "Supernaturel" to "supernaturel", + "Surnaturel" to "surnaturel", + "Tragédie" to "tragedie", + "Vengeance" to "vengeance", + "Webcomic" to "webcomic", + "Yuri" to "yuri", + "École" to "ecole", ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), -// availableContentTypes = EnumSet.of( -// ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA -// ), - availableStates = EnumSet.of( - MangaState.ONGOING, MangaState.FINISHED - ), - ) - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val slug = manga.publicUrl.toMangaSlug() - val fullUrl = "https://$domain/oeuvre/$slug/" - val doc = webClient.httpGet(fullUrl).parseHtml() - - val title = doc.selectFirst("h1.ori-sr-title")?.text() ?: manga.title - val cover = doc.selectFirst("div.ori-sr-cover img")?.attr("abs:src") ?: manga.coverUrl.orEmpty() - val description = buildString { - doc.select("div.ori-sr-syn-texte p").eachText().forEach { appendLine(it) } - }.trim() - - val infos = doc.select("div.ori-sr-infos dt").associate { dt -> - dt.text().lowercase(Locale.FRENCH) to dt.nextElementSibling()?.text().orEmpty() - } - - val state = when (infos["statut"]?.lowercase(Locale.FRENCH)) { - "en cours" -> MangaState.ONGOING - "terminé" -> MangaState.FINISHED - "en pause" -> MangaState.PAUSED - "abandonné", "annulé" -> MangaState.ABANDONED - else -> null - } - - val altTitle = infos["nom alternatif"]?.takeIf { it.isNotBlank() } - val authors = listOfNotNull( - infos["auteur"]?.takeIf { it.isNotBlank() }, - infos["scénario"]?.takeIf { it.isNotBlank() }, - infos["artiste"]?.takeIf { it.isNotBlank() }, - infos["dessin"]?.takeIf { it.isNotBlank() } - ).distinct().toSet() - - val tags = doc.select("div.ori-sr-genres a.ori-sr-genre").map { - MangaTag(it.text(), it.text(), source) - }.toSet() - - val chaptersDeferred = async { fetchChapters(slug) } - - manga.copy( - url = slug, - title = title, - altTitles = setOfNotNull(altTitle), - coverUrl = cover, - description = description, - authors = authors, - tags = tags, - state = state, - chapters = chaptersDeferred.await(), - ) - } - - private suspend fun fetchChapters(slug: String): List<MangaChapter> { - val doc = webClient.httpPost( - "https://$domain/oeuvre/$slug/ajax/chapters/", - emptyMap() - ).parseHtml() - return doc.select("div.ori-chl-row").map { element -> - val link = element.selectFirst("a.ori-chl-corps")!! - val href = link.attr("href") - val name = element.selectFirst("span.ori-chl-nom")?.text() ?: link.text() - val dateText = element.selectFirst("span.ori-chl-date")?.text() - MangaChapter( - id = generateUid(href.toChapterSlug()), - title = name, - number = name.extractChapterNumber(), - volume = 0, - url = href.toChapterSlug(), - uploadDate = parseChapterDate(dateText), - source = source, - scanlator = null, - branch = null, - ) - }.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = "https://$domain/oeuvre/${chapter.url}/" - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.reading-content img.wp-manga-chapter-img").mapIndexed { _, img -> - val url = when { - img.hasAttr("data-src") -> img.absUrl("data-src").trim() - else -> img.absUrl("src").trim() - } - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - - private fun String.toMangaSlug(): String = trimEnd('/').substringAfterLast("/").substringBefore("?") - - private fun String.toChapterSlug(): String { - val parts = split("/").filter { it.isNotBlank() } - return if (parts.size >= 2) parts.takeLast(2).joinToString("/") else this - } - - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrBlank()) return 0L - val regex = Regex("""(\d{1,2})\s+(\p{L}+)\.?\s+(\d{4})""") - val match = regex.find(date) ?: return 0L - val (day, month, year) = match.destructured - val monthNumber = monthNumber(month) ?: return 0L - return try { - LocalDate.of(year.toInt(), monthNumber, day.toInt()) - .atStartOfDay(ZoneId.of("Europe/Paris")) - .toInstant() - .toEpochMilli() - } catch (_: Exception) { 0L } - } - - private fun monthNumber(month: String): Int? { - val name = month.lowercase(Locale.FRENCH) - return when { - name.startsWith("jan") -> 1 - name.startsWith("fev") || name.startsWith("fév") -> 2 - name.startsWith("mar") -> 3 - name.startsWith("avr") -> 4 - name.startsWith("mai") -> 5 - name.startsWith("juin") -> 6 - name.startsWith("juil") -> 7 - name.startsWith("ao") -> 8 - name.startsWith("sep") -> 9 - name.startsWith("oct") -> 10 - name.startsWith("nov") -> 11 - name.startsWith("dec") || name.startsWith("déc") -> 12 - else -> null - } - } } From 4c440ae66397e63560aba41849abf5177d03bc3a Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 15:42:35 +0200 Subject: [PATCH 389/434] Create HentaiOrigines.kt --- .../tsuki/site/fr/nsfw/HentaiOrigines.kt | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt diff --git a/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt b/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt new file mode 100644 index 00000000..2c0b7284 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt @@ -0,0 +1,86 @@ +package tsuki.site.fr.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.OriginesParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("HENTAIORIGINES", "Hentai Origines", "fr", ContentType.HENTAI) +class HentaiOrigines(context: MangaLoaderContext) : + OriginesParser(context, MangaParserSource.HENTAIORIGINES, "hentai-origines.com") { + + override val mangaPath = "manga" + + override val origins = listOf( + "Doujinshi" to "doujinshi", + "Pornhwa" to "pornhwa", + "Pornhua" to "pornhua", + "Hentai" to "hentai", + ) + + override val genres = listOf( + "Action" to "action", + "Alien" to "alien", + "Alpha" to "alpha", + "Amitié" to "amitie", + "Art martiaux" to "art-martiaux", + "Aventure" to "aventure", + "Belle-mère" to "belle-mere", + "Boy's Love" to "yaoi", + "Campus" to "campus", + "Comédie" to "comedie", + "Domination" to "domination", + "Drame" to "drame", + "Démon" to "demon", + "Ecchi" to "ecchi", + "Fantasy" to "fantasy", + "Futanari" to "futanari", + "Furry" to "furry", + "Fétichisme" to "fetichisme", + "Gallerie" to "gallerie", + "Gangster" to "gangster", + "Gofast" to "gofast", + "Gore" to "gore", + "Guideverse" to "guideverse", + "Hardcore" to "hardcore", + "Harem" to "harem", + "Historique" to "historique", + "Horreur" to "horreur", + "Humiliation" to "humiliation", + "Inceste" to "inceste", + "Isekai" to "isekai", + "Josei" to "josei", + "Loli" to "loli", + "Love" to "love", + "Magie" to "magie", + "Mature" to "mature", + "Milf" to "milf", + "Mini-série" to "mini-serie", + "Monsters girls" to "monsters-girls", + "Ntr" to "ntr", + "Office" to "office", + "Omégaverse" to "omegaverse", + "Oneshot" to "oneshot", + "Parodie" to "parodie", + "Professeur" to "professeur", + "Psychologie" to "psychologie", + "Rape" to "rape", + "Romance" to "romance", + "Réincarnation" to "reincarnation", + "School life" to "school-life", + "Sci-fi" to "sci-fi", + "Shonen-ai" to "shonen-ai", + "Slice of life" to "slice-of-life", + "Smut" to "smut", + "Soft" to "soft", + "Sport" to "sport", + "Surnaturel" to "surnaturel", + "Tomgirl" to "tomgirl", + "Tragédie" to "tragedie", + "Triangle amoureux" to "triangle-amoureux", + "Uncensored" to "uncensored", + "Yuri" to "yuri", + ) +} From a5ca889fc4d84c8aa2e5e5725067ed2df81aab8d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:21:03 +0200 Subject: [PATCH 390/434] Update OriginesParser.kt --- .../kotlin/tsuki/parsers/OriginesParser.kt | 79 +++++++++++++++---- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/OriginesParser.kt b/src/main/kotlin/tsuki/parsers/OriginesParser.kt index 5fbb9171..407bba74 100644 --- a/src/main/kotlin/tsuki/parsers/OriginesParser.kt +++ b/src/main/kotlin/tsuki/parsers/OriginesParser.kt @@ -21,6 +21,11 @@ import tsuki.model.SortOrder import tsuki.util.extractChapterNumber import tsuki.util.generateUid import tsuki.util.parseHtml +import tsuki.util.parseFailed +import tsuki.util.removeSuffix +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.mapNotNullToSet import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -41,18 +46,24 @@ abstract class OriginesParser( override val configKeyDomain = ConfigKey.Domain(domain) - /** Path prefix for series, e.g. "manga" or "oeuvre". */ + /** Path prefix for series */ protected abstract val mangaPath: String /** Legacy paths used by older URLs. */ protected open val legacyMangaPaths: Set<String> = emptySet() - /** Genres as label to slug. */ - protected abstract val genres: List<Pair<String, String>> - /** Origins as label to slug (optional). */ protected open val origins: List<Pair<String, String>> = emptyList() + /** Path used to scrape genres (relative to domain). */ + protected open val listUrl = "oeuvre/" + + /** Check madara. */ + protected open val tagPrefix = "manga-genres/" + + /** Hardcoded genres. */ + protected open val genres: List<Pair<String, String>> = emptyList() + override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.UPDATED, SortOrder.POPULARITY, @@ -166,19 +177,53 @@ abstract class OriginesParser( ) } - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = buildSet { - genres.forEach { (title, slug) -> - add(MangaTag(key = slug, title = title, source = source)) + protected open suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet<String>(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix('/').substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null } - }, - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - ), - ) + MangaTag( + key = href, + title = a.ownText().ifEmpty { + a.selectFirst(".menu-image-title")?.textOrNull() + }?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null, + source = source, + ) + } + } + override suspend fun getFilterOptions(): MangaListFilterOptions { + val allTags = mutableSetOf<MangaTag>() + try { + allTags.addAll(fetchAvailableTags()) + } catch (_: Exception) { + } + genres.forEach { (title, slug) -> + allTags.add(MangaTag(key = slug, title = title, source = source)) + } + origins.forEach { (title, slug) -> + allTags.add(MangaTag(key = "$ORIGIN_PREFIX$slug", title = "Origine: $title", source = source)) + } + return MangaListFilterOptions( + availableTags = allTags, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + ) + } override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val slug = manga.url.toMangaSlug() val fullUrl = "https://$domain/$mangaPath/$slug/" @@ -261,7 +306,7 @@ abstract class OriginesParser( ) }.reversed() } - + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val fullUrl = "https://$domain/$mangaPath/${chapter.url}/" val doc = webClient.httpGet(fullUrl).parseHtml() @@ -278,7 +323,7 @@ abstract class OriginesParser( ) } } - + private fun parseChapterDate(date: String?): Long { if (date.isNullOrBlank()) return 0L val regex = Regex("""(\d{1,2})\s+(\p{L}+)\.?(?:\s+(\d{4}))?""") @@ -317,7 +362,7 @@ abstract class OriginesParser( else -> null } } - + private fun okhttp3.Response.parseJson(): JSONObject { val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) return JSONObject(body) From cefb4e0b453534a430e15550a13bd359fb6386c6 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 16:21:47 +0200 Subject: [PATCH 391/434] Update MangasOriginesFr.kt --- .../kotlin/tsuki/site/fr/MangasOriginesFr.kt | 66 ------------------- 1 file changed, 66 deletions(-) diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt index 69cfac1d..9a5982a7 100644 --- a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -9,72 +9,6 @@ import tsuki.model.MangaParserSource @MangaSourceParser("MANGASORIGINESFR", "Mangas Origines", "fr") class MangasOriginesFr(context: MangaLoaderContext) : OriginesParser(context, MangaParserSource.MANGASORIGINESFR, "mangas-origines.fr") { - override val mangaPath = "oeuvre" - override val legacyMangaPaths = setOf("catalogues") - - override val genres = listOf( - "Action" to "action", - "Adventure" to "adventure", - "Amitié" to "amitie", - "Amour" to "amour", - "Art Martiaux" to "art-martiaux", - "Aventure" to "aventure", - "BL" to "bl", - "Boys" to "boys", - "Combat" to "combat", - "Comedy" to "comedy", - "Comédie" to "comedie", - "Dark Fantasy" to "dark-fantasy", - "Drama" to "drama", - "Drame" to "drame", - "Dystopie" to "dystopie", - "Démon" to "demon", - "Ecchi" to "ecchi", - "Erotique" to "erotique", - "Fantastique" to "fantastique", - "Fantasy" to "fantasy", - "Guerre" to "guerre", - "Harem" to "harem", - "Historique" to "historique", - "Horreur" to "horreur", - "Isekai" to "isekai", - "Jeu" to "jeu", - "Josei" to "josei", - "Magie" to "magie", - "Malédiction" to "malediction", - "Mature" to "mature", - "Moderne" to "moderne", - "Mort" to "mort", - "Murim" to "murim", - "Musique" to "musique", - "Mystère" to "mystere", - "Novel" to "novel", - "Post-Apo" to "post-apo", - "Prison" to "prison", - "Psychologique" to "psychologique", - "Religion" to "religion", - "Returner" to "returner", - "Romance" to "romance", - "Réincarnation" to "reincarnation", - "Régression" to "regression", - "School life" to "school-life", - "Sci-fi" to "sci-fi", - "Seinen" to "seinen", - "Shojo" to "shojo", - "Shonen" to "shonen", - "Slice of Life" to "slice-of-life", - "Société" to "societe", - "Sorcellerie" to "sorcellerie", - "Sport" to "sport", - "Steampunk" to "steampunk", - "Supernaturel" to "supernaturel", - "Surnaturel" to "surnaturel", - "Tragédie" to "tragedie", - "Vengeance" to "vengeance", - "Webcomic" to "webcomic", - "Yuri" to "yuri", - "École" to "ecole", - ) } From 297e4dc88ce6a0ee1d08f2501295bc9682fd5102 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sat, 5 Sep 2026 17:55:39 +0200 Subject: [PATCH 392/434] fix voratoon --- src/main/kotlin/tsuki/site/id/VoraToon.kt | 136 +++++++++++----------- 1 file changed, 68 insertions(+), 68 deletions(-) diff --git a/src/main/kotlin/tsuki/site/id/VoraToon.kt b/src/main/kotlin/tsuki/site/id/VoraToon.kt index abc1b18d..13288505 100644 --- a/src/main/kotlin/tsuki/site/id/VoraToon.kt +++ b/src/main/kotlin/tsuki/site/id/VoraToon.kt @@ -1,6 +1,5 @@ package tsuki.site.id -import org.json.JSONArray import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey @@ -26,8 +25,6 @@ import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import org.json.JSONObject import java.net.URLEncoder -import java.text.DecimalFormat -import java.text.DecimalFormatSymbols import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -38,7 +35,7 @@ internal class VoraToon(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("v1.voratoon.com") - private val apiBase = "https://api.voratoon.com" + private val apiBase = "https://$domain/backend" override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { super.onCreateConfig(keys) @@ -186,30 +183,31 @@ internal class VoraToon(context: MangaLoaderContext) : override suspend fun getDetails(manga: Manga): Manga { val slug = manga.url.removeSuffix("/").substringAfterLast("/") - val url = "$apiBase/series?includeMeta=true&take=1&page=1&takeChapter=9999&filter=slug%3D%3D$slug" - val json = webClient.httpGet(url).body?.string().orEmpty() + + val seriesUrl = "$apiBase/series/$slug" + val json = webClient.httpGet(seriesUrl).body?.string().orEmpty() return try { val root = JSONObject(json) - val dataArray = root.getJSONArray("data") - if (dataArray.length() == 0) return manga - - val seriesItem = dataArray.getJSONObject(0) - val data = seriesItem.getJSONObject("data") - - val title = data.getString("title") - val nativeTitle = data.nonNullString("nativeTitle") - val synopsis = data.optString("synopsis", "") - val author = data.nonNullString("author") - val coverImage = data.optString("coverImage", "") - val backgroundImage = data.nonNullString("backgroundImage") - val format = data.nonNullString("format")?.lowercase(Locale.ROOT) - - val genres = parseGenres(data) - val chapters = parseChapters(seriesItem, slug) - - val releaseYear = data.nonNullString("releaseDate") + val data = root.getJSONObject("data") + val seriesData = data.getJSONObject("data") + + val title = seriesData.getString("title") + val nativeTitle = seriesData.nonNullString("nativeTitle") + val synopsis = seriesData.optString("synopsis", "") + val author = seriesData.nonNullString("author") + val coverImage = seriesData.optString("coverImage", "") + val backgroundImage = seriesData.nonNullString("backgroundImage") + val format = seriesData.nonNullString("format")?.lowercase(Locale.ROOT) + + val genres = mutableSetOf<MangaTag>() + seriesData.optJSONArray("genres")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.toGenreTagOrNull()?.let(genres::add) + } + } + val releaseYear = seriesData.nonNullString("releaseDate") val extraTags = buildSet { releaseYear?.let { add(MangaTag(it, it, source)) } format?.let { @@ -217,16 +215,20 @@ internal class VoraToon(context: MangaLoaderContext) : } } + val chaptersUrl = "$apiBase/series/$slug/chapters" + val chaptersJson = webClient.httpGet(chaptersUrl).body?.string().orEmpty() + val chapters = parseChapters(chaptersJson, slug) + manga.copy( title = title, altTitles = setOfNotNull(nativeTitle), description = synopsis, - state = data.toMangaState(), + state = seriesData.toMangaState(), authors = setOfNotNull(author), tags = genres + extraTags, coverUrl = coverImage, largeCoverUrl = backgroundImage ?: coverImage, - rating = data.toRating(), + rating = seriesData.toRating(), chapters = chapters, ) } catch (_: Exception) { @@ -234,46 +236,48 @@ internal class VoraToon(context: MangaLoaderContext) : } } - private fun parseGenres(data: JSONObject): Set<MangaTag> { - val genres = mutableSetOf<MangaTag>() - data.optJSONArray("genres")?.let { arr -> - for (i in 0 until arr.length()) { - arr.optJSONObject(i)?.toGenreTagOrNull()?.let(genres::add) - } - } - return genres - } - - private fun parseChapters(seriesItem: JSONObject, slug: String): List<MangaChapter> { + private fun parseChapters(json: String, slug: String): List<MangaChapter> { val chapters = mutableListOf<MangaChapter>() - val chaptersArray = seriesItem.optJSONArray("chapters") ?: JSONArray() - - for (i in 0 until chaptersArray.length()) { - val ch = chaptersArray.getJSONObject(i) - val chapterIndex = ch.getInt("chapterIndex") - val formattedIndex = chapterNumberFormatter.format(chapterIndex.toDouble()) - val chapterUrl = "/series/$slug/chapter/$formattedIndex" - val titleText = ch.optJSONObject("data")?.nonNullString("title") - ?: "Chapter $formattedIndex" - val uploadDate = parseChapterDate(ch.optString("createdAt", "")) - - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = titleText, - url = chapterUrl, - number = chapterIndex.toFloat(), - volume = 0, - scanlator = null, - uploadDate = uploadDate, - branch = null, - source = source, - ), - ) - } + try { + val root = JSONObject(json) + val chaptersArray = root.getJSONArray("data") ?: return emptyList() + + for (i in 0 until chaptersArray.length()) { + val ch = chaptersArray.getJSONObject(i) + val chapterData = ch.optJSONObject("data") + + val chapterIndex = chapterData?.optDouble("index", 0.0) ?: 0.0 + val formattedIndex = chapterIndex.toString().removeSuffix(".0") + + val chapterUrl = "/series/$slug/chapter/$formattedIndex" - chapters.sortBy { it.number } - return chapters + val title = chapterData?.nonNullString("title") + val titleText = if (title != null) { + "Chapter $formattedIndex: $title" + } else { + "Chapter $formattedIndex" + } + + val uploadDate = parseChapterDate(ch.optString("createdAt", "")) + + chapters.add( + MangaChapter( + id = generateUid(chapterUrl), + title = titleText, + url = chapterUrl, + number = chapterIndex.toFloat(), + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + ) + } + } catch (index: Exception) { + throw Exception("Error parsing chapters.", index) + } + return chapters.reversed() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { @@ -398,9 +402,5 @@ internal class VoraToon(context: MangaLoaderContext) : companion object { private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT) - private val chapterNumberFormatter = DecimalFormat( - "#.##", - DecimalFormatSymbols.getInstance(Locale.US), - ) } } From 78c45440d1442b0aa7e40bff8ed011eccf030592 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:46:34 +0200 Subject: [PATCH 393/434] Add support for year filtering in Chikari.kt --- src/main/kotlin/tsuki/site/en/Chikari.kt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/main/kotlin/tsuki/site/en/Chikari.kt b/src/main/kotlin/tsuki/site/en/Chikari.kt index 9016f92d..a8f0e9f8 100644 --- a/src/main/kotlin/tsuki/site/en/Chikari.kt +++ b/src/main/kotlin/tsuki/site/en/Chikari.kt @@ -18,9 +18,11 @@ import tsuki.model.MangaState import tsuki.model.MangaTag import tsuki.model.RATING_UNKNOWN import tsuki.model.SortOrder +import tsuki.model.YEAR_UNKNOWN import tsuki.util.generateUid import tsuki.util.parseJson +import tsuki.util.oneOrThrowIfMany import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -49,6 +51,7 @@ class Chikari(context: MangaLoaderContext) : isMultipleTagsSupported = true, isTagsExclusionSupported = true, isSearchWithFiltersSupported = true, + isYearSupported = true, ) private val tags by lazy { @@ -71,6 +74,7 @@ class Chikari(context: MangaLoaderContext) : ContentType.MANHUA, ContentType.OTHER, // OEL ), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { @@ -94,6 +98,7 @@ class Chikari(context: MangaLoaderContext) : addQueryParameter("sort", sortParam) addQueryParameter("limit", pageSize.toString()) addQueryParameter("offset", offset.toString()) + addQueryParameter("adult", (filter.contentRating.oneOrThrowIfMany() == ContentRating.ADULT).toString()) filter.query?.takeIf { it.isNotBlank() }?.let { addQueryParameter("q", it) @@ -114,6 +119,10 @@ class Chikari(context: MangaLoaderContext) : filter.states.firstOrNull()?.let { state -> addQueryParameter("status", state.toQueryParam()) } + + if (filter.year != YEAR_UNKNOWN) { + addQueryParameter("year", filter.year.toString()) + } }.build().toString() } From 88b86e148886785368eb643e9df7e675e8d53add Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 00:53:27 +0200 Subject: [PATCH 394/434] fix search --- src/main/kotlin/tsuki/site/en/Batcave.kt | 51 ++++++++++++++++++++---- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/src/main/kotlin/tsuki/site/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt index d04757e7..d6972123 100644 --- a/src/main/kotlin/tsuki/site/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -25,15 +25,16 @@ import tsuki.util.attrAsRelativeUrl import tsuki.util.generateUid import tsuki.util.parseHtml import tsuki.util.toAbsoluteUrl -import tsuki.util.urlEncoded import okhttp3.FormBody +import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.MediaType.Companion.toMediaTypeOrNull import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody.Companion.toRequestBody import okhttp3.Response import org.json.JSONObject +import org.jsoup.Jsoup import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -57,6 +58,13 @@ internal class Batcave(context: MangaLoaderContext) : .build() } + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Sec-Fetch-Dest", "document") + .set("Sec-Fetch-Mode", "navigate") + .set("Sec-Fetch-Site", "none") + .set("Sec-Fetch-User", "?1") + .build() + private val apiClient: OkHttpWebClient by lazy { OkHttpWebClient(rawHttpClient, source) } @@ -147,8 +155,9 @@ internal class Batcave(context: MangaLoaderContext) : } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - if (query != null) return searchManga(query, page) + if (!filter.query.isNullOrBlank()) { + return searchManga(page, filter.query!!) + } val urlBuilder = StringBuilder().apply { append("https://$domain/ComicList/") @@ -180,11 +189,37 @@ internal class Batcave(context: MangaLoaderContext) : } } - private suspend fun searchManga(query: String, page: Int): List<Manga> { - val encoded = query.urlEncoded() - val url = if (page == 1) "https://$domain/search/$encoded/" - else "https://$domain/search/$encoded/page/$page/" - return parseMangaList(apiClient.httpGet(url).parseHtml()) + private suspend fun searchManga(page: Int, query: String): List<Manga> { + val url = "https://$domain".toHttpUrl().newBuilder() + .addPathSegment("search") + .addPathSegment(query) + .apply { + if (page > 1) { + addPathSegment("page") + addPathSegment(page.toString()) + addPathSegment("") + } + } + .build() + val html = apiClient.httpGet(url.toString()).body?.string() ?: return emptyList() + val doc = Jsoup.parse(html, "https://$domain") + return doc.select("#dle-content > .readed").map { element -> + val a = element.selectFirst(".readed__title > a") ?: return@map null + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.absUrl("href"), + title = a.ownText(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = element.selectFirst(".readed__img img")?.attrAsAbsoluteUrl("data-src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() } private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { From 1aabf693f735ba0df6f7e50793bb178b842ef5bc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 01:43:00 +0200 Subject: [PATCH 395/434] Create Pixiv.kt --- src/main/kotlin/tsuki/site/all/Pixiv.kt | 459 ++++++++++++++++++++++++ 1 file changed, 459 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/all/Pixiv.kt diff --git a/src/main/kotlin/tsuki/site/all/Pixiv.kt b/src/main/kotlin/tsuki/site/all/Pixiv.kt new file mode 100644 index 00000000..d06a7e53 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Pixiv.kt @@ -0,0 +1,459 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONObject +import java.util.EnumSet + +private val KNOWN_LOCALES = listOf("en", "ja", "zh", "zh-tw", "ko") + +interface PixivTargetCompanion<out T : PixivTarget> { + val searchPrefix: String + fun fromSearchQuery(query: String): T? { + if (!query.startsWith(searchPrefix)) return null + val id = query.removePrefix(searchPrefix) + if (!id.matches(Regex("\\d+"))) return null + return fromSearchQueryId(id) + } + fun fromSearchQueryId(id: String): T? +} + +sealed class PixivTarget { + companion object { + val BASE_URI = "https://www.pixiv.net".toHttpUrl() + + fun fromSearchQuery(query: String) = sequenceOf(User, Series, Illustration) + .firstNotNullOfOrNull { it.fromSearchQuery(query) } + + fun fromUri(uri: String) = uri.toHttpUrlOrNull()?.let { fromUri(it) } + fun fromUri(uri: HttpUrl): PixivTarget? { + if (!(uri.scheme in listOf(null, "http", "https") && uri.host.let { "pixiv.net" == it.removePrefix("www.") }) + ) return null + var pathSegments = uri.pathSegments.ifEmpty { null } ?: return null + if (KNOWN_LOCALES.contains(pathSegments[0])) { + pathSegments = pathSegments.subList(1, pathSegments.size) + } + if (pathSegments.size < 2) return null + with(pathSegments[0]) { + return when { + equals("artworks") -> Illustration(pathSegments[1]) + equals("users") -> User(pathSegments[1]) + equals("user") && (pathSegments.size >= 4 && pathSegments[2] == "series") -> + Series(pathSegments[3], pathSegments[1]) + else -> null + } + } + } + } + + abstract fun toHttpUrl(): HttpUrl + abstract fun toSearchQuery(): String + + data class User(val userId: String) : PixivTarget() { + companion object : PixivTargetCompanion<User> { + override val searchPrefix = "user:" + override fun fromSearchQueryId(id: String) = User(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("users").addPathSegment(userId).build() + override fun toSearchQuery(): String = searchPrefix + userId + } + + data class Illustration(val illustId: String) : PixivTarget() { + companion object : PixivTargetCompanion<Illustration> { + override val searchPrefix = "aid:" + override fun fromSearchQueryId(id: String) = Illustration(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("artworks").addPathSegment(illustId).build() + override fun toSearchQuery(): String = searchPrefix + illustId + } + + data class Series(val seriesId: String, val authorUserId: String? = null) : PixivTarget() { + companion object : PixivTargetCompanion<Series> { + override val searchPrefix = "sid:" + override fun fromSearchQueryId(id: String) = Series(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("user").addPathSegment(authorUserId ?: "") + .addPathSegment("series").addPathSegment(seriesId).build() + override fun toSearchQuery(): String = searchPrefix + seriesId + } +} + +@MangaSourceParser("PIXIV_EN", "Pixiv (English)", "en") +internal class PixivEn(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_EN) + +@MangaSourceParser("PIXIV_JA", "Pixiv (Japanese)", "ja") +internal class PixivJa(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_JA) + +@MangaSourceParser("PIXIV_ZH", "Pixiv (Chinese)", "zh") +internal class PixivZh(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_ZH) + +@MangaSourceParser("PIXIV_KO", "Pixiv (Korean)", "ko") +internal class PixivKo(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_KO) + +abstract class PixivParser( + context: MangaLoaderContext, + source: MangaParserSource, +) : PagedMangaParser(context, source, 18) { + + override val configKeyDomain = ConfigKey.Domain("pixiv.net") + + private val baseUrl = "https://$domain" + + private val imageQualityKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "thumb_mini" to "Thumb Mini", + "small" to "Small", + "regular" to "Regular", + "original" to "Original", + ), + defaultValue = "original", + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(imageQualityKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = + MangaListFilterOptions() + + private suspend fun apiGet(path: String): JSONObject { + val url = (baseUrl + path).toHttpUrlOrNull() + ?: throw IllegalArgumentException("Invalid URL: $path") + val response = webClient.httpGet(url.toString()) + val body = response.body?.string() ?: throw Exception("Empty response") + return JSONObject(body) + } + + private suspend fun fetchRanking(page: Int): List<Manga> { + val rankingJson = apiGet("/touch/ajax/ranking/illust?mode=daily&type=manga&page=$page") + val rankingArray = rankingJson.optJSONObject("body")?.optJSONArray("ranking") + ?: return emptyList() + val illustIds = mutableListOf<String>() + for (i in 0 until rankingArray.length()) { + val entry = rankingArray.getJSONObject(i) + entry.optString("illustId").takeIf { it.isNotEmpty() }?.let { illustIds.add(it) } + } + if (illustIds.isEmpty()) return emptyList() + val detailsUrl = "/touch/ajax/illust/details/many?" + + illustIds.joinToString("&") { "illust_ids[]=$it" } + val detailsJson = apiGet(detailsUrl) + val illustsArray = detailsJson.optJSONObject("body")?.optJSONArray("illust_details") + ?: return emptyList() + return (0 until illustsArray.length()).mapNotNull { i -> + illustsArray.getJSONObject(i).toMangaIllust() + } + } + + private suspend fun fetchLatest(page: Int): List<Manga> { + val json = apiGet("/touch/ajax/latest?type=manga&p=$page") + val illustsArray = json.optJSONObject("body")?.optJSONArray("illusts") + ?: return emptyList() + return (0 until illustsArray.length()).mapNotNull { i -> + illustsArray.getJSONObject(i).toMangaIllust() + } + } + + private suspend fun searchManga(query: String, page: Int): List<Manga> { + val target = PixivTarget.fromUri(query) ?: PixivTarget.fromSearchQuery(query) + when (target) { + is PixivTarget.Illustration -> { + val illust = getIllustCached(target.illustId) + return listOfNotNull(illust?.toMangaIllust()) + } + is PixivTarget.Series -> { + val series = fetchSeries(target.seriesId) + return listOfNotNull(series?.toMangaSeries()) + } + is PixivTarget.User -> { + val user = getUserCached(target.userId) + return listOf(user.toMangaUser(target.userId)) + } + else -> {} + } + + val word = query.trim() + val url = "/touch/ajax/search/illusts?word=${word.urlEncoded()}&s_mode=s_tc&p=$page" + val json = apiGet(url) + val illustsArray = json.optJSONObject("body")?.optJSONArray("illusts") + ?: return emptyList() + val results = mutableListOf<JSONObject>() + for (i in 0 until illustsArray.length()) { + val obj = illustsArray.getJSONObject(i) + if (obj.optInt("is_ad_container") == 1) continue + if (obj.optString("type") == "2") continue + results.add(obj) + } + return results.mapNotNull { it.toMangaIllust() } + } + + private fun String.urlEncoded(): String = java.net.URLEncoder.encode(this, "UTF-8") + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim() ?: "" + if (query.isNotBlank()) { + return searchManga(query, page) + } + return when (order) { + SortOrder.POPULARITY -> fetchRanking(page) + SortOrder.NEWEST -> fetchLatest(page) + else -> fetchLatest(page) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val target = PixivTarget.fromUri(baseUrl + manga.url) ?: return@coroutineScope manga + + val detailsDeferred = async { fetchMangaDetails(target, manga) } + val chaptersDeferred = async { fetchMangaChapters(target) } + + val details = detailsDeferred.await() + val chapters = chaptersDeferred.await() + + details.copy(chapters = chapters) + } + + private suspend fun fetchMangaDetails(target: PixivTarget, manga: Manga): Manga { + return when (target) { + is PixivTarget.User -> { + val user = getUserCached(target.userId) + manga.copy( + title = user.optString("name", manga.title), + authors = setOf(user.optString("name")), + description = user.optString("comment", manga.description ?: ""), + coverUrl = user.optString("imageBig", manga.coverUrl), + ) + } + is PixivTarget.Series -> { + val series = fetchSeries(target.seriesId) ?: return manga + val illusts = getSeriesIllustsCached(target.seriesId) ?: emptyList() + val first = illusts.firstOrNull() + val cover = series.optString("coverImage", null) + ?: first?.optString("url") + ?: manga.coverUrl + val tags = illusts.flatMap { obj -> + val arr = obj.optJSONArray("tags") ?: return@flatMap emptyList() + (0 until arr.length()).map { arr.getString(it) } + }.distinct().map { MangaTag(it, it, source) }.toSet() + manga.copy( + title = series.optString("title", manga.title), + authors = setOfNotNull(first?.optJSONObject("author_details")?.optString("user_name")), + description = series.optString("caption", manga.description ?: ""), + coverUrl = cover, + tags = tags, + ) + } + is PixivTarget.Illustration -> { + val illust = getIllustCached(target.illustId) ?: return manga + val tags = (0 until (illust.optJSONArray("tags")?.length() ?: 0)) + .map { illust.getJSONArray("tags").getString(it) } + .map { MangaTag(it, it, source) }.toSet() + manga.copy( + title = illust.optString("title", manga.title), + authors = setOfNotNull(illust.optJSONObject("author_details")?.optString("user_name")), + description = illust.optString("comment", manga.description ?: ""), + coverUrl = illust.optString("url", manga.coverUrl), + tags = tags, + ) + } + } + } + + private suspend fun fetchMangaChapters(target: PixivTarget): List<MangaChapter> { + val illusts = when (target) { + is PixivTarget.User -> { + val json = apiGet("/touch/ajax/user/illusts?id=${target.userId}&p=1") + val arr = json.optJSONObject("body")?.optJSONArray("illusts") ?: return emptyList() + (0 until arr.length()).map { arr.getJSONObject(it) } + } + is PixivTarget.Series -> getSeriesIllustsCached(target.seriesId) ?: emptyList() + is PixivTarget.Illustration -> listOfNotNull(getIllustCached(target.illustId)) + } + return illusts.mapIndexed { i, obj -> + val id = obj.optString("id", i.toString()) + MangaChapter( + id = generateUid(id), + title = obj.optString("title", "(null)"), + number = (illusts.size - i).toFloat(), + volume = 0, + url = "/artworks/$id", + scanlator = null, + uploadDate = obj.optLongOrNull("upload_timestamp")?.let { it * 1000 } ?: 0, + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val illustId = chapter.url.substringAfterLast('/') + val json = apiGet("/ajax/illust/$illustId/pages") + val pagesArray = json.optJSONArray("body") ?: return emptyList() + val quality = config[imageQualityKey] ?: "original" + return (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + val urls = pageObj.optJSONObject("urls") ?: JSONObject() + val imageUrl = getImageUrl(urls, quality) + MangaPage( + id = generateUid("${chapter.id}-$i"), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private fun getImageUrl(urls: JSONObject, quality: String): String { + val sizeOrder = listOf("thumb_mini", "small", "regular", "original") + val startIndex = sizeOrder.indexOf(quality).takeIf { it >= 0 } ?: sizeOrder.lastIndex + return sizeOrder.drop(startIndex).firstNotNullOf { size -> + urls.optString(size).takeIf { it.isNotEmpty() } + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getIllustCached(illustId: String): JSONObject? { + val json = apiGet("/touch/ajax/illust/details?illust_id=$illustId") + return json.optJSONObject("body")?.optJSONObject("illust_details") + } + + private suspend fun getUserCached(userId: String): JSONObject { + val json = apiGet("/ajax/user/$userId?full=1") + return json.optJSONObject("body") ?: json + } + + private suspend fun getSeriesIllustsCached(seriesId: String): List<JSONObject>? { + val result = mutableListOf<JSONObject>() + var lastOrder = 0 + while (true) { + val json = apiGet("/touch/ajax/illust/series_content/$seriesId?last_order=$lastOrder") + val arr = json.optJSONObject("body")?.optJSONArray("series_contents") ?: break + if (arr.length() == 0) break + for (i in 0 until arr.length()) { + result.add(arr.getJSONObject(i)) + } + lastOrder += arr.length() + } + return result.takeIf { it.isNotEmpty() } + } + + private suspend fun fetchSeries(seriesId: String): JSONObject? { + val json = apiGet("/touch/ajax/illust/series/$seriesId") + return json.optJSONObject("body")?.optJSONObject("series") + } + + private fun JSONObject.optLongOrNull(key: String): Long? = + try { optLong(key) } catch (_: Exception) { null } + + private fun JSONObject.toMangaIllust(): Manga? { + val id = optString("id").takeIf { it.isNotEmpty() } ?: return null + val series = optJSONObject("series") + val url = if (series != null) { + val userId = series.optString("userId", optJSONObject("author_details")?.optString("user_id") ?: "") + "/user/$userId/series/${series.optString("id")}" + } else { + "/artworks/$id" + } + val tags = optJSONArray("tags")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) }.map { MangaTag(it, it, source) }.toSet() + } ?: emptySet() + val authors = setOfNotNull(optJSONObject("author_details")?.optString("user_name")) + return Manga( + id = generateUid(url), + title = optString("title", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = if (optString("x_restrict") == "1") ContentRating.ADULT else ContentRating.SAFE, + coverUrl = optString("url", ""), + tags = tags, + state = null, + authors = authors, + largeCoverUrl = null, + description = optString("comment"), + source = source, + ) + } + + private fun JSONObject.toMangaSeries(): Manga? { + val id = optString("id").takeIf { it.isNotEmpty() } ?: return null + val userId = optString("userId").takeIf { it.isNotEmpty() } ?: return null + val url = "/user/$userId/series/$id" + return Manga( + id = generateUid(url), + title = optString("title", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + coverUrl = optString("coverImage", ""), + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = optString("caption"), + source = source, + ) + } + + private fun JSONObject.toMangaUser(userId: String): Manga { + val url = "/users/$userId" + return Manga( + id = generateUid(url), + title = optString("name", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + coverUrl = optString("imageBig", ""), + tags = emptySet(), + state = null, + authors = setOf(optString("name")), + largeCoverUrl = null, + description = optString("comment"), + source = source, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} From e4d6fe22fc459c82dc9197dfd5a163528f1cda06 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:12:08 +0200 Subject: [PATCH 396/434] trash website --- src/main/kotlin/tsuki/site/id/Softkomik.kt | 495 --------------------- 1 file changed, 495 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/id/Softkomik.kt diff --git a/src/main/kotlin/tsuki/site/id/Softkomik.kt b/src/main/kotlin/tsuki/site/id/Softkomik.kt deleted file mode 100644 index 8e407288..00000000 --- a/src/main/kotlin/tsuki/site/id/Softkomik.kt +++ /dev/null @@ -1,495 +0,0 @@ -package tsuki.site.id - -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser - -import tsuki.model.ContentRating -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder - -import tsuki.util.generateUid -import tsuki.util.getCookies -import tsuki.util.parseHtml -import tsuki.util.parseJson -import tsuki.util.toAbsoluteUrl -import tsuki.util.urlEncoded - -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.OkHttpClient -import okhttp3.Response -import org.json.JSONArray -import org.json.JSONObject -import java.net.URLDecoder -import java.util.concurrent.ConcurrentHashMap -import java.util.concurrent.TimeUnit - -import tsuki.Broken -@Broken -@MangaSourceParser("SOFTKOMIK", "Softkomik", "id") -internal class Softkomik(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.SOFTKOMIK, pageSize = 24), MangaParserAuthProvider { - - override val authUrl: String get() = "https://$domain/akun/login" - override suspend fun isAuthorized(): Boolean { - val cookies = context.cookieJar.getCookies(domain) - return cookies.any { it.name == "tokkey" } - } - override suspend fun getUsername(): String = "" - - override val configKeyDomain = ConfigKey.Domain("softkomik.co") - - private val apiUrl = "https://v2.softdevices.my.id" - private val coverUrl = "https://cover.softdevices.my.id/softkomik-cover" - - private val cdnUrls = listOf( - "https://psy1.komik.im", - "https://image.komik.im/softkomik", - "https://cdn1.softkomik.online/softkomik", - "https://cd1.softkomik.online/softkomik", - "https://f1.softkomik.com/file/softkomik-image", - "https://img.softdevices.my.id/softkomik-image", - "https://image.softkomik.com/softkomik", - ) - - private val loginRequiredGenres = setOf("ecchi", "mature") - private val requiredLoginSuffix = "login-required" - private val requiredLoginFragment = "#$requiredLoginSuffix" - - override val availableSortOrders = setOf(SortOrder.NEWEST, SortOrder.POPULARITY) - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - ) - - private val customClient: OkHttpClient by lazy { - context.httpClient.newBuilder() - .addInterceptor(::apiAuthInterceptor) - .addInterceptor(::imageInterceptor) - .build() - } - - override val webClient by lazy { - tsuki.network.OkHttpWebClient(customClient, source) - } - - private fun pageHeaders() = getRequestHeaders().newBuilder() - .add("rsc", "1") - .build() - - private fun apiHeaders(session: SessionDto): Headers { - val builder = Headers.Builder() - .add("Accept", "application/json") - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - if (session.sign.isNotEmpty()) { - builder.add("X-Token", session.token.cleanB64()) - builder.add("X-Sign", session.sign.take(64)) - } else { - builder.add("Authorization", "Bearer ${session.token}") - } - return builder.build() - } - - private data class SessionDto(val ex: Long, val token: String, val sign: String) - private val sessionsByKey = ConcurrentHashMap<String, SessionDto>() - private val sessionKeyChapterList = "chapter-list" - private val sessionKeyChapterImage = "chapter-image" - - private suspend fun getSessionAsync(route: SessionRoute): SessionDto { - sessionsByKey[route.key]?.takeIf { it.ex > System.currentTimeMillis() + 30_000L }?.let { return it } - - try { - val apiHeaders = Headers.Builder() - .add("Accept", "application/json") - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("X-Requested-With", "XMLHttpRequest") - .build() - val res = webClient.httpGet(route.sessionApiUrl, apiHeaders).parseJson() - val rawSign = res.optString("sign", "") - val session = SessionDto( - ex = res.optLong("ex", System.currentTimeMillis() + 60_000L), - token = res.optString("token", ""), - sign = rawSign.substringBefore('|'), - ) - sessionsByKey[route.key] = session - return session - } catch (_: Exception) { - val token = getBearerTokenFromCookie() - if (token != null) { - val session = SessionDto( - ex = System.currentTimeMillis() + TimeUnit.HOURS.toMillis(2), - token = token, - sign = "", - ) - sessionsByKey[route.key] = session - return session - } - throw Exception("Gagal mendapatkan session. Coba lagi.") - } - } - - private fun getSessionBlocking(route: SessionRoute): SessionDto = runBlocking { - getSessionAsync(route) - } - - private fun getBearerTokenFromCookie(): String? { - val cookies = context.cookieJar.getCookies(domain) - val tokkey = cookies.firstOrNull { it.name == "tokkey" }?.value ?: return null - val decoded = runCatching { URLDecoder.decode(tokkey, Charsets.UTF_8.name()) }.getOrDefault(tokkey) - return decoded.cleanB64() - } - - private fun String.cleanB64(): String = - substringBefore('=').let { it + "=".repeat((4 - (it.length % 4)) % 4) } - - private fun apiAuthInterceptor(chain: Interceptor.Chain): Response { - val request = chain.request() - if (!request.url.toString().startsWith(apiUrl)) { - return chain.proceed(request) - } - val route = resolveSessionRoute(request.url) - val session = getSessionBlocking(route) - val newRequest = request.newBuilder() - .apply { - if (session.sign.isNotEmpty()) { - header("X-Token", session.token.cleanB64()) - header("X-Sign", session.sign.take(64)) - } else { - header("Authorization", "Bearer ${session.token}") - } - } - .build() - return chain.proceed(newRequest) - } - - private fun imageInterceptor(chain: Interceptor.Chain): Response { - val request = chain.request() - val userAgent = request.header("User-Agent") - val normalizedUserAgent = userAgent?.replace( - Regex("""\s*Mobile Safari/\d+(?:\.\d+)*""", RegexOption.IGNORE_CASE), - "", - )?.trim()?.ifEmpty { null } - - val modRequest = if (normalizedUserAgent != userAgent) { - request.newBuilder().header("User-Agent", normalizedUserAgent ?: "").build() - } else request - - val response: Response? = try { - chain.proceed(modRequest) - } catch (_: java.net.UnknownHostException) { - null - } - - if (response?.isSuccessful == true) return response - - val currentHost = cdnUrls.firstOrNull { request.url.toString().startsWith(it) } - if (currentHost == null) return response ?: throw java.net.UnknownHostException(request.url.host) - - response?.close() - val imagePath = request.url.toString().removePrefix(currentHost).removePrefix("/") - for (newHost in cdnUrls) { - if (newHost == currentHost) continue - val newUrl = "$newHost/$imagePath".toHttpUrl() - try { - val newResp = chain.proceed(request.newBuilder().url(newUrl).build()) - if (newResp.isSuccessful) return newResp - newResp.close() - } catch (_: Exception) {} - } - throw java.net.UnknownHostException("All CDN hosts failed for $imagePath") - } - - private data class SessionRoute( - val key: String, - val sessionApiUrl: String, - val slug: String?, - val isChapterListRequest: Boolean, - val isChapterImageRequest: Boolean, - ) - - private fun resolveSessionRoute(url: okhttp3.HttpUrl): SessionRoute { - val segments = url.pathSegments - val komikIndex = segments.indexOf("komik") - val slug = if (komikIndex != -1) segments.getOrNull(komikIndex + 1) else null - val isChapterListRequest = komikIndex != -1 && segments.getOrNull(komikIndex + 2) == "chapter" - val isChapterImageRequest = isChapterListRequest && (segments.contains("imgs") || segments.contains("img")) - - val key = if (isChapterImageRequest) sessionKeyChapterImage else sessionKeyChapterList - val sessionApiUrl = if (isChapterImageRequest) { - "https://$domain/api/session/chapter/oioa" - } else { - "https://$domain/api/session/iuiuiwqw" - } - return SessionRoute( - key = key, - sessionApiUrl = sessionApiUrl, - slug = slug, - isChapterListRequest = isChapterListRequest, - isChapterImageRequest = isChapterImageRequest, - ) - } - - @Volatile - private var genresCache: Set<MangaTag>? = null - private val genresMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - ) - - private suspend fun getOrFetchGenres(): Set<MangaTag> { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: fetchGenres().also { genresCache = it } - } - } - - private suspend fun fetchGenres(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/komik/library", pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) ?: return emptySet() - val arr = pageProps.optJSONArray("genre") ?: return emptySet() - return LinkedHashSet<MangaTag>(arr.length()).apply { - for (i in 0 until arr.length()) { - val jo = arr.optJSONObject(i) ?: continue - val value = jo.optString("value", "").ifEmpty { jo.optString("label", "") } - val label = jo.optString("label", "").ifEmpty { value } - if (value.isNotEmpty() && label.isNotEmpty()) { - add(MangaTag(title = label, key = value, source = source)) - } - } - } - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim().orEmpty() - if (query.isNotEmpty()) { - return searchByQuery(query, page) - } - val url = buildString { - append("https://$domain/komik/library") - append("?page=").append(page) - append("&sortBy=").append(if (order == SortOrder.POPULARITY) "popular" else "newKomik") - filter.tags.firstOrNull()?.key?.let { - append("&genre=").append(it.urlEncoded()) - } - } - val doc = webClient.httpGet(url, pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) ?: return emptyList() - val libData = pageProps.optJSONObject("libData") ?: pageProps - val dataArray = libData.optJSONArray("data") ?: return emptyList() - return parseListData(dataArray) - } - - private suspend fun searchByQuery(query: String, page: Int): List<Manga> { - val urlPath = "$apiUrl/komik?name=${query.urlEncoded()}&search=true&limit=24&page=$page" - val session = getSessionAsync(resolveSessionRoute(urlPath.toHttpUrl())) - val headers = apiHeaders(session) - val json = webClient.httpGet(urlPath, headers).parseJson() - val dataArray = json.optJSONArray("data") ?: return emptyList() - return parseListData(dataArray) - } - - private fun parseListData(dataArray: JSONArray): List<Manga> { - val result = ArrayList<Manga>(dataArray.length()) - for (i in 0 until dataArray.length()) { - val jo = dataArray.optJSONObject(i) ?: continue - val slug = jo.optString("title_slug", "").ifEmpty { jo.optString("id", "") } - if (slug.isEmpty()) continue - val gambar = jo.optString("gambar", "").removePrefix("/") - result.add( - Manga( - id = generateUid(slug), - title = jo.optString("title", "Untitled"), - altTitles = emptySet(), - url = "/$slug", - publicUrl = "https://$domain/$slug", - rating = RATING_UNKNOWN, - contentRating = ContentRating.SAFE, - coverUrl = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else "", - tags = emptySet(), - state = parseStatus(jo.optString("status")), - authors = emptySet(), - source = source, - ), - ) - } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.publicUrl, pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) - val detail = pageProps?.optJSONObject("data") - - val title = detail?.optString("title")?.takeIf { it.isNotBlank() } ?: manga.title - val gambar = detail?.optString("gambar", "")?.removePrefix("/") ?: "" - val coverImg = if (gambar.isNotEmpty()) "$coverUrl/$gambar" else manga.coverUrl - val description = detail?.optString("sinopsis")?.takeIf { it.isNotBlank() } - val author = detail?.optString("author")?.takeIf { it.isNotBlank() } - val state = parseStatus(detail?.optString("status")) - - val tags = LinkedHashSet<MangaTag>() - detail?.optJSONArray("Genre")?.let { arr -> - for (i in 0 until arr.length()) { - val name = arr.optString(i, "").trim() - if (name.isNotEmpty()) tags.add(MangaTag(name, name.lowercase(), source)) - } - } - - val isLoginRequired = tags.any { it.title.lowercase() in loginRequiredGenres } - val slug = manga.url.trim('/').substringBefore('/') - val chapters = fetchChapterList(slug, isLoginRequired) - - return manga.copy( - title = title, - coverUrl = coverImg, - description = description, - tags = tags, - authors = setOfNotNull(author), - state = state, - chapters = chapters, - ) - } - - private suspend fun fetchChapterList(slug: String, isLoginRequired: Boolean): List<MangaChapter> { - val apiPath = "$apiUrl/komik/$slug/chapter?limit=9999999" - val headers = if (isLoginRequired) { - val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan untuk mengakses chapter") - Headers.Builder() - .add("Authorization", "Bearer $token") - .add("Referer", "https://$domain/$slug") - .add("Origin", "https://$domain") - .build() - } else { - val session = getSessionAsync(resolveSessionRoute(apiPath.toHttpUrl())) - apiHeaders(session) - } - - val json = webClient.httpGet(apiPath, headers).parseJson() - val chapterArray = json.optJSONArray("chapter") ?: return emptyList() - val chapters = ArrayList<MangaChapter>(chapterArray.length()) - for (i in 0 until chapterArray.length()) { - val ch = chapterArray.optJSONObject(i) ?: continue - val chStr = ch.optString("chapter", "") - if (chStr.isEmpty()) continue - val number = chStr.substringBefore(".").toFloatOrNull() ?: continue - var chapterUrl = "/$slug/chapter/$chStr" - if (isLoginRequired) chapterUrl += requiredLoginFragment - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = "Chapter ${formatChapterDisplay(chStr)}", - url = chapterUrl, - number = number, - volume = 0, - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ), - ) - } - chapters.sortBy { it.number } - return chapters - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val isLoginRequired = chapter.url.contains(requiredLoginFragment) - val cleanUrl = chapter.url.substringBefore('#') - - val doc = webClient.httpGet(cleanUrl.toAbsoluteUrl(domain), pageHeaders()).parseHtml() - val pageProps = extractNextJs(doc) - val data = pageProps?.optJSONObject("data") - val komik = data?.optJSONObject("komik") - val chapterData = data?.optJSONObject("data") - - val slug = cleanUrl.trim('/').substringBefore("/chapter/") - val chNum = cleanUrl.substringAfterLast("/chapter/").trim('/') - - var imageSrc = chapterData?.optJSONArray("imageSrc") ?: JSONArray() - val storageInter2 = chapterData?.optBoolean("storageInter2", false) ?: false - - if (imageSrc.length() == 0) { - val id = chapterData?.optString("_id") ?: komik?.optString("_id") ?: return emptyList() - imageSrc = fetchChapterImages(slug, chNum, id, isLoginRequired) - } - - if (imageSrc.length() == 0) return emptyList() - - val imageBaseUrl = if (storageInter2) cdnUrls[2] else cdnUrls[0] - return (0 until imageSrc.length()).mapNotNull { i -> - val path = imageSrc.optString(i, "").removePrefix("/") - if (path.isEmpty()) return@mapNotNull null - MangaPage( - id = generateUid(path), - url = "$imageBaseUrl/$path", - preview = null, - source = source, - ) - } - } - - private suspend fun fetchChapterImages( - slug: String, - chapter: String, - id: String, - isLoginRequired: Boolean, - ): JSONArray { - val url = "$apiUrl/komik/$slug/chapter/$chapter/imgs/$id" - val headers = if (isLoginRequired) { - val token = getBearerTokenFromCookie() ?: throw Exception("Login diperlukan") - Headers.Builder() - .add("Authorization", "Bearer $token") - .add("Referer", "https://$domain/$slug/chapter/$chapter") - .add("Origin", "https://$domain") - .build() - } else { - val session = getSessionAsync(resolveSessionRoute(url.toHttpUrl())) - apiHeaders(session) - } - val json = webClient.httpGet(url, headers).parseJson() - return json.optJSONArray("imageSrc") ?: JSONArray() - } - - private fun extractNextJs(doc: org.jsoup.nodes.Document): JSONObject? { - val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() ?: return null - return runCatching { - JSONObject(raw).optJSONObject("props")?.optJSONObject("pageProps") - }.getOrNull() - } - - private fun parseStatus(text: String?): MangaState? = when { - text == null -> null - text.contains("ongoing", ignoreCase = true) -> MangaState.ONGOING - text.contains("tamat", ignoreCase = true) || text.contains("completed", ignoreCase = true) -> MangaState.FINISHED - else -> null - } - - private fun formatChapterDisplay(chStr: String): String { - val parts = chStr.split(".") - val numPart = parts[0].toFloatOrNull() ?: return chStr - val formatted = if (numPart == numPart.toLong().toFloat()) numPart.toLong().toString() - else numPart.toString().trimEnd('0').trimEnd('.') - val suffix = parts.drop(1).joinToString(".") - return if (suffix.isNotEmpty()) "$formatted.$suffix" else formatted - } -} From 303493f586c74d9874374006b36189d83e459938 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:33:37 +0200 Subject: [PATCH 397/434] Update Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 113 +++++++++++++++-------- 1 file changed, 74 insertions(+), 39 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index 52324673..2151404c 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -91,7 +91,7 @@ internal class Kagane(context: MangaLoaderContext) : isSearchSupported = true, isSearchWithFiltersSupported = true, isMultipleTagsSupported = true, - isTagsExclusionSupported = true, + isTagsExclusionSupported = false, ) private fun Locale.toKaganeLangCode(): String { @@ -134,33 +134,10 @@ internal class Kagane(context: MangaLoaderContext) : private var genresCache: Set<MangaTag>? = null private val genresMutex = Mutex() - private suspend fun fetchGenres(): Set<MangaTag> { - val headers = getRequestHeaders().newBuilder() - .add("Origin", "https://$domain") - .add("Referer", "https://$domain/") - .build() - return try { - val raw = webClient.httpGet("$apiUrl/api/v2/genres/list", headers).parseRaw() - val genres = runCatching { JSONArray(raw) }.getOrElse { - val wrapper = runCatching { JSONObject(raw) }.getOrNull() - wrapper?.optJSONArray("content") - ?: wrapper?.optJSONArray("genres") - ?: JSONArray() - } - buildSet { - for (i in 0 until genres.length()) { - val item = genres.optJSONObject(i) ?: continue - val id = item.optGenreId() - val title = item.optGenreName() - if (id.isNotBlank() && title.isNotBlank() && UUID_REGEX.matches(id)) { - add(MangaTag(title, id, source)) - } - } - } - } catch (_: Exception) { - emptySet() - } - } + private fun fetchGenres(): Set<MangaTag> = GENRES + .filter { UUID_REGEX.matches(it.first) } + .map { MangaTag(it.second, it.first, source) } + .toSet() private fun parseContentRating(value: String?): ContentRating? { return when (value?.lowercase(Locale.ROOT)) { @@ -224,17 +201,6 @@ internal class Kagane(context: MangaLoaderContext) : genresObj.put("match_all", false) jsonBody.put("genres", genresObj) } - if (filter.tagsExclude.isNotEmpty()) { - val excludedGenreIds = filter.tagsExclude.map { it.key }.filter { UUID_REGEX.matches(it) } - if (excludedGenreIds.isNotEmpty()) { - val genresObj = jsonBody.optJSONObject("genres") ?: JSONObject().also { - jsonBody.put("genres", it) - } - genresObj.put("exclude", JSONArray().apply { - excludedGenreIds.forEach(::put) - }) - } - } jsonBody.put("content_rating", JSONArray().apply { val ratings = filter.contentRating.ifEmpty { EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) @@ -908,5 +874,74 @@ internal class Kagane(context: MangaLoaderContext) : .ifBlank { optString("genreName") } .ifBlank { optString("name") } .ifBlank { optString("title") } + + private val GENRES = listOf( + "019bef86-56a6-71a1-af75-c1e43f48fba4" to "4-koma", + "019c3a02-5288-777a-b5d8-1bf9f79163ae" to "AI", + "019c1fe4-fabf-7242-a0a8-7866f98f705e" to "Action", + "019c1b6f-b682-7fee-9a9b-86e648b77e9e" to "Adventure", + "019c1fea-1b14-72d5-b2f8-c3194c4e1efe" to "Award Winning", + "019c1fe5-51fe-7f26-aee3-f057ac50f81d" to "Comedy", + "019c3a02-b00d-731d-884c-41460acc81e4" to "Coming of Age", + "019c2043-f726-7f78-9c58-7caf4662bf1f" to "Cooking", + "019c2043-d238-744b-8d5a-21bbc5166bd7" to "Crime", + "019c1fe8-cad1-7842-82a7-db5a87fc0a5a" to "Demons", + "019c1fea-2ddc-7e74-9935-a8b1b7782dd2" to "Doujinshi", + "019c1b6f-3d3b-7528-a3e2-a791cfdcc4f7" to "Drama", + "019c200d-6238-7364-9422-c50bd75e4e0d" to "Ecchi", + "019c1fe9-da2f-7938-a795-2b862b221cb3" to "Fan Colored", + "019c1b6f-51ed-7991-a796-a8ca26e86666" to "Fantasy", + "019c1fe9-9527-7f47-bf28-36ab2d6d4acd" to "Full Color", + "019c1fe7-5189-7619-9094-6e4b85c139ec" to "Gender Bender", + "019c1fe6-6717-77fa-8205-85d8e8a8755c" to "Gore", + "019c1fe8-74eb-73e6-840e-7916f145012e" to "Harem", + "019c3a01-d241-7d44-836d-095b98d4dc1e" to "Hentai", + "019c1fe6-7ea3-7741-9191-5ba94f6e9552" to "Historical", + "019c1b6f-c2e3-7909-93cf-0a48a2f2278f" to "Horror", + "019c1fe6-3ba5-7e76-aa0d-8756c314e2fa" to "Isekai", + "019c1b70-101d-7c9c-a28c-cc2439104443" to "Josei", + "019c1feb-8388-7c89-8c2d-ddfb530065e6" to "LGBTQIA+", + "019c1fe8-2221-7d69-9cd2-b67d64924994" to "Magic", + "019c1fe7-13fa-7019-a5a6-e7bbceb200f2" to "Magical Girls", + "019c1fe8-4c93-7f12-a6f0-e805fe4ade21" to "Martial Arts", + "019c2044-1fd9-7e84-b9c9-cbf20945a664" to "Mecha", + "019c1fe7-2c1e-730c-af33-ca3bef3a161d" to "Medical", + "019c1fe8-e822-7938-a04c-dee0a24f022c" to "Military", + "019c1fe8-3261-7c3d-9103-428f9a54eb41" to "Monsters", + "019c200f-1825-738c-b7c1-67266b9520b1" to "Music", + "019c1b6f-7991-7edb-8c7f-df6876f09a94" to "Mystery", + "019c1fe8-a048-7453-a06d-212678bf777a" to "Office Workers", + "019c1fe9-f0a1-71c6-bbe0-6afc3822647a" to "Official Colored", + "019c22cd-bb85-7541-9db6-7e6ea3394d0b" to "Omegaverse", + "019c1fe9-ae8d-7294-bcc8-9bd4269b94d7" to "Oneshot", + "019c1fe7-380c-72a5-8668-930c5e90ec44" to "Philosophical", + "019c1fe9-4fe3-776a-b11c-a14d76a52e7d" to "Police", + "019c1fe9-0e7c-7782-8265-c29ee6c19e79" to "Post-Apocalyptic", + "019c1ee5-dfa9-7b69-86a8-16831c1d81fe" to "Psychological", + "019c1fe8-5a6d-73bd-835d-16ae059bce6e" to "Reincarnation", + "019c1fe8-8837-78de-a911-f54754c2e713" to "Reverse Harem", + "019beb9a-a36a-718c-b551-d917a0818cc7" to "Romance", + "019c1fe7-db44-77a1-acc2-28c2bbb49ae1" to "School Life", + "019c1fe5-b704-72a6-a676-a8ad87ebcdd8" to "Sci-Fi", + "019c1fea-983f-7279-944e-8fb6e2cccdbc" to "Seinen", + "019c1b6f-fe1c-7eed-b93b-e1bf6dff63fd" to "Shoujo", + "019c1ee5-626f-7284-b5e8-8f3df5c5f76b" to "Shoujo Ai", + "019c1fea-85c7-76a7-996a-b41ce0cb05f8" to "Shounen", + "019c1ee5-4be9-766d-b545-5b87d7cdff96" to "Shounen Ai", + "019c1fe5-84d9-7a67-9240-1f303dc231bc" to "Slice of Life", + "019c2042-2a6f-7bcb-9c29-6098b8426aee" to "Smut", + "019c1fe6-bc40-7aa4-bac6-29a6cc29b559" to "Sports", + "019c1b6f-9c7b-77a0-8b54-e62925422b91" to "Supernatural", + "019c1fe8-b2ab-7e15-a55b-9fb28901cc9d" to "Survival", + "019c1fe6-0228-7052-98a2-fe6df7994d93" to "Thriller", + "019c1fe8-f932-7a19-a502-4ad6a2adfae4" to "Time Travel", + "019c1fe6-211d-7e73-927d-3c54b66a9055" to "Tragedy", + "019c1fe9-78cc-7d2e-be56-cc3f8442ccf0" to "Vampires", + "019c1fe9-21a7-7bbd-9763-10a5855a8e5c" to "Video Games", + "019c1fe9-3609-73d2-b4bf-8420cc15c5b2" to "Villainess", + "019c1fe7-0344-755b-b4df-c019f4cdbc21" to "Wuxia", + "019c1ee5-38f9-7324-9f7d-07a01dbf88a6" to "Yaoi", + "019c1ee5-7e79-7fd2-8fcc-de739858c851" to "Yuri", + ) } } From c517395edd14dbbc3bf179f02a2fa6b68f1e0582 Mon Sep 17 00:00:00 2001 From: biglag <dmitriy.gudkov@dotcross.ai> Date: Sun, 6 Sep 2026 23:49:01 +0800 Subject: [PATCH 398/434] Fix: fix nsfw sources --- .github/summary.yaml | 2 +- .../kotlin/tsuki/parsers/GroupleParser.kt | 4 +- .../tsuki/site/ru/nsfw/AllHentaiParser.kt | 1 - .../tsuki/site/ru/nsfw/HenChanParser.kt | 1 - .../tsuki/site/ru/nsfw/HentaiLibParser.kt | 17 -- .../kotlin/tsuki/site/ru/nsfw/Mangazavr.kt | 16 -- .../tsuki/site/ru/nsfw/NudeMoonParser.kt | 198 ------------------ 7 files changed, 3 insertions(+), 236 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/HentaiLibParser.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/Mangazavr.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/NudeMoonParser.kt diff --git a/.github/summary.yaml b/.github/summary.yaml index 0e21b9bf..edeeedd8 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 236 \ No newline at end of file +total: 233 \ No newline at end of file diff --git a/src/main/kotlin/tsuki/parsers/GroupleParser.kt b/src/main/kotlin/tsuki/parsers/GroupleParser.kt index 0e0c851e..e22cb743 100644 --- a/src/main/kotlin/tsuki/parsers/GroupleParser.kt +++ b/src/main/kotlin/tsuki/parsers/GroupleParser.kt @@ -157,8 +157,8 @@ internal abstract class GroupleParser( largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("src"), coverUrl = manga.coverUrl ?: coverImg?.attrAsAbsoluteUrlOrNull("src"), - tags = root.selectFirstOrThrow("div.creation-element-tags") - .getElementsByAttributeValueContaining("href", "/list/genre/").mapTo(manga.tags.toMutableSet()) { a -> + tags = root.select("a[href*='/list/genre/']") + .mapTo(manga.tags.toMutableSet()) { a -> MangaTag( title = a.text().toTitleCase(), key = a.attr("href").removeSuffix('/').substringAfterLast('/'), diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt index 2076e13c..610aba5b 100644 --- a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt +++ b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt @@ -10,7 +10,6 @@ import tsuki.model.ContentType import tsuki.model.MangaParserSource import tsuki.util.* -@Broken @MangaSourceParser("ALLHENTAI", "AllHentai", "ru", type = ContentType.HENTAI) internal class AllHentaiParser( context: MangaLoaderContext, diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt index 5c2f0bb2..32cfc545 100644 --- a/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt +++ b/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt @@ -10,7 +10,6 @@ import tsuki.model.* import tsuki.util.* import java.util.* -@Broken @MangaSourceParser("HENCHAN", "Хентай-тян", "ru", type = ContentType.HENTAI) internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.HENCHAN) { diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/HentaiLibParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/HentaiLibParser.kt deleted file mode 100644 index 032693ed..00000000 --- a/src/main/kotlin/tsuki/site/ru/nsfw/HentaiLibParser.kt +++ /dev/null @@ -1,17 +0,0 @@ -package tsuki.site.ru.nsfw - -import tsuki.parsers.LibSocialParser -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@Broken -@MangaSourceParser("HENTAILIB", "HentaiLib", "ru", type = ContentType.HENTAI) -internal class HentaiLibParser(context: MangaLoaderContext) : LibSocialParser( - context = context, - source = MangaParserSource.HENTAILIB, - siteId = 4, - siteDomains = arrayOf("v1.hentailib.org", "hentailib.me"), -) diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/Mangazavr.kt b/src/main/kotlin/tsuki/site/ru/nsfw/Mangazavr.kt deleted file mode 100644 index 3c7c731d..00000000 --- a/src/main/kotlin/tsuki/site/ru/nsfw/Mangazavr.kt +++ /dev/null @@ -1,16 +0,0 @@ -package tsuki.site.ru.nsfw - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.parsers.MadaraParser - -@Broken -@MangaSourceParser("MANGAZAVR", "Mangazavr", "ru", ContentType.HENTAI) -internal class Mangazavr(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAZAVR, "mangazavr.ru") { - override val listUrl = "/?s=&post_type=wp-manga" - override val datePattern = "dd.MM.yyyy" -} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/NudeMoonParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/NudeMoonParser.kt deleted file mode 100644 index 2d31fe59..00000000 --- a/src/main/kotlin/tsuki/site/ru/nsfw/NudeMoonParser.kt +++ /dev/null @@ -1,198 +0,0 @@ -package tsuki.site.ru.nsfw - -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.AbstractMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.util.* -import java.text.SimpleDateFormat -import java.util.* - -@Broken -@MangaSourceParser("NUDEMOON", "Nude-Moon", "ru", type = ContentType.HENTAI) -internal class NudeMoonParser( - context: MangaLoaderContext, -) : AbstractMangaParser(context, MangaParserSource.NUDEMOON), MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain( - "b.nude-moon.fun", - "nude-moon.org", - "nude-moon.net", - ) - override val authUrl: String - get() = "https://${domain}/index.php" - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name == "fusion_user" - } - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - init { - context.cookieJar.insertCookies( - domain, - "NMfYa=1;", - "nm_mobile=1;", - ) - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val domain = domain - - val url = when { - !filter.query.isNullOrEmpty() -> { - if (!isAuthorized()) { - throw AuthRequiredException(source) - } - "https://$domain/search?stext=${filter.query.urlEncoded()}&rowstart=$offset" - } - - else -> { - if (filter.tags.isNotEmpty()) { - filter.tags.joinToString( - separator = "_", - prefix = "https://$domain/tags/", - postfix = "&rowstart=$offset", - transform = { it.key.urlEncoded() }, - ) - } else { - val order = when (order) { - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "date" - SortOrder.RATING -> "like" - else -> "like" - } - "https://$domain/all_manga?$order&rowstart=$offset" - } - } - - } - - val doc = webClient.httpGet(url).parseHtml() - return doc.body().select("table.news_pic2").mapNotNull { row -> - val a = row.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = a.attr("title") - val author = row.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.textOrNull() - Manga( - id = generateUid(href), - url = href, - title = title.substringAfter(" / "), - altTitles = setOfNotNull(title.substringBefore(" / ", "").takeUnless { it.isBlank() }), - authors = setOfNotNull(author), - coverUrl = row.selectFirst("img")?.absUrl("src").orEmpty(), - tags = row.selectFirst(".tag-links")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - }.orEmpty(), - source = source, - publicUrl = a.absUrl("href"), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - description = row.selectFirst("div.description")?.html(), - state = null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val body = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml().body() - val root = body.selectFirstOrThrow("table.news_pic2") - val dateFormat = SimpleDateFormat("dd MMMM yyyy", sourceLocale) - val author = root.getElementsByAttributeValueContaining("href", "/mangaka/").firstOrNull()?.text() - return manga.copy( - largeCoverUrl = body.selectFirstOrThrow("img[data-src]").attrAsAbsoluteUrl("data-src"), - description = root.selectFirst(".description")?.html() ?: manga.description, - tags = root.getElementsByAttributeValueContaining("href", "/tag/").mapToSet { - MangaTag( - title = it.text().toTitleCase(), - key = it.attr("href").substringAfterLast('/'), - source = source, - ) - } + manga.tags, - authors = author?.let { setOf(it) } ?: manga.authors, - chapters = listOf( - MangaChapter( - id = manga.id, - url = manga.url, - source = source, - number = 0f, - volume = 0, - title = manga.title, - scanlator = root.getElementsByAttributeValueContaining("href", "/perevod/").firstOrNull() - ?.textOrNull(), - uploadDate = dateFormat.parseSafe( - root.getElementsByAttributeValueEnding("src", "ico/time.png").firstOrNull() - ?.nextElementSibling()?.text(), - ), - branch = null, - ), - ), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val pages = doc.select("img[data-src]") - return pages.map { img -> - val url = img.attrAsRelativeUrl("data-src") - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - }.toList() - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val domain = domain - val doc = webClient.httpGet("https://$domain/tags").parseHtml() - val root = doc.body().getElementsByAttributeValue("name", "multitags").first() - ?: doc.parseFailed("Tags form not found") - return root.select("input").mapToSet { - val value = it.attr("value").trim() - MangaTag( - title = value.toTitleCase(sourceLocale), - key = value.replace(' ', '_'), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val body = webClient.httpGet("https://${domain}/").parseHtml().body() - return body.getElementsContainingOwnText("Профиль").firstOrNull()?.attr("href")?.substringAfterLast('/') - ?: run { - throw if (body.selectFirst("form[name=\"loginform\"]") != null) { - AuthRequiredException(source) - } else { - body.parseFailed("Cannot find username") - } - } - } -} From 00225022b13de27d48bfb9d193bf7a8cbb11fe43 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 17:54:24 +0200 Subject: [PATCH 399/434] Update Kagane.kt --- src/main/kotlin/tsuki/site/all/Kagane.kt | 154 ++++++++++++----------- 1 file changed, 84 insertions(+), 70 deletions(-) diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt index 2151404c..9d11e5f6 100644 --- a/src/main/kotlin/tsuki/site/all/Kagane.kt +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -91,7 +91,7 @@ internal class Kagane(context: MangaLoaderContext) : isSearchSupported = true, isSearchWithFiltersSupported = true, isMultipleTagsSupported = true, - isTagsExclusionSupported = false, + isTagsExclusionSupported = true, ) private fun Locale.toKaganeLangCode(): String { @@ -201,6 +201,20 @@ internal class Kagane(context: MangaLoaderContext) : genresObj.put("match_all", false) jsonBody.put("genres", genresObj) } + if (filter.tagsExclude.isNotEmpty()) { + val excludedGenreIds = filter.tagsExclude.map { it.key }.filter { UUID_REGEX.matches(it) } + if (excludedGenreIds.isNotEmpty()) { + val genresObj = jsonBody.optJSONObject("genres") ?: JSONObject().also { + jsonBody.put("genres", it) + } + if (!genresObj.has("values")) { + genresObj.put("values", JSONArray()) + } + genresObj.put("exclude", JSONArray().apply { + excludedGenreIds.forEach(::put) + }) + } + } jsonBody.put("content_rating", JSONArray().apply { val ratings = filter.contentRating.ifEmpty { EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) @@ -795,7 +809,7 @@ internal class Kagane(context: MangaLoaderContext) : return response } - companion object { + companion object { private const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L private val UUID_REGEX = Regex( @@ -875,73 +889,73 @@ internal class Kagane(context: MangaLoaderContext) : .ifBlank { optString("name") } .ifBlank { optString("title") } - private val GENRES = listOf( - "019bef86-56a6-71a1-af75-c1e43f48fba4" to "4-koma", - "019c3a02-5288-777a-b5d8-1bf9f79163ae" to "AI", - "019c1fe4-fabf-7242-a0a8-7866f98f705e" to "Action", - "019c1b6f-b682-7fee-9a9b-86e648b77e9e" to "Adventure", - "019c1fea-1b14-72d5-b2f8-c3194c4e1efe" to "Award Winning", - "019c1fe5-51fe-7f26-aee3-f057ac50f81d" to "Comedy", - "019c3a02-b00d-731d-884c-41460acc81e4" to "Coming of Age", - "019c2043-f726-7f78-9c58-7caf4662bf1f" to "Cooking", - "019c2043-d238-744b-8d5a-21bbc5166bd7" to "Crime", - "019c1fe8-cad1-7842-82a7-db5a87fc0a5a" to "Demons", - "019c1fea-2ddc-7e74-9935-a8b1b7782dd2" to "Doujinshi", - "019c1b6f-3d3b-7528-a3e2-a791cfdcc4f7" to "Drama", - "019c200d-6238-7364-9422-c50bd75e4e0d" to "Ecchi", - "019c1fe9-da2f-7938-a795-2b862b221cb3" to "Fan Colored", - "019c1b6f-51ed-7991-a796-a8ca26e86666" to "Fantasy", - "019c1fe9-9527-7f47-bf28-36ab2d6d4acd" to "Full Color", - "019c1fe7-5189-7619-9094-6e4b85c139ec" to "Gender Bender", - "019c1fe6-6717-77fa-8205-85d8e8a8755c" to "Gore", - "019c1fe8-74eb-73e6-840e-7916f145012e" to "Harem", - "019c3a01-d241-7d44-836d-095b98d4dc1e" to "Hentai", - "019c1fe6-7ea3-7741-9191-5ba94f6e9552" to "Historical", - "019c1b6f-c2e3-7909-93cf-0a48a2f2278f" to "Horror", - "019c1fe6-3ba5-7e76-aa0d-8756c314e2fa" to "Isekai", - "019c1b70-101d-7c9c-a28c-cc2439104443" to "Josei", - "019c1feb-8388-7c89-8c2d-ddfb530065e6" to "LGBTQIA+", - "019c1fe8-2221-7d69-9cd2-b67d64924994" to "Magic", - "019c1fe7-13fa-7019-a5a6-e7bbceb200f2" to "Magical Girls", - "019c1fe8-4c93-7f12-a6f0-e805fe4ade21" to "Martial Arts", - "019c2044-1fd9-7e84-b9c9-cbf20945a664" to "Mecha", - "019c1fe7-2c1e-730c-af33-ca3bef3a161d" to "Medical", - "019c1fe8-e822-7938-a04c-dee0a24f022c" to "Military", - "019c1fe8-3261-7c3d-9103-428f9a54eb41" to "Monsters", - "019c200f-1825-738c-b7c1-67266b9520b1" to "Music", - "019c1b6f-7991-7edb-8c7f-df6876f09a94" to "Mystery", - "019c1fe8-a048-7453-a06d-212678bf777a" to "Office Workers", - "019c1fe9-f0a1-71c6-bbe0-6afc3822647a" to "Official Colored", - "019c22cd-bb85-7541-9db6-7e6ea3394d0b" to "Omegaverse", - "019c1fe9-ae8d-7294-bcc8-9bd4269b94d7" to "Oneshot", - "019c1fe7-380c-72a5-8668-930c5e90ec44" to "Philosophical", - "019c1fe9-4fe3-776a-b11c-a14d76a52e7d" to "Police", - "019c1fe9-0e7c-7782-8265-c29ee6c19e79" to "Post-Apocalyptic", - "019c1ee5-dfa9-7b69-86a8-16831c1d81fe" to "Psychological", - "019c1fe8-5a6d-73bd-835d-16ae059bce6e" to "Reincarnation", - "019c1fe8-8837-78de-a911-f54754c2e713" to "Reverse Harem", - "019beb9a-a36a-718c-b551-d917a0818cc7" to "Romance", - "019c1fe7-db44-77a1-acc2-28c2bbb49ae1" to "School Life", - "019c1fe5-b704-72a6-a676-a8ad87ebcdd8" to "Sci-Fi", - "019c1fea-983f-7279-944e-8fb6e2cccdbc" to "Seinen", - "019c1b6f-fe1c-7eed-b93b-e1bf6dff63fd" to "Shoujo", - "019c1ee5-626f-7284-b5e8-8f3df5c5f76b" to "Shoujo Ai", - "019c1fea-85c7-76a7-996a-b41ce0cb05f8" to "Shounen", - "019c1ee5-4be9-766d-b545-5b87d7cdff96" to "Shounen Ai", - "019c1fe5-84d9-7a67-9240-1f303dc231bc" to "Slice of Life", - "019c2042-2a6f-7bcb-9c29-6098b8426aee" to "Smut", - "019c1fe6-bc40-7aa4-bac6-29a6cc29b559" to "Sports", - "019c1b6f-9c7b-77a0-8b54-e62925422b91" to "Supernatural", - "019c1fe8-b2ab-7e15-a55b-9fb28901cc9d" to "Survival", - "019c1fe6-0228-7052-98a2-fe6df7994d93" to "Thriller", - "019c1fe8-f932-7a19-a502-4ad6a2adfae4" to "Time Travel", - "019c1fe6-211d-7e73-927d-3c54b66a9055" to "Tragedy", - "019c1fe9-78cc-7d2e-be56-cc3f8442ccf0" to "Vampires", - "019c1fe9-21a7-7bbd-9763-10a5855a8e5c" to "Video Games", - "019c1fe9-3609-73d2-b4bf-8420cc15c5b2" to "Villainess", - "019c1fe7-0344-755b-b4df-c019f4cdbc21" to "Wuxia", - "019c1ee5-38f9-7324-9f7d-07a01dbf88a6" to "Yaoi", - "019c1ee5-7e79-7fd2-8fcc-de739858c851" to "Yuri", - ) + private val GENRES = listOf( + "019bef86-56a6-71a1-af75-c1e43f48fba4" to "4-koma", + "019c3a02-5288-777a-b5d8-1bf9f79163ae" to "AI", + "019c1fe4-fabf-7242-a0a8-7866f98f705e" to "Action", + "019c1b6f-b682-7fee-9a9b-86e648b77e9e" to "Adventure", + "019c1fea-1b14-72d5-b2f8-c3194c4e1efe" to "Award Winning", + "019c1fe5-51fe-7f26-aee3-f057ac50f81d" to "Comedy", + "019c3a02-b00d-731d-884c-41460acc81e4" to "Coming of Age", + "019c2043-f726-7f78-9c58-7caf4662bf1f" to "Cooking", + "019c2043-d238-744b-8d5a-21bbc5166bd7" to "Crime", + "019c1fe8-cad1-7842-82a7-db5a87fc0a5a" to "Demons", + "019c1fea-2ddc-7e74-9935-a8b1b7782dd2" to "Doujinshi", + "019c1b6f-3d3b-7528-a3e2-a791cfdcc4f7" to "Drama", + "019c200d-6238-7364-9422-c50bd75e4e0d" to "Ecchi", + "019c1fe9-da2f-7938-a795-2b862b221cb3" to "Fan Colored", + "019c1b6f-51ed-7991-a796-a8ca26e86666" to "Fantasy", + "019c1fe9-9527-7f47-bf28-36ab2d6d4acd" to "Full Color", + "019c1fe7-5189-7619-9094-6e4b85c139ec" to "Gender Bender", + "019c1fe6-6717-77fa-8205-85d8e8a8755c" to "Gore", + "019c1fe8-74eb-73e6-840e-7916f145012e" to "Harem", + "019c3a01-d241-7d44-836d-095b98d4dc1e" to "Hentai", + "019c1fe6-7ea3-7741-9191-5ba94f6e9552" to "Historical", + "019c1b6f-c2e3-7909-93cf-0a48a2f2278f" to "Horror", + "019c1fe6-3ba5-7e76-aa0d-8756c314e2fa" to "Isekai", + "019c1b70-101d-7c9c-a28c-cc2439104443" to "Josei", + "019c1feb-8388-7c89-8c2d-ddfb530065e6" to "LGBTQIA+", + "019c1fe8-2221-7d69-9cd2-b67d64924994" to "Magic", + "019c1fe7-13fa-7019-a5a6-e7bbceb200f2" to "Magical Girls", + "019c1fe8-4c93-7f12-a6f0-e805fe4ade21" to "Martial Arts", + "019c2044-1fd9-7e84-b9c9-cbf20945a664" to "Mecha", + "019c1fe7-2c1e-730c-af33-ca3bef3a161d" to "Medical", + "019c1fe8-e822-7938-a04c-dee0a24f022c" to "Military", + "019c1fe8-3261-7c3d-9103-428f9a54eb41" to "Monsters", + "019c200f-1825-738c-b7c1-67266b9520b1" to "Music", + "019c1b6f-7991-7edb-8c7f-df6876f09a94" to "Mystery", + "019c1fe8-a048-7453-a06d-212678bf777a" to "Office Workers", + "019c1fe9-f0a1-71c6-bbe0-6afc3822647a" to "Official Colored", + "019c22cd-bb85-7541-9db6-7e6ea3394d0b" to "Omegaverse", + "019c1fe9-ae8d-7294-bcc8-9bd4269b94d7" to "Oneshot", + "019c1fe7-380c-72a5-8668-930c5e90ec44" to "Philosophical", + "019c1fe9-4fe3-776a-b11c-a14d76a52e7d" to "Police", + "019c1fe9-0e7c-7782-8265-c29ee6c19e79" to "Post-Apocalyptic", + "019c1ee5-dfa9-7b69-86a8-16831c1d81fe" to "Psychological", + "019c1fe8-5a6d-73bd-835d-16ae059bce6e" to "Reincarnation", + "019c1fe8-8837-78de-a911-f54754c2e713" to "Reverse Harem", + "019beb9a-a36a-718c-b551-d917a0818cc7" to "Romance", + "019c1fe7-db44-77a1-acc2-28c2bbb49ae1" to "School Life", + "019c1fe5-b704-72a6-a676-a8ad87ebcdd8" to "Sci-Fi", + "019c1fea-983f-7279-944e-8fb6e2cccdbc" to "Seinen", + "019c1b6f-fe1c-7eed-b93b-e1bf6dff63fd" to "Shoujo", + "019c1ee5-626f-7284-b5e8-8f3df5c5f76b" to "Shoujo Ai", + "019c1fea-85c7-76a7-996a-b41ce0cb05f8" to "Shounen", + "019c1ee5-4be9-766d-b545-5b87d7cdff96" to "Shounen Ai", + "019c1fe5-84d9-7a67-9240-1f303dc231bc" to "Slice of Life", + "019c2042-2a6f-7bcb-9c29-6098b8426aee" to "Smut", + "019c1fe6-bc40-7aa4-bac6-29a6cc29b559" to "Sports", + "019c1b6f-9c7b-77a0-8b54-e62925422b91" to "Supernatural", + "019c1fe8-b2ab-7e15-a55b-9fb28901cc9d" to "Survival", + "019c1fe6-0228-7052-98a2-fe6df7994d93" to "Thriller", + "019c1fe8-f932-7a19-a502-4ad6a2adfae4" to "Time Travel", + "019c1fe6-211d-7e73-927d-3c54b66a9055" to "Tragedy", + "019c1fe9-78cc-7d2e-be56-cc3f8442ccf0" to "Vampires", + "019c1fe9-21a7-7bbd-9763-10a5855a8e5c" to "Video Games", + "019c1fe9-3609-73d2-b4bf-8420cc15c5b2" to "Villainess", + "019c1fe7-0344-755b-b4df-c019f4cdbc21" to "Wuxia", + "019c1ee5-38f9-7324-9f7d-07a01dbf88a6" to "Yaoi", + "019c1ee5-7e79-7fd2-8fcc-de739858c851" to "Yuri", + ) } } From c03f2215a049d3f09f61173e2513244e96976ba0 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:07:49 +0200 Subject: [PATCH 400/434] Update and rename YaoiChanParser.kt to YaoiChan.kt --- src/main/kotlin/tsuki/site/ru/YaoiChan.kt | 58 +++++++++++++++++++ .../kotlin/tsuki/site/ru/YaoiChanParser.kt | 51 ---------------- 2 files changed, 58 insertions(+), 51 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/YaoiChan.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/YaoiChanParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/YaoiChan.kt b/src/main/kotlin/tsuki/site/ru/YaoiChan.kt new file mode 100644 index 00000000..5bdbd8ab --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/YaoiChan.kt @@ -0,0 +1,58 @@ +package tsuki.site.ru + +import tsuki.parsers.ChanParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.toAbsoluteUrl + +@MangaSourceParser("YAOICHAN", "Яой-тян", "ru") +internal class YaoiChan(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "v9.yaoi-chan.me", + "v10.yaoi-chan.me", + "v3.yaoi-chan.me", + "v2.yaoi-chan.me", + "v1.yaoi-chan.me", + "yaoi-chan.me", + ) + + override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + chapters = root.select("table.table_cha").flatMap { table -> + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = a.text(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/YaoiChanParser.kt b/src/main/kotlin/tsuki/site/ru/YaoiChanParser.kt deleted file mode 100644 index 03cebbd8..00000000 --- a/src/main/kotlin/tsuki/site/ru/YaoiChanParser.kt +++ /dev/null @@ -1,51 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.ChanParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaParserSource -import tsuki.model.SortOrder -import tsuki.util.* - -@MangaSourceParser("YAOICHAN", "Яой-тян", "ru") -internal class YaoiChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { - - override val configKeyDomain = ConfigKey.Domain( - "v9.yaoi-chan.me", - "v10.yaoi-chan.me", - "v3.yaoi-chan.me", - "v2.yaoi-chan.me", - "v1.yaoi-chan.me", - "yaoi-chan.me", - ) - - override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - chapters = root.select("table.table_cha").flatMap { table -> - table.select("div.manga") - }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> - val href = a.attrAsRelativeUrl("href") - MangaChapter( - id = generateUid(href), - title = a.text(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = 0L, - source = source, - scanlator = null, - branch = null, - ) - }, - ) - } -} From af7beafd64fdddfeb2da9b58171cc7b354bdb957 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:09:26 +0200 Subject: [PATCH 401/434] Update and rename WaMangaParser.kt to WaManga.kt --- src/main/kotlin/tsuki/site/ru/WaManga.kt | 179 +++++++++++++++++ .../kotlin/tsuki/site/ru/WaMangaParser.kt | 182 ------------------ 2 files changed, 179 insertions(+), 182 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/WaManga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/WaMangaParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/WaManga.kt b/src/main/kotlin/tsuki/site/ru/WaManga.kt new file mode 100644 index 00000000..05db5e34 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/WaManga.kt @@ -0,0 +1,179 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.json.asTypedList +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe + +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("WAMANGA", "WaManga", "ru") +internal class WaManga(context: MangaLoaderContext) : + SinglePageMangaParser(context, MangaParserSource.WAMANGA) { + + override val configKeyDomain = ConfigKey.Domain("wamanga.ru") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalog() + .flatMapTo(linkedSetOf()) { it.parseWaMangaTags() }, + ) + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + val requiredGenres = filter.tags.mapTo(mutableSetOf()) { it.key } + + return fetchCatalog() + .asSequence() + .filter { item -> + query == null || item.parseWaMangaSearchText().any { value -> + value.contains(query, ignoreCase = true) + } + } + .filter { item -> + requiredGenres.isEmpty() || item.parseWaMangaGenres().containsAll(requiredGenres) + } + .map(::parseManga) + .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.title }) + .toList() + } + + override suspend fun getDetails(manga: Manga): Manga { + val details = webClient.httpGet(apiUrl("manga/${manga.url}")).parseJson() + val chapters = webClient.httpGet(apiUrl("manga/${manga.url}/chapters")) + .parseJsonArray() + .mapJSON(::parseChapter) + .sortedByDescending { it.number } + + return parseManga(details).copy( + id = manga.id, + url = manga.url, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + return webClient.httpGet(apiUrl("chapters/${chapter.url}")) + .parseJson() + .getJSONArray("files") + .asTypedList<JSONObject>() + .sortedBy { it.getString("position").toDouble() } + .map { file -> + val imageUrl = file.getString("diskFile").toWaMangaAbsoluteUrl(domain) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchCatalog(): List<JSONObject> = webClient + .httpGet(apiUrl("manga")) + .parseJsonArray() + .asTypedList() + + private fun parseManga(item: JSONObject): Manga { + val id = item.getString("id") + val slug = item.getString("slug") + return Manga( + id = generateUid(id), + url = id, + title = item.getString("title"), + altTitles = item.parseWaMangaAlternativeTitles(), + publicUrl = "https://$domain/manga/$slug", + rating = RATING_UNKNOWN, + coverUrl = item.getString("coverUrl").toWaMangaAbsoluteUrl(domain), + largeCoverUrl = item.getString("imageUrl").toWaMangaAbsoluteUrl(domain), + tags = item.parseWaMangaTags(), + state = item.getString("statusTitle").toWaMangaState(), + authors = item.getJSONArray("authors").asTypedList<String>().toSet(), + source = source, + contentRating = if (item.getBoolean("isAdult")) ContentRating.ADULT else ContentRating.SAFE, + description = item.getStringOrNull("description"), + ) + } + + private fun parseChapter(item: JSONObject): MangaChapter { + val id = item.getString("id") + return MangaChapter( + id = generateUid(id), + url = id, + source = source, + number = item.getDouble("position").toFloat(), + volume = 0, + title = item.getStringOrNull("title")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = WA_MANGA_DATE_FORMAT.get().parseSafe(item.getString("createdAt")), + branch = null, + ) + } + + private fun apiUrl(path: String): String = "https://$domain/api/v1/$path" +} + +internal fun JSONObject.parseWaMangaAlternativeTitles(): Set<String> = buildSet { + getStringOrNull("titleEnglish")?.takeIf { it.isNotBlank() }?.let(::add) + getJSONArray("alternateTitles").asTypedList<String>().filterTo(this) { it.isNotBlank() } +} + +internal fun JSONObject.parseWaMangaGenres(): Set<String> = + getJSONArray("genres").asTypedList<String>().toSet() + +internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { + add(getString("title")) + addAll(parseWaMangaAlternativeTitles()) +} + +private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> + MangaTag(genre, genre, source) +} + +private fun String.toWaMangaState(): MangaState = when (this) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "abandoned" -> MangaState.ABANDONED + else -> error("Unsupported WaManga status: $this") +} + +private fun String.toWaMangaAbsoluteUrl(domain: String): String = when { + startsWith("https://") || startsWith("http://") -> this + startsWith('/') -> "https://$domain$this" + else -> error("Unsupported WaManga asset URL: $this") +} + +private val WA_MANGA_DATE_FORMAT = ThreadLocal.withInitial { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US) +} diff --git a/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt b/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt deleted file mode 100644 index 4addd136..00000000 --- a/src/main/kotlin/tsuki/site/ru/WaMangaParser.kt +++ /dev/null @@ -1,182 +0,0 @@ -package tsuki.site.ru - -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.SinglePageMangaParser -import tsuki.model.ContentRating -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder -import tsuki.util.json.asTypedList -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.generateUid -import tsuki.util.parseJson -import tsuki.util.parseJsonArray -import tsuki.util.parseSafe -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("WAMANGA", "WaManga", "ru", type = ContentType.MANGA) -internal class WaMangaParser( - context: MangaLoaderContext, -) : SinglePageMangaParser(context, MangaParserSource.WAMANGA) { - - override val configKeyDomain = ConfigKey.Domain("wamanga.ru") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchCatalog() - .flatMapTo(linkedSetOf()) { it.parseWaMangaTags() }, - ) - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - val requiredGenres = filter.tags.mapTo(mutableSetOf()) { it.key } - - return fetchCatalog() - .asSequence() - .filter { item -> - query == null || item.parseWaMangaSearchText().any { value -> - value.contains(query, ignoreCase = true) - } - } - .filter { item -> - requiredGenres.isEmpty() || item.parseWaMangaGenres().containsAll(requiredGenres) - } - .map(::parseManga) - .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.title }) - .toList() - } - - override suspend fun getDetails(manga: Manga): Manga { - val details = webClient.httpGet(apiUrl("manga/${manga.url}")).parseJson() - val chapters = webClient.httpGet(apiUrl("manga/${manga.url}/chapters")) - .parseJsonArray() - .mapJSON(::parseChapter) - .sortedByDescending { it.number } - - return parseManga(details).copy( - id = manga.id, - url = manga.url, - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - return webClient.httpGet(apiUrl("chapters/${chapter.url}")) - .parseJson() - .getJSONArray("files") - .asTypedList<JSONObject>() - .sortedBy { it.getString("position").toDouble() } - .map { file -> - val imageUrl = file.getString("diskFile").toWaMangaAbsoluteUrl(domain) - MangaPage( - id = generateUid(imageUrl), - url = imageUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchCatalog(): List<JSONObject> = webClient - .httpGet(apiUrl("manga")) - .parseJsonArray() - .asTypedList() - - private fun parseManga(item: JSONObject): Manga { - val id = item.getString("id") - val slug = item.getString("slug") - return Manga( - id = generateUid(id), - url = id, - title = item.getString("title"), - altTitles = item.parseWaMangaAlternativeTitles(), - publicUrl = "https://$domain/manga/$slug", - rating = RATING_UNKNOWN, - coverUrl = item.getString("coverUrl").toWaMangaAbsoluteUrl(domain), - largeCoverUrl = item.getString("imageUrl").toWaMangaAbsoluteUrl(domain), - tags = item.parseWaMangaTags(), - state = item.getString("statusTitle").toWaMangaState(), - authors = item.getJSONArray("authors").asTypedList<String>().toSet(), - source = source, - contentRating = if (item.getBoolean("isAdult")) ContentRating.ADULT else ContentRating.SAFE, - description = item.getStringOrNull("description"), - ) - } - - private fun parseChapter(item: JSONObject): MangaChapter { - val id = item.getString("id") - return MangaChapter( - id = generateUid(id), - url = id, - source = source, - number = item.getDouble("position").toFloat(), - volume = 0, - title = item.getStringOrNull("title")?.takeIf { it.isNotBlank() }, - scanlator = null, - uploadDate = WA_MANGA_DATE_FORMAT.get().parseSafe(item.getString("createdAt")), - branch = null, - ) - } - - private fun apiUrl(path: String): String = "https://$domain/api/v1/$path" -} - -internal fun JSONObject.parseWaMangaAlternativeTitles(): Set<String> = buildSet { - getStringOrNull("titleEnglish")?.takeIf { it.isNotBlank() }?.let(::add) - getJSONArray("alternateTitles").asTypedList<String>().filterTo(this) { it.isNotBlank() } -} - -internal fun JSONObject.parseWaMangaGenres(): Set<String> = - getJSONArray("genres").asTypedList<String>().toSet() - -internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { - add(getString("title")) - addAll(parseWaMangaAlternativeTitles()) -} - -private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> - MangaTag( - title = genre, - key = genre, - source = MangaParserSource.WAMANGA, - ) -} - -private fun String.toWaMangaState(): MangaState = when (this) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "abandoned" -> MangaState.ABANDONED - else -> error("Unsupported WaManga status: $this") -} - -private fun String.toWaMangaAbsoluteUrl(domain: String): String = when { - startsWith("https://") || startsWith("http://") -> this - startsWith('/') -> "https://$domain$this" - else -> error("Unsupported WaManga asset URL: $this") -} - -private val WA_MANGA_DATE_FORMAT = ThreadLocal.withInitial { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US) -} From 388c5aa3076aff4b566fb0576f0117444150e704 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:11:36 +0200 Subject: [PATCH 402/434] Update and rename UsagiParser.kt to Usagi.kt --- src/main/kotlin/tsuki/site/ru/Usagi.kt | 14 ++++++++++++ src/main/kotlin/tsuki/site/ru/UsagiParser.kt | 24 -------------------- 2 files changed, 14 insertions(+), 24 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/Usagi.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/UsagiParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/Usagi.kt b/src/main/kotlin/tsuki/site/ru/Usagi.kt new file mode 100644 index 00000000..00fb2c68 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Usagi.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("USAGI", "Usagi", "ru") +internal class UsagiParser(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.USAGI, 1) { + override val configKeyDomain = ConfigKey.Domain("web.usagi.one") +} diff --git a/src/main/kotlin/tsuki/site/ru/UsagiParser.kt b/src/main/kotlin/tsuki/site/ru/UsagiParser.kt deleted file mode 100644 index 2155bb1e..00000000 --- a/src/main/kotlin/tsuki/site/ru/UsagiParser.kt +++ /dev/null @@ -1,24 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.GroupleParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("USAGI", "Usagi", "ru") -internal class UsagiParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.USAGI, 1) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf("web.usagi.one") - } -} From 54e03b1bf0986799f161a2325e3a618ce83504fc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:12:54 +0200 Subject: [PATCH 403/434] Update and rename SlashLibParser.kt to SlashLib.kt --- .../tsuki/site/ru/{SlashLibParser.kt => SlashLib.kt} | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) rename src/main/kotlin/tsuki/site/ru/{SlashLibParser.kt => SlashLib.kt} (52%) diff --git a/src/main/kotlin/tsuki/site/ru/SlashLibParser.kt b/src/main/kotlin/tsuki/site/ru/SlashLib.kt similarity index 52% rename from src/main/kotlin/tsuki/site/ru/SlashLibParser.kt rename to src/main/kotlin/tsuki/site/ru/SlashLib.kt index c5b25258..bd0eab89 100644 --- a/src/main/kotlin/tsuki/site/ru/SlashLibParser.kt +++ b/src/main/kotlin/tsuki/site/ru/SlashLib.kt @@ -3,12 +3,9 @@ package tsuki.site.ru import tsuki.parsers.LibSocialParser import tsuki.MangaLoaderContext import tsuki.MangaSourceParser + import tsuki.model.MangaParserSource @MangaSourceParser("YAOILIB", "SlashLib", "ru") -internal class SlashLibParser(context: MangaLoaderContext) : LibSocialParser( - context = context, - source = MangaParserSource.YAOILIB, - siteId = 2, - siteDomains = arrayOf("v2.slashlib.me"), -) +internal class SlashLibParser(context: MangaLoaderContext) : + LibSocialParser(context, MangaParserSource.YAOILIB, 2, "v2.slashlib.me") From e2834116365f4feefbb253d9548cb21e24fa932c Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:13:47 +0200 Subject: [PATCH 404/434] Update and rename SelfMangaParser.kt to SelfManga.kt --- src/main/kotlin/tsuki/site/ru/SelfManga.kt | 15 +++++++++++ .../kotlin/tsuki/site/ru/SelfMangaParser.kt | 27 ------------------- 2 files changed, 15 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/SelfManga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/SelfMangaParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/SelfManga.kt b/src/main/kotlin/tsuki/site/ru/SelfManga.kt new file mode 100644 index 00000000..9edea585 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/SelfManga.kt @@ -0,0 +1,15 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("SELFMANGA", "SelfManga", "ru", ContentType.OTHER) +internal class SelfMangaParser(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.SELFMANGA, 3) { + override val configKeyDomain = ConfigKey.Domain("1.selfmanga.live") +} diff --git a/src/main/kotlin/tsuki/site/ru/SelfMangaParser.kt b/src/main/kotlin/tsuki/site/ru/SelfMangaParser.kt deleted file mode 100644 index 16d3e8a0..00000000 --- a/src/main/kotlin/tsuki/site/ru/SelfMangaParser.kt +++ /dev/null @@ -1,27 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.GroupleParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.ContentType -import tsuki.model.MangaParserSource - -@MangaSourceParser("SELFMANGA", "SelfManga", "ru", type = ContentType.OTHER) -internal class SelfMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.SELFMANGA, 3) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "1.selfmanga.live", - ) - } -} From b540ef693751703d0dd223ba521d78486129bbb3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:14:54 +0200 Subject: [PATCH 405/434] Update Usagi.kt --- src/main/kotlin/tsuki/site/ru/Usagi.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/ru/Usagi.kt b/src/main/kotlin/tsuki/site/ru/Usagi.kt index 00fb2c68..e463f179 100644 --- a/src/main/kotlin/tsuki/site/ru/Usagi.kt +++ b/src/main/kotlin/tsuki/site/ru/Usagi.kt @@ -8,7 +8,7 @@ import tsuki.config.ConfigKey import tsuki.model.MangaParserSource @MangaSourceParser("USAGI", "Usagi", "ru") -internal class UsagiParser(context: MangaLoaderContext) : +internal class Usagi(context: MangaLoaderContext) : GroupleParser(context, MangaParserSource.USAGI, 1) { override val configKeyDomain = ConfigKey.Domain("web.usagi.one") } From 1371de4a5c26b94e6804945c323cf59552a96285 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:15:22 +0200 Subject: [PATCH 406/434] Update SlashLib.kt --- src/main/kotlin/tsuki/site/ru/SlashLib.kt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ru/SlashLib.kt b/src/main/kotlin/tsuki/site/ru/SlashLib.kt index bd0eab89..b5a7ef40 100644 --- a/src/main/kotlin/tsuki/site/ru/SlashLib.kt +++ b/src/main/kotlin/tsuki/site/ru/SlashLib.kt @@ -6,6 +6,6 @@ import tsuki.MangaSourceParser import tsuki.model.MangaParserSource -@MangaSourceParser("YAOILIB", "SlashLib", "ru") -internal class SlashLibParser(context: MangaLoaderContext) : - LibSocialParser(context, MangaParserSource.YAOILIB, 2, "v2.slashlib.me") +@MangaSourceParser("SLASHLIB", "SlashLib", "ru") +internal class SlashLib(context: MangaLoaderContext) : + LibSocialParser(context, MangaParserSource.SLASHLIB, 2, "v2.slashlib.me") From 7c72f3d8226cc4227ce9ad50856952278f860f1f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:15:58 +0200 Subject: [PATCH 407/434] Rename SelfMangaParser to SelfManga --- src/main/kotlin/tsuki/site/ru/SelfManga.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/ru/SelfManga.kt b/src/main/kotlin/tsuki/site/ru/SelfManga.kt index 9edea585..92ed3dc3 100644 --- a/src/main/kotlin/tsuki/site/ru/SelfManga.kt +++ b/src/main/kotlin/tsuki/site/ru/SelfManga.kt @@ -9,7 +9,7 @@ import tsuki.model.ContentType import tsuki.model.MangaParserSource @MangaSourceParser("SELFMANGA", "SelfManga", "ru", ContentType.OTHER) -internal class SelfMangaParser(context: MangaLoaderContext) : +internal class SelfManga(context: MangaLoaderContext) : GroupleParser(context, MangaParserSource.SELFMANGA, 3) { override val configKeyDomain = ConfigKey.Domain("1.selfmanga.live") } From f539f4594979aa8a0779d8b6877836901334795e Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:16:12 +0200 Subject: [PATCH 408/434] Update and rename SeiMangaParser.kt to SeiManga.kt --- src/main/kotlin/tsuki/site/ru/SeiManga.kt | 14 ++++++++++ .../kotlin/tsuki/site/ru/SeiMangaParser.kt | 27 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/SeiManga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/SeiMangaParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/SeiManga.kt b/src/main/kotlin/tsuki/site/ru/SeiManga.kt new file mode 100644 index 00000000..eb5dfd7a --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/SeiManga.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("SEIMANGA", "SeiManga", "ru") +internal class SeiManga(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.SEIMANGA, 21) { + override val configKeyDomain = ConfigKey.Domain("1.seimanga.me", "seimanga.me") +} diff --git a/src/main/kotlin/tsuki/site/ru/SeiMangaParser.kt b/src/main/kotlin/tsuki/site/ru/SeiMangaParser.kt deleted file mode 100644 index 641e9cd1..00000000 --- a/src/main/kotlin/tsuki/site/ru/SeiMangaParser.kt +++ /dev/null @@ -1,27 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.GroupleParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("SEIMANGA", "SeiManga", "ru") -internal class SeiMangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.SEIMANGA, 21) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "1.seimanga.me", - "seimanga.me", - ) - } -} From 51d9fe4fbf053493a539cf364322d856cf0be2a1 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:22:51 +0200 Subject: [PATCH 409/434] Add Remanga.kt file --- src/main/kotlin/tsuki/site/ru/Remanga.kt | 410 ++++++++++++++++++ .../kotlin/tsuki/site/ru/RemangaParser.kt | 378 ---------------- 2 files changed, 410 insertions(+), 378 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/Remanga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/RemangaParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/Remanga.kt b/src/main/kotlin/tsuki/site/ru/Remanga.kt new file mode 100644 index 00000000..ed0b2d47 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Remanga.kt @@ -0,0 +1,410 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ContentUnavailableException +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.YEAR_UNKNOWN + +import tsuki.util.LinkResolver +import tsuki.util.copyCookies +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.getDomain +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONTo +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapChapters +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode +import tsuki.util.find + +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.json.JSONArray +import org.json.JSONObject +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Date +import java.util.EnumSet +import java.util.Locale + +private const val PAGE_SIZE = 20 +private const val TOO_MANY_REQUESTS = 429 + +@MangaSourceParser("REMANGA", "Реманга", "ru") +internal class Remanga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { + + private val baseHeaders: Headers + get() = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .build() + + override fun getRequestHeaders() = getApiHeaders() + + override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг") + + override val authUrl: String + get() = "https://${domain}" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } + + private val regexLastUrlPath = Regex("/[^/]+/?$") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isYearRangeSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.allOf(MangaState::class.java), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.code == TOO_MANY_REQUESTS) { + response.closeQuietly() + Thread.sleep(1000) + return chain.proceed(chain.request().newBuilder().build()) + } + return response + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + copyCookies() + val urlBuilder = urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("search") + .addQueryParameter("page", page.toString()) + .addQueryParameter("count", PAGE_SIZE.toString()) + .addQueryParameter("ordering", getSortKey(order)) + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("query", filter.query) + } else { + urlBuilder.addPathSegment("catalog") + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("genres", tag.key) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("exclude_genres", tag.key) + } + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_gte", filter.yearFrom.toString()) + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_lte", filter.yearFrom.toString()) + } + for (age in filter.contentRating) { + when (age) { + ContentRating.SAFE -> urlBuilder.addQueryParameter("age_limit", "0") + ContentRating.SUGGESTIVE -> { + urlBuilder.addQueryParameter("age_limit", "1") + urlBuilder.addQueryParameter("age_limit", "2") + } + + else -> Unit + } + } + for (state in filter.states) { + urlBuilder.addQueryParameter( + "status", + when (state) { + MangaState.ONGOING -> "2" + MangaState.FINISHED -> "1" + MangaState.ABANDONED -> "4" + MangaState.PAUSED -> "3" + MangaState.UPCOMING -> "5" + MangaState.RESTRICTED -> "6" + }, + ) + } + val content = webClient.httpGet(urlBuilder.build()).parseJson() + .getJSONArray("results") + return content.mapJSON { jo -> + parseManga(jo) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + return getDetails(slug, manga.publicUrl) + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val slug = link.pathSegments.getOrNull(1) ?: return super.resolveLink(resolver, link) + return getDetails(slug, link.toString()) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + copyCookies() + val slug = seed.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${seed.url}", seed.publicUrl) + val json = webClient.httpGet( + // https://api.remanga.org/api/v2/titles/the_beginning_after_the_end/relations/ + urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("titles") + .addPathSegment(slug) + .addPathSegment("relations") + .build(), + ).parseJson() + .optJSONArray("titles") ?: return emptyList() + return json.mapJSON { jo -> parseManga(jo) } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain(this, "api"))) + .parseJson() + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).parseSafe(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава станет доступной $at") + } else { + throw ContentUnavailableException("Глава недоступна") + } + } + val result = ArrayList<MangaPage>(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item) + is JSONArray -> item.mapJSONTo(result) { parsePage(it) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).parseJson() + return jo.getJSONObject("content").getString("username") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private suspend fun getDetails(slug: String, publicUrl: String): Manga { + copyCookies() + val jo = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/$slug/", + ).parseJson() + jo.optJSONObject("detail")?.getStringOrNull("message")?.let { msg -> + throw ParseException(msg, publicUrl) + } + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + val branches = jo.getJSONArray("branches").mapJSONToSet { + it.getLong("id") to it.optJSONObject("publishers")?.getStringOrNull("name") + } + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = jo.getStringOrNull("another_name")?.split(" / ") + ?.toSet().orEmpty() + setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = when (jo.optJSONObject("age_limit")?.getIntOrDefault("id", -1)) { + 0 -> ContentRating.SAFE + 1, 2 -> ContentRating.SUGGESTIVE + else -> null + }, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + description = jo.getStringOrNull("description"), + chapters = branches.flatMap { (branchId, branchName) -> + grabChapters(branchId).mapChapters(reversed = true) { _, cjo -> + if ( + cjo.getBooleanOrDefault("is_paid", false) && + !cjo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = cjo.getLong("id") + val name = cjo.getStringOrNull("name")?.toTitleCase(Locale.ROOT) + val publishers = cjo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/v2/titles/chapters/$id/", + number = cjo.getFloatOrDefault("chapter", 0f), + volume = cjo.getIntOrDefault("tome", 0), + title = name, + uploadDate = dateFormat.parseSafe(cjo.getStringOrNull("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = source, + branch = branchName, + ) + } + }, + source = source, + ) + } + + private fun parseManga(jo: JSONObject): Manga { + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = null, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + source = source, + ) + } + + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(userCookie.value.urlDecode()) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } + + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } + + private fun parsePage(jo: JSONObject) = MangaPage( + // id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone + id = generateUid(jo.getString("link")), + url = jo.getString("link"), + preview = null, + source = source, + ) + + private suspend fun grabChapters(branchId: Long): List<JSONObject> { + val result = ArrayList<JSONObject>(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/chapters/?branch_id=$branchId&page=$page&count=500", + ).parseJson().getJSONArray("results") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } +} diff --git a/src/main/kotlin/tsuki/site/ru/RemangaParser.kt b/src/main/kotlin/tsuki/site/ru/RemangaParser.kt deleted file mode 100644 index efb82b26..00000000 --- a/src/main/kotlin/tsuki/site/ru/RemangaParser.kt +++ /dev/null @@ -1,378 +0,0 @@ -package tsuki.site.ru - -import okhttp3.Headers -import okhttp3.HttpUrl -import okhttp3.Interceptor -import okhttp3.Response -import okhttp3.internal.closeQuietly -import org.json.JSONArray -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ContentUnavailableException -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.* -import java.text.DateFormat -import java.text.SimpleDateFormat -import java.util.* - -private const val PAGE_SIZE = 20 -private const val TOO_MANY_REQUESTS = 429 - -@MangaSourceParser("REMANGA", "Реманга", "ru") -internal class RemangaParser( - context: MangaLoaderContext, -) : PagedMangaParser(context, MangaParserSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { - - private val baseHeaders: Headers - get() = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .build() - - override fun getRequestHeaders() = getApiHeaders() - - override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг") - - override val authUrl: String - get() = "https://${domain}" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - ) - - override suspend fun isAuthorized(): Boolean { - return context.cookieJar.getCookies(domain).any { - it.name == "user" - } - } - - private val regexLastUrlPath = Regex("/[^/]+/?$") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isYearRangeSupported = true, - isTagsExclusionSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.allOf(MangaState::class.java), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE), - ) - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (response.code == TOO_MANY_REQUESTS) { - response.closeQuietly() - Thread.sleep(1000) - return chain.proceed(chain.request().newBuilder().build()) - } - return response - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - copyCookies() - val domain = domain - val urlBuilder = urlBuilder(subdomain = "api") - .addPathSegment("api") - .addPathSegment("v2") - .addPathSegment("search") - .addQueryParameter("page", page.toString()) - .addQueryParameter("count", PAGE_SIZE.toString()) - .addQueryParameter("ordering", getSortKey(order)) - if (!filter.query.isNullOrEmpty()) { - urlBuilder.addQueryParameter("query", filter.query) - } else { - urlBuilder.addPathSegment("catalog") - } - for (tag in filter.tags) { - urlBuilder.addQueryParameter("genres", tag.key) - } - for (tag in filter.tagsExclude) { - urlBuilder.addQueryParameter("exclude_genres", tag.key) - } - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("issue_year_gte", filter.yearFrom.toString()) - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.addQueryParameter("issue_year_lte", filter.yearFrom.toString()) - } - for (age in filter.contentRating) { - when (age) { - ContentRating.SAFE -> urlBuilder.addQueryParameter("age_limit", "0") - ContentRating.SUGGESTIVE -> { - urlBuilder.addQueryParameter("age_limit", "1") - urlBuilder.addQueryParameter("age_limit", "2") - } - - else -> Unit - } - } - for (state in filter.states) { - urlBuilder.addQueryParameter( - "status", - when (state) { - MangaState.ONGOING -> "2" - MangaState.FINISHED -> "1" - MangaState.ABANDONED -> "4" - MangaState.PAUSED -> "3" - MangaState.UPCOMING -> "5" - MangaState.RESTRICTED -> "6" - }, - ) - } - val content = webClient.httpGet(urlBuilder.build()).parseJson() - .getJSONArray("results") - return content.mapJSON { jo -> - parseManga(jo) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.find(regexLastUrlPath)?.removePrefix("/") - ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) - return getDetails(slug, manga.publicUrl) - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val slug = link.pathSegments.getOrNull(1) ?: return super.resolveLink(resolver, link) - return getDetails(slug, link.toString()) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - copyCookies() - val slug = seed.url.find(regexLastUrlPath)?.removePrefix("/") - ?: throw ParseException("Cannot obtain slug from ${seed.url}", seed.publicUrl) - val json = webClient.httpGet( - // https://api.remanga.org/api/v2/titles/the_beginning_after_the_end/relations/ - urlBuilder(subdomain = "api") - .addPathSegment("api") - .addPathSegment("v2") - .addPathSegment("titles") - .addPathSegment(slug) - .addPathSegment("relations") - .build(), - ).parseJson() - .optJSONArray("titles") ?: return emptyList() - return json.mapJSON { jo -> parseManga(jo) } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain(this, "api"))) - .parseJson() - val pages = content.optJSONArray("pages") - if (pages == null) { - val pubDate = content.getStringOrNull("pub_date")?.let { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).parseSafe(it) - } - if (pubDate != null && pubDate > System.currentTimeMillis()) { - val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) - throw ContentUnavailableException("Глава станет доступной $at") - } else { - throw ContentUnavailableException("Глава недоступна") - } - } - val result = ArrayList<MangaPage>(pages.length()) - for (i in 0 until pages.length()) { - when (val item = pages.get(i)) { - is JSONObject -> result += parsePage(item) - is JSONArray -> item.mapJSONTo(result) { parsePage(it) } - else -> throw ParseException("Unknown json item $item", chapter.url) - } - } - return result - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") - .parseJson().getJSONObject("content").getJSONArray("genres") - return content.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(), - key = jo.getInt("id").toString(), - source = source, - ) - } - } - - override suspend fun getUsername(): String { - val jo = webClient.httpGet( - url = "https://api.${domain}/api/users/current/", - ).parseJson() - return jo.getJSONObject("content").getString("username") - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - private suspend fun getDetails(slug: String, publicUrl: String): Manga { - copyCookies() - val jo = webClient.httpGet( - url = "https://api.$domain/api/v2/titles/$slug/", - ).parseJson() - jo.optJSONObject("detail")?.getStringOrNull("message")?.let { msg -> - throw ParseException(msg, publicUrl) - } - val url = "/manga/${jo.getString("dir")}" - val cover = jo.getJSONObject("cover") - val branches = jo.getJSONArray("branches").mapJSONToSet { - it.getLong("id") to it.optJSONObject("publishers")?.getStringOrNull("name") - } - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) - return Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url/main", - title = jo.getString("main_name"), - altTitles = jo.getStringOrNull("another_name")?.split(" / ") - ?.toSet().orEmpty() + setOfNotNull(jo.getStringOrNull("secondary_name")), - rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, - coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), - largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), - authors = emptySet(), - contentRating = when (jo.optJSONObject("age_limit")?.getIntOrDefault("id", -1)) { - 0 -> ContentRating.SAFE - 1, 2 -> ContentRating.SUGGESTIVE - else -> null - }, - state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { - 1 -> MangaState.FINISHED - 2 -> MangaState.ONGOING - 3 -> MangaState.PAUSED - 4 -> MangaState.ABANDONED - 5 -> MangaState.UPCOMING - 6 -> MangaState.RESTRICTED - else -> null - }, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(sourceLocale), - key = g.getInt("id").toString(), - source = source, - ) - }.orEmpty(), - description = jo.getStringOrNull("description"), - chapters = branches.flatMap { (branchId, branchName) -> - grabChapters(branchId).mapChapters(reversed = true) { _, cjo -> - if ( - cjo.getBooleanOrDefault("is_paid", false) && - !cjo.getBooleanOrDefault("is_bought", false) - ) { - return@mapChapters null - } - val id = cjo.getLong("id") - val name = cjo.getStringOrNull("name")?.toTitleCase(Locale.ROOT) - val publishers = cjo.optJSONArray("publishers") - MangaChapter( - id = generateUid(id), - url = "/api/v2/titles/chapters/$id/", - number = cjo.getFloatOrDefault("chapter", 0f), - volume = cjo.getIntOrDefault("tome", 0), - title = name, - uploadDate = dateFormat.parseSafe(cjo.getStringOrNull("upload_date")), - scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), - source = source, - branch = branchName, - ) - } - }, - source = source, - ) - } - - private fun parseManga(jo: JSONObject): Manga { - val url = "/manga/${jo.getString("dir")}" - val cover = jo.getJSONObject("cover") - return Manga( - id = generateUid(url), - url = url, - publicUrl = "https://$domain$url/main", - title = jo.getString("main_name"), - altTitles = setOfNotNull(jo.getStringOrNull("secondary_name")), - rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, - coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), - largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), - authors = emptySet(), - contentRating = null, - state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { - 1 -> MangaState.FINISHED - 2 -> MangaState.ONGOING - 3 -> MangaState.PAUSED - 4 -> MangaState.ABANDONED - 5 -> MangaState.UPCOMING - 6 -> MangaState.RESTRICTED - else -> null - }, - tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> - MangaTag( - title = g.getString("name").toTitleCase(sourceLocale), - key = g.getInt("id").toString(), - source = source, - ) - }.orEmpty(), - source = source, - ) - } - - private fun getApiHeaders(): Headers { - val userCookie = context.cookieJar.getCookies(domain).find { - it.name == "user" - } ?: return baseHeaders - val jo = JSONObject(userCookie.value.urlDecode()) - val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders - return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() - } - - private fun copyCookies() { - val domain = domain - context.cookieJar.copyCookies(domain, "api.$domain") - } - - private fun getSortKey(order: SortOrder?) = when (order) { - SortOrder.UPDATED -> "-chapter_date" - SortOrder.POPULARITY -> "-rating" - SortOrder.RATING -> "-votes" - SortOrder.NEWEST -> "-id" - else -> "-chapter_date" - } - - private fun parsePage(jo: JSONObject) = MangaPage( - // id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone - id = generateUid(jo.getString("link")), - url = jo.getString("link"), - preview = null, - source = source, - ) - - private suspend fun grabChapters(branchId: Long): List<JSONObject> { - val result = ArrayList<JSONObject>(100) - var page = 1 - while (true) { - val content = webClient.httpGet( - url = "https://api.$domain/api/v2/titles/chapters/?branch_id=$branchId&page=$page&count=500", - ).parseJson().getJSONArray("results") - val len = content.length() - if (len == 0) { - break - } - result.ensureCapacity(result.size + len) - for (i in 0 until len) { - result.add(content.getJSONObject(i)) - } - page++ - } - return result - } -} From 7fe50c3f04bedcf995051852a7f92b8c08aa0066 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:23:49 +0200 Subject: [PATCH 410/434] Update and rename ReadmangaParser.kt to Readmanga.kt --- src/main/kotlin/tsuki/site/ru/Readmanga.kt | 14 ++++++++++ .../kotlin/tsuki/site/ru/ReadmangaParser.kt | 27 ------------------- 2 files changed, 14 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/Readmanga.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/ReadmangaParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/Readmanga.kt b/src/main/kotlin/tsuki/site/ru/Readmanga.kt new file mode 100644 index 00000000..db3dca70 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Readmanga.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("READMANGA_RU", "ReadManga", "ru") +internal class Readmanga(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.READMANGA_RU, 1) { + override val configKeyDomain = ConfigKey.Domain("a.zazaza.me", "3.readmanga.ru") +} diff --git a/src/main/kotlin/tsuki/site/ru/ReadmangaParser.kt b/src/main/kotlin/tsuki/site/ru/ReadmangaParser.kt deleted file mode 100644 index 4c30e4aa..00000000 --- a/src/main/kotlin/tsuki/site/ru/ReadmangaParser.kt +++ /dev/null @@ -1,27 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.GroupleParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.MangaParserSource - -@MangaSourceParser("READMANGA_RU", "ReadManga", "ru") -internal class ReadmangaParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.READMANGA_RU, 1) { - - override val configKeyDomain = ConfigKey.Domain(*domains) - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("referer", "https://$domain/") - .build() - - companion object { - - val domains = arrayOf( - "a.zazaza.me", - "3.readmanga.ru", - ) - } -} From b0cfa93d14c07eb115a29b96c2d2dcbe343c5605 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:24:18 +0200 Subject: [PATCH 411/434] Refactor MangaoneLove class initialization --- src/main/kotlin/tsuki/site/ru/MangaoneLove.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt index a9021673..c87596d8 100644 --- a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt +++ b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt @@ -2,12 +2,13 @@ package tsuki.site.ru import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource import tsuki.parsers.MadaraParser +import tsuki.model.MangaParserSource + @MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") internal class MangaoneLove(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { - override val datePattern = "dd.MM.yyyy" - override val postReq = true + MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { + override val datePattern = "dd.MM.yyyy" + override val postReq = true } From c4d7d7a9361c047ae2757ec35c698fd83fbd30dc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:26:12 +0200 Subject: [PATCH 412/434] Update and rename MangaWtfParser.kt to MangaWtf.kt --- src/main/kotlin/tsuki/site/ru/MangaWtf.kt | 286 ++++++++++++++++++ .../kotlin/tsuki/site/ru/MangaWtfParser.kt | 263 ---------------- 2 files changed, 286 insertions(+), 263 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/MangaWtf.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaWtf.kt b/src/main/kotlin/tsuki/site/ru/MangaWtf.kt new file mode 100644 index 00000000..6c035da8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaWtf.kt @@ -0,0 +1,286 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.asTypedList +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapNotNullToSet +import tsuki.util.nl2br +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder + +import androidx.collection.ArrayMap +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAWTF", "MangaWtf", "ru") +internal class MangaWtf(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAWTF, pageSize = 20) { + + override val availableSortOrders: Set<SortOrder> = + EnumSet.of( + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) + + override val configKeyDomain = ConfigKey.Domain("inkstory.net") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.UPCOMING, + MangaState.PAUSED, + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentRating = EnumSet.allOf(ContentRating::class.java), + ) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("type", "COMIC") + when { + filter.query.isNullOrEmpty() -> { + url.addQueryParameter( + "sort", + when (order) { + SortOrder.UPDATED -> "updatedAt,desc" + SortOrder.POPULARITY -> "viewsCount,desc" + SortOrder.RATING -> "likesCount,desc" + SortOrder.NEWEST -> "createdAt,desc" + else -> throw IllegalArgumentException("Unsupported $order") + }, + ) + if (filter.tags.isNotEmpty()) { + url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) + } + if (filter.tagsExclude.isNotEmpty()) { + url.addQueryParameter("labelsExclude", filter.tagsExclude.joinToString(",") { it.key }) + } + if (filter.states.isNotEmpty()) { + url.addQueryParameter( + "status", + filter.states.joinToString(",") { + when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "DONE" + MangaState.ABANDONED -> "" + MangaState.PAUSED -> "FROZEN" + MangaState.UPCOMING -> "ANNOUNCE" + else -> throw IllegalArgumentException("$it not supported") + } + }, + ) + } + if (filter.contentRating.isNotEmpty()) { + url.addQueryParameter( + "contentStatus", + filter.contentRating.joinToString(",") { + when (it) { + ContentRating.SAFE -> "SAFE" + ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" + ContentRating.ADULT -> "PORNOGRAPHIC" + } + }, + ) + } + } + + else -> { + url.addQueryParameter("search", filter.query) + } + } + val ja = webClient.httpGet(url.build()).parseJsonArray() + return ja.mapJSON { jo -> jo.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga = + coroutineScope { + val chaptersDeferred = async { getChapters(manga.url) } + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addPathSegment(manga.url) + val jo = webClient.httpGet(url.build()).parseJson() + val isNsfwSource = jo.getStringOrNull("contentStatus").isNsfw() + Manga( + id = generateUid(jo.getString("id")), + title = jo.getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(jo.getJSONObject("name").getStringOrNull("en")), + url = jo.getString("id"), + publicUrl = "https://$domain/manga/${jo.getString("slug")}", + rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = jo.getString("poster"), + tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, + state = jo.getStringOrNull("status")?.toMangaState(), + authors = jo.getJSONArray("relations").asTypedList<JSONObject>().mapNotNullToSet { + if (it.getStringOrNull("type") == "AUTHOR") { + it.getJSONObject("publisher").getStringOrNull("name") + } else { + null + } + }, + source = source, + largeCoverUrl = null, + description = jo.getString("description").nl2br(), + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addPathSegment(chapter.url) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("pages").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getString("id")), + url = jo.getString("image"), + preview = null, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val url = urlBuilder("api").addPathSegment("v2").addPathSegment("labels") + val json = webClient.httpGet(url.build()).parseJsonArray() + return json.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getChapters(mangaId: String): List<MangaChapter> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addQueryParameter("bookId", mangaId) + val ja = webClient.httpGet(url.build()).parseJsonArray() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) + val branches = ArrayMap<String, String>() + return ja + .mapJSON { jo -> + val number = jo.getFloatOrDefault("number", 0f) + val volume = jo.getIntOrDefault("volume", 0) + val branchId = jo.getString("branchId") + MangaChapter( + id = generateUid(jo.getString("id")), + title = jo.getStringOrNull("name"), + number = number, + volume = volume, + url = jo.getString("id"), + scanlator = null, + uploadDate = dateFormat.parseSafe(jo.getString("createdAt")), + branch = branches.getOrPut(branchId) { getBranchName(branchId) }, + source = source, + ) + }.reversed() + } + + private suspend fun getBranchName(id: String): String? { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("branches") + .addPathSegment(id) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() + } + + private fun String.toMangaState() = + when (this.uppercase(Locale.ROOT)) { + "DONE" -> MangaState.FINISHED + "ONGOING" -> MangaState.ONGOING + "FROZEN" -> MangaState.PAUSED + "ANNOUNCE" -> MangaState.UPCOMING + else -> null + } + + private fun String?.isNsfw() = + this.equals("EROTIC", ignoreCase = true) || + this.equals("PORNOGRAPHIC", ignoreCase = true) + + private fun JSONObject.toMangaTag() = + MangaTag( + title = getString("name").toTitleCase(sourceLocale), + key = getString("slug"), + source = source, + ) + + private fun JSONObject.toManga(): Manga { + val isNsfwSource = getStringOrNull("contentStatus").isNsfw() + return Manga( + id = generateUid(getString("id")), + title = getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(getJSONObject("name").getStringOrNull("en")), + url = getString("id"), + publicUrl = "https://$domain/manga/${getString("slug")}", + rating = getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = getString("poster"), + tags = setOf(), + state = getStringOrNull("status")?.toMangaState(), + authors = emptySet(), + source = source, + ) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt b/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt deleted file mode 100644 index 1e67bc3d..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaWtfParser.kt +++ /dev/null @@ -1,263 +0,0 @@ -package tsuki.site.ru - -import androidx.collection.ArrayMap -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import org.json.JSONObject -import tsuki.InternalParsersApi -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGA_WTF", "MangaWtf", "ru") -internal class MangaWtfParser( - context: MangaLoaderContext, -) : PagedMangaParser(context, MangaParserSource.MANGA_WTF, pageSize = 20) { - - override val availableSortOrders: Set<SortOrder> = - EnumSet.of( - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.UPDATED, - SortOrder.NEWEST, - ) - - @InternalParsersApi - override val configKeyDomain = ConfigKey.Domain("inkstory.net") - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.UPCOMING, - MangaState.PAUSED, - MangaState.ONGOING, - MangaState.FINISHED, - ), - availableContentRating = EnumSet.allOf(ContentRating::class.java), - ) - - init { - paginator.firstPage = 0 - searchPaginator.firstPage = 0 - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("books") - .addQueryParameter("page", page.toString()) - .addQueryParameter("size", pageSize.toString()) - .addQueryParameter("type", "COMIC") - when { - filter.query.isNullOrEmpty() -> { - url.addQueryParameter( - "sort", - when (order) { - SortOrder.UPDATED -> "updatedAt,desc" - SortOrder.POPULARITY -> "viewsCount,desc" - SortOrder.RATING -> "likesCount,desc" - SortOrder.NEWEST -> "createdAt,desc" - else -> throw IllegalArgumentException("Unsupported $order") - }, - ) - if (filter.tags.isNotEmpty()) { - url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) - } - if (filter.tagsExclude.isNotEmpty()) { - url.addQueryParameter("labelsExclude", filter.tagsExclude.joinToString(",") { it.key }) - } - if (filter.states.isNotEmpty()) { - url.addQueryParameter( - "status", - filter.states.joinToString(",") { - when (it) { - MangaState.ONGOING -> "ONGOING" - MangaState.FINISHED -> "DONE" - MangaState.ABANDONED -> "" - MangaState.PAUSED -> "FROZEN" - MangaState.UPCOMING -> "ANNOUNCE" - else -> throw IllegalArgumentException("$it not supported") - } - }, - ) - } - if (filter.contentRating.isNotEmpty()) { - url.addQueryParameter( - "contentStatus", - filter.contentRating.joinToString(",") { - when (it) { - ContentRating.SAFE -> "SAFE" - ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" - ContentRating.ADULT -> "PORNOGRAPHIC" - } - }, - ) - } - } - - else -> { - url.addQueryParameter("search", filter.query) - } - } - val ja = webClient.httpGet(url.build()).parseJsonArray() - return ja.mapJSON { jo -> jo.toManga() } - } - - override suspend fun getDetails(manga: Manga): Manga = - coroutineScope { - val chaptersDeferred = async { getChapters(manga.url) } - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("books") - .addPathSegment(manga.url) - val jo = webClient.httpGet(url.build()).parseJson() - val isNsfwSource = jo.getStringOrNull("contentStatus").isNsfw() - Manga( - id = generateUid(jo.getString("id")), - title = jo.getJSONObject("name").getString("ru"), - altTitles = setOfNotNull(jo.getJSONObject("name").getStringOrNull("en")), - url = jo.getString("id"), - publicUrl = "https://$domain/manga/${jo.getString("slug")}", - rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = jo.getString("poster"), - tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, - state = jo.getStringOrNull("status")?.toMangaState(), - authors = jo.getJSONArray("relations").asTypedList<JSONObject>().mapNotNullToSet { - if (it.getStringOrNull("type") == "AUTHOR") { - it.getJSONObject("publisher").getStringOrNull("name") - } else { - null - } - }, - source = source, - largeCoverUrl = null, - description = jo.getString("description").nl2br(), - chapters = chaptersDeferred.await(), - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("chapters") - .addPathSegment(chapter.url) - val json = webClient.httpGet(url.build()).parseJson() - return json.getJSONArray("pages").mapJSON { jo -> - MangaPage( - id = generateUid(jo.getString("id")), - url = jo.getString("image"), - preview = null, - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val url = urlBuilder("api").addPathSegment("v2").addPathSegment("labels") - val json = webClient.httpGet(url.build()).parseJsonArray() - return json.mapJSONToSet { jo -> - MangaTag( - title = jo.getString("name").toTitleCase(sourceLocale), - key = jo.getString("slug"), - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - override suspend fun getPageUrl(page: MangaPage): String = page.url - - private suspend fun getChapters(mangaId: String): List<MangaChapter> { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("chapters") - .addQueryParameter("bookId", mangaId) - val ja = webClient.httpGet(url.build()).parseJsonArray() - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) - val branches = ArrayMap<String, String>() - return ja - .mapJSON { jo -> - val number = jo.getFloatOrDefault("number", 0f) - val volume = jo.getIntOrDefault("volume", 0) - val branchId = jo.getString("branchId") - MangaChapter( - id = generateUid(jo.getString("id")), - title = jo.getStringOrNull("name"), - number = number, - volume = volume, - url = jo.getString("id"), - scanlator = null, - uploadDate = dateFormat.parseSafe(jo.getString("createdAt")), - branch = branches.getOrPut(branchId) { getBranchName(branchId) }, - source = source, - ) - }.reversed() - } - - private suspend fun getBranchName(id: String): String? { - val url = - urlBuilder("api") - .addPathSegment("v2") - .addPathSegment("branches") - .addPathSegment(id) - val json = webClient.httpGet(url.build()).parseJson() - return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() - } - - private fun String.toMangaState() = - when (this.uppercase(Locale.ROOT)) { - "DONE" -> MangaState.FINISHED - "ONGOING" -> MangaState.ONGOING - "FROZEN" -> MangaState.PAUSED - "ANNOUNCE" -> MangaState.UPCOMING - else -> null - } - - private fun String?.isNsfw() = - this.equals("EROTIC", ignoreCase = true) || - this.equals("PORNOGRAPHIC", ignoreCase = true) - - private fun JSONObject.toMangaTag() = - MangaTag( - title = getString("name").toTitleCase(sourceLocale), - key = getString("slug"), - source = source, - ) - - private fun JSONObject.toManga(): Manga { - val isNsfwSource = getStringOrNull("contentStatus").isNsfw() - return Manga( - id = generateUid(getString("id")), - title = getJSONObject("name").getString("ru"), - altTitles = setOfNotNull(getJSONObject("name").getStringOrNull("en")), - url = getString("id"), - publicUrl = "https://$domain/manga/${getString("slug")}", - rating = getFloatOrDefault("averageRating", -10f) / 10f, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = getString("poster"), - tags = setOf(), - state = getStringOrNull("status")?.toMangaState(), - authors = emptySet(), - source = source, - ) - } -} From a88cd2cf9de1077536d6613ac50f4f982108f70d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:26:30 +0200 Subject: [PATCH 413/434] Update MangaMammy.kt --- src/main/kotlin/tsuki/site/ru/MangaMammy.kt | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ru/MangaMammy.kt b/src/main/kotlin/tsuki/site/ru/MangaMammy.kt index e0b65077..b1b01cdf 100644 --- a/src/main/kotlin/tsuki/site/ru/MangaMammy.kt +++ b/src/main/kotlin/tsuki/site/ru/MangaMammy.kt @@ -2,12 +2,13 @@ package tsuki.site.ru import tsuki.MangaLoaderContext import tsuki.MangaSourceParser -import tsuki.model.MangaParserSource import tsuki.parsers.MadaraParser +import tsuki.model.MangaParserSource + @MangaSourceParser("MANGAMAMMY", "Nimanga", "ru") internal class MangaMammy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAMAMMY, "p.nimanga.com") { - override val datePattern = "dd.MM.yyyy" - override val postReq = true + MadaraParser(context, MangaParserSource.MANGAMAMMY, "p.nimanga.com") { + override val datePattern = "dd.MM.yyyy" + override val postReq = true } From 1b396625221c96e6e8fafe7972b095cdf861b469 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:29:00 +0200 Subject: [PATCH 414/434] Update and rename MangaLibParser.kt to MangaLib.kt --- src/main/kotlin/tsuki/site/ru/MangaLib.kt | 25 +++++++++++++++++ .../kotlin/tsuki/site/ru/MangaLibParser.kt | 27 ------------------- 2 files changed, 25 insertions(+), 27 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/MangaLib.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaLibParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaLib.kt b/src/main/kotlin/tsuki/site/ru/MangaLib.kt new file mode 100644 index 00000000..b7526e02 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaLib.kt @@ -0,0 +1,25 @@ +package tsuki.site.ru + +import tsuki.parsers.LibSocialParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.exception.AuthRequiredException +import tsuki.exception.NotFoundException + +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGALIB", "MangaLib", "ru") +internal class MangaLib(context: MangaLoaderContext, ) : + LibSocialParser(context, MangaParserSource.MANGALIB, 1, "mangalib.org") { + + override val configKeyDomain = ConfigKey.Domain("mangalib.org", "mangalib.me") + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { + super.getPages(chapter) + } catch (e: NotFoundException) { + throw AuthRequiredException(source, e) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/MangaLibParser.kt b/src/main/kotlin/tsuki/site/ru/MangaLibParser.kt deleted file mode 100644 index b6d934b4..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaLibParser.kt +++ /dev/null @@ -1,27 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.LibSocialParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.exception.AuthRequiredException -import tsuki.exception.NotFoundException -import tsuki.model.MangaChapter -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGALIB", "MangaLib", "ru") -internal class MangaLibParser( - context: MangaLoaderContext, -) : LibSocialParser( - context = context, - source = MangaParserSource.MANGALIB, - siteId = 1, - siteDomains = arrayOf("mangalib.org", "mangalib.me"), -) { - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { - super.getPages(chapter) - } catch (e: NotFoundException) { - throw AuthRequiredException(source, e) - } -} From e47391f51b300067bb86c7d69f05d40f1e289e11 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:29:35 +0200 Subject: [PATCH 415/434] Update and rename MangaChanParser.kt to MangaChan.kt --- .../tsuki/site/ru/{MangaChanParser.kt => MangaChan.kt} | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) rename src/main/kotlin/tsuki/site/ru/{MangaChanParser.kt => MangaChan.kt} (57%) diff --git a/src/main/kotlin/tsuki/site/ru/MangaChanParser.kt b/src/main/kotlin/tsuki/site/ru/MangaChan.kt similarity index 57% rename from src/main/kotlin/tsuki/site/ru/MangaChanParser.kt rename to src/main/kotlin/tsuki/site/ru/MangaChan.kt index 7a1dd832..0f7704f5 100644 --- a/src/main/kotlin/tsuki/site/ru/MangaChanParser.kt +++ b/src/main/kotlin/tsuki/site/ru/MangaChan.kt @@ -4,9 +4,11 @@ import tsuki.parsers.ChanParser import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey + import tsuki.model.MangaParserSource @MangaSourceParser("MANGACHAN", "Манга-тян", "ru") -internal class MangaChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.MANGACHAN) { - override val configKeyDomain = ConfigKey.Domain("manga-chan.me") +internal class MangaChan(context: MangaLoaderContext) : + ChanParser(context, MangaParserSource.MANGACHAN) { + override val configKeyDomain = ConfigKey.Domain("manga-chan.me") } From 122f33efa9df6e1abb8981cd963dfdbf9624dc19 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:31:15 +0200 Subject: [PATCH 416/434] Update and rename DesuMeParser.kt to DesuMe.kt --- src/main/kotlin/tsuki/site/ru/DesuMe.kt | 282 ++++++++++++++++++ src/main/kotlin/tsuki/site/ru/DesuMeParser.kt | 280 ----------------- 2 files changed, 282 insertions(+), 280 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/DesuMe.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/DesuMeParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/DesuMe.kt b/src/main/kotlin/tsuki/site/ru/DesuMe.kt new file mode 100644 index 00000000..ba248dde --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/DesuMe.kt @@ -0,0 +1,282 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.UserAgents +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("DESUME", "Desu", "ru") +internal class DesuMe(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DESUME, 20) { + + override val configKeyDomain = ConfigKey.Domain("desu.uno") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalogDocument().parseDesuTags(), + ) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("User-Agent", UserAgents.KOTATSU) + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + if (query != null) { + if (page != searchPaginator.firstPage) return emptyList() + val document = webClient.httpPost( + "https://$domain/manga/search/", + mapOf("q" to query), + ).parseHtml() + return document.parseDesuSearchResults().map(::parseSearchManga) + } + + val url = buildCatalogUrl(page, order, filter) + return webClient.httpGet(url) + .parseHtml() + .parseDesuCatalogItems() + .map(::parseCatalogManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + .parseHtml() + .selectFirstOrThrow("#animeView") + val title = root.selectFirstOrThrow("meta[itemprop=headline]").attr("content") + val alternativeTitles = buildSet { + root.selectFirst("meta[itemprop=alternativeHeadline]") + ?.attr("content") + ?.takeIf { it.isNotBlank() } + ?.let(::add) + root.selectFirst(".alternativeHeadline") + ?.text() + ?.split(',') + ?.map(String::trim) + ?.filterTo(this) { it.isNotEmpty() } + } + val mangaId = root.selectFirst("ul.chlist[data-manga_id]") + ?.attr("data-manga_id") + ?.toLongOrNull() + ?: error("Cannot find Desu manga id") + val chapters = root.select("ul.chlist > li").map { item -> + val link = item.selectFirstOrThrow("h4 > a") + val chapterUrl = link.attrAsRelativeUrl("href") + val match = DESU_CHAPTER_PATH.find(chapterUrl) + ?: error("Unsupported Desu chapter URL: $chapterUrl") + val chapterId = item.selectFirst("[data-chapters_id]") + ?.attr("data-chapters_id") + ?.toLongOrNull() + ?: error("Cannot find Desu chapter id") + MangaChapter( + id = generateUid(chapterId), + url = "/api/manga/$mangaId/chapters/$chapterId", + source = source, + number = match.groupValues[2].toFloat(), + volume = match.groupValues[1].toInt(), + title = link.selectFirst(".title")?.text()?.removePrefix("- ")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = 0L, + branch = null, + ) + } + + return manga.copy( + title = title, + altTitles = alternativeTitles, + publicUrl = root.selectFirstOrThrow("link[itemprop=url]").attr("href"), + largeCoverUrl = root.selectFirstOrThrow("img[itemprop=image]").attr("src"), + description = root.selectFirst("#description .russian")?.html(), + tags = root.select("a[itemprop=genre]").mapTo(linkedSetOf()) { link -> + MangaTag( + key = link.attr("href").substringAfter("genres="), + title = link.text().removePrefix("# ").toTitleCase(), + source = source, + ) + }, + state = root.selectFirst(".b-anime_status_tag")?.classNames()?.toDesuState(), + rating = root.selectFirst("meta[itemprop=ratingValue]") + ?.attr("content") + ?.toFloatOrNull() + ?.div(10f) + ?: RATING_UNKNOWN, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val pages = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)) + .parseJson() + .getJSONObject("chapter") + .getJSONArray("pages") + + return (0 until pages.length()).map { index -> + val imageUrl = pages.getJSONObject(index).getString("url") + MangaPage( + id = generateUid(imageUrl), + preview = null, + source = chapter.source, + url = imageUrl, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val document = webClient.httpGet(link).parseHtml() + val root = document.selectFirst("#animeView") ?: return null + val title = root.selectFirst("meta[itemprop=headline]")?.attr("content") ?: return null + val url = root.selectFirst("link[itemprop=url]")?.attr("href") ?: return null + return resolver.resolveManga(this, id = generateUid(url), url = url, title = title) + } + + private suspend fun fetchCatalogDocument(): Document = webClient + .httpGet("https://$domain/manga/") + .parseHtml() + + private fun buildCatalogUrl(page: Int, order: SortOrder, filter: MangaListFilter): String = buildString { + append("https://") + append(domain) + append("/manga/?page=") + append(page) + val sortKey = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "popular" + SortOrder.NEWEST -> "id" + SortOrder.ALPHABETICAL -> "name" + else -> error("Unsupported Desu sort order: $order") + } + if (sortKey != "updated") { + append("&order_by=") + append(sortKey) + } + if (filter.tags.isNotEmpty()) { + append("&genres=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + private fun parseCatalogManga(item: Element): Manga { + val link = item.selectFirstOrThrow("a.animeTitle") + val url = link.attrAsRelativeUrl("href") + val cover = item.selectFirstOrThrow("a.avatar .img") + .attr("style") + .let { style -> DESU_BACKGROUND_IMAGE.find(style)?.groupValues?.get(1) } + ?: error("Cannot find Desu catalog cover") + val russianTitle = item.selectFirstOrThrow(".dimmed.oTitle [itemprop=title]").text() + val originalTitle = link.text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = cover, + largeCoverUrl = null, + state = null, + rating = item.parseDesuCatalogRating(), + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } + + private fun parseSearchManga(item: Element): Manga { + val url = item.attrAsRelativeUrl("href") + val russianTitle = item.selectFirstOrThrow(".itemSubTitle").text() + val originalTitle = item.selectFirstOrThrow(".itemTitle").text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = item.selectFirstOrThrow("img").attr("src"), + largeCoverUrl = null, + state = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } +} + +internal fun Document.parseDesuCatalogItems(): List<Element> = + select("ol.memberList > li.memberListItem") + +internal fun Document.parseDesuSearchResults(): List<Element> = + select("ol.section > li > a[href^=manga/]") + +internal fun Document.parseDesuTags(): Set<MangaTag> = + select("#animeFilter input[data-genre-id][data-genre-slug][data-genre-name]").mapTo(linkedSetOf()) { input -> + MangaTag( + "${input.attr("data-genre-id")}-${input.attr("data-genre-slug")}", + input.attr("data-genre-name").toTitleCase(), + MangaParserSource.DESUME, + ) + } + +private fun Element.parseDesuCatalogRating(): Float { + val rating = select(".animeInfo dl").asSequence() + .flatMap { it.children().asSequence() } + .windowed(2) + .firstOrNull { it[0].tagName() == "dt" && it[0].text() == "Рейтинг:" && it[1].tagName() == "dd" } + ?.get(1) + ?.text() + ?.toFloatOrNull() + return rating?.div(10f) ?: RATING_UNKNOWN +} + +private fun Set<String>.toDesuState(): MangaState? = when { + "ongoing" in this -> MangaState.ONGOING + "released" in this -> MangaState.FINISHED + "anons" in this -> MangaState.UPCOMING + isEmpty() -> null + else -> error("Unsupported Desu status classes: $this") +} + +private val DESU_BACKGROUND_IMAGE = Regex("url\\(['\"]?([^'\")]+)") +private val DESU_CHAPTER_PATH = Regex("/vol(\\d+)/ch([0-9.]+)/") diff --git a/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt b/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt deleted file mode 100644 index d827b163..00000000 --- a/src/main/kotlin/tsuki/site/ru/DesuMeParser.kt +++ /dev/null @@ -1,280 +0,0 @@ -package tsuki.site.ru - -import okhttp3.Headers -import okhttp3.HttpUrl -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.model.ContentRating -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder -import tsuki.network.UserAgents -import tsuki.util.LinkResolver -import tsuki.util.attrAsRelativeUrl -import tsuki.util.generateUid -import tsuki.util.parseHtml -import tsuki.util.parseJson -import tsuki.util.selectFirstOrThrow -import tsuki.util.toAbsoluteUrl -import tsuki.util.toTitleCase -import java.util.EnumSet - -@MangaSourceParser("DESUME", "Desu", "ru") -internal class DesuMeParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.DESUME, 20) { - - override val configKeyDomain = ConfigKey.Domain("desu.uno") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchCatalogDocument().parseDesuTags(), - ) - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("User-Agent", UserAgents.KOTATSU) - .build() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - if (query != null) { - if (page != searchPaginator.firstPage) return emptyList() - val document = webClient.httpPost( - "https://$domain/manga/search/", - mapOf("q" to query), - ).parseHtml() - return document.parseDesuSearchResults().map(::parseSearchManga) - } - - val url = buildCatalogUrl(page, order, filter) - return webClient.httpGet(url) - .parseHtml() - .parseDesuCatalogItems() - .map(::parseCatalogManga) - } - - override suspend fun getDetails(manga: Manga): Manga { - val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) - .parseHtml() - .selectFirstOrThrow("#animeView") - val title = root.selectFirstOrThrow("meta[itemprop=headline]").attr("content") - val alternativeTitles = buildSet { - root.selectFirst("meta[itemprop=alternativeHeadline]") - ?.attr("content") - ?.takeIf { it.isNotBlank() } - ?.let(::add) - root.selectFirst(".alternativeHeadline") - ?.text() - ?.split(',') - ?.map(String::trim) - ?.filterTo(this) { it.isNotEmpty() } - } - val mangaId = root.selectFirst("ul.chlist[data-manga_id]") - ?.attr("data-manga_id") - ?.toLongOrNull() - ?: error("Cannot find Desu manga id") - val chapters = root.select("ul.chlist > li").map { item -> - val link = item.selectFirstOrThrow("h4 > a") - val chapterUrl = link.attrAsRelativeUrl("href") - val match = DESU_CHAPTER_PATH.find(chapterUrl) - ?: error("Unsupported Desu chapter URL: $chapterUrl") - val chapterId = item.selectFirst("[data-chapters_id]") - ?.attr("data-chapters_id") - ?.toLongOrNull() - ?: error("Cannot find Desu chapter id") - MangaChapter( - id = generateUid(chapterId), - url = "/api/manga/$mangaId/chapters/$chapterId", - source = source, - number = match.groupValues[2].toFloat(), - volume = match.groupValues[1].toInt(), - title = link.selectFirst(".title")?.text()?.removePrefix("- ")?.takeIf { it.isNotBlank() }, - scanlator = null, - uploadDate = 0L, - branch = null, - ) - } - - return manga.copy( - title = title, - altTitles = alternativeTitles, - publicUrl = root.selectFirstOrThrow("link[itemprop=url]").attr("href"), - largeCoverUrl = root.selectFirstOrThrow("img[itemprop=image]").attr("src"), - description = root.selectFirst("#description .russian")?.html(), - tags = root.select("a[itemprop=genre]").mapTo(linkedSetOf()) { link -> - MangaTag( - key = link.attr("href").substringAfter("genres="), - title = link.text().removePrefix("# ").toTitleCase(), - source = source, - ) - }, - state = root.selectFirst(".b-anime_status_tag")?.classNames()?.toDesuState(), - rating = root.selectFirst("meta[itemprop=ratingValue]") - ?.attr("content") - ?.toFloatOrNull() - ?.div(10f) - ?: RATING_UNKNOWN, - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val pages = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)) - .parseJson() - .getJSONObject("chapter") - .getJSONArray("pages") - - return (0 until pages.length()).map { index -> - val imageUrl = pages.getJSONObject(index).getString("url") - MangaPage( - id = generateUid(imageUrl), - preview = null, - source = chapter.source, - url = imageUrl, - ) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val document = webClient.httpGet(link).parseHtml() - val root = document.selectFirst("#animeView") ?: return null - val title = root.selectFirst("meta[itemprop=headline]")?.attr("content") ?: return null - val url = root.selectFirst("link[itemprop=url]")?.attr("href") ?: return null - return resolver.resolveManga(this, id = generateUid(url), url = url, title = title) - } - - private suspend fun fetchCatalogDocument(): Document = webClient - .httpGet("https://$domain/manga/") - .parseHtml() - - private fun buildCatalogUrl(page: Int, order: SortOrder, filter: MangaListFilter): String = buildString { - append("https://") - append(domain) - append("/manga/?page=") - append(page) - val sortKey = when (order) { - SortOrder.UPDATED -> "updated" - SortOrder.POPULARITY -> "popular" - SortOrder.NEWEST -> "id" - SortOrder.ALPHABETICAL -> "name" - else -> error("Unsupported Desu sort order: $order") - } - if (sortKey != "updated") { - append("&order_by=") - append(sortKey) - } - if (filter.tags.isNotEmpty()) { - append("&genres=") - append(filter.tags.joinToString(",") { it.key }) - } - } - - private fun parseCatalogManga(item: Element): Manga { - val link = item.selectFirstOrThrow("a.animeTitle") - val url = link.attrAsRelativeUrl("href") - val cover = item.selectFirstOrThrow("a.avatar .img") - .attr("style") - .let { style -> DESU_BACKGROUND_IMAGE.find(style)?.groupValues?.get(1) } - ?: error("Cannot find Desu catalog cover") - val russianTitle = item.selectFirstOrThrow(".dimmed.oTitle [itemprop=title]").text() - val originalTitle = link.text() - return Manga( - id = generateUid(url), - url = url, - publicUrl = url.toAbsoluteUrl(domain), - source = source, - title = russianTitle, - altTitles = setOf(originalTitle), - coverUrl = cover, - largeCoverUrl = null, - state = null, - rating = item.parseDesuCatalogRating(), - contentRating = null, - tags = emptySet(), - authors = emptySet(), - description = null, - ) - } - - private fun parseSearchManga(item: Element): Manga { - val url = item.attrAsRelativeUrl("href") - val russianTitle = item.selectFirstOrThrow(".itemSubTitle").text() - val originalTitle = item.selectFirstOrThrow(".itemTitle").text() - return Manga( - id = generateUid(url), - url = url, - publicUrl = url.toAbsoluteUrl(domain), - source = source, - title = russianTitle, - altTitles = setOf(originalTitle), - coverUrl = item.selectFirstOrThrow("img").attr("src"), - largeCoverUrl = null, - state = null, - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - description = null, - ) - } -} - -internal fun Document.parseDesuCatalogItems(): List<Element> = - select("ol.memberList > li.memberListItem") - -internal fun Document.parseDesuSearchResults(): List<Element> = - select("ol.section > li > a[href^=manga/]") - -internal fun Document.parseDesuTags(): Set<MangaTag> = - select("#animeFilter input[data-genre-id][data-genre-slug][data-genre-name]").mapTo(linkedSetOf()) { input -> - MangaTag( - key = "${input.attr("data-genre-id")}-${input.attr("data-genre-slug")}", - title = input.attr("data-genre-name").toTitleCase(), - source = MangaParserSource.DESUME, - ) - } - -private fun Element.parseDesuCatalogRating(): Float { - val rating = select(".animeInfo dl").asSequence() - .flatMap { it.children().asSequence() } - .windowed(2) - .firstOrNull { it[0].tagName() == "dt" && it[0].text() == "Рейтинг:" && it[1].tagName() == "dd" } - ?.get(1) - ?.text() - ?.toFloatOrNull() - return rating?.div(10f) ?: RATING_UNKNOWN -} - -private fun Set<String>.toDesuState(): MangaState? = when { - "ongoing" in this -> MangaState.ONGOING - "released" in this -> MangaState.FINISHED - "anons" in this -> MangaState.UPCOMING - isEmpty() -> null - else -> error("Unsupported Desu status classes: $this") -} - -private val DESU_BACKGROUND_IMAGE = Regex("url\\(['\"]?([^'\")]+)") -private val DESU_CHAPTER_PATH = Regex("/vol(\\d+)/ch([0-9.]+)/") From 6b2eec8dedb9651dfd60319aa2b2943f677a4f4d Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:35:22 +0200 Subject: [PATCH 417/434] Refactor AComics.kt by adding imports and updating class --- src/main/kotlin/tsuki/site/ru/AComics.kt | 39 +++++++++++++++++++----- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ru/AComics.kt b/src/main/kotlin/tsuki/site/ru/AComics.kt index b64f02f9..666651d5 100644 --- a/src/main/kotlin/tsuki/site/ru/AComics.kt +++ b/src/main/kotlin/tsuki/site/ru/AComics.kt @@ -1,16 +1,41 @@ package tsuki.site.ru -import androidx.collection.ArrayMap -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.jsoup.nodes.Document import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey import tsuki.core.PagedMangaParser -import tsuki.model.* -import tsuki.util.* -import java.util.* + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.insertCookies +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import java.util.EnumSet +import kotlin.collections.toSet @MangaSourceParser("ACOMICS", "AComics", "ru", ContentType.COMICS) internal class AComics(context: MangaLoaderContext) : From 8e42db79d26ec1be2cadcd7221ef4faf5cb08df3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:35:56 +0200 Subject: [PATCH 418/434] Update and rename ComXParser.kt to ComX.kt --- src/main/kotlin/tsuki/site/ru/ComX.kt | 270 ++++++++++++++++++++ src/main/kotlin/tsuki/site/ru/ComXParser.kt | 238 ----------------- 2 files changed, 270 insertions(+), 238 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/ComX.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/ComXParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/ComX.kt b/src/main/kotlin/tsuki/site/ru/ComX.kt new file mode 100644 index 00000000..5c356baa --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/ComX.kt @@ -0,0 +1,270 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.YEAR_UNKNOWN + +import tsuki.util.Set +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.insertCookies +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectFirstOrThrow +import tsuki.util.splitByWhitespace +import tsuki.util.suspendlazy.getOrNull +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("COMX", "Com-X", "ru", ContentType.COMICS) +internal class ComX(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.COMX, 20) { + + override val configKeyDomain = ConfigKey.Domain("com-x.life", "comx.life") + + private val availableTags = suspendLazy(initializer = ::fetchTags) + private val cdnImageUrl = "img.com-x.life/comix/" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + init { + context.cookieJar.insertCookies(domain, "adt-accepted", "1") + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags.get(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val urlBuilder = StringBuilder() + when { + !filter.query.isNullOrEmpty() -> { + val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "%20") { part -> + part.urlEncoded() + } + urlBuilder.append("/search/") + urlBuilder.append(encodedQuery) + if (page > 1) { + urlBuilder.append("/page/$page/") + } + } + + else -> { + urlBuilder.append("/ComicList") + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.append("/y[from]=${filter.yearFrom}") + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.append("/y[to]=${filter.yearTo}") + } + if (filter.tags.isNotEmpty()) { + urlBuilder.append("/g=") + urlBuilder.append(filter.tags.joinToString(",") { it.key }) + } + urlBuilder.append("/sort") + if (page > 1) { + urlBuilder.append("/page/$page/") + } + } + } + + val fullUrl = urlBuilder.toString().toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.readed.d-flex.short").map { item -> + val a = item.selectFirstOrThrow("a.readed__img.img-fit-cover.anim") + val img = item.selectFirst("img[data-src]") + val href = a.attrAsRelativeUrl("href") + val titleElement = item.selectFirstOrThrow("h3.readed__title a") + val (mainTitle, altTitle) = titleElement.text() + .split("\\s*/\\s*".toRegex()) + .map { it.trim() } + .let { parts -> + when { + parts.size >= 2 -> parts[1] to parts[0] + parts.isNotEmpty() -> parts[0] to "" + else -> "" to "" + } + } + + Manga( + id = generateUid(href), + url = href, + publicUrl = a.attrAsAbsoluteUrl("href"), + title = mainTitle, + altTitles = if (altTitle.isNotEmpty()) setOf(altTitle) else emptySet(), + authors = emptySet(), + description = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + coverUrl = img?.attrAsAbsoluteUrlOrNull("data-src"), + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) + + val scriptData = doc.selectFirst("script:containsData(__DATA__)")?.data() + ?.substringAfter("window.__DATA__ = ") + ?.substringBefore(";</script>") + ?.trim() + ?: throw ParseException("Script data not found", manga.url) + + val jsonData = JSONObject(scriptData) + val chaptersJson = jsonData.getJSONArray("chapters") + val newsId = jsonData.getLong("news_id") + + val chapters = List(chaptersJson.length()) { i -> + val chapter = chaptersJson.getJSONObject(i) + val chapterId = chapter.getLong("id") + + MangaChapter( + id = generateUid("$newsId/$chapterId"), + url = "/reader/$newsId/$chapterId", + number = chapter.getFloatOrDefault("posi", 0f), + title = decodeText(chapter.getStringOrNull("title")), + uploadDate = dateFormat.parseSafe(chapter.getStringOrNull("date")), + source = source, + scanlator = null, + branch = null, + volume = chapter.getIntOrDefault("volume", 0), + ) + }.reversed() + + val author = doc.selectFirst("li:contains(Publisher:)") + ?.textOrNull() + ?.substringAfter("Publisher:") + ?.trim() + ?.nullIfEmpty() + val state = when ( + doc.selectFirst("li:contains(Release type:)")?.text()?.substringAfter("Release type:")?.trim() + ) { + "Ongoing" -> MangaState.ONGOING + else -> MangaState.FINISHED + } + + val tagLinks = doc.getElementsByAttributeValueContaining("href", "/genres/") + val tags = if (tagLinks.isNotEmpty()) { + availableTags.getOrNull()?.let { allTags -> + tagLinks.mapNotNullToSet { a -> + val tagName = a.text() + allTags.find { it.title.equals(tagName, ignoreCase = true) } + } + } + } else { + null + } + + return manga.copy( + authors = setOfNotNull(author), + state = state, + chapters = chapters, + description = doc.select("div.page__text.full-text.clearfix").textOrNull(), + tags = tags ?: manga.tags, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val newsId = chapter.url.substringAfter("/reader/").substringBefore("/") + context.cookieJar.insertCookies(domain, "adult=$newsId") + + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val data = doc.selectFirst("script:containsData(__DATA__)")?.data() + ?.substringAfter("=") + ?.trim() + ?.removeSuffix(";") + ?.substringAfter("\"images\":[") + ?.substringBefore("]") + ?.split(",") + ?.map { it.trim().removeSurrounding("\"").replace("\\", "") } + ?: throw ParseException("Image data not found", chapter.url) + + return data.map { imageUrl -> + val finalUrl = "https://$cdnImageUrl$imageUrl" + MangaPage( + id = generateUid(imageUrl), + url = finalUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/comix/").parseHtml() + val scriptData = doc.selectFirstOrThrow("script:containsData(__XFILTER__)").data() + + val genresJson = scriptData + .substringAfter("\"g\":{") + .substringBefore("}}}") + "}" + + val genresObj = JSONObject("{$genresJson}") + val valuesArray = genresObj.getJSONArray("values") + + return Set(valuesArray.length()) { i -> + val genre = valuesArray.getJSONObject(i) + MangaTag( + key = genre.getInt("id").toString(), + title = genre.getString("value").toTitleCase(sourceLocale), + source = source, + ) + } + } + + private fun decodeText(text: String?): String? { + if (text == null) return null + return text.replace("\\u([0-9a-fA-F]{4})".toRegex()) { matchResult -> + val codePoint = matchResult.groupValues[1].toInt(16) + codePoint.toChar().toString() + } + } +} diff --git a/src/main/kotlin/tsuki/site/ru/ComXParser.kt b/src/main/kotlin/tsuki/site/ru/ComXParser.kt deleted file mode 100644 index 8b2d7001..00000000 --- a/src/main/kotlin/tsuki/site/ru/ComXParser.kt +++ /dev/null @@ -1,238 +0,0 @@ -package tsuki.site.ru - -import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.getFloatOrDefault -import tsuki.util.json.getIntOrDefault -import tsuki.util.json.getStringOrNull -import tsuki.util.suspendlazy.getOrNull -import tsuki.util.suspendlazy.suspendLazy -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("COMX", "Com-X", "ru", ContentType.COMICS) -internal class ComXParser(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.COMX, 20) { - - override val configKeyDomain = ConfigKey.Domain("com-x.life", "comx.life") - - private val availableTags = suspendLazy(initializer = ::fetchTags) - private val cdnImageUrl = "img.com-x.life/comix/" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - init { - context.cookieJar.insertCookies(domain, "adt-accepted", "1") - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isYearRangeSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags.get(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val urlBuilder = StringBuilder() - when { - !filter.query.isNullOrEmpty() -> { - val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "%20") { part -> - part.urlEncoded() - } - urlBuilder.append("/search/") - urlBuilder.append(encodedQuery) - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - - else -> { - urlBuilder.append("/ComicList") - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.append("/y[from]=${filter.yearFrom}") - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.append("/y[to]=${filter.yearTo}") - } - if (filter.tags.isNotEmpty()) { - urlBuilder.append("/g=") - urlBuilder.append(filter.tags.joinToString(",") { it.key }) - } - urlBuilder.append("/sort") - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - } - - val fullUrl = urlBuilder.toString().toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.readed.d-flex.short").map { item -> - val a = item.selectFirstOrThrow("a.readed__img.img-fit-cover.anim") - val img = item.selectFirst("img[data-src]") - val href = a.attrAsRelativeUrl("href") - val titleElement = item.selectFirstOrThrow("h3.readed__title a") - val (mainTitle, altTitle) = titleElement.text() - .split("\\s*/\\s*".toRegex()) - .map { it.trim() } - .let { parts -> - when { - parts.size >= 2 -> parts[1] to parts[0] - parts.isNotEmpty() -> parts[0] to "" - else -> "" to "" - } - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = mainTitle, - altTitles = if (altTitle.isNotEmpty()) setOf(altTitle) else emptySet(), - authors = emptySet(), - description = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - coverUrl = img?.attrAsAbsoluteUrlOrNull("data-src"), - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - - val scriptData = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("window.__DATA__ = ") - ?.substringBefore(";</script>") - ?.trim() - ?: throw ParseException("Script data not found", manga.url) - - val jsonData = JSONObject(scriptData) - val chaptersJson = jsonData.getJSONArray("chapters") - val newsId = jsonData.getLong("news_id") - - val chapters = List(chaptersJson.length()) { i -> - val chapter = chaptersJson.getJSONObject(i) - val chapterId = chapter.getLong("id") - - MangaChapter( - id = generateUid("$newsId/$chapterId"), - url = "/reader/$newsId/$chapterId", - number = chapter.getFloatOrDefault("posi", 0f), - title = decodeText(chapter.getStringOrNull("title")), - uploadDate = dateFormat.parseSafe(chapter.getStringOrNull("date")), - source = source, - scanlator = null, - branch = null, - volume = chapter.getIntOrDefault("volume", 0), - ) - }.reversed() - - val author = doc.selectFirst("li:contains(Publisher:)") - ?.textOrNull() - ?.substringAfter("Publisher:") - ?.trim() - ?.nullIfEmpty() - val state = when ( - doc.selectFirst("li:contains(Release type:)")?.text()?.substringAfter("Release type:")?.trim() - ) { - "Ongoing" -> MangaState.ONGOING - else -> MangaState.FINISHED - } - - val tagLinks = doc.getElementsByAttributeValueContaining("href", "/genres/") - val tags = if (tagLinks.isNotEmpty()) { - availableTags.getOrNull()?.let { allTags -> - tagLinks.mapNotNullToSet { a -> - val tagName = a.text() - allTags.find { it.title.equals(tagName, ignoreCase = true) } - } - } - } else { - null - } - - return manga.copy( - authors = setOfNotNull(author), - state = state, - chapters = chapters, - description = doc.select("div.page__text.full-text.clearfix").textOrNull(), - tags = tags ?: manga.tags, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val newsId = chapter.url.substringAfter("/reader/").substringBefore("/") - context.cookieJar.insertCookies(domain, "adult=$newsId") - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val data = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("=") - ?.trim() - ?.removeSuffix(";") - ?.substringAfter("\"images\":[") - ?.substringBefore("]") - ?.split(",") - ?.map { it.trim().removeSurrounding("\"").replace("\\", "") } - ?: throw ParseException("Image data not found", chapter.url) - - return data.map { imageUrl -> - val finalUrl = "https://$cdnImageUrl$imageUrl" - MangaPage( - id = generateUid(imageUrl), - url = finalUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/comix/").parseHtml() - val scriptData = doc.selectFirstOrThrow("script:containsData(__XFILTER__)").data() - - val genresJson = scriptData - .substringAfter("\"g\":{") - .substringBefore("}}}") + "}" - - val genresObj = JSONObject("{$genresJson}") - val valuesArray = genresObj.getJSONArray("values") - - return Set(valuesArray.length()) { i -> - val genre = valuesArray.getJSONObject(i) - MangaTag( - key = genre.getInt("id").toString(), - title = genre.getString("value").toTitleCase(sourceLocale), - source = source, - ) - } - } - - private fun decodeText(text: String?): String? { - if (text == null) return null - return text.replace("\\u([0-9a-fA-F]{4})".toRegex()) { matchResult -> - val codePoint = matchResult.groupValues[1].toInt(16) - codePoint.toChar().toString() - } - } -} From b93fec7a91740248f77eb8642655a4e96605237f Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:37:05 +0200 Subject: [PATCH 419/434] Update and rename HenChanParser.kt to HenChan.kt --- src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt | 104 ++++++++++++++++++ .../tsuki/site/ru/nsfw/HenChanParser.kt | 90 --------------- 2 files changed, 104 insertions(+), 90 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt b/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt new file mode 100644 index 00000000..d45a8754 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt @@ -0,0 +1,104 @@ +package tsuki.site.ru.nsfw + +import tsuki.parsers.ChanParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode + +import java.util.EnumSet +import okhttp3.HttpUrl + +@MangaSourceParser("HENCHAN", "Хентай-тян", "ru", ContentType.HENTAI) +internal class HenChan(context: MangaLoaderContext) : + ChanParser(context, MangaParserSource.HENCHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "x5.h-chan.me", + "xxxx.henchan.pro", + "xxl.hentaichan.live", + "xxx.henchan.pro", + "y.hentaichan.live", + "xx.hentaichan.live", + "x.henchan.pro", + "hentaichan.live", + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val readLink = manga.url.replace("manga", "online") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { + val a = it.children().last() ?: doc.parseFailed("Invalid tag") + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } ?: manga.tags, + chapters = listOf( + MangaChapter( + id = generateUid(readLink), + url = readLink, + source = source, + number = 0f, + volume = 0, + uploadDate = 0L, + title = null, + scanlator = null, + branch = null, + ), + ), + ) + } + + override fun buildUrl(offset: Int, order: SortOrder, filter: MangaListFilter): HttpUrl = when { + filter.query.isNullOrEmpty() && filter.tags.isEmpty() && filter.tagsExclude.isEmpty() -> { + val builder = urlBuilder().addQueryParameter("offset", offset.toString()) + when (order) { + SortOrder.POPULARITY -> { + builder.addPathSegment("mostviews") + builder.addQueryParameter("sort", "manga") + } + SortOrder.RATING -> { + builder.addPathSegment("mostfavorites") + builder.addQueryParameter("sort", "manga") + } + else -> { + builder.addPathSegment("manga") + builder.addPathSegment("newest") + } + } + builder.build() + } + else -> { + super.buildUrl(offset, order, filter) + } + } +} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt deleted file mode 100644 index 32cfc545..00000000 --- a/src/main/kotlin/tsuki/site/ru/nsfw/HenChanParser.kt +++ /dev/null @@ -1,90 +0,0 @@ -package tsuki.site.ru.nsfw - -import tsuki.parsers.ChanParser -import okhttp3.HttpUrl -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.model.* -import tsuki.util.* -import java.util.* - -@MangaSourceParser("HENCHAN", "Хентай-тян", "ru", type = ContentType.HENTAI) -internal class HenChanParser(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.HENCHAN) { - - override val configKeyDomain = ConfigKey.Domain( - "x5.h-chan.me", - "xxxx.henchan.pro", - "xxl.hentaichan.live", - "xxx.henchan.pro", - "y.hentaichan.live", - "xx.hentaichan.live", - "x.henchan.pro", - "hentaichan.live", - ) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - val readLink = manga.url.replace("manga", "online") - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { - val a = it.children().last() ?: doc.parseFailed("Invalid tag") - MangaTag( - title = a.text().toTitleCase(), - key = a.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - } ?: manga.tags, - chapters = listOf( - MangaChapter( - id = generateUid(readLink), - url = readLink, - source = source, - number = 0f, - volume = 0, - uploadDate = 0L, - title = null, - scanlator = null, - branch = null, - ), - ), - ) - } - - override fun buildUrl(offset: Int, order: SortOrder, filter: MangaListFilter): HttpUrl = when { - filter.query.isNullOrEmpty() && filter.tags.isEmpty() && filter.tagsExclude.isEmpty() -> { - val builder = urlBuilder().addQueryParameter("offset", offset.toString()) - when (order) { - SortOrder.POPULARITY -> { - builder.addPathSegment("mostviews") - builder.addQueryParameter("sort", "manga") - } - - SortOrder.RATING -> { - builder.addPathSegment("mostfavorites") - builder.addQueryParameter("sort", "manga") - } - - else -> { // SortOrder.NEWEST - builder.addPathSegment("manga") - builder.addPathSegment("newest") - } - } - builder.build() - } - - else -> { - super.buildUrl(offset, order, filter) - } - } -} From ee1c41194299a2d60423eb4888619ae7fed6bbd4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:37:43 +0200 Subject: [PATCH 420/434] Update and rename AllHentaiParser.kt to AllHentai.kt --- .../kotlin/tsuki/site/ru/nsfw/AllHentai.kt | 47 +++++++++++++++++++ .../tsuki/site/ru/nsfw/AllHentaiParser.kt | 44 ----------------- 2 files changed, 47 insertions(+), 44 deletions(-) create mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt delete mode 100644 src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt new file mode 100644 index 00000000..c0caf7be --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt @@ -0,0 +1,47 @@ +package tsuki.site.ru.nsfw + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.exception.AuthRequiredException + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import tsuki.util.getCookies +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", ContentType.HENTAI) +internal class AllHentai(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.ALLHENTAI, 1) { + + override val configKeyDomain = ConfigKey.Domain( + "20.allhen.online", + "z.ahen.me", + "24.allhen.online", + "2023.allhen.online", + ) + + override suspend fun isAuthorized(): Boolean = + super.isAuthorized() || context.cookieJar.getCookies(domain).any { it.name == "remember_me" } + + override val authUrl: String + get() { + val targetUri = "https://$domain/".urlEncoded() + return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://qawa.org/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else { + res + } + } +} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt deleted file mode 100644 index 610aba5b..00000000 --- a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentaiParser.kt +++ /dev/null @@ -1,44 +0,0 @@ -package tsuki.site.ru.nsfw - -import tsuki.parsers.GroupleParser -import tsuki.Broken -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.exception.AuthRequiredException -import tsuki.model.ContentType -import tsuki.model.MangaParserSource -import tsuki.util.* - -@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", type = ContentType.HENTAI) -internal class AllHentaiParser( - context: MangaLoaderContext, -) : GroupleParser(context, MangaParserSource.ALLHENTAI, 1) { - - override val configKeyDomain = ConfigKey.Domain( - "20.allhen.online", - "z.ahen.me", - "24.allhen.online", - "2023.allhen.online", - ) - - override suspend fun isAuthorized(): Boolean = - super.isAuthorized() || context.cookieJar.getCookies(domain).any { it.name == "remember_me" } - - override val authUrl: String - get() { - val targetUri = "https://$domain/".urlEncoded() - return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" - } - - override suspend fun getUsername(): String { - val root = webClient.httpGet("https://qawa.org/").parseHtml().body() - val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) - val res = element.parent()?.text() - return if (res.isNullOrEmpty()) { - root.parseFailed("Cannot find username") - } else { - res - } - } -} From bb4abc6c60e3f9c4fa33ba070a0a246e3e41399b Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:39:19 +0200 Subject: [PATCH 421/434] Update LibSocialParser.kt --- .../kotlin/tsuki/parsers/LibSocialParser.kt | 929 +++++++++--------- 1 file changed, 483 insertions(+), 446 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/LibSocialParser.kt b/src/main/kotlin/tsuki/parsers/LibSocialParser.kt index 84d86013..d197bf81 100644 --- a/src/main/kotlin/tsuki/parsers/LibSocialParser.kt +++ b/src/main/kotlin/tsuki/parsers/LibSocialParser.kt @@ -1,6 +1,54 @@ package tsuki.parsers -import androidx.collection.* +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.network.UserAgents + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.LinkResolver +import tsuki.util.SCHEME_HTTPS +import tsuki.util.WebViewHelper +import tsuki.util.concatUrl +import tsuki.util.formatSimple +import tsuki.util.generateUid +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNull +import tsuki.util.json.mapJSONNotNullToSet +import tsuki.util.json.unescapeJson +import tsuki.util.nl2br +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.removeSurrounding +import tsuki.util.suspendlazy.suspendLazy + +import androidx.collection.ArraySet +import androidx.collection.IntObjectMap +import androidx.collection.IntSet +import androidx.collection.MutableIntSet +import androidx.collection.MutableScatterMap +import androidx.collection.ScatterMap +import androidx.collection.intObjectMapOf import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import kotlinx.coroutines.runBlocking @@ -10,464 +58,453 @@ import okhttp3.Response import org.json.JSONArray import org.json.JSONException import org.json.JSONObject -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.network.UserAgents -import tsuki.util.* -import tsuki.util.json.* -import tsuki.util.suspendlazy.suspendLazy import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale internal abstract class LibSocialParser( - context: MangaLoaderContext, - source: MangaParserSource, - protected val siteId: Int, - siteDomains: Array<String>, + context: MangaLoaderContext, + source: MangaParserSource, + protected val siteId: Int, + siteDomains: String, ) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { - protected val apiHost = "api.cdnlibs.org" - - override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE) - - override val authUrl: String - get() = "https://$domain/ru/front/auth" - - override suspend fun isAuthorized(): Boolean { - val token = getAuthData()?.optJSONObject("token")?.getStringOrNull("access_token") - return !token.isNullOrEmpty() - } - - override suspend fun getUsername(): String = getAuthData() - ?.getJSONObject("auth") - ?.getString("username") - ?: throw AuthRequiredException(source) - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - ) - - final override val configKeyDomain = ConfigKey.Domain(*siteDomains) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - MangaState.PAUSED, - MangaState.UPCOMING, - ), - ) - - final override fun intercept(chain: Interceptor.Chain): Response { - val token = runBlocking { getAuthData() }?.optJSONObject("token")?.getStringOrNull("access_token") - val requestBuilder = chain.request().newBuilder() - if (!token.isNullOrEmpty()) { - requestBuilder.header("Authorization", "Bearer $token") - } - requestBuilder.header("Site-Id", siteId.toString()) - return chain.proceed(requestBuilder.build()) - } - - private val statesMap = intObjectMapOf( - 1, MangaState.ONGOING, - 2, MangaState.FINISHED, - 3, MangaState.UPCOMING, - 4, MangaState.PAUSED, - 5, MangaState.ABANDONED, - ) - private val imageServers = suspendLazy(initializer = ::fetchServers) - private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - null to null, - SERVER_MAIN to "Первый", - SERVER_SECONDARY to "Второй", - SERVER_COMPRESS to "Сжатия", - SERVER_DOWNLOAD to "Загрузки", - SERVER_CROP to "Обрезки", - ), - defaultValue = null, - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val urlBuilder = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addQueryParameter("site_id[]", siteId.toString()) - .addQueryParameter("fields[]", "rate") - .addQueryParameter("fields[]", "rate_avg") - .addQueryParameter("page", page.toString()) - for (state in filter.states) { - urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString()) - } - for (tag in filter.tags) { - urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1)) - } - for (tag in filter.tagsExclude) { - urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1)) - } - if (!filter.query.isNullOrEmpty()) { - urlBuilder.addQueryParameter("q", filter.query) - } - urlBuilder.addQueryParameter( - "sort_by", - when (order) { - SortOrder.UPDATED -> "last_chapter_at" - SortOrder.POPULARITY -> "views" - SortOrder.RATING -> "rate_avg" - SortOrder.NEWEST -> "created_at" - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - -> "rus_name" - - else -> null - }, - ) - urlBuilder.addQueryParameter( - "sort_type", - when (order) { - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL_DESC, - -> "desc" - - SortOrder.ALPHABETICAL -> "asc" - else -> null - }, - ) - val json = webClient.httpGet(urlBuilder.build()).parseJson() - val data = json.getJSONArray("data") - return data.mapJSON(::parseManga) - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val chaptersDeferred = async { fetchChapters(manga) } - val url = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(manga.url) - .addQueryParameter("fields[]", "summary") - .addQueryParameter("fields[]", "genres") - .addQueryParameter("fields[]", "tags") - .addQueryParameter("fields[]", "authors") - .addQueryParameter("fields[]", "close_view") - .build() - val json = webClient.httpGet(url).parseJson().getJSONObject("data") - val genres = json.getJSONArray("genres").mapJSON { jo -> - MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source) - } - val tags = json.getJSONArray("tags").mapJSON { jo -> - MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source) - } - val authors = json.getJSONArray("authors").mapJSONNotNullToSet { - it.getStringOrNull("name") - } - val chapters = chaptersDeferred.await() - val isRestricted = json.getIntOrDefault("close_view", 0) > 0 - manga.copy( - title = json.getStringOrNull("rus_name") ?: manga.title, - altTitles = setOfNotNull(json.getStringOrNull("name")), - tags = tagsSetOf(tags, genres), - state = if (chapters.isEmpty() && isRestricted) { - MangaState.RESTRICTED - } else { - manga.state - }, - authors = authors, - contentRating = json.optJSONObject("ageRestriction")?.let { - val id = it.getIntOrDefault("id", -1) - if (id >= 4) ContentRating.SUGGESTIVE else sourceContentRating - } ?: manga.contentRating, - description = json.parseLibSocialSummary()?.nl2br(), - chapters = chapters, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope { - val pages = async { - webClient.httpGet( - concatUrl("https://$apiHost/api/manga/", chapter.url), - ).parseJson().getJSONObject("data") - } - val servers = imageServers.get() - val json = pages.await() - val primaryServer = getPrimaryImageServer(servers) - json.getJSONArray("pages").mapJSON { jo -> - val url = jo.getString("url") - MangaPage( - id = generateUid(jo.getLong("id")), - url = concatUrl(primaryServer, url), - preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) }, - source = source, - ) - } - } - - private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { - val tags = async { fetchTags("tags") } - val genres = async { fetchTags("genres") } - tagsSetOf(tags.await(), genres.await()) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val json = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(seed.url) - .addPathSegment("similar") - .build(), - ).parseJson().getJSONArray("data") - return json.mapJSON { jo -> - parseManga(jo.getJSONObject("media")) - } - } - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - return resolver.resolveManga(this, link.pathSegments.lastOrNull() ?: return null) - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(splitTranslationsKey) - keys.add(preferredServerKey) - } - - private fun parseManga(jo: JSONObject): Manga { - val cover = jo.getJSONObject("cover") - val isNsfwSource = jo.getJSONObject("ageRestriction").getIntOrDefault("id", 0) >= 3 - return Manga( - id = generateUid(jo.getLong("id")), - title = jo.getString("rus_name").ifEmpty { jo.getString("name") }, - altTitles = setOfNotNull(jo.getString("name")), - url = jo.getString("slug_url"), - publicUrl = "https://$domain/ru/manga/" + jo.getString("slug_url"), - rating = jo.optJSONObject("rating") - ?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN, - contentRating = sourceContentRating, - coverUrl = cover.getString("thumbnail"), - tags = setOf(), - state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1], - authors = emptySet(), - largeCoverUrl = cover.getString("default"), - source = source, - ) - } - - private fun getPrimaryImageServer(servers: ScatterMap<String, String>): String { - val preferred = config[preferredServerKey] - if (preferred != null) { - servers[preferred]?.let { return it } - } - return checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) { - "No available images servers" - } - } - - private suspend fun fetchChapters(manga: Manga): List<MangaChapter> { - val url = HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("manga") - .addPathSegment(manga.url) - .addPathSegment("chapters") - .build() - val json = webClient.httpGet(url).parseJson().getJSONArray("data") - val builder = ChaptersListBuilder(json.length()) - val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) - val useBranching = config[splitTranslationsKey] - for (i in 0 until json.length()) { - val jo = json.getJSONObject(i) - val volume = jo.getIntOrDefault("volume", 0) - val number = jo.getFloatOrDefault("number", 0f) - val numberString = number.formatSimple() - val name = jo.getStringOrNull("name") - val branches = jo.getJSONArray("branches") - for (j in 0 until branches.length()) { - val bjo = branches.getJSONObject(j) - val isRestricted = bjo.optJSONObject("restricted_view")?.let { - !it.getBooleanOrDefault("is_open", true) - } ?: false - if (isRestricted) { - continue - } - val id = bjo.getLong("id") - val branchId = bjo.getLongOrDefault("branch_id", 0L) - val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name") - builder += MangaChapter( - id = generateUid(id), - title = name, - number = number, - volume = volume, - url = buildString { - append(manga.url) - append("/chapter?number=") - append(numberString) - append("&volume=") - append(volume) - if (branchId != 0L) { - append("&branch_id=") - append(branchId) - } - }, - scanlator = team, - uploadDate = dateFormat.parseSafe(bjo.getStringOrNull("created_at")), - branch = if (useBranching) team else null, - source = source, - ) - } - } - return builder.toList() - } - - private suspend fun fetchTags(type: String): List<MangaTag> { - val data = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api").addPathSegment(type).build(), - ).parseJson().getJSONArray("data") - val prefix = type.first().toString() - return data.mapJSONNotNull { jo -> - val sites = jo.getJSONArray("site_ids").toIntSet() - if (siteId !in sites) { - return@mapJSONNotNull null - } - MangaTag( - title = jo.getString("name"), - key = prefix + jo.getInt("id"), - source = source, - ) - } - } - - private suspend fun fetchServers(): ScatterMap<String, String> { - val json = webClient.httpGet( - HttpUrl.Builder() - .scheme(SCHEME_HTTPS) - .host(apiHost) - .addPathSegment("api") - .addPathSegment("constants") - .addQueryParameter("fields[]", "imageServers") - .build(), - ).parseJson().getJSONObject("data").getJSONArray("imageServers") - val result = MutableScatterMap<String, String>() - for (i in 0 until json.length()) { - val jo = json.getJSONObject(i) - val sites = jo.getJSONArray("site_ids").toIntSet() - if (siteId !in sites) { - continue - } - result[jo.getString("id")] = jo.getString("url") - } - return result - } - - private fun <V> IntObjectMap<V>.keyOf(value: V): Int { - forEach { k, v -> - if (v == value) { - return k - } - } - throw NoSuchElementException("No key associated with value $value") - } - - private fun JSONArray.toIntSet(): IntSet { - val result = MutableIntSet(length()) - for (i in 0 until length()) { - result.add(getInt(i)) - } - return result - } - - private fun MangaTag.typeKey() = when (key.firstOrNull()) { - 'g' -> "genres" - 't' -> "tags" - else -> throw IllegalArgumentException("Tag $key($title) is of unknown type") - } - - private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> { - val result = ArraySet<MangaTag>(tags.size + genres.size) - val names = HashSet<String>(tags.size + genres.size) - genres.forEach { x -> if (names.add(x.title)) result.add(x) } - tags.forEach { x -> if (names.add(x.title)) result.add(x) } - return result - } - - private suspend fun getAuthData(): JSONObject? { - val raw = WebViewHelper(context).getLocalStorageValue(domain, "auth") ?: return null - return JSONObject(raw.unescapeJson().removeSurrounding('"')) - } - - protected companion object { - - const val SERVER_MAIN = "main" - const val SERVER_SECONDARY = "secondary" - const val SERVER_COMPRESS = "compress" - const val SERVER_DOWNLOAD = "download" - const val SERVER_CROP = "crop" - } + protected val apiHost = "api.cdnlibs.org" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE) + + override val authUrl: String + get() = "https://$domain/ru/front/auth" + + override suspend fun isAuthorized(): Boolean { + val token = getAuthData()?.optJSONObject("token")?.getStringOrNull("access_token") + return !token.isNullOrEmpty() + } + + override suspend fun getUsername(): String = getAuthData() + ?.getJSONObject("auth") + ?.getString("username") + ?: throw AuthRequiredException(source) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val configKeyDomain = ConfigKey.Domain(siteDomains) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + MangaState.UPCOMING, + ), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val token = runBlocking { getAuthData() }?.optJSONObject("token")?.getStringOrNull("access_token") + val requestBuilder = chain.request().newBuilder() + if (!token.isNullOrEmpty()) { + requestBuilder.header("Authorization", "Bearer $token") + } + requestBuilder.header("Site-Id", siteId.toString()) + return chain.proceed(requestBuilder.build()) + } + + private val statesMap = intObjectMapOf( + 1, MangaState.ONGOING, + 2, MangaState.FINISHED, + 3, MangaState.UPCOMING, + 4, MangaState.PAUSED, + 5, MangaState.ABANDONED, + ) + private val imageServers = suspendLazy(initializer = ::fetchServers) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + null to null, + SERVER_MAIN to "Первый", + SERVER_SECONDARY to "Второй", + SERVER_COMPRESS to "Сжатия", + SERVER_DOWNLOAD to "Загрузки", + SERVER_CROP to "Обрезки", + ), + defaultValue = null, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val urlBuilder = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addQueryParameter("site_id[]", siteId.toString()) + .addQueryParameter("fields[]", "rate") + .addQueryParameter("fields[]", "rate_avg") + .addQueryParameter("page", page.toString()) + for (state in filter.states) { + urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString()) + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1)) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1)) + } + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("q", filter.query) + } + urlBuilder.addQueryParameter( + "sort_by", + when (order) { + SortOrder.UPDATED -> "last_chapter_at" + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "rate_avg" + SortOrder.NEWEST -> "created_at" + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + -> "rus_name" + + else -> null + }, + ) + urlBuilder.addQueryParameter( + "sort_type", + when (order) { + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL_DESC, + -> "desc" + + SortOrder.ALPHABETICAL -> "asc" + else -> null + }, + ) + val json = webClient.httpGet(urlBuilder.build()).parseJson() + val data = json.getJSONArray("data") + return data.mapJSON(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { fetchChapters(manga) } + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addQueryParameter("fields[]", "summary") + .addQueryParameter("fields[]", "genres") + .addQueryParameter("fields[]", "tags") + .addQueryParameter("fields[]", "authors") + .addQueryParameter("fields[]", "close_view") + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + val genres = json.getJSONArray("genres").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source) + } + val tags = json.getJSONArray("tags").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source) + } + val authors = json.getJSONArray("authors").mapJSONNotNullToSet { + it.getStringOrNull("name") + } + val chapters = chaptersDeferred.await() + val isRestricted = json.getIntOrDefault("close_view", 0) > 0 + manga.copy( + title = json.getStringOrNull("rus_name") ?: manga.title, + altTitles = setOfNotNull(json.getStringOrNull("name")), + tags = tagsSetOf(tags, genres), + state = if (chapters.isEmpty() && isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = authors, + contentRating = json.optJSONObject("ageRestriction")?.let { + val id = it.getIntOrDefault("id", -1) + if (id >= 4) ContentRating.SUGGESTIVE else sourceContentRating + } ?: manga.contentRating, + description = json.parseLibSocialSummary()?.nl2br(), + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = coroutineScope { + val pages = async { + webClient.httpGet( + concatUrl("https://$apiHost/api/manga/", chapter.url), + ).parseJson().getJSONObject("data") + } + val servers = imageServers.get() + val json = pages.await() + val primaryServer = getPrimaryImageServer(servers) + json.getJSONArray("pages").mapJSON { jo -> + val url = jo.getString("url") + MangaPage( + id = generateUid(jo.getLong("id")), + url = concatUrl(primaryServer, url), + preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) }, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { + val tags = async { fetchTags("tags") } + val genres = async { fetchTags("genres") } + tagsSetOf(tags.await(), genres.await()) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(seed.url) + .addPathSegment("similar") + .build(), + ).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + parseManga(jo.getJSONObject("media")) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + return resolver.resolveManga(this, link.pathSegments.lastOrNull() ?: return null) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(splitTranslationsKey) + keys.add(preferredServerKey) + } + + private fun parseManga(jo: JSONObject): Manga { + val cover = jo.getJSONObject("cover") + return Manga( + id = generateUid(jo.getLong("id")), + title = jo.getString("rus_name").ifEmpty { jo.getString("name") }, + altTitles = setOfNotNull(jo.getString("name")), + url = jo.getString("slug_url"), + publicUrl = "https://$domain/ru/manga/" + jo.getString("slug_url"), + rating = jo.optJSONObject("rating") + ?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN, + contentRating = sourceContentRating, + coverUrl = cover.getString("thumbnail"), + tags = setOf(), + state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1], + authors = emptySet(), + largeCoverUrl = cover.getString("default"), + source = source, + ) + } + + private fun getPrimaryImageServer(servers: ScatterMap<String, String>): String { + val preferred = config[preferredServerKey] + if (preferred != null) { + servers[preferred]?.let { return it } + } + return checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) { + "No available images servers" + } + } + + private suspend fun fetchChapters(manga: Manga): List<MangaChapter> { + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addPathSegment("chapters") + .build() + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + val builder = ChaptersListBuilder(json.length()) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + val useBranching = config[splitTranslationsKey] + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val volume = jo.getIntOrDefault("volume", 0) + val number = jo.getFloatOrDefault("number", 0f) + val numberString = number.formatSimple() + val name = jo.getStringOrNull("name") + val branches = jo.getJSONArray("branches") + for (j in 0 until branches.length()) { + val bjo = branches.getJSONObject(j) + val isRestricted = bjo.optJSONObject("restricted_view")?.let { + !it.getBooleanOrDefault("is_open", true) + } ?: false + if (isRestricted) { + continue + } + val id = bjo.getLong("id") + val branchId = bjo.getLongOrDefault("branch_id", 0L) + val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name") + builder += MangaChapter( + id = generateUid(id), + title = name, + number = number, + volume = volume, + url = buildString { + append(manga.url) + append("/chapter?number=") + append(numberString) + append("&volume=") + append(volume) + if (branchId != 0L) { + append("&branch_id=") + append(branchId) + } + }, + scanlator = team, + uploadDate = dateFormat.parseSafe(bjo.getStringOrNull("created_at")), + branch = if (useBranching) team else null, + source = source, + ) + } + } + return builder.toList() + } + + private suspend fun fetchTags(type: String): List<MangaTag> { + val data = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api").addPathSegment(type).build(), + ).parseJson().getJSONArray("data") + val prefix = type.first().toString() + return data.mapJSONNotNull { jo -> + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + return@mapJSONNotNull null + } + MangaTag( + title = jo.getString("name"), + key = prefix + jo.getInt("id"), + source = source, + ) + } + } + + private suspend fun fetchServers(): ScatterMap<String, String> { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("constants") + .addQueryParameter("fields[]", "imageServers") + .build(), + ).parseJson().getJSONObject("data").getJSONArray("imageServers") + val result = MutableScatterMap<String, String>() + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + continue + } + result[jo.getString("id")] = jo.getString("url") + } + return result + } + + private fun <V> IntObjectMap<V>.keyOf(value: V): Int { + forEach { k, v -> + if (v == value) { + return k + } + } + throw NoSuchElementException("No key associated with value $value") + } + + private fun JSONArray.toIntSet(): IntSet { + val result = MutableIntSet(length()) + for (i in 0 until length()) { + result.add(getInt(i)) + } + return result + } + + private fun MangaTag.typeKey() = when (key.firstOrNull()) { + 'g' -> "genres" + 't' -> "tags" + else -> throw IllegalArgumentException("Tag $key($title) is of unknown type") + } + + private fun tagsSetOf(tags: Collection<MangaTag>, genres: Collection<MangaTag>): Set<MangaTag> { + val result = ArraySet<MangaTag>(tags.size + genres.size) + val names = HashSet<String>(tags.size + genres.size) + genres.forEach { x -> if (names.add(x.title)) result.add(x) } + tags.forEach { x -> if (names.add(x.title)) result.add(x) } + return result + } + + private suspend fun getAuthData(): JSONObject? { + val raw = WebViewHelper(context).getLocalStorageValue(domain, "auth") ?: return null + return JSONObject(raw.unescapeJson().removeSurrounding('"')) + } + + protected companion object { + + const val SERVER_MAIN = "main" + const val SERVER_SECONDARY = "secondary" + const val SERVER_COMPRESS = "compress" + const val SERVER_DOWNLOAD = "download" + const val SERVER_CROP = "crop" + } } internal fun JSONObject.parseLibSocialSummary(): String? = when (val value = opt("summary")) { - null, - JSONObject.NULL, - -> null + null, + JSONObject.NULL, + -> null - is String -> value - is JSONObject -> value.parseLibSocialRichText() - else -> throw JSONException("Unsupported summary value: ${value.javaClass.name}") + is String -> value + is JSONObject -> value.parseLibSocialRichText() + else -> throw JSONException("Unsupported summary value: ${value.javaClass.name}") } private fun JSONObject.parseLibSocialRichText(): String { - val type = getString("type") - return when (type) { - "text" -> getString("text") - "hardBreak" -> "\n" - "paragraph", - "heading", - -> parseLibSocialChildren(separator = "") - - "doc", - "blockquote", - "bulletList", - "orderedList", - "listItem", - -> parseLibSocialChildren(separator = "\n") - - else -> throw JSONException("Unsupported summary node type: $type") - } + return when (val type = getString("type")) { + "text" -> getString("text") + "hardBreak" -> "\n" + "paragraph", + "heading", + -> parseLibSocialChildren(separator = "") + + "doc", + "blockquote", + "bulletList", + "orderedList", + "listItem", + -> parseLibSocialChildren(separator = "\n") + + else -> throw JSONException("Unsupported summary node type: $type") + } } private fun JSONObject.parseLibSocialChildren(separator: String): String { - val children = optJSONArray("content") ?: return "" - return children.mapJSON { child -> child.parseLibSocialRichText() }.joinToString(separator) + val children = optJSONArray("content") ?: return "" + return children.mapJSON { child -> child.parseLibSocialRichText() }.joinToString(separator) } From f9dd08a1bb529034b66bea4b64736bf831ff20cc Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:40:05 +0200 Subject: [PATCH 422/434] Refactor ChanParser for improved readability and structure --- src/main/kotlin/tsuki/parsers/ChanParser.kt | 520 +++++++++++--------- 1 file changed, 277 insertions(+), 243 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/ChanParser.kt b/src/main/kotlin/tsuki/parsers/ChanParser.kt index 448a2c5d..488bfee0 100644 --- a/src/main/kotlin/tsuki/parsers/ChanParser.kt +++ b/src/main/kotlin/tsuki/parsers/ChanParser.kt @@ -1,271 +1,305 @@ package tsuki.parsers -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.jsoup.internal.StringUtil -import org.jsoup.nodes.Element import tsuki.MangaLoaderContext import tsuki.MangaParserAuthProvider import tsuki.core.AbstractMangaParser import tsuki.exception.AuthRequiredException -import tsuki.model.* -import tsuki.util.* + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.host +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.removeSurrounding +import tsuki.util.requireElementById +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Element import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale +import kotlin.text.isBlank internal abstract class ChanParser( - context: MangaLoaderContext, - source: MangaParserSource, + context: MangaLoaderContext, + source: MangaParserSource, ) : AbstractMangaParser(context, source), MangaParserAuthProvider { - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - ) + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) - override val authUrl: String - get() = "https://${domain}" + override val authUrl: String + get() = "https://${domain}" - override suspend fun isAuthorized(): Boolean = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } + override suspend fun isAuthorized(): Boolean = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - ) + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val domain = domain - val doc = webClient.httpGet(buildUrl(offset, order, filter)).parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") - ?: doc.parseFailed("Cannot find root") - return root.select("div.content_row").mapNotNull { row -> - val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") - ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.text().parseTitle() - val author = row.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text() - Manga( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitles = setOfNotNull(title.second), - title = title.first, - authors = setOfNotNull(author), - coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") - ?.absUrl("src").orEmpty(), - tags = row.selectFirst("div.genre")?.select("a")?.mapToSet { - MangaTag( - title = it.text().toTagName(), - key = it.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - }.orEmpty(), - rating = RATING_UNKNOWN, - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val domain = domain + val doc = webClient.httpGet(buildUrl(offset, order, filter)).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + }.orEmpty(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("dle-content") - val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) - return manga.copy( - description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), - largeCoverUrl = root.getElementById("cover")?.absUrl("src"), - chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> - val a = tr.selectFirst("a") - val href = a?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - title = tr.selectFirst("a")?.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - scanlator = null, - branch = null, - uploadDate = dateFormat.parseSafe(tr.selectFirst("div.date")?.text()), - source = source, - ) - }, - ) - } + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + chapters = root.select("table.table_cha tr:gt(1)").mapChapters(reversed = true) { i, tr -> + val a = tr.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + title = tr.selectFirst("a")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.parseSafe(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val scripts = doc.select("script") - for (script in scripts) { - val data = script.html() - val pos = data.indexOf("\"fullimg") - if (pos == -1) { - continue - } - val json = data.substring(pos).substringAfter('[').substringBefore(';') - .substringBeforeLast(']') - val domain = domain - return json.split(",").mapNotNull { - it.trim() - .removeSurrounding('"', '\'') - .toRelativeUrl(domain) - .takeUnless(String::isBlank) - }.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } - doc.parseFailed("Pages list not found at ${chapter.url}") - } + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val domain = domain - val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() - val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") - ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") - return root.select("li.sidetag").mapToSet { li -> - val a = li.children().lastOrNull() ?: li.parseFailed("a is null") - MangaTag( - title = a.text().toTagName(), - key = a.attr("href").substringAfterLast('/').urlDecode(), - source = source, - ) - } - } + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } + } - override suspend fun getUsername(): String { - val doc = webClient.httpGet("https://${domain}").parseHtml().body() - val root = doc.requireElementById("top_user") - val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() - ?: throw AuthRequiredException(source) - return a.attr("href").removeSuffix('/').substringAfterLast('/') - } + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } - override suspend fun getRelatedManga(seed: Manga): List<Manga> { - val doc = webClient.httpGet(seed.url.replace("/manga/", "/related/").toAbsoluteUrl(domain)).parseHtml() - val root = doc.body().requireElementById("right") - return root.select("div.related").mapNotNull { div -> - val info = div.selectFirst(".related_info") ?: return@mapNotNull null - val a = info.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val title = a.text().parseTitle() - val author = info.getElementsByAttributeValueStarting( - "href", - "/mangaka", - ).firstOrNull()?.text() - Manga( - id = generateUid(a.attrAsRelativeUrlAnyHost("href")), - url = href, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - altTitles = setOfNotNull(title.second), - title = title.first, - authors = setOfNotNull(author), - coverUrl = div.selectFirst("img")?.absUrl("src").orEmpty(), - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val doc = webClient.httpGet(seed.url.replace("/manga/", "/related/").toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("right") + return root.select("div.related").mapNotNull { div -> + val info = div.selectFirst(".related_info") ?: return@mapNotNull null + val a = info.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = info.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = div.selectFirst("img")?.absUrl("src").orEmpty(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } - protected open fun buildUrl( - offset: Int, - order: SortOrder, - filter: MangaListFilter, - ): HttpUrl { - val builder = urlBuilder() - builder.addQueryParameter("offset", offset.toString()) - when { - !filter.query.isNullOrEmpty() -> { - builder.addQueryParameter("do", "search") - builder.addQueryParameter("subaction", "search") - builder.addQueryParameter("search_start", ((offset / 40) + 1).toString()) - builder.addQueryParameter("full_search", "0") - builder.addQueryParameter("result_from", (offset + 1).toString()) - builder.addQueryParameter("result_num", "40") - builder.addQueryParameter("story", filter.query) - builder.addQueryParameter("need_sort_date", "false") - } + protected open fun buildUrl( + offset: Int, + order: SortOrder, + filter: MangaListFilter, + ): HttpUrl { + val builder = urlBuilder() + builder.addQueryParameter("offset", offset.toString()) + when { + !filter.query.isNullOrEmpty() -> { + builder.addQueryParameter("do", "search") + builder.addQueryParameter("subaction", "search") + builder.addQueryParameter("search_start", ((offset / 40) + 1).toString()) + builder.addQueryParameter("full_search", "0") + builder.addQueryParameter("result_from", (offset + 1).toString()) + builder.addQueryParameter("result_num", "40") + builder.addQueryParameter("story", filter.query) + builder.addQueryParameter("need_sort_date", "false") + } - else -> { - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { - builder.addPathSegment("tags") - val joiner = StringUtil.StringJoiner("+") - filter.tags.forEach { joiner.add(it.key) } - filter.tagsExclude.forEach { joiner.add("-"); joiner.append(it.key) } - builder.addPathSegment(joiner.complete()) - builder.addQueryParameter( - "n", - when (order) { - SortOrder.RATING, - SortOrder.POPULARITY, - -> "favdesc" + else -> { + if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { + builder.addPathSegment("tags") + val joiner = StringUtil.StringJoiner("+") + filter.tags.forEach { joiner.add(it.key) } + filter.tagsExclude.forEach { joiner.add("-"); joiner.append(it.key) } + builder.addPathSegment(joiner.complete()) + builder.addQueryParameter( + "n", + when (order) { + SortOrder.RATING, + SortOrder.POPULARITY, + -> "favdesc" - SortOrder.ALPHABETICAL -> "abcasc" - else -> "" // SortOrder.NEWEST - }, - ) - } else { - when (order) { - SortOrder.POPULARITY -> builder.addPathSegment("mostviews") - SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog") - SortOrder.RATING -> builder.addPathSegment("mostfavorites") - else -> { // SortOrder.NEWEST - builder.addPathSegment("manga") - builder.addPathSegment("new") - } - } - } - } - } - return builder.build() - } + SortOrder.ALPHABETICAL -> "abcasc" + else -> "" // SortOrder.NEWEST + }, + ) + } else { + when (order) { + SortOrder.POPULARITY -> builder.addPathSegment("mostviews") + SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog") + SortOrder.RATING -> builder.addPathSegment("mostfavorites") + else -> { // SortOrder.NEWEST + builder.addPathSegment("manga") + builder.addPathSegment("new") + } + } + } + } + } + return builder.build() + } - private fun String.toTagName() = replace('_', ' ').toTitleCase() + private fun String.toTagName() = replace('_', ' ').toTitleCase() - private fun String.parseTitle(): Pair<String, String?> { - var depth = 0 - for (i in indices.reversed()) { - val c = this[i] - if (c == '(') { - depth-- - if (depth == 0 && (i + 2) < lastIndex && i > 0) { - return substring(i + 1, lastIndex).trim() to substring(0, i).trim().nullIfEmpty() - } - } else if (c == ')') { - depth++ - } - } - return this to null - } + private fun String.parseTitle(): Pair<String, String?> { + var depth = 0 + for (i in indices.reversed()) { + val c = this[i] + if (c == '(') { + depth-- + if (depth == 0 && (i + 2) < lastIndex && i > 0) { + return substring(i + 1, lastIndex).trim() to substring(0, i).trim().nullIfEmpty() + } + } else if (c == ')') { + depth++ + } + } + return this to null + } - private fun Element.attrAsRelativeUrlAnyHost(attributeKey: String): String { - val attr = attr(attributeKey) - if (attr.isEmpty() || attr.startsWith("data:")) { - throw IllegalArgumentException("Wrong url $attr") - } - if (attr.startsWith('/')) { - return attr - } - val host = attr.toHttpUrl().host - return attr.substringAfter(host) - } + private fun Element.attrAsRelativeUrlAnyHost(attributeKey: String): String { + val attr = attr(attributeKey) + if (attr.isEmpty() || attr.startsWith("data:")) { + throw IllegalArgumentException("Wrong url $attr") + } + if (attr.startsWith('/')) { + return attr + } + val host = attr.toHttpUrl().host + return attr.substringAfter(host) + } } From 72059cf593921ba0258fdb0c9099537ff7e034d3 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:43:47 +0200 Subject: [PATCH 423/434] Update GroupleParser.kt --- .../kotlin/tsuki/parsers/GroupleParser.kt | 89 +++++++++++++------ 1 file changed, 61 insertions(+), 28 deletions(-) diff --git a/src/main/kotlin/tsuki/parsers/GroupleParser.kt b/src/main/kotlin/tsuki/parsers/GroupleParser.kt index e22cb743..cf14bd4e 100644 --- a/src/main/kotlin/tsuki/parsers/GroupleParser.kt +++ b/src/main/kotlin/tsuki/parsers/GroupleParser.kt @@ -1,16 +1,66 @@ package tsuki.parsers -import tsuki.site.ru.SeiMangaParser -import tsuki.site.ru.ReadmangaParser -import tsuki.site.ru.SelfMangaParser -import tsuki.site.ru.UsagiParser +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.exception.ParseException +import tsuki.network.CommonHeaders + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.YEAR_MAX +import tsuki.model.YEAR_MIN + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.concatUrl +import tsuki.util.findGroupValue +import tsuki.util.flatMapChapters +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.headersContentLength +import tsuki.util.ifZero +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.mapToSet +import tsuki.util.metaValue +import tsuki.util.nullIfEmpty +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.runCatchingCancellable +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectFirstParent +import tsuki.util.substringBetween +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.upBy +import tsuki.util.urlBuilder +import tsuki.util.urlEncoded + import androidx.collection.MutableScatterMap import androidx.collection.ScatterMap import kotlinx.coroutines.flow.channelFlow import kotlinx.coroutines.flow.firstOrNull import kotlinx.coroutines.launch import okhttp3.Headers -import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import okhttp3.Interceptor @@ -19,20 +69,10 @@ import okhttp3.internal.closeQuietly import okio.IOException import org.json.JSONArray import org.jsoup.nodes.Element -import tsuki.MangaLoaderContext -import tsuki.MangaParserAuthProvider -import tsuki.config.ConfigKey -import tsuki.core.AbstractMangaParser -import tsuki.exception.AuthRequiredException -import tsuki.exception.ParseException -import tsuki.model.* -import tsuki.util.* -import tsuki.util.json.getStringOrNull -import tsuki.util.json.mapJSON -import tsuki.util.suspendlazy.suspendLazy import java.net.HttpURLConnection import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale private const val PAGE_SIZE = 70 private const val NSFW_ALERT = "сексуальные сцены" @@ -59,8 +99,9 @@ internal abstract class GroupleParser( private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) override fun getRequestHeaders(): Headers = Headers.Builder() - .add("User-Agent", config[userAgentKey]) - .add("Accept-Language", "ru,en-US;q=0.7,en;q=0.3") + .add(CommonHeaders.REFERER, "https://$domain/") + .add(CommonHeaders.USER_AGENT, config[userAgentKey]) + .add(CommonHeaders.ACCEPT_LANGUAGE, "ru,en-US;q=0.7,en;q=0.3") .build() override val availableSortOrders: Set<SortOrder> = EnumSet.of( @@ -131,7 +172,7 @@ internal abstract class GroupleParser( } else { null } - val newSource = getSource(response.request.url) + val newSource = source val chaptersList = root.getElementById("chapters-list") var isRestricted = false if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { @@ -330,14 +371,6 @@ internal abstract class GroupleParser( return root.select("div.tile").mapNotNull(::parseManga) } - protected open fun getSource(url: HttpUrl): MangaSource = when (url.host) { - in SeiMangaParser.domains -> MangaParserSource.SEIMANGA - in ReadmangaParser.domains -> MangaParserSource.READMANGA_RU - in SelfMangaParser.domains -> MangaParserSource.SELFMANGA - in UsagiParser.domains -> MangaParserSource.USAGI - else -> source - } - private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { SortOrder.ALPHABETICAL -> "name" SortOrder.POPULARITY -> "rate" From 5b2b7f74433bf3de84429875d1ad9b5cb7a8d608 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:44:39 +0200 Subject: [PATCH 424/434] Fix MangaTag source reference in WaManga parser --- src/main/kotlin/tsuki/site/ru/WaManga.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/kotlin/tsuki/site/ru/WaManga.kt b/src/main/kotlin/tsuki/site/ru/WaManga.kt index 05db5e34..20fdef7f 100644 --- a/src/main/kotlin/tsuki/site/ru/WaManga.kt +++ b/src/main/kotlin/tsuki/site/ru/WaManga.kt @@ -32,7 +32,7 @@ import java.util.EnumSet import java.util.Locale @MangaSourceParser("WAMANGA", "WaManga", "ru") -internal class WaManga(context: MangaLoaderContext) : +internal class WaManga(context: MangaLoaderContext) : SinglePageMangaParser(context, MangaParserSource.WAMANGA) { override val configKeyDomain = ConfigKey.Domain("wamanga.ru") @@ -158,7 +158,7 @@ internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { } private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> - MangaTag(genre, genre, source) + MangaTag(genre, genre, MangaParserSource.WAMANGA) } private fun String.toWaMangaState(): MangaState = when (this) { From 271b67854e1265fefa6ef5c9941923e55c2313ed Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:49:03 +0200 Subject: [PATCH 425/434] Delete src/main/kotlin/tsuki/site/ru/MangaMammy.kt --- src/main/kotlin/tsuki/site/ru/MangaMammy.kt | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaMammy.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaMammy.kt b/src/main/kotlin/tsuki/site/ru/MangaMammy.kt deleted file mode 100644 index b1b01cdf..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaMammy.kt +++ /dev/null @@ -1,14 +0,0 @@ -package tsuki.site.ru - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGAMAMMY", "Nimanga", "ru") -internal class MangaMammy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAMAMMY, "p.nimanga.com") { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} From e80c8637f91c3064699d5f710b8eb6dd9eba5121 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:51:28 +0200 Subject: [PATCH 426/434] Update DesuMe.kt --- src/main/kotlin/tsuki/site/ru/DesuMe.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/ru/DesuMe.kt b/src/main/kotlin/tsuki/site/ru/DesuMe.kt index ba248dde..a7ef5ff9 100644 --- a/src/main/kotlin/tsuki/site/ru/DesuMe.kt +++ b/src/main/kotlin/tsuki/site/ru/DesuMe.kt @@ -139,7 +139,7 @@ internal class DesuMe(context: MangaLoaderContext) : ?.toFloatOrNull() ?.div(10f) ?: RATING_UNKNOWN, - chapters = chapters, + chapters = chapters.reversed(), ) } From 31a36fd13b107adeaf4de353e413d723501189fd Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:53:47 +0200 Subject: [PATCH 427/434] Reverse chapters order in parseManga return --- src/main/kotlin/tsuki/site/ru/WaManga.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main/kotlin/tsuki/site/ru/WaManga.kt b/src/main/kotlin/tsuki/site/ru/WaManga.kt index 20fdef7f..12398472 100644 --- a/src/main/kotlin/tsuki/site/ru/WaManga.kt +++ b/src/main/kotlin/tsuki/site/ru/WaManga.kt @@ -79,7 +79,7 @@ internal class WaManga(context: MangaLoaderContext) : return parseManga(details).copy( id = manga.id, url = manga.url, - chapters = chapters, + chapters = chapters.reversed(), ) } From b5811582934668a8c7811c0d31cb70188dc82640 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:58:01 +0200 Subject: [PATCH 428/434] Add YaoiChan.kt to nsfw directory --- src/main/kotlin/tsuki/site/ru/{ => nsfw}/YaoiChan.kt | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename src/main/kotlin/tsuki/site/ru/{ => nsfw}/YaoiChan.kt (93%) diff --git a/src/main/kotlin/tsuki/site/ru/YaoiChan.kt b/src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt similarity index 93% rename from src/main/kotlin/tsuki/site/ru/YaoiChan.kt rename to src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt index 5bdbd8ab..ccae5233 100644 --- a/src/main/kotlin/tsuki/site/ru/YaoiChan.kt +++ b/src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt @@ -1,10 +1,11 @@ -package tsuki.site.ru +package tsuki.site.ru.nsfw -import tsuki.parsers.ChanParser import tsuki.MangaLoaderContext import tsuki.MangaSourceParser import tsuki.config.ConfigKey +import tsuki.parsers.ChanParser +import tsuki.model.ContentType import tsuki.model.Manga import tsuki.model.MangaChapter import tsuki.model.MangaParserSource @@ -17,7 +18,7 @@ import tsuki.util.parseHtml import tsuki.util.requireElementById import tsuki.util.toAbsoluteUrl -@MangaSourceParser("YAOICHAN", "Яой-тян", "ru") +@MangaSourceParser("YAOICHAN", "Яой-тян", "ru", ContentType.HENTAI) internal class YaoiChan(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { override val configKeyDomain = ConfigKey.Domain( From 315b2c2db22b8508981377a5108c1358ab1e8456 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 18:59:15 +0200 Subject: [PATCH 429/434] Delete src/main/kotlin/tsuki/site/ru/MangaoneLove.kt --- src/main/kotlin/tsuki/site/ru/MangaoneLove.kt | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaoneLove.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt deleted file mode 100644 index c87596d8..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt +++ /dev/null @@ -1,14 +0,0 @@ -package tsuki.site.ru - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") -internal class MangaoneLove(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} From a338c92983d4a2253d706062a43ded28f69f4658 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:00:08 +0200 Subject: [PATCH 430/434] Create MangaoneLove.kt --- src/main/kotlin/tsuki/site/ru/MangaoneLove.kt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/ru/MangaoneLove.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt new file mode 100644 index 00000000..c87596d8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") +internal class MangaoneLove(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { + override val datePattern = "dd.MM.yyyy" + override val postReq = true +} From ec7fc24651c8f6dcc368146e83741c5276864a72 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:02:58 +0200 Subject: [PATCH 431/434] Delete src/main/kotlin/tsuki/site/ru/MangaoneLove.kt --- src/main/kotlin/tsuki/site/ru/MangaoneLove.kt | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/MangaoneLove.kt diff --git a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt b/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt deleted file mode 100644 index c87596d8..00000000 --- a/src/main/kotlin/tsuki/site/ru/MangaoneLove.kt +++ /dev/null @@ -1,14 +0,0 @@ -package tsuki.site.ru - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.parsers.MadaraParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("MANGAONELOVE", "MangaOneLove", "ru") -internal class MangaoneLove(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAONELOVE, "mangaonelove.su", 10) { - override val datePattern = "dd.MM.yyyy" - override val postReq = true -} From 029173585e68b6edde3739962d2c019a6b35df77 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:03:52 +0200 Subject: [PATCH 432/434] Delete src/main/kotlin/tsuki/site/ru/ComX.kt --- src/main/kotlin/tsuki/site/ru/ComX.kt | 270 -------------------------- 1 file changed, 270 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/ComX.kt diff --git a/src/main/kotlin/tsuki/site/ru/ComX.kt b/src/main/kotlin/tsuki/site/ru/ComX.kt deleted file mode 100644 index 5c356baa..00000000 --- a/src/main/kotlin/tsuki/site/ru/ComX.kt +++ /dev/null @@ -1,270 +0,0 @@ -package tsuki.site.ru - -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser -import tsuki.config.ConfigKey -import tsuki.core.PagedMangaParser -import tsuki.exception.ParseException - -import tsuki.model.ContentRating -import tsuki.model.ContentType -import tsuki.model.Manga -import tsuki.model.MangaChapter -import tsuki.model.MangaListFilter -import tsuki.model.MangaListFilterCapabilities -import tsuki.model.MangaListFilterOptions -import tsuki.model.MangaPage -import tsuki.model.MangaParserSource -import tsuki.model.MangaState -import tsuki.model.MangaTag -import tsuki.model.RATING_UNKNOWN -import tsuki.model.SortOrder -import tsuki.model.YEAR_UNKNOWN - -import tsuki.util.Set -import tsuki.util.attrAsAbsoluteUrl -import tsuki.util.attrAsAbsoluteUrlOrNull -import tsuki.util.attrAsRelativeUrl -import tsuki.util.generateUid -import tsuki.util.insertCookies -import tsuki.util.json.getFloatOrDefault -import tsuki.util.json.getIntOrDefault -import tsuki.util.json.getStringOrNull -import tsuki.util.mapNotNullToSet -import tsuki.util.nullIfEmpty -import tsuki.util.parseHtml -import tsuki.util.parseSafe -import tsuki.util.selectFirstOrThrow -import tsuki.util.splitByWhitespace -import tsuki.util.suspendlazy.getOrNull -import tsuki.util.suspendlazy.suspendLazy -import tsuki.util.textOrNull -import tsuki.util.toAbsoluteUrl -import tsuki.util.toTitleCase -import tsuki.util.urlEncoded - -import org.json.JSONObject -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -@MangaSourceParser("COMX", "Com-X", "ru", ContentType.COMICS) -internal class ComX(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.COMX, 20) { - - override val configKeyDomain = ConfigKey.Domain("com-x.life", "comx.life") - - private val availableTags = suspendLazy(initializer = ::fetchTags) - private val cdnImageUrl = "img.com-x.life/comix/" - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - init { - context.cookieJar.insertCookies(domain, "adt-accepted", "1") - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isYearRangeSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags.get(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val urlBuilder = StringBuilder() - when { - !filter.query.isNullOrEmpty() -> { - val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "%20") { part -> - part.urlEncoded() - } - urlBuilder.append("/search/") - urlBuilder.append(encodedQuery) - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - - else -> { - urlBuilder.append("/ComicList") - if (filter.yearFrom != YEAR_UNKNOWN) { - urlBuilder.append("/y[from]=${filter.yearFrom}") - } - if (filter.yearTo != YEAR_UNKNOWN) { - urlBuilder.append("/y[to]=${filter.yearTo}") - } - if (filter.tags.isNotEmpty()) { - urlBuilder.append("/g=") - urlBuilder.append(filter.tags.joinToString(",") { it.key }) - } - urlBuilder.append("/sort") - if (page > 1) { - urlBuilder.append("/page/$page/") - } - } - } - - val fullUrl = urlBuilder.toString().toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - return doc.select("div.readed.d-flex.short").map { item -> - val a = item.selectFirstOrThrow("a.readed__img.img-fit-cover.anim") - val img = item.selectFirst("img[data-src]") - val href = a.attrAsRelativeUrl("href") - val titleElement = item.selectFirstOrThrow("h3.readed__title a") - val (mainTitle, altTitle) = titleElement.text() - .split("\\s*/\\s*".toRegex()) - .map { it.trim() } - .let { parts -> - when { - parts.size >= 2 -> parts[1] to parts[0] - parts.isNotEmpty() -> parts[0] to "" - else -> "" to "" - } - } - - Manga( - id = generateUid(href), - url = href, - publicUrl = a.attrAsAbsoluteUrl("href"), - title = mainTitle, - altTitles = if (altTitle.isNotEmpty()) setOf(altTitle) else emptySet(), - authors = emptySet(), - description = null, - tags = emptySet(), - rating = RATING_UNKNOWN, - state = null, - coverUrl = img?.attrAsAbsoluteUrlOrNull("data-src"), - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - - val scriptData = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("window.__DATA__ = ") - ?.substringBefore(";</script>") - ?.trim() - ?: throw ParseException("Script data not found", manga.url) - - val jsonData = JSONObject(scriptData) - val chaptersJson = jsonData.getJSONArray("chapters") - val newsId = jsonData.getLong("news_id") - - val chapters = List(chaptersJson.length()) { i -> - val chapter = chaptersJson.getJSONObject(i) - val chapterId = chapter.getLong("id") - - MangaChapter( - id = generateUid("$newsId/$chapterId"), - url = "/reader/$newsId/$chapterId", - number = chapter.getFloatOrDefault("posi", 0f), - title = decodeText(chapter.getStringOrNull("title")), - uploadDate = dateFormat.parseSafe(chapter.getStringOrNull("date")), - source = source, - scanlator = null, - branch = null, - volume = chapter.getIntOrDefault("volume", 0), - ) - }.reversed() - - val author = doc.selectFirst("li:contains(Publisher:)") - ?.textOrNull() - ?.substringAfter("Publisher:") - ?.trim() - ?.nullIfEmpty() - val state = when ( - doc.selectFirst("li:contains(Release type:)")?.text()?.substringAfter("Release type:")?.trim() - ) { - "Ongoing" -> MangaState.ONGOING - else -> MangaState.FINISHED - } - - val tagLinks = doc.getElementsByAttributeValueContaining("href", "/genres/") - val tags = if (tagLinks.isNotEmpty()) { - availableTags.getOrNull()?.let { allTags -> - tagLinks.mapNotNullToSet { a -> - val tagName = a.text() - allTags.find { it.title.equals(tagName, ignoreCase = true) } - } - } - } else { - null - } - - return manga.copy( - authors = setOfNotNull(author), - state = state, - chapters = chapters, - description = doc.select("div.page__text.full-text.clearfix").textOrNull(), - tags = tags ?: manga.tags, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val newsId = chapter.url.substringAfter("/reader/").substringBefore("/") - context.cookieJar.insertCookies(domain, "adult=$newsId") - - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - val data = doc.selectFirst("script:containsData(__DATA__)")?.data() - ?.substringAfter("=") - ?.trim() - ?.removeSuffix(";") - ?.substringAfter("\"images\":[") - ?.substringBefore("]") - ?.split(",") - ?.map { it.trim().removeSurrounding("\"").replace("\\", "") } - ?: throw ParseException("Image data not found", chapter.url) - - return data.map { imageUrl -> - val finalUrl = "https://$cdnImageUrl$imageUrl" - MangaPage( - id = generateUid(imageUrl), - url = finalUrl, - preview = null, - source = source, - ) - } - } - - private suspend fun fetchTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain/comix/").parseHtml() - val scriptData = doc.selectFirstOrThrow("script:containsData(__XFILTER__)").data() - - val genresJson = scriptData - .substringAfter("\"g\":{") - .substringBefore("}}}") + "}" - - val genresObj = JSONObject("{$genresJson}") - val valuesArray = genresObj.getJSONArray("values") - - return Set(valuesArray.length()) { i -> - val genre = valuesArray.getJSONObject(i) - MangaTag( - key = genre.getInt("id").toString(), - title = genre.getString("value").toTitleCase(sourceLocale), - source = source, - ) - } - } - - private fun decodeText(text: String?): String? { - if (text == null) return null - return text.replace("\\u([0-9a-fA-F]{4})".toRegex()) { matchResult -> - val codePoint = matchResult.groupValues[1].toInt(16) - codePoint.toChar().toString() - } - } -} From 01c696b76e530baf641304d2b8545d247d0ce2e4 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:05:10 +0200 Subject: [PATCH 433/434] Delete src/main/kotlin/tsuki/site/ru/SlashLib.kt --- src/main/kotlin/tsuki/site/ru/SlashLib.kt | 11 ----------- 1 file changed, 11 deletions(-) delete mode 100644 src/main/kotlin/tsuki/site/ru/SlashLib.kt diff --git a/src/main/kotlin/tsuki/site/ru/SlashLib.kt b/src/main/kotlin/tsuki/site/ru/SlashLib.kt deleted file mode 100644 index b5a7ef40..00000000 --- a/src/main/kotlin/tsuki/site/ru/SlashLib.kt +++ /dev/null @@ -1,11 +0,0 @@ -package tsuki.site.ru - -import tsuki.parsers.LibSocialParser -import tsuki.MangaLoaderContext -import tsuki.MangaSourceParser - -import tsuki.model.MangaParserSource - -@MangaSourceParser("SLASHLIB", "SlashLib", "ru") -internal class SlashLib(context: MangaLoaderContext) : - LibSocialParser(context, MangaParserSource.SLASHLIB, 2, "v2.slashlib.me") From 3312d16acb0713e8a0c2b5d40184da47a5faea30 Mon Sep 17 00:00:00 2001 From: InvalidDavid <118831950+InvalidDavid@users.noreply.github.com> Date: Sun, 6 Sep 2026 19:33:35 +0200 Subject: [PATCH 434/434] Create DragonTea.kt --- src/main/kotlin/tsuki/site/en/DragonTea.kt | 25 ++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 src/main/kotlin/tsuki/site/en/DragonTea.kt diff --git a/src/main/kotlin/tsuki/site/en/DragonTea.kt b/src/main/kotlin/tsuki/site/en/DragonTea.kt new file mode 100644 index 00000000..4f727a71 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/DragonTea.kt @@ -0,0 +1,25 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.MadaraParser +import tsuki.network.UserAgents + +import tsuki.model.MangaParserSource + +@MangaSourceParser("DRAGONTEA", "DragonTea", "en") +internal class DragonTea(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.DRAGONTEA, "dragontea.ink") { + override val datePattern = "MM/dd/yyyy" + override val postReq = false + override val listUrl = "novel/" + override val tagPrefix = "novel-genre/" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } +}