diff --git a/.github/ISSUE_TEMPLATE/01_report_issue.yml b/.github/ISSUE_TEMPLATE/01_report_issue.yml deleted file mode 100644 index 07dad314..00000000 --- a/.github/ISSUE_TEMPLATE/01_report_issue.yml +++ /dev/null @@ -1,106 +0,0 @@ -name: 🞠Issue report -description: Report an issue in UMA -labels: [Bug] -body: - - - type: input - id: component - attributes: - label: Source name - description: | - You can find the extension name and version in source settings. - placeholder: | - Example: "Not Real Scans" - validations: - required: true - - - type: input - id: language - attributes: - label: Source language - placeholder: | - Example: "English" - validations: - required: true - - - type: textarea - id: reproduce-steps - attributes: - label: Steps to reproduce - description: Provide an example of the issue. - placeholder: | - Example: - 1. First step - 2. Second step - 3. Issue here - validations: - required: true - - - type: textarea - id: error-logs - attributes: - label: Error logs or stack trace - description: | - If applicable, paste any error messages or stack traces. - placeholder: | - Paste error output here... - - - type: input - id: app-version - attributes: - label: Usagi / Kotatsu fork version - description: | - You can find your Mihon/Mihon fork version number in **About**. - placeholder: | - Example: "App 0.0.1v" - validations: - required: true - - - type: input - id: android-version - attributes: - label: Android version - description: | - You can find this somewhere in your Android settings. - placeholder: | - Example: "Android 11" - validations: - required: true - - - type: textarea - id: other-details - attributes: - label: Other details - placeholder: | - Additional details and attachments. - - - type: checkboxes - id: acknowledgements - attributes: - label: Acknowledgements - description: Your issue will be closed if you haven't done these steps. - options: - - label: I have searched [existing issues](https://github.com/InvalidDavid/UMA/issues) both open & closed, and confirm that this is a new unreported issue. - required: true - - label: I have written a short but informative title. - required: true - - label: I have updated the app to to the latest version. - required: true - - label: I have provided clear steps to reproduce the issue. - required: true - - label: I have included relevant error logs or stack traces. - required: true - - label: I will correctly fill out all of the requested information in this form. - required: true - - - type: textarea - attributes: - label: - description: Do **not** modify. This is a reminder for other users to vote. - value: | - --- - - Add a :+1: [reaction] to [issues you find important]. - - [reaction]: https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/ - [issues you find important]: https://github.com/InvalidDavid/UMA/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc diff --git a/.github/ISSUE_TEMPLATE/02_source_requests.yml b/.github/ISSUE_TEMPLATE/02_source_requests.yml deleted file mode 100644 index 6a1cd967..00000000 --- a/.github/ISSUE_TEMPLATE/02_source_requests.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: 🌠Source request -description: Suggest a new source in UMA -labels: [Source request] -body: - - - type: input - id: name - attributes: - label: Source name - placeholder: | - Example: "Not a real scan" - validations: - required: true - - - type: input - id: link - attributes: - label: Source link - placeholder: | - Example: "https://google.net" - validations: - required: true - - - type: input - id: category - attributes: - label: Source language - placeholder: | - Example: "English / EN", "spanish / ES" - validations: - required: true - - - type: textarea - id: other-details - attributes: - label: Other details - placeholder: | - Additional details and attachments. - Example: - "18+/NSFW = yes" - - - type: checkboxes - id: acknowledgements - attributes: - label: Acknowledgements - description: Your issue will be closed if you haven't done these steps. - options: - - label: I have searched [existing issues](https://github.com/InvalidDavid/UMA/issues) both open & closed, and confirm that this is a new unreported feature request. - required: true - - label: I have written a meaningful title describing the feature. - required: true - - label: I have written a meaningful title with the source name. - required: true - - label: I will correctly fill out all of the requested information in this form. - required: true - - - type: textarea - attributes: - label: - description: Do **not** modify. This is a reminder for other users to vote. - value: | - --- - - Add a :+1: [reaction] to [issues you find important]. - - [reaction]: https://github.blog/2016-03-10-add-reactions-to-pull-requests-issues-and-comments/ - [issues you find important]: https://github.com/InvalidDavid/UMA/issues?q=is%3Aissue+is%3Aopen+sort%3Areactions-%2B1-desc diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 3ba13e0c..00000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1 +0,0 @@ -blank_issues_enabled: false diff --git a/.github/summary.yaml b/.github/summary.yaml index f0d18c2f..edeeedd8 100644 --- a/.github/summary.yaml +++ b/.github/summary.yaml @@ -1 +1 @@ -total: 76 \ No newline at end of file +total: 233 \ No newline at end of file diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 2349a736..ff63bacb 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -6,7 +6,7 @@ on: branches: - master paths: - - 'src/main/kotlin/org/koitharu/kotatsu/parsers/**' + - 'src/main/kotlin/tsuki/**' jobs: publish: @@ -46,4 +46,4 @@ jobs: tag_name: ${{ steps.meta.outputs.sha_short }} name: "${{ steps.meta.outputs.sha_short }}" files: build/libs/uma.jar - token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file + token: ${{ secrets.GITHUB_TOKEN }} diff --git a/.idea/runConfigurations/buildJar.xml b/.idea/runConfigurations/buildJar.xml index 09571555..3eb34e73 100644 --- a/.idea/runConfigurations/buildJar.xml +++ b/.idea/runConfigurations/buildJar.xml @@ -21,4 +21,4 @@ false - \ No newline at end of file + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a39b726..96941724 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,7 @@ This is a living document that evolves over time. If you find issues or have sug - [Common Patterns & Best Practices](#common-patterns--best-practices) - [URL Handling](#url-handling) - [Error Handling](#error-handling) + - [Testing Your Parser](#testing-your-parser) - [Building & Testing](#building--testing) - [Submitting a Pull Request](#submitting-a-pull-request) - [Troubleshooting](#troubleshooting) diff --git a/build.gradle.kts b/build.gradle.kts index 81ed09e1..cfd7984a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ plugins { } group = "org.usagi" -version = "1.0.1" +version = "1.0.0" tasks.test { useJUnitPlatform() @@ -34,7 +34,7 @@ tasks.withType().configureEach "-opt-in=kotlin.RequiresOptIn", "-opt-in=kotlin.contracts.ExperimentalContracts", "-opt-in=kotlinx.coroutines.ExperimentalCoroutinesApi", - "-opt-in=org.koitharu.kotatsu.parsers.InternalParsersApi", + "-opt-in=tsuki.InternalParsersApi", ) } } @@ -54,18 +54,14 @@ publishing { dependencies { implementation(libs.kotlinx.coroutines.core) - implementation(libs.kotlinx.serialization.json) implementation(libs.okhttp) implementation(libs.okio) implementation(libs.json) implementation(libs.androidx.collection) - implementation("com.squareup.okhttp3:okhttp-brotli:5.4.0") - api(libs.core.parsers) + api(libs.tsuki) api(libs.jsoup) - compileOnly(libs.android.stubs) - ksp(project(":plugins-ksp")) testImplementation(libs.junit.api) @@ -73,18 +69,21 @@ dependencies { testImplementation(libs.junit.params) testRuntimeOnly(libs.junit.launcher) testImplementation(libs.kotlinx.coroutines.test) - testImplementation(libs.quickjs) } tasks.register("buildJar") { + description = "Build all sources to a JAR file" dependsOn("dexJar") } tasks.register("dexJar") { + description = "Dex classes after build" dependsOn(tasks.jar) inputJar.set(tasks.jar.flatMap { it.archiveFile }) outputJar.set(layout.projectDirectory.file("build/libs/uma.jar")) classpath.from(configurations.runtimeClasspath) } -tasks.register("generateTestsReport") +tasks.register("generateTestsReport") { + description = "Generate a HTML file to get tests report" +} diff --git a/gradle.properties b/gradle.properties index df06b5a1..83f3a9a7 100644 --- a/gradle.properties +++ b/gradle.properties @@ -22,3 +22,6 @@ org.gradle.vfs.watch=true org.gradle.workers.max=4 systemProp.org.gradle.unsafe.configuration-cache=false Xannotation-default-target=param-property + +# Enabled parallel sync for Gradle 9.4+ +org.gradle.tooling.parallel=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e1bb9889..e1d8f1c6 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -1,6 +1,5 @@ [versions] collection = "1.6.0" -core = "9b51187e96" kotlin = "2.2.10" ksp = "2.2.10-2.0.2" coroutines = "1.10.2" @@ -11,11 +10,9 @@ json = "20251224" # Keep Android 5, 6 compatible (with SDK 21 - 23) #noinspection NewerVersionAvailable jsoup = "1.17.2" -quickjs = "1.1.0" korte = "4.0.10" -serialization = "1.9.0" simplexml = "2.7.1" -stubs = "4.1.1.4" +tsuki = "1.0.5" [plugins] kotlin-jvm = { id = "org.jetbrains.kotlin.jvm", version.ref = "kotlin" } @@ -23,12 +20,10 @@ kotlin-serialization = { id = "org.jetbrains.kotlin.plugin.serialization", versi ksp = { id = "com.google.devtools.ksp", version.ref = "ksp" } [libraries] -android-stubs = { module = "com.google.android:android", version.ref = "stubs" } -core-parsers = { module = "com.github.UsagiApp:core-parsers", version.ref = "core"} +tsuki = { module = "com.github.UsagiApp:Tsuki", version.ref = "tsuki"} ksp-symbol-processing-api = { module = "com.google.devtools.ksp:symbol-processing-api", version.ref = "ksp" } kotlinx-coroutines-core = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-core", version.ref = "coroutines" } kotlinx-coroutines-test = { group = "org.jetbrains.kotlinx", name = "kotlinx-coroutines-test", version.ref = "coroutines" } -kotlinx-serialization-json = { module = "org.jetbrains.kotlinx:kotlinx-serialization-json", version.ref = "serialization" } junit-launcher = { group = "org.junit.platform", name = "junit-platform-launcher" } junit-api = { group = "org.junit.jupiter", name = "junit-jupiter-api", version.ref = "junit" } junit-engine = { group = "org.junit.jupiter", name = "junit-jupiter-engine", version.ref = "junit" } @@ -38,6 +33,5 @@ okio = { module = "com.squareup.okio:okio", version.ref = "okio" } json = { module = "org.json:json", version.ref = "json" } androidx-collection = { module = "androidx.collection:collection", version.ref = "collection" } jsoup = { module = "org.jsoup:jsoup", version.ref = "jsoup" } -quickjs = { module = "io.webfolder:quickjs", version.ref = "quickjs" } korte = { module = "com.soywiz.korlibs.korte:korte-jvm", version.ref = "korte" } -simplexml = { module = "org.simpleframework:simple-xml", version.ref = "simplexml" } \ No newline at end of file +simplexml = { module = "org.simpleframework:simple-xml", version.ref = "simplexml" } diff --git a/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt b/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt index bacd2079..bfb3674c 100644 --- a/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt +++ b/plugins-ksp/src/main/kotlin/org/koitharu/kotatsu/parsers/ksp/ParserProcessor.kt @@ -20,21 +20,17 @@ class ParserProcessor( private val sourceNamePattern = Regex("[A-Z_][A-Z0-9_]{3,}") override fun process(resolver: Resolver): List { - val kotatsuSymbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.MangaSourceParser") - val tachiyomiSymbols = resolver.getSymbolsWithAnnotation("org.koitharu.kotatsu.parsers.TachiyomiSource") - + val kotatsuSymbols = resolver.getSymbolsWithAnnotation("tsuki.MangaSourceParser") val kotatsuRet = kotatsuSymbols.filterNot { it.validate() }.toList() - val tachiyomiRet = tachiyomiSymbols.filterNot { it.validate() }.toList() - - if (!kotatsuSymbols.iterator().hasNext() && !tachiyomiSymbols.iterator().hasNext()) { - return kotatsuRet + tachiyomiRet + if (!kotatsuSymbols.iterator().hasNext()) { + return kotatsuRet } val dependencies = Dependencies.ALL_FILES val factoryFile = try { codeGenerator.createNewFile( dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers", + packageName = "tsuki", fileName = "MangaParserFactory", ) } catch (e: FileAlreadyExistsException) { @@ -45,7 +41,7 @@ class ParserProcessor( try { codeGenerator.createNewFile( dependencies = dependencies, - packageName = "org.koitharu.kotatsu.parsers.model", + packageName = "tsuki.model", fileName = "MangaSource", ) } catch (e: FileAlreadyExistsException) { @@ -54,29 +50,27 @@ class ParserProcessor( } val totalCount = sourcesFile?.writer().use { sourcesWriter -> factoryFile?.writer().use { factoryWriter -> - writeContent(sourcesWriter, factoryWriter, kotatsuSymbols, tachiyomiSymbols) + writeContent(sourcesWriter, factoryWriter, kotatsuSymbols) } } writeSummary(totalCount) - return kotatsuRet + tachiyomiRet + return kotatsuRet } private fun writeContent( sourcesWriter: Writer?, factoryWriter: Writer?, kotatsuSymbols: Sequence, - tachiyomiSymbols: Sequence, ): Int { if (sourcesWriter == null && factoryWriter == null) { return 0 } factoryWriter?.write( """ - package org.koitharu.kotatsu.parsers + package tsuki - import org.koitharu.kotatsu.parsers.model.MangaParserSource - import org.koitharu.kotatsu.parsers.core.MangaParserWrapper - import org.koitharu.kotatsu.parsers.compat.TachiyomiSourceAdapter + import tsuki.model.MangaParserSource + import tsuki.core.MangaParserWrapper internal fun MangaParserSource.newParser(context: MangaLoaderContext): MangaParser = when (this) { @@ -84,7 +78,7 @@ class ParserProcessor( ) sourcesWriter?.write( """ - package org.koitharu.kotatsu.parsers.model + package tsuki.model public enum class MangaParserSource( override val title: String, @@ -103,18 +97,11 @@ class ParserProcessor( .onEach { it.accept(kotatsuVisitor, Unit) } .count() - // Process Tachiyomi source bridges - val tachiyomiVisitor = TachiyomiVisitor(sourcesWriter, factoryWriter, kotatsuVisitor.titles) - val tachiyomiCount = tachiyomiSymbols - .filter { it is KSClassDeclaration && it.validate() } - .onEach { it.accept(tachiyomiVisitor, Unit) } - .count() - factoryWriter?.write( - """ + $$""" }.let { require(it.source == this) { - "Cannot instantiate manga parser: ${'$'}name mapped to ${'$'}{it.source}" + "Cannot instantiate manga parser: $name mapped to ${it.source}" } MangaParserWrapper(it) } @@ -126,7 +113,7 @@ class ParserProcessor( } """.trimIndent(), ) - return kotatsuCount + tachiyomiCount + return kotatsuCount } private fun writeSummary(totalCount: Int) { @@ -197,91 +184,4 @@ class ParserProcessor( ) } } - - /** - * Visitor for @TachiyomiSource annotations. - * Generates factory entries that instantiate the Tachiyomi HttpSource and wrap it with TachiyomiSourceAdapter. - */ - private inner class TachiyomiVisitor( - private val sourcesWriter: Writer?, - private val factoryWriter: Writer?, - private val titles: HashMap, - ) : KSVisitorVoid() { - - override fun visitClassDeclaration( - classDeclaration: KSClassDeclaration, - data: Unit, - ) { - val annotation = classDeclaration.annotations.single { it.shortName.asString() == "TachiyomiSource" } - - // Get annotation arguments with defaults - val nameArg = annotation.arguments.find { it.name?.asString() == "name" }?.value as? String ?: "" - val titleArg = annotation.arguments.find { it.name?.asString() == "title" }?.value as? String ?: "" - val localeArg = annotation.arguments.find { it.name?.asString() == "locale" }?.value as? String ?: "" - val typeArg = annotation.arguments.find { it.name?.asString() == "type" }?.value?.toString() - val type = if (typeArg.isNullOrBlank() || typeArg == "null") "ContentType.MANGA" else typeArg - - // Auto-detect values from class if not provided - val className = classDeclaration.simpleName.asString() - val qualifiedClassName = checkNotNull(classDeclaration.qualifiedName?.asString()) { - "Class qualified name is null" - } - - // Generate source name from class name (UPPER_CASE) - val sourceName = nameArg.takeIf { it.isNotBlank() } - ?: className.uppercase().replace(Regex("[^A-Z0-9_]"), "_").let { - // Ensure it starts with a letter - if (it.firstOrNull()?.isLetter() == true) it else "SOURCE_$it" - } - - // Use provided title or class name - val sourceTitle = titleArg.takeIf { it.isNotBlank() } ?: className - - // Auto-detect locale from package name if not provided - val locale = localeArg.takeIf { it.isNotBlank() } - ?: detectLocaleFromPackage(qualifiedClassName) - - val localeString = "\"$locale\"" - val localeObj = if (locale.isEmpty()) null else Locale(locale) - val localeTitle = localeObj?.getDisplayLanguage(localeObj) - - if (localeObj != null && localeObj !in availableLocales) { - logger.error("Tachiyomi source $sourceName has wrong locale: $localeTitle") - } - - if (!sourceNamePattern.matches(sourceName)) { - logger.error("Tachiyomi source name must be uppercase: $sourceName") - } - - val prevTitleClass = titles.put(sourceTitle, qualifiedClassName) - if (prevTitleClass != null) { - logger.warn("Source title duplication: \"$sourceTitle\" is assigned to both $prevTitleClass and $qualifiedClassName") - } - - // Factory: wrap the Tachiyomi source with TachiyomiSourceAdapter - factoryWriter?.write( - "\tMangaParserSource.$sourceName -> TachiyomiSourceAdapter(context, MangaParserSource.$sourceName, $qualifiedClassName())\n" - ) - - // Enum entry - val localeComment = localeTitle?.toTitleCase(localeObj)?.let { " /* $it */" }.orEmpty() - sourcesWriter?.write( - "\t$sourceName(\"$sourceTitle\", $localeString$localeComment, $type, false),\n", - ) - } - - private fun detectLocaleFromPackage(qualifiedName: String): String { - // Common locale patterns in package names - val localePatterns = mapOf( - ".vi." to "vi", ".en." to "en", ".ja." to "ja", ".ko." to "ko", - ".zh." to "zh", ".es." to "es", ".fr." to "fr", ".de." to "de", - ".it." to "it", ".pt." to "pt", ".ru." to "ru", ".ar." to "ar", - ".th." to "th", ".id." to "id", ".tr." to "tr", ".pl." to "pl", - ".all." to "", ".multi." to "" - ) - - return localePatterns.entries.find { qualifiedName.contains(it.key) }?.value - ?: "" - } - } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt deleted file mode 100644 index 3f418bae..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/compat/TachiyomiSourceAdapter.kt +++ /dev/null @@ -1,560 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.compat - -import android.app.Application -import eu.kanade.tachiyomi.network.NetworkHelper -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.withContext -import okhttp3.Headers -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.InternalParsersApi -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import uy.kohesive.injekt.Injekt - -/** - * Adapter that wraps a Tachiyomi [HttpSource] and exposes it as a parser via [PagedMangaParser]. - * Enables Tachiyomi extension sources to be used within Usagi and Kotatsu ecosystem (if possible). - */ -@OptIn(InternalParsersApi::class) -open class TachiyomiSourceAdapter( - context: MangaLoaderContext, - source: MangaParserSource, - protected val tachiyomiSource: HttpSource, -) : PagedMangaParser(context, source, DEFAULT_PAGE_SIZE) { - - /** Disable login button, it's useless */ - override val authorizationProvider: MangaParserAuthProvider? = null - - /** Default for source settings. */ - protected open val splitByTranslationsKey = ConfigKey.SplitByTranslations(false) - protected open val showSuspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) - protected open val preferredImageServerKey = ConfigKey.PreferredImageServer(emptyMap(), null) - - init { - /** Register Application context so Injekt.get() works. */ - runCatching { - val activityThreadClass = Class.forName("android.app.ActivityThread") - val currentApplicationMethod = activityThreadClass.getMethod("currentApplication") - val app = currentApplicationMethod.invoke(null) as? Application - if (app != null) { - Injekt.addSingleton(app) - } else { - Injekt.addSingleton(Application()) - } - Injekt.addSingleton(context) - }.onFailure { - Injekt.addSingleton(Application()) - Injekt.addSingleton(context) - } - - /** Setup intercepted client for domain and User-Agent swapping */ - val originalDomain = runCatching { java.net.URI(tachiyomiSource.baseUrl).host }.getOrNull() - val interceptedClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - var request = chain.request() - val builder = request.newBuilder() - val customUa = config[userAgentKey] - if (originalDomain != null && request.url.host == originalDomain) { - val newUrl = request.url.newBuilder() - .host(domain) - .build() - builder.url(newUrl) - } - if (customUa.isNotBlank()) { - builder.header("User-Agent", customUa) - } - chain.proceed(builder.build()) - } - .build() - - runCatching { - tachiyomiSource.javaClass.getMethod("setClient", okhttp3.OkHttpClient::class.java) - .invoke(tachiyomiSource, interceptedClient) - } - - NetworkHelper.setClient(interceptedClient) - } - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val configKeyDomain: ConfigKey.Domain by lazy { - val host = runCatching { - java.net.URI(tachiyomiSource.baseUrl).host - ?: tachiyomiSource.baseUrl.toHostOnly() - }.getOrDefault(tachiyomiSource.baseUrl.toHostOnly()) - ConfigKey.Domain(host) - } - - // ========================== Sort / Order =============================== - override val availableSortOrders: Set = buildSet { - add(SortOrder.UPDATED) - add(SortOrder.POPULARITY) - if (tachiyomiSource.supportsLatest) { - add(SortOrder.NEWEST) - } - } - - /** - * Lazily computed: true if the source's filter list contains any Filter.Text entry, - * which typically means author/artist/year text inputs are supported. - */ - private val hasTextFilter: Boolean by lazy { - tachiyomiSource.getFilterList().any { it is eu.kanade.tachiyomi.source.model.Filter.Text } - } - - /** Mapping filters, allow to search + use all filters through tags */ - @InternalParsersApi - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isAuthorSearchSupported = hasTextFilter, - ) - - - // ============================== List =================================== - - open override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - - val mangasPage: MangasPage = withContext(Dispatchers.IO) { - when { - !query.isNullOrEmpty() || filter.hasNonSearchOptions() -> { - val tachiyomiFilters = tachiyomiSource.getFilterList() - - // --- Map Kotatsu tag filters → Tachiyomi CheckBox / Group --- - filter.tags.forEach { tag -> - tachiyomiFilters.forEach { f -> - if (f is eu.kanade.tachiyomi.source.model.Filter.CheckBox && f.name == tag.title) { - f.state = true - } else if (f is eu.kanade.tachiyomi.source.model.Filter.Group<*>) { - f.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox && sub.name == tag.title) { - sub.state = true - } - } - } else if (f is eu.kanade.tachiyomi.source.model.Filter.Select<*> && !isStatusFilterSelect(f)) { - // For generic Select filters, select the matching option by name - val idx = f.values.indexOfFirst { it.toString() == tag.title } - if (idx >= 0) f.state = idx - } - } - } - - // --- Map Kotatsu state filters → Tachiyomi StatusFilter (Group or Select) --- - val states = filter.states - if (states.isNotEmpty()) { - tachiyomiFilters.forEach { f -> - when { - f is eu.kanade.tachiyomi.source.model.Filter.Group<*> && isStatusFilterGroup(f) -> { - f.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox) { - val mappedState = mapNameToState(sub.name) - if (mappedState != null && mappedState in states) { - sub.state = true - } - } - } - } - f is eu.kanade.tachiyomi.source.model.Filter.Select<*> && isStatusFilterSelect(f) -> { - // Pick the first matching option for single-select status - val values = f.values - for (i in 1 until values.size) { - val mappedState = mapNameToState(values[i].toString()) - if (mappedState != null && mappedState in states) { - f.state = i - break - } - } - } - else -> {} - } - } - } - - tachiyomiSource.fetchSearchManga(page, query ?: "", tachiyomiFilters) - .toBlocking().single() - } - order == SortOrder.NEWEST && tachiyomiSource.supportsLatest -> { - tachiyomiSource.fetchLatestUpdates(page).toBlocking().single() - } - else -> { - tachiyomiSource.fetchPopularManga(page).toBlocking().single() - } - } - } - - return mangasPage.mangas.map { it.toManga() } - } - - // ============================== Details ================================ - - open override suspend fun getDetails(manga: Manga): Manga { - val sManga = manga.toSManga() - - return withContext(Dispatchers.IO) { - val details = tachiyomiSource.fetchMangaDetails(sManga).toBlocking().single() - val chapters = tachiyomiSource.fetchChapterList(sManga).toBlocking().single() - - /** Auto-detect chapter ordering: Tachiyomi default is newest-first, - but some sources return oldest-first. Detect by comparing chapter_number - of first two entries. If not available, fall back to date comparison. */ - val isNewestFirst = detectNewestFirst(chapters) - - manga.copy( - title = details.title.ifEmpty { manga.title }, - altTitles = emptySet(), - coverUrl = details.thumbnail_url ?: manga.coverUrl, - largeCoverUrl = details.thumbnail_url, - description = details.description, - authors = setOfNotNull(details.author, details.artist) - .filter { it.isNotBlank() }.toSet(), - tags = parseGenreTags(details.genre), - state = mapStatus(details.status), - chapters = chapters.mapChapters { i, sChapter -> - sChapter.toMangaChapter(i, chapters.size, isNewestFirst) - }.sortedBy { it.number }, - ) - } - } - - /** - * Detects whether [chapters] are ordered newest-first (index 0 = latest). - * Priority: - * 1. chapter_number: if first > second → newest-first - * 2. date_upload: if first > second → newest-first - * 3. Default: true (Tachiyomi convention) - * It helps rearrange the chapter list in asc order, avoiding confusion. - */ - private fun detectNewestFirst(chapters: List): Boolean { - if (chapters.size < 2) return true - val a = chapters[0] - val b = chapters[1] - if (a.chapter_number > 0f && b.chapter_number > 0f) { - return a.chapter_number > b.chapter_number - } - if (a.date_upload > 0L && b.date_upload > 0L) { - return a.date_upload > b.date_upload - } - return true - } - - // ============================== Pages ================================== - - open override suspend fun getPages(chapter: MangaChapter): List { - val sChapter = SChapter.create().apply { - url = chapter.url - name = chapter.title ?: "" - chapter_number = chapter.number - } - - val pages = withContext(Dispatchers.IO) { - tachiyomiSource.fetchPageList(sChapter).toBlocking().single() - } - - return pages.map { page -> - val imgUrl = page.imageUrl ?: page.url - MangaPage( - id = generateUid(imgUrl.ifEmpty { "${chapter.url}#${page.index}" }), - url = imgUrl, - preview = null, - source = source, - ) - } - } - - open override suspend fun getPageUrl(page: MangaPage): String { - val url = page.url - if (url.startsWith("http://") || url.startsWith("https://")) return url - return "https://$domain$url" - } - - // ============================== Filter Options ========================= - - open override suspend fun getFilterOptions(): MangaListFilterOptions = withContext(Dispatchers.IO) { - val tachiyomiFilters = tachiyomiSource.getFilterList() - val tags = mutableSetOf() - val states = mutableSetOf() - - tachiyomiFilters.forEach { filter -> - when (filter) { - is eu.kanade.tachiyomi.source.model.Filter.CheckBox -> { - tags.add(MangaTag(title = filter.name, key = filter.name.lowercase(), source = source)) - } - - // --- Group (multi-checkbox list, e.g. GenreList or StatusFilter) --- - is eu.kanade.tachiyomi.source.model.Filter.Group<*> -> { - if (isStatusFilterGroup(filter)) { - // Map each child checkbox to a MangaState - filter.state.forEach { sub -> - if (sub is eu.kanade.tachiyomi.source.model.Filter.CheckBox) { - mapNameToState(sub.name)?.let { states.add(it) } - } - } - } else { - // Generic group → flatten children as tags - filter.state.forEach { sub -> - when (sub) { - is eu.kanade.tachiyomi.source.model.Filter.CheckBox -> { - tags.add( - MangaTag( - title = sub.name, - key = sub.name.lowercase(), - source = source, - ), - ) - } - - else -> {} // ignore nested selects / text inside groups - } - } - } - } - - // --- Select (single-choice dropdown, e.g. GenreFilter / StatusFilter in fastscan) --- - is eu.kanade.tachiyomi.source.model.Filter.Select<*> -> { - if (isStatusFilterSelect(filter)) { - // Map each option to MangaState (skip index 0 which is usually "All") - val values = filter.values - for (i in 1 until values.size) { - mapNameToState(values[i].toString())?.let { states.add(it) } - } - } else { - // Generic select → extract non-"All" options as tags - val values = filter.values - for (i in 1 until values.size) { - val name = values[i].toString() - if (name.isNotBlank()) { - tags.add(MangaTag(title = name, key = name.lowercase(), source = source)) - } - } - } - } - - // Filter.Text / Header / Separator — intentionally ignored for tag/state population - else -> {} - } - } - - MangaListFilterOptions(availableTags = tags, availableStates = states) - } - - /** - * Heuristic: returns true when a [eu.kanade.tachiyomi.source.model.Filter.Group] represents a status/state selector. - * Matches group names containing words like "status", "tình trạng", "trang thai", etc. - */ - private fun isStatusFilterGroup(filter: eu.kanade.tachiyomi.source.model.Filter.Group<*>): Boolean { - val name = filter.name.lowercase() - return STATUS_KEYWORDS.any { name.contains(it) } - } - - /** - * Heuristic: returns true when a [eu.kanade.tachiyomi.source.model.Filter.Select] represents a status/state selector. - */ - private fun isStatusFilterSelect(filter: eu.kanade.tachiyomi.source.model.Filter.Select<*>): Boolean { - val name = filter.name.lowercase() - return STATUS_KEYWORDS.any { name.contains(it) } - } - - /** - * Maps common status label strings (in multiple languages) to a [MangaState]. - * Returns null if no mapping is found. - */ - protected open fun mapNameToState(name: String): MangaState? { - val lower = name.lowercase() - return when { - ONGOING_KEYWORDS.any { lower.contains(it) } -> MangaState.ONGOING - FINISHED_KEYWORDS.any { lower.contains(it) } -> MangaState.FINISHED - PAUSED_KEYWORDS.any { lower.contains(it) } -> MangaState.PAUSED - ABANDONED_KEYWORDS.any { lower.contains(it) } -> MangaState.ABANDONED - else -> null - } - } - - // ============================== Headers ================================ - - open override fun getRequestHeaders(): Headers = tachiyomiSource.headers - - // ============================== Interceptor ============================ - - open override fun intercept(chain: Interceptor.Chain): Response = chain.proceed(chain.request()) - - // ============================== Private Helpers ======================== - - protected open fun SManga.toManga(): Manga { - val mangaUrl = this.url - val publicUrl = if (mangaUrl.startsWith("http")) mangaUrl - else "${tachiyomiSource.baseUrl}$mangaUrl" - - return Manga( - id = generateUid(mangaUrl), - title = this.title, - altTitles = emptySet(), - url = mangaUrl, - publicUrl = publicUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = this.thumbnail_url, - tags = parseGenreTags(this.genre), - state = mapStatus(this.status), - authors = setOfNotNull(this.author, this.artist) - .filter { it.isNotBlank() }.toSet(), - source = source, - ) - } - - protected open fun Manga.toSManga(): SManga = SManga.create().apply { - url = this@toSManga.url - title = this@toSManga.title - thumbnail_url = this@toSManga.coverUrl - } - - protected open fun SChapter.toMangaChapter(index: Int, total: Int, isNewestFirst: Boolean = true): MangaChapter { - // When chapter_number is set by the source, use it directly. - // Otherwise derive from position: newest-first → total-index, oldest-first → index+1 - val number = if (this.chapter_number > 0f) { - this.chapter_number - } else if (isNewestFirst) { - (total - index).toFloat() // index 0 = newest = highest number - } else { - (index + 1).toFloat() // index 0 = oldest = chapter 1 - } - return MangaChapter( - id = generateUid(this.url), - title = this.name.ifBlank { null }, - number = number, - volume = 0, - url = this.url, - scanlator = this.scanlator, - uploadDate = this.date_upload, - branch = null, - source = source, - ) - } - - protected open fun parseGenreTags(genreStr: String?): Set { - if (genreStr.isNullOrBlank()) return emptySet() - return genreStr.split(",") - .map { it.trim() } - .filter { it.isNotBlank() } - .mapTo(mutableSetOf()) { genre -> - MangaTag(title = genre, key = genre.lowercase(), source = source) - } - } - - protected open fun mapStatus(status: Int): MangaState? = when (status) { - SManga.ONGOING -> MangaState.ONGOING - SManga.COMPLETED, SManga.PUBLISHING_FINISHED -> MangaState.FINISHED - SManga.ON_HIATUS -> MangaState.PAUSED - SManga.CANCELLED -> MangaState.ABANDONED - else -> null - } - - protected open fun String.toHostOnly(): String = - removePrefix("https://").removePrefix("http://").substringBefore("/") - - companion object { - const val DEFAULT_PAGE_SIZE = 20 - - /** Filter names (lowercased) that indicate a status/state filter. */ - private val STATUS_KEYWORDS = listOf( - "status", - "tình trạng", - "tinh trang", - "trạng thái", - "trang thai", - "durum", - "état", - "estado", - "zustand", - ) - - /** Label fragments (lowercased) that map to [MangaState.ONGOING]. */ - private val ONGOING_KEYWORDS = listOf( - "ongoing", - "on-going", - "on going", - "updating", - "publishing", - "airing", - "Ä‘ang", // Vietnamese "Ä‘ang tiến hành" - "dang tien hanh", - "devam", // Turkish - "en cours", // French - "en curso", // Spanish - "lançando", // Portuguese - "выходит", // Russian - "продолжаетÑÑ", - ) - - /** Label fragments (lowercased) that map to [MangaState.FINISHED]. */ - private val FINISHED_KEYWORDS = listOf( - "completed", - "complete", - "finished", - "end", - "full", - "hoàn", // Vietnamese "hoàn thành" / "hoàn tất" - "hoan thanh", - "trá»n bá»™", - "tron bo", - "tamamland", // Turkish - "achevé", - "terminé", - "завершено", // Russian - ) - - /** Label fragments (lowercased) that map to [MangaState.PAUSED]. */ - private val PAUSED_KEYWORDS = listOf( - "hiatus", - "on hold", - "on-hold", - "paused", - "beklemede", // Turkish - "durduruldu", - "en pause", - "заморожено", // Russian - ) - - /** Label fragments (lowercased) that map to [MangaState.ABANDONED]. */ - private val ABANDONED_KEYWORDS = listOf( - "canceled", - "cancelled", - "dropped", - "drop", - "abandoned", - "discontinued", - "iptal", // Turkish - "đã hủy", - "da huy", - "abandonné", - "заброшено", // Russian - ) - } -} - diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt deleted file mode 100644 index c9f84c20..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaFire.kt +++ /dev/null @@ -1,504 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import okhttp3.Interceptor -import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.parseJson -import java.util.EnumSet -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope - -internal abstract class MangaFireParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val siteLang: String, -) : PagedMangaParser(context, source, 30), Interceptor, MangaParserAuthProvider { - - override val configKeyDomain = ConfigKey.Domain("mangafire.to") - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, // chapter update - SortOrder.POPULARITY, // most views - SortOrder.RATING, // rating score - SortOrder.NEWEST, // created manga - SortOrder.ALPHABETICAL, // title asc - SortOrder.RELEVANCE, // relevance sc - SortOrder.POPULARITY_WEEK, - SortOrder.POPULARITY_MONTH, - ) - - private val apiClient by lazy { - val newHttpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request().newBuilder() - .header("Referer", "https://$domain/") - .header("Accept", "application/json") - .build() - chain.proceed(request) - } - .build() - OkHttpWebClient(newHttpClient, source) - } - - override fun intercept(chain: Interceptor.Chain) = chain.proceed( - chain.request().newBuilder() - .header("Referer", "https://$domain/") - .build() - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearRangeSupported = true, - ) - - companion object { - val GENRE_MAP = mapOf( - "Action" to "1", - "Adult" to "268929", - "Adventure" to "78", - "Avant Garde" to "3", - "Boys Love" to "4", - "Comedy" to "5", - "Crime" to "268921", - "Demons" to "77", - "Drama" to "6", - "Ecchi" to "7", - "Fantasy" to "79", - "Girls Love" to "9", - "Gourmet" to "10", - "Harem" to "11", - "Hentai" to "268930", - "Historical" to "268922", - "Horror" to "530", - "Isekai" to "13", - "Iyashikei" to "531", - "Josei" to "15", - "Kids" to "532", - "Magic" to "539", - "Magical Girls" to "268923", - "Mahou Shoujo" to "533", - "Martial Arts" to "534", - "Mature" to "268931", - "Mecha" to "19", - "Medical" to "268924", - "Military" to "535", - "Music" to "21", - "Mystery" to "22", - "Parody" to "23", - "Philosophical" to "268925", - "Psychological" to "536", - "Reverse Harem" to "25", - "Romance" to "26", - "School" to "73", - "Sci-Fi" to "28", - "Seinen" to "537", - "Shoujo" to "30", - "Shounen" to "31", - "Slice of Life" to "538", - "Smut" to "268932", - "Space" to "33", - "Sports" to "34", - "Super Power" to "75", - "Superhero" to "268926", - "Supernatural" to "76", - "Suspense" to "37", - "Thriller" to "38", - "Tragedy" to "268927", - "Vampire" to "39", - "Wuxia" to "268928" - ) - } - - private val tags by lazy { - GENRE_MAP.entries.map { (title, id) -> - MangaTag(title, id, source) - }.toSet() - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of( - MangaState.ONGOING, MangaState.FINISHED, - MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER, - ), - availableDemographics = EnumSet.of(Demographic.SHOUNEN, Demographic.SHOUJO, Demographic.SEINEN, Demographic.JOSEI), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { - val urlBuilder = okhttp3.HttpUrl.Builder() - .scheme("https") - .host(domain) - .addPathSegments("api/titles") - .addQueryParameter("page", page.toString()) - .addQueryParameter("limit", "50") - - if (!filter.query.isNullOrBlank()) { - urlBuilder.addQueryParameter("keyword", filter.query) - } - - if (filter.yearFrom > 0) { - urlBuilder.addQueryParameter("year_from", filter.yearFrom.toString()) - } - if (filter.yearTo > 0) { - urlBuilder.addQueryParameter("year_to", filter.yearTo.toString()) - } - - filter.types.forEach { type -> - val value = when (type) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.OTHER -> "other" - else -> null - } - value?.let { urlBuilder.addQueryParameter("types[]", it) } - } - - filter.demographics.forEach { demo -> - val id = when (demo) { - Demographic.JOSEI -> "268919" - Demographic.SEINEN -> "268920" - Demographic.SHOUJO -> "268917" - Demographic.SHOUNEN -> "268918" - else -> null - } - id?.let { urlBuilder.addQueryParameter("demographics[]", it) } - } - - filter.tags.forEach { urlBuilder.addQueryParameter("genres_in[]", it.key) } - filter.tagsExclude.forEach { urlBuilder.addQueryParameter("genres_ex[]", it.key) } - - filter.states.forEach { state -> - val apiState = when (state) { - MangaState.ONGOING -> "releasing" - MangaState.FINISHED -> "finished" - MangaState.ABANDONED -> "discontinued" - MangaState.PAUSED -> "on_hiatus" - MangaState.UPCOMING -> "not_yet_released" - else -> null - } - apiState?.let { urlBuilder.addQueryParameter("statuses[]", it) } - } - - val sortParam = when (order) { - SortOrder.UPDATED -> "chapter_updated_at" to "desc" - SortOrder.POPULARITY -> "views_total" to "desc" - SortOrder.RATING -> "score" to "desc" - SortOrder.NEWEST -> "created_at" to "desc" - SortOrder.ALPHABETICAL -> "title" to "asc" - SortOrder.RELEVANCE -> "relevance" to "desc" - SortOrder.POPULARITY_WEEK -> "views_7d" to "desc" - SortOrder.POPULARITY_MONTH -> "views_30d" to "desc" - else -> null - } - - sortParam?.let { (field, dir) -> - urlBuilder.addQueryParameter("order[$field]", dir) - } - - val url = urlBuilder.build().toString() - - val response = apiClient.httpGet(url).parseJson() - val items = response.getJSONArray("items") - val mangas = mutableListOf() - for (i in 0 until items.length()) { - val obj = items.getJSONObject(i) - val hid = obj.getString("hid") - val slug = obj.optString("slug", null) - val title = obj.getString("title") - val poster = obj.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") - ?: poster?.optString("small") ?: "" - val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" - mangas.add( - Manga( - id = generateUid(urlPath), - url = urlPath, - publicUrl = "https://$domain$urlPath", - title = title, - coverUrl = cover, - source = source, - altTitles = emptySet(), - largeCoverUrl = null, - authors = emptySet(), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - tags = emptySet(), - ) - ) - } - return mangas - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val result = coroutineScope { - val hid = extractHid(manga.url) - - val detailsJson = apiClient.httpGet("https://$domain/api/titles/$hid").parseJson() - val data = detailsJson.getJSONObject("data") - - val hasVolumes = data.optBoolean("hasVolumes", false) - - val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } - - val title = data.getString("title") - val poster = data.optJSONObject("poster") - val cover = poster?.optString("large") - ?: poster?.optString("medium") - ?: poster?.optString("small") - val synopsisHtml = data.optString("synopsisHtml", null) - val status = data.optString("status", null) - val type = data.optString("type", null) - val authorsList = data.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val artistsList = data.optJSONArray("artists")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - }.orEmpty() - val genres = data.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val themes = data.optJSONArray("themes")?.let { arr -> - (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } - } - val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - - val rawRating = data.optDouble("rating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" - - val genreList = buildList { - type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } - genres?.let { addAll(it) } - themes?.let { addAll(it) } - } - val genreTags = genreList.mapNotNull { name -> - tags.find { it.title == name } - }.toSet() - - val chapters = chaptersDeferred.await() - - manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - authors = (authorsList + artistsList).toSet(), - description = synopsisText.trim(), - rating = rating, - state = when (status?.lowercase()) { - "releasing" -> MangaState.ONGOING - "finished" -> MangaState.FINISHED - "discontinued" -> MangaState.ABANDONED - "on_hiatus" -> MangaState.PAUSED - "not_yet_released" -> MangaState.UPCOMING - else -> null - }, - tags = genreTags, - altTitles = altTitlesArray.toSet(), - chapters = chapters, - ) - } - - detailsCache.put(manga.url, result) - return result - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean { - return size > 10 - } - } - - private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List { - val chapters = mutableListOf() - val base = "https://$domain/api/titles/$hid" - - var page = 1 - var lastPage: Int - do { - val json = apiClient.httpGet( - "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" - ).parseJson() - val items = json.getJSONArray("items") - val meta = json.optJSONObject("meta") - lastPage = meta?.optInt("lastPage", 1) ?: 1 - - for (i in 0 until items.length()) { - val ch = items.getJSONObject(i) - if (ch.getString("language") != siteLang) continue - - val id = ch.getInt("id") - val number = ch.getDouble("number").toFloat() - val name = ch.optString("name", null) - val createdAt = ch.optLong("createdAt", 0L) * 1000L - val type = ch.getString("type") - val chapterUrl = "/title/$hid/$id" - val displayName = buildString { - append("Ch. ") - append(number.toString().removeSuffix(".0")) - if (!name.isNullOrBlank()) append(" - $name") - } - chapters.add( - MangaChapter( - id = generateUid(chapterUrl), - title = displayName, - number = number, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = createdAt, - branch = type, - source = source, - ) - ) - } - page++ - } while (page <= lastPage) - - if (hasVolumes) { - try { - val volJson = apiClient.httpGet("$base/volumes?language=$siteLang").parseJson() - val volItems = volJson.getJSONArray("items") - for (i in 0 until volItems.length()) { - val vol = volItems.getJSONObject(i) - if (vol.getString("language") != siteLang) continue - - val volId = vol.getInt("id") - val volNumber = vol.getDouble("number").toFloat() - val volName = vol.optString("name", "").takeIf { it.isNotBlank() } - val chapterCount = vol.optInt("chapterCount", 0) - - val title = buildString { - append("Vol. ") - append(volNumber.toString().removeSuffix(".0")) - if (volName != null) append(" - $volName") - } - val name = if (chapterCount > 0) "$chapterCount chapters" else "" - - chapters.add( - MangaChapter( - id = generateUid("/title/$hid/vol/$volId"), - title = title, - number = volNumber, - volume = 0, - url = "/title/$hid/vol/$volId", - scanlator = name, - uploadDate = 0L, - branch = "Volume", - source = source, - ) - ) - } - } catch (_: Exception) { - } - } - - val distinctBranches = chapters.map { it.branch }.distinct() - val useGroups = distinctBranches.size > 1 - - return chapters - .map { chapter -> - chapter.copy( - branch = if (useGroups) (chapter.branch ?: "").replaceFirstChar { it.uppercase() } else null - ) - } - .sortedBy { it.number } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chapterId = chapter.url.substringAfterLast("/") // numeric ID - val isVolume = chapter.url.contains("/vol/") - val endpoint = if (isVolume) "volumes" else "chapters" - - val response = apiClient.httpGet("https://$domain/api/$endpoint/$chapterId").parseJson() - val pagesArray = response.getJSONObject("data").getJSONArray("pages") - val pages = ArrayList(pagesArray.length()) - for (i in 0 until pagesArray.length()) { - val pageObj = pagesArray.getJSONObject(i) - val url = pageObj.getString("url") - pages.add( - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - ) - } - return pages - } - - override suspend fun getRelatedManga(seed: Manga): List = emptyList() - - override val authUrl: String get() = "https://$domain" - override suspend fun isAuthorized(): Boolean = true - override suspend fun getUsername(): String = "" - - - private fun extractHid(url: String): String { - val lastPart = url.removeSuffix("/").substringAfterLast("/") - return when { - lastPart.contains(".") -> lastPart.substringAfterLast(".") - lastPart.contains("-") -> lastPart.substringBefore("-") - else -> lastPart - } - } - - @MangaSourceParser("MANGAFIRE_EN", "MangaFire (English)", "en") - class English(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_EN, "en") - - @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") - - @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire Spanish (Latin)", "es") - class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") - - @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") - class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") - - @MangaSourceParser("MANGAFIRE_JA", "MangaFire (Japanese)", "ja") - class Japanese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_JA, "ja") - - @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") - - @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire Portuguese (Brazil)", "pt") - class PortugueseBR(context: MangaLoaderContext) : - MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt deleted file mode 100644 index bf7ea2fc..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaPlus.kt +++ /dev/null @@ -1,293 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Interceptor -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.Response -import okhttp3.ResponseBody.Companion.toResponseBody -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.SinglePageMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNull -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy -import java.util.* - -internal abstract class MangaPlusParser( - context: MangaLoaderContext, - source: MangaParserSource, - private val sourceLang: String -): - SinglePageMangaParser(context, source), Interceptor { - - private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" - override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - private val extraHeaders = Headers.headersOf("Session-Token", UUID.randomUUID().toString()) - - override suspend fun getList(order: SortOrder, filter: MangaListFilter): List { - return when { - filter.query.isNullOrEmpty() -> { - when (order) { - SortOrder.POPULARITY -> getPopularList() - SortOrder.UPDATED -> getLatestList() - else -> getAllTitleList() - } - } - else -> getAllTitleList(filter.query) - } - } - - private suspend fun getPopularList(): List { - val json = apiCall("/title_list/ranking") - - return json.getJSONObject("titleRankingView") - .getJSONArray("titles") - .asTypedList() - .toMangaList() - } - - private suspend fun getLatestList(): List { - val json = apiCall("/title_list/updated") - - return json.getJSONObject("titleUpdatedView") - .getJSONArray("latestTitle") - .mapJSON { it.getJSONObject("title") } - .toMangaList() - } - - private val allTitleCache = suspendLazy { - apiCall("/title_list/allV2") - .getJSONObject("allTitlesViewV2") - .getJSONArray("AllTitlesGroup") - .mapJSON { it.getJSONArray("titles").asTypedList() } - .flatten() - } - - private suspend fun getAllTitleList(query: String? = null): List { - return allTitleCache.get().toMangaList(query) - } - - private fun List.toMangaList(query: String? = null): List { - return mapNotNull { - val language = it.getStringOrNull("language") ?: "ENGLISH" - - if (language != sourceLang) { - return@mapNotNull null - } - - val name = it.getString("name") - val author = it.getString("author") - .split('/') - .joinToString(transform = String::trim) - - if (query != null && !(name.contains(query, true) || author.contains(query, true))) { - return@mapNotNull null - } - - val titleId = it.getInt("titleId").toString() - - Manga( - id = generateUid(titleId), - url = titleId, - publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), - title = name, - coverUrl = it.getString("portraitImageUrl"), - altTitles = emptySet(), - authors = setOf(author), - contentRating = null, - rating = RATING_UNKNOWN, - state = null, - source = source, - tags = emptySet(), - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val json = apiCall("/title_detailV3?title_id=${manga.url}") - .getJSONObject("titleDetailView") - val title = json.getJSONObject("title") - - val completed = json.getJSONObject("titleLabels") - .getString("releaseSchedule").let { - it == "DISABLED" || it == "COMPLETED" - } - - val hiatus = json.getStringOrNull("nonAppearanceInfo")?.contains("on a hiatus") == true - val author = title.getString("author") - .split("/").joinToString(transform = String::trim) - - return manga.copy( - title = title.getString("name"), - publicUrl = "/titles/${title.getInt("titleId")}".toAbsoluteUrl(domain), - coverUrl = title.getString("portraitImageUrl"), - authors = setOf(author), - description = buildString { - json.getString("overview").let(::append) - json.getStringOrNull("viewingPeriodDescription") - ?.takeIf { !completed } - ?.let { append("

", it) } - }, - chapters = parseChapters( - json.getJSONArray("chapterListGroup"), - title.getStringOrNull("language") ?: "ENGLISH", - ), - state = when { - completed -> MangaState.FINISHED - hiatus -> MangaState.PAUSED - else -> MangaState.ONGOING - }, - ) - } - - private fun parseChapters(chapterListGroup: JSONArray, language: String): List { - val chapterList = chapterListGroup - .asTypedList() - .flatMap { - it.optJSONArray("firstChapterList")?.asTypedList().orEmpty() + - it.optJSONArray("lastChapterList")?.asTypedList().orEmpty() - } - - return chapterList.mapChapters { _, chapter -> - val chapterId = chapter.getInt("chapterId").toString() - val subtitle = chapter.getStringOrNull("subTitle") ?: return@mapChapters null - - MangaChapter( - id = generateUid(chapterId), - url = chapterId, - title = subtitle, - number = chapter.getString("name") - .substringAfter("#") - .toFloatOrNull() ?: -1f, - volume = 0, - uploadDate = chapter.getInt("startTimeStamp") * 1000L, - branch = when (language) { - "PORTUGUESE_BR" -> "Portuguese (Brazil)" - else -> language.lowercase().toTitleCase() - }, - scanlator = null, - source = source, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val pages = apiCall("/manga_viewer?chapter_id=${chapter.url}&split=yes&img_quality=super_high") - .getJSONObject("mangaViewer") - .getJSONArray("pages") - - return pages.mapJSONNotNull { - val mangaPage = it.optJSONObject("mangaPage") - ?: return@mapJSONNotNull null - val url = mangaPage.getString("imageUrl") - val encryptionKey = mangaPage.getStringOrNull("encryptionKey") - MangaPage( - id = generateUid(url), - url = url + if (encryptionKey == null) "" else "#$encryptionKey", - preview = null, - source = source, - ) - } - } - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val response = chain.proceed(request) - val encryptionKey = request.url.fragment - - if (encryptionKey.isNullOrEmpty()) { - return response - } - - return response.map { responseBody -> - val contentType = response.headers["Content-Type"] ?: "image/jpeg" - val image = responseBody.bytes().decodeXorCipher(encryptionKey) - image.toResponseBody(contentType.toMediaTypeOrNull()) - } - } - - private fun ByteArray.decodeXorCipher(key: String): ByteArray { - val keyStream = key.chunked(2) - .map { it.toInt(16) } - - return mapIndexed { i, byte -> byte.toInt() xor keyStream[i % keyStream.size] } - .map(Int::toByte) - .toByteArray() - } - - private suspend fun apiCall(url: String): JSONObject { - val newUrl = "$apiUrl$url".toHttpUrl().newBuilder() - .addQueryParameter("format", "json") - .build() - val response = webClient.httpGet(newUrl, extraHeaders).parseJson() - - val success = response.optJSONObject("success") - - return checkNotNull(success) { - val error = response.getJSONObject("error") - val reason = error.getJSONArray("popups") - .asTypedList() - .firstOrNull { it.getStringOrNull("language") == null } - - if (reason?.getStringOrNull("subject") == "Not Found" && url.contains("manga_viewer")) { - "This chapter has expired" - } else { - reason?.getStringOrNull("body") ?: "Unknown Error" - } - } - } - - @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") - class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH") - - @MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus (Spanish)", "es") - class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH") - - @MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus (French)", "fr") - class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH") - - @MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus (Indonesian)", "id") - class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN") - - @MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt") - class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR") - - @MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus (Russian)", "ru") - class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN") - - @MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus (Thai)", "th") - class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI") - - @MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus (Vietnamese)", "vi") - class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE") - - @MangaSourceParser("MANGAPLUSPARSER_DE", "MANGA Plus (German)", "de") - class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN") -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt deleted file mode 100644 index 4e555f96..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/BuonDua.kt +++ /dev/null @@ -1,18 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.GalleryParser - -@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) -internal class BuonDua(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { - - override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( - "buondua.com", - "buondua.us", - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt deleted file mode 100644 index fa042ea0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/MangaCrazy.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import java.util.Locale - -@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) -internal class MangaCrazy(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { - override val sourceLocale: Locale = Locale.ENGLISH -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt deleted file mode 100644 index be1dfa16..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Novelcrow.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) -internal class Novelcrow(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { - override val tagPrefix = "comic-genre/" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt deleted file mode 100644 index 86488f68..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/CoffeeManga.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("COFFEEMANGA", "CoffeeManga", "en") -internal class CoffeeManga(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.COFFEEMANGA, "coffeemanga.ink") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt deleted file mode 100644 index 03153da3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/EnThunderScans.kt +++ /dev/null @@ -1,24 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MangaReaderParser - -@MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") -internal class EnThunderScans(context: MangaLoaderContext) : - MangaReaderParser( - context, - MangaParserSource.ENTHUNDERSCANS, - "en-thunderscans.com", - pageSize = 30, - searchPageSize = 10, - ) { - override val listUrl = "/comics" - - override val filterCapabilities: MangaListFilterCapabilities - get() = super.filterCapabilities.copy( - isTagsExclusionSupported = false, - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt deleted file mode 100644 index 7e31556c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KingofShojo.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MangaThemesia - -@MangaSourceParser("KINGOFSHOJO", "King of Shojo", "en") -internal class KingofShojo(context: MangaLoaderContext) : - MangaThemesia(context, MangaParserSource.KINGOFSHOJO, "kingofshojo.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt deleted file mode 100644 index 7b3f82d0..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangaread.kt +++ /dev/null @@ -1,13 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANGAREAD", "MangaRead", "en") -internal class MangaRead(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGAREAD, "www.mangaread.org") { - override val tagPrefix = "genres/" - override val datePattern = "dd.MM.yyyy" -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt deleted file mode 100644 index ffc91246..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaNex.kt +++ /dev/null @@ -1,14 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHWANEX", "ManhwaNex", "en") -internal class ManhwaNex(context: MangaLoaderContext): - MadaraParser(context, MangaParserSource.MANHWANEX, "manhwanex.com") { - - override val postReq = false - -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt deleted file mode 100644 index b264ee08..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaTop.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHWATOP", "ManhwaTop", "en") -internal class ManhwaTop(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANHWATOP, "manhwatop.com") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt deleted file mode 100644 index 6a3530ea..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Novel24h.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource - -@MangaSourceParser("TWENTYFOURHNOVEL", "24HNovel", "en") -internal class Novel24hParser(context: MangaLoaderContext) : - MadaraParser( - context = context, - source = MangaParserSource.TWENTYFOURHNOVEL, - domain = "24hnovel.com" - ) { - override val withoutAjax = true -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt deleted file mode 100644 index 19cc6732..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/ComicLand.kt +++ /dev/null @@ -1,286 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.urlEncoded - -@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) -internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( - context, MangaParserSource.COMICLAND, 20) { - - override val configKeyDomain = - ConfigKey.Domain("comicland.org") - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - - val newRequest = request.newBuilder() - .header( - "Origin", - "https://comicland.org" - ) - .header( - "Referer", - "https://comicland.org/" - ) - .build() - - return chain.proceed(newRequest) - } - - override val availableSortOrders = - setOf( - SortOrder.UPDATED, // Recommended - SortOrder.POPULARITY, // Popular - SortOrder.NEWEST // Official/Ongoing equivalent - ) - - override val filterCapabilities = - MangaListFilterCapabilities( - isMultipleTagsSupported = false, - isTagsExclusionSupported = false, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - isAuthorSearchSupported = false - ) - - private val api = - "https://api.comicland.org/api" - - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter - ): List { - - - val offset = page * 20 - val endpoint = - if (!filter.query.isNullOrBlank()) { - "$api/comic/search?q=${filter.query!!.urlEncoded()}" - } else { - val category = - filter.tags - .firstOrNull() - ?.key - when(category) { - "official" -> - "$api/comics/official" - "ongoing" -> - "$api/comics" - "popular" -> - "$api/comics/popular" - else -> - "$api/comics" - } - } - - val separator = - if (endpoint.contains("?")) "&" else "?" - - val response = - webClient.httpGet( - "$endpoint${separator}offset=$offset&limit=20" - ) - - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") - ?: return emptyList() - - val array = - data.optJSONArray("list") - ?: data.optJSONArray("items") - ?: return emptyList() - - val result = ArrayList() - for (i in 0 until array.length()) { - val comic = - array.getJSONObject(i) - val slug = - comic.optString("slug") - result.add( - Manga( - id = generateUid(slug), - title = comic.optString("title"), - url = slug, - publicUrl = "https://comicland.org/comic/$slug", - coverUrl = comic.optString("cover_url"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - contentRating = ContentRating.ADULT, - source = source - ) - ) - } - return result - } - - override suspend fun getDetails( - manga: Manga - ): Manga { - val response = - webClient.httpGet( - "$api/comic/detail?slug=${manga.url}" - ) - - val root = - JSONObject(response.body.string()) - - val comic = - root.optJSONObject("data") - ?: return manga - - val chapters = - ArrayList() - - val chapterArray = - comic.optJSONArray("chapters") - - if (chapterArray != null) { - for (i in 0 until chapterArray.length()) { - val chapter = - chapterArray.getJSONObject(i) - - val index = - chapter.optDouble( - "chapter_index", - 0.0 - ) - chapters.add( - MangaChapter( - id = generateUid("${comic.optString("slug")}/$index"), - title = chapter.optString("title"), - number = index.toFloat(), - volume = 0, - url ="/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", - scanlator = null, - uploadDate = 0, - branch = null, - source = source - ) - ) - } - } - return manga.copy( - title = - comic.optString( - "title", - manga.title - ), - description = - comic.optString( - "description", - "" - ), - coverUrl = - comic.optString( - "cover_url", - manga.coverUrl - ), - authors = - parseNames( - comic.optJSONArray("authors") - ), - tags = emptySet(), - chapters = chapters - ) - } - - - private fun parseNames( - array: org.json.JSONArray? - ): Set { - if (array == null) - return emptySet() - return buildSet { - for (i in 0 until array.length()) { - add( - array.getJSONObject(i) - .optString("name") - ) - } - } - } - - - override suspend fun getPages( - chapter: MangaChapter - ): List { - - val slug = - chapter.url - .substringAfter("/comic/") - .substringBefore("/chapter/") - - val index = - chapter.url - .substringAfter("/chapter/") - - val response = - webClient.httpGet( - "$api/chapter/pages_by_index?slug=$slug&index=$index" - ) - - val root = - JSONObject(response.body.string()) - - val data = - root.optJSONObject("data") - ?: return emptyList() - - val pages = - data.optJSONArray("pages") - ?: return emptyList() - - return buildList { - for (i in 0 until pages.length()) { - val image = - pages.getString(i) - add( - MangaPage( - id = generateUid(image), - url = image, - preview = null, - source = source - ) - ) - } - } - } - - private val categories = mapOf( - "recommended" to "/comics", - "official" to "/comics/official", - "ongoing" to "/comics?status=ongoing", - "popular" to "/comics/popular" - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = categories.map { (key, _) -> - MangaTag( - title = key.replaceFirstChar { it.uppercase() }, - key = key, - source = source - ) - }.toSet(), - - availableContentRating = emptySet() - ) - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt deleted file mode 100644 index 620224f9..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Hiperdex.kt +++ /dev/null @@ -1,19 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import okhttp3.Headers -import org.koitharu.kotatsu.parsers.Broken -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser - -@Broken -@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) -internal class Hiperdex(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .set("x-svc-gate", "f5pabmx7sdek") - .build() -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt deleted file mode 100644 index b6ae5367..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manga18.kt +++ /dev/null @@ -1,11 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.Manga18Parser - -@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) -internal class Manga18(context: MangaLoaderContext) : - Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt deleted file mode 100644 index 20995789..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MangaDistrict.kt +++ /dev/null @@ -1,106 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.EnumSet - -@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) -internal class MangaDistrict(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { - - override val tagPrefix = "publication-genre/" - override val withoutAjax: Boolean = true - override val datePattern: String = "MMMM d, yyyy" - override val stylePage: String = "?style=list" - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.RELEVANCE, - SortOrder.NEWEST, - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) - - override suspend fun getChapters(manga: Manga, doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - - val chapters = doc.body() - .select("li.wp-manga-chapter") - .mapNotNull { li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val link = href + stylePage - - val name = a.selectFirst("p")?.text() - ?: a.ownText() - - val dateText = - li.selectFirst("a.c-new-tag")?.attr("title") - ?: li.selectFirst("span.chapter-release-date i")?.text() - - MangaChapter( - id = generateUid(href), - title = name, - number = extractChapterNumber(name) ?: 0f, - volume = 0, - url = link, - uploadDate = parseChapterDate(dateFormat, dateText), - scanlator = null, - branch = null, - source = source, - ) - } - .sortedWith( - compareBy { it.number } - .thenBy { it.title } - ) - - return chapters - } - - private fun extractChapterNumber(name: String): Float? { - val match = Regex( - """(?:chapter|ch\.?)\s*(\d+(?:\.\d+)?)""", - RegexOption.IGNORE_CASE - ).find(name) - - return match - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - - override suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/series/").parseHtml() - val elements = doc.select("header ul.second-menu li a, div.genres_wrap ul li a") - - return elements.mapNotNullToSet { a -> - val href = a.attr("href") - .removeSuffix("/") - .substringAfterLast(tagPrefix, "") - - if (href.isBlank()) return@mapNotNullToSet null - - MangaTag( - key = href, - title = a.text() - .replace(Regex("""\s*\(\d+\)"""), "") - .replace(Regex("""[^\x20-\x7E]"""), "") - .trim() - .toTitleCase(), - source = source, - ) - } - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt deleted file mode 100644 index bff2c95b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/OmegaScan.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.HeanCmsParser - -@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) -internal class OmegaScans(context: MangaLoaderContext) : - HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt deleted file mode 100644 index b6c01ea9..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/DoujinDesu.kt +++ /dev/null @@ -1,21 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.DoujinDesuParser -import java.util.EnumSet - -@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) -internal class DoujinDesu(context: MangaLoaderContext) : - DoujinDesuParser(context, MangaParserSource.DOUJINDESU) { - - override val defaultTypes: String = "doujinshi,manga,manhwa" - - override val availableContentTypes: Set = EnumSet.of( - ContentType.MANGA, - ContentType.DOUJINSHI, - ContentType.MANHWA - ) -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt deleted file mode 100644 index b5dbcac1..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/HiperToon.kt +++ /dev/null @@ -1,10 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser - -@MangaSourceParser("HIPERTOON", "Hipertoon", "pt") -internal class HiperToon(context: MangaLoaderContext) : - HiperParser(context, MangaParserSource.HIPERTOON, "hipertoon.com") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt deleted file mode 100644 index 17c4fc08..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/zh/hentai/Xiutaku.kt +++ /dev/null @@ -1,16 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.zh.hentai - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.parsers.GalleryParser - -@MangaSourceParser("XIUTAKU", "Xiutaku", "zh", type = ContentType.OTHER) -internal class Xiutaku(context: MangaLoaderContext) : - GalleryParser(context, MangaParserSource.XIUTAKU, "xiutaku.com") { - - override suspend fun getFilterOptions(): MangaListFilterOptions = - MangaListFilterOptions(availableTags = fetchTags()) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt deleted file mode 100644 index 795f0bcf..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/mistscans/MistScans.kt +++ /dev/null @@ -1,26 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.mistscans - -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.Keyoapp -import eu.kanade.tachiyomi.source.model.SManga -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.TachiyomiSource - -@TachiyomiSource("MISTSCANS", "Mist Scans", "en") -class MistScans : Keyoapp() { - - override fun popularMangaSelector() = ".series-splide .splide__slide" - - override fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - val a = element.selectFirst("a[href]")!! - title = a.attr("title") - setUrlWithoutDomain(a.attr("abs:href")) - thumbnail_url = element.getImageUrl("*[style*=background-image]") - } - - override val name: String - get() = "Mist Scans" - override val lang: String - get() = "en" - override val baseUrl: String - get() = "https://mistscans.com" -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt deleted file mode 100644 index 522caf2b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScan.kt +++ /dev/null @@ -1,58 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.qiscan - -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwa -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaSortFilter -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaStatusFilter -import org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa.EZManhwaTypeFilter -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import org.koitharu.kotatsu.parsers.TachiyomiSource - - -@TachiyomiSource("QISCANS", "QiScans", "en") -class QiScans : EZManhwa("QiScans", "https://qimanga.com", "https://api.qimanga.com/api/v1") { - - override fun headersBuilder() = super.headersBuilder() - .set("Origin", baseUrl) - .set("Sec-Fetch-Dest", "empty") - .set("Sec-Fetch-Mode", "cors") - .set("Sec-Fetch-Site", "same-site") - - // QiScans search endpoint ignores filters — only send the query when searching. - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val isSearch = query.isNotBlank() - val endpoint = if (isSearch) "$apiUrl/series/search" else "$apiUrl/series" - val url = endpoint.toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "20") - if (isSearch) { - addQueryParameter("q", query) - } else { - var sortAdded = false - for (filter in filters) { - when (filter) { - is EZManhwaSortFilter -> { - addQueryParameter("sort", filter.value) - sortAdded = true - } - is EZManhwaStatusFilter -> if (filter.value.isNotBlank()) addQueryParameter("status", filter.value) - is EZManhwaTypeFilter -> if (filter.value.isNotBlank()) addQueryParameter("type", filter.value) - is QiScansGenreFilter -> if (filter.value.isNotBlank()) addQueryParameter("genre", filter.value) - else -> {} - } - } - if (!sortAdded) addQueryParameter("sort", "latest") - } - }.build() - return GET(url, headers) - } - - override fun getFilterList() = FilterList( - EZManhwaSortFilter(), - EZManhwaStatusFilter(), - EZManhwaTypeFilter(), - QiScansGenreFilter(), - ) -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt deleted file mode 100644 index d4c29a0b..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/qiscan/QiScansGenreFilter.kt +++ /dev/null @@ -1,68 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.qiscan - -import eu.kanade.tachiyomi.source.model.Filter - -class QiScansGenreFilter : Filter.Select("Genre", DISPLAY) { - val value get() = VALUES[state] - - companion object { - private val ENTRIES = arrayOf( - "All" to "", - "Acting" to "acting", - "Action" to "action", - "Action (Alt)" to "action-582", - "Adventure" to "adventure", - "Adventure (Alt)" to "adventure-589", - "Apocalypse" to "apocalypce", - "Comedy" to "comedy", - "Cooking" to "cooking", - "Crazy MC" to "crazy-mc", - "Cultivation" to "cultivation", - "Drama" to "drama", - "Ecchi" to "ecchi", - "Fantasy" to "fantasy", - "Fantasy (Alt)" to "fantasy-747", - "Fight" to "fight", - "Gender Bender" to "gender-bender", - "Harem" to "harem", - "Hidden" to "hidden", - "Historical" to "historical", - "Horror" to "horror", - "Josei" to "josei", - "Live" to "live", - "Magic" to "magic", - "Manhua" to "manhua", - "Martial Arts" to "martial-arts", - "Mature" to "mature", - "Mecha" to "mecha", - "Medieval Area" to "medieval-area", - "Munchkin" to "munchkin", - "Murim" to "murim", - "Mystery" to "mystery", - "Myth" to "myth", - "Politics" to "politics", - "Psychological" to "psychological", - "Reincarnation" to "reincarnation", - "Revenge" to "revenge", - "Romance" to "romance", - "School Life" to "school-life", - "Sci-Fi" to "sci-fi", - "Seinen" to "seinen", - "Shounen" to "shounen", - "Slice of Life" to "slice-of-life", - "Sports" to "sports", - "Supernatural" to "supernatural", - "Superpower" to "superpower", - "System" to "system", - "Taming" to "taming", - "Tower" to "tower", - "Tragedy" to "tragedy", - "Urban" to "urban", - "Vampires" to "vampiers", - "Virtual Reality" to "virtual-reality", - "Wuxia" to "wuxia", - ) - val DISPLAY = ENTRIES.map { it.first }.toTypedArray() - val VALUES = ENTRIES.map { it.second }.toTypedArray() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt deleted file mode 100644 index 031a927c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Dto.kt +++ /dev/null @@ -1,104 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import java.text.SimpleDateFormat - -@Serializable -class ResponseDto( - val data: T, - val meta: MetaDto? = null, -) - -@Serializable -class MetaDto( - @SerialName("has_more") val hasMore: Boolean = false, -) - -@Serializable -class SeriesDto( - private val id: Int, - private val title: String, - private val summary: String? = null, - private val cover: String? = null, - private val author: List? = null, - private val artist: List? = null, - private val tags: List? = null, - private val status: Int? = null, -) { - fun toSManga(cdnUrl: String, tagsMap: Map? = null) = SManga.create().apply { - url = id.toString() - this.title = this@SeriesDto.title - thumbnail_url = cover?.let { "$cdnUrl/covers/$it" } - if (tagsMap != null) { - description = summary - author = this@SeriesDto.author?.joinToString() - artist = this@SeriesDto.artist?.joinToString() - genre = tags?.mapNotNull { tagsMap[it] }?.joinToString() - this.status = when (this@SeriesDto.status) { - 1 -> SManga.ONGOING - 2 -> SManga.COMPLETED - 3, 4, 5 -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - } -} - -@Serializable -class ChapterDto( - private val id: Int, - private val number: Float, - private val title: String? = null, - @SerialName("created_at") private val createdAt: String? = null, - @SerialName("group_id") private val groupId: Int? = null, - private val group: GroupDto? = null, -) { - fun toSChapter(seriesIdStr: String, dateFormat: SimpleDateFormat) = SChapter.create().apply { - // SChapter.url is used as a full API path to avoid passing multiple fields later - url = "/chapter-navigation?series_id=$seriesIdStr&chapter_id=$id&group_id=${groupId ?: 0}" - name = buildString { - append("Chapter ") - append(number.toString().removeSuffix(".0")) - if (!title.isNullOrEmpty()) { - append(" - ") - append(title) - } - } - date_upload = dateFormat.tryParse(createdAt) - scanlator = group?.title - } -} - -@Serializable -class GroupDto( - val title: String? = null, -) - -@Serializable -class PageListDto( - private val chapter: ChapterDataDto? = null, -) { - fun toPages(cdnUrl: String): List { - val chapterId = chapter?.id ?: return emptyList() - return chapter.pages?.map { - Page(it.position, imageUrl = "$cdnUrl/pages/$chapterId/${it.path}") - } ?: emptyList() - } -} - -@Serializable -class ChapterDataDto( - val id: Int? = null, // Capturing the chapter id to use it in the image path - val pages: List? = null, -) - -@Serializable -class PageDto( - val position: Int, - val path: String, -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt deleted file mode 100644 index 947dddf6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/Filters.kt +++ /dev/null @@ -1,44 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.source.model.Filter - -class TypeFilter : Filter.Group("Type", types.map { TypeCheckBox(it.first, it.second) }) -class TypeCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -private val types = listOf( - Pair("Comic", 1), - Pair("Manga", 2), - Pair("Manhwa", 3), - Pair("Manhua", 4), - Pair("Webtoon", 5), -) - -class StatusFilter : Filter.Group("Status", statuses.map { StatusCheckBox(it.first, it.second) }) -class StatusCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -private val statuses = listOf( - Pair("Ongoing", 1), - Pair("Completed", 2), - Pair("Hiatus", 3), - Pair("Cancelled", 4), - Pair("Dropped", 5), -) - -class TagFilter : Filter.Group("Tags", tagsMap.map { TagCheckBox(it.value, it.key) }) -class TagCheckBox(name: String, val id: Int) : Filter.CheckBox(name) - -val tagsMap = mapOf( - 49 to "Regression", 48 to "Male Protagonist", 47 to "Survival", - 46 to "Avant Garde", 45 to "Award Winning", 44 to "Lolicon", - 43 to "Mahou Shoujo", 42 to "Doujinshi", 41 to "Girls Love", - 40 to "Hentai", 39 to "Mecha", 38 to "Shotacon", 37 to "Ecchi", - 36 to "Music", 35 to "Smut", 34 to "Erotica", 33 to "Adult", - 32 to "Gourmet", 31 to "Yuri", 30 to "Shoujo Ai", 29 to "Yaoi", - 28 to "Shounen Ai", 27 to "Boys Love", 26 to "Harem", 25 to "Tragedy", - 24 to "Gender Bender", 23 to "Suspense", 22 to "Psychological", - 21 to "Mature", 20 to "Horror", 19 to "Mystery", 18 to "Martial Arts", - 17 to "Sci-fi", 16 to "Adventure", 15 to "Supernatural", 14 to "Sports", - 13 to "Shounen", 12 to "Historical", 11 to "Seinen", 10 to "Action", - 9 to "Josei", 8 to "Thriller", 7 to "School Life", 6 to "Slice Of Life", - 5 to "Drama", 4 to "Comedy", 3 to "Shoujo", 2 to "Romance", 1 to "Fantasy", -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt deleted file mode 100644 index 840c6eee..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/scansgg/ScansGG.kt +++ /dev/null @@ -1,185 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.scansgg - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone -import eu.kanade.tachiyomi.network.asObservable - -@TachiyomiSource("SCANSGG", "ScansGG", "en") -class ScansGG : HttpSource() { - - override val name = "ScansGG" - override val baseUrl = "https://scans.gg" - override val lang = "en" - override val supportsLatest = true - - private val apiUrl = "https://api.scans.gg" - private val cdnUrl = "https://cdn.scans.gg/uploads" - - override fun headersBuilder() = super.headersBuilder() - .add("Origin", baseUrl) - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - // ============================== Popular ============================== - - override fun popularMangaRequest(page: Int): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder() - .addQueryParameter("limit", POPULAR_LIMIT.toString()) - .addQueryParameter("offset", ((page - 1) * POPULAR_LIMIT).toString()) - .build() - return GET(url, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val dto = response.parseAs>>() - val mangas = dto.data.map { it.toSManga(cdnUrl) } - - // The /series endpoint doesn't return the meta pagination object, - // so we check if the returned items match our limit - return MangasPage(mangas, mangas.size == POPULAR_LIMIT) - } - - // ============================== Latest =============================== - - override fun latestUpdatesRequest(page: Int): Request { - val url = "$apiUrl/chapters".toHttpUrl().newBuilder() - .addQueryParameter("page", page.toString()) - .addQueryParameter("limit", LATEST_LIMIT.toString()) - .addQueryParameter("chapters", "true") - .addQueryParameter("series_details", "true") - .addQueryParameter("group_details", "true") - .addQueryParameter("sort", "date") - .build() - return GET(url, headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val dto = response.parseAs>>() - val mangas = dto.data.map { it.toSManga(cdnUrl) } - return MangasPage(mangas, dto.meta?.hasMore == true) - } - - // ============================== Search =============================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder().apply { - addQueryParameter("limit", POPULAR_LIMIT.toString()) - addQueryParameter("offset", ((page - 1) * POPULAR_LIMIT).toString()) - if (query.isNotEmpty()) { - addQueryParameter("q", query) - } - - val types = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - val statuses = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - val tags = filters.firstInstanceOrNull()?.state?.filter { it.state }?.map { it.id } ?: emptyList() - - addQueryParameter("q_type", types.joinToString(",", "[", "]")) - addQueryParameter("q_status", statuses.joinToString(",", "[", "]")) - addQueryParameter("q_tags", tags.joinToString(",", "[", "]")) - }.build() - - return GET(url, headers) - } - - override fun searchMangaParse(response: Response) = popularMangaParse(response) - - // ============================== Details ============================== - - override fun getMangaUrl(manga: SManga): String = "$baseUrl/series/${manga.url}" - - override fun mangaDetailsRequest(manga: SManga): Request { - val url = "$apiUrl/series".toHttpUrl().newBuilder() - .addQueryParameter("id", manga.url) - .addQueryParameter("trackers", "true") - .addQueryParameter("sources", "true") - .build() - return GET(url, headers) - } - - override fun mangaDetailsParse(response: Response): SManga { - val dto = response.parseAs>() - return dto.data.toSManga(cdnUrl, tagsMap) - } - - // ============================= Chapters ============================== - -// ============================= Chapters ============================== - - override fun getChapterUrl(chapter: SChapter): String { - val url = (apiUrl + chapter.url).toHttpUrl() - val seriesId = url.queryParameter("series_id") ?: return baseUrl - return "$baseUrl/series/$seriesId" - } - - override fun fetchChapterList(manga: SManga): Observable> = - client.newCall(chapterListRequest(manga)) - .asObservable() - .map(::chapterListParseApi) - - override fun chapterListRequest(manga: SManga): Request { - val url = "$apiUrl/chapters".toHttpUrl().newBuilder() - .addQueryParameter("series_id", manga.url) - .addQueryParameter("limit", CHAPTER_LIMIT.toString()) - .addQueryParameter("page", "1") - .addQueryParameter("group_details", "true") - .build() - - return GET(url, headers) - } - - private fun chapterListParseApi(response: Response): List { - val mangaId = response.request.url.queryParameter("series_id") - ?: throw Exception("Missing series id") - - val dto = response.parseAs>>() - - return dto.data.map { - it.toSChapter(mangaId, dateFormat) - } - } - - override fun chapterListParse(response: Response): List = - throw UnsupportedOperationException() - // =============================== Pages =============================== - - override fun pageListRequest(chapter: SChapter): Request = GET(apiUrl + chapter.url, headers) - - override fun pageListParse(response: Response): List { - val dto = response.parseAs>() - return dto.data.toPages(cdnUrl) - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ============================== Filters ============================== - - override fun getFilterList() = FilterList( - TypeFilter(), - StatusFilter(), - TagFilter(), - ) - - companion object { - private const val POPULAR_LIMIT = 21 - private const val LATEST_LIMIT = 14 - private const val CHAPTER_LIMIT = 100 - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt deleted file mode 100644 index 0dfe0ea5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Constants.kt +++ /dev/null @@ -1,125 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import java.util.Locale - -const val PER_PAGE = 18 - -const val SERIES_PATH_SEGMENT = "series" - -const val API_QUERY_PATH = "/api/query" -const val API_POST_PATH = "/api/post" -const val API_CHAPTER_PATH = "/api/chapter" - -const val STATUS_FILTER_KEY = "seriesStatus" -val STATUS_OPTIONS: Options = listOf( - "All" to "", - "Ongoing" to "ONGOING", - "Completed" to "COMPLETED", - "Cancelled" to "CANCELLED", - "Dropped" to "DROPPED", - "Coming soon" to "COMING_SOON", - "Mass released" to "MASS_RELEASED", -) - -const val TYPE_FILTER_KEY = "seriesType" -val TYPE_OPTIONS: Options = listOf( - "All" to "", - "Manga" to "MANGA", - "Manhua" to "MANHUA", - "Manhwa" to "MANHWA", -) - -const val SORT_FILTER_KEY = "orderBy" -val SORT_OPTIONS: Options = listOf( - "Last chapter added" to "lastChapterAddedAt", - "Views" to "totalViews", - "Created at" to "createdAt", - "Chapter count" to "chaptersCount", - "Alphabetical" to "postTitle", -) - -const val SORT_DIRECTION_FILTER_KEY = "orderDirection" -val SORT_DIRECTION_OPTIONS: Options = listOf( - "Descending" to "desc", - "Ascending" to "asc", -) - -const val GENRE_INCLUDE_FILTER_KEY = "genreIds" -const val GENRE_EXCLUDE_FILTER_KEY = "excludedGenreIds" - -val GENRE_OPTIONS: Options = listOf( - "Action" to "1", - "Drama" to "2", - "Shounen" to "3", - "Sports" to "4", - "Manhwa" to "5", - "Martial Arts" to "6", - "Comedy" to "7", - "Fantasy" to "8", - "Horror" to "9", - "Seinen" to "10", - "Supernatural" to "11", - "Mature" to "12", - "Adventure" to "13", - "Monsters" to "14", - "System" to "15", - "Reincarnation" to "16", - "Revenge" to "17", - "Slice Of Life" to "18", - "Historical" to "19", - "Romance" to "20", - "Josei" to "21", - "Shoujo" to "22", - "School Life" to "23", - "terror" to "24", - "elf" to "25", - "shojo" to "26", - "Video Games" to "27", - "Fantas" to "28", - "WEB COMIC" to "29", - "Webtoons" to "30", - "Murim" to "31", - "Restaurant" to "32", - "Webtoon" to "33", - "+100 Chapter" to "34", - "Tower" to "35", - "Legendary " to "36", - "Dungeons" to "37", - "bully" to "38", - "orphan" to "39", - "Sci-Fi" to "40", - "Gore" to "41", - "Isekai" to "42", - "magic" to "43", - "blood" to "44", - "war" to "45", - "magic and sword" to "46", - "academy" to "47", - "violence" to "48", - "Harem" to "49", - "Myth" to "50", - "OverpoweredMC" to "51", - "TerritoryManagement" to "52", - "Swordsman" to "53", - "Necromancer" to "54", - "Mage" to "55", - "JackOfAllTrades" to "56", - "Artifacts" to "57", - "CharacterGrowth" to "58", - "Mercenary" to "59", - "Elementals" to "60", - "Genius" to "61", - "Psychological" to "62", - "Tragedy" to "63", - "Gender Bender" to "64", -).sortedBy { it.first.trim().lowercase(Locale.ROOT) } - -const val POPULAR_ORDER_BY = "totalViews" -const val LATEST_ORDER_BY = "lastChapterAddedAt" -const val ORDER_DESC = "desc" - -const val SHOW_LOCKED_CHAPTERS_PREF_KEY = "pref_show_locked_chapters" -const val SHOW_LOCKED_CHAPTERS_DEFAULT = false - -const val MISSING_TITLE_MESSAGE = "Series title is missing" -const val LOCKED_CHAPTER_MESSAGE = "Unlock chapter in WebView" \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt deleted file mode 100644 index 6b296ff6..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Dto.kt +++ /dev/null @@ -1,75 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import kotlinx.serialization.Serializable -import kotlinx.serialization.json.JsonPrimitive - -@Serializable -class SearchResponseDto( - val posts: List, - val totalCount: Int, -) - -@Serializable -class PostSummaryDto( - val id: Int, - val slug: String, - val postTitle: String, - val featuredImage: String? = null, - val seriesStatus: String? = null, - val genres: List = emptyList(), -) - -@Serializable -class GenreDto( - val name: String, -) - -@Serializable -class PostResponseDto( - val post: PostDto, -) - -@Serializable -class PostDto( - val id: Int, - val slug: String, - val postTitle: String, - val postContent: String? = null, - val alternativeTitles: String? = null, - val author: String? = null, - val artist: String? = null, - val featuredImage: String? = null, - val seriesType: String? = null, - val seriesStatus: String? = null, - val genres: List = emptyList(), - val chapters: List = emptyList(), -) - -@Serializable -class PostChapterDto( - val id: Int, - val slug: String, - val number: JsonPrimitive, - val title: String? = null, - val createdAt: String, - val isLocked: Boolean? = null, - val isAccessible: Boolean? = null, -) - -@Serializable -class ChapterResponseDto( - val chapter: ChapterDto, -) - -@Serializable -class ChapterDto( - val isLocked: Boolean? = null, - val isAccessible: Boolean? = null, - val images: List = emptyList(), -) - -@Serializable -class ChapterImageDto( - val url: String, - val order: Int? = null, -) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt deleted file mode 100644 index 217083bc..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Filters.kt +++ /dev/null @@ -1,83 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import eu.kanade.tachiyomi.source.model.Filter -import okhttp3.HttpUrl - -typealias Options = List> - -interface UrlPartFilter { - fun addUrlParameter(url: HttpUrl.Builder) -} - -abstract class SelectFilter( - name: String, - private val urlParameter: String, - private val options: Options, - defaultValue: String? = null, -) : Filter.Select( - name, - options.map { it.first }.toTypedArray(), - options.indexOfFirst { it.second == defaultValue }.takeIf { it != -1 } ?: 0, -), - UrlPartFilter { - override fun addUrlParameter(url: HttpUrl.Builder) { - options[state].second.takeIf { it.isNotBlank() }?.let { - url.addQueryParameter(urlParameter, it) - } - } -} - -class StatusFilter( - title: String, - key: String, - options: Options, -) : SelectFilter(title, key, options) - -class TypeFilter( - title: String, - key: String, - options: Options, -) : SelectFilter(title, key, options) - -class SortFilter( - title: String, - key: String, - options: Options, - defaultValue: String? = null, -) : SelectFilter(title, key, options, defaultValue) - -class SortDirectionFilter( - title: String, - key: String, - options: Options, - defaultValue: String? = null, -) : SelectFilter(title, key, options, defaultValue) - -class GenreFilter( - name: String, - genres: Options, -) : Filter.Group( - name, - genres.map { GenreValue(it.first, it.second) }, -), - UrlPartFilter { - override fun addUrlParameter(url: HttpUrl.Builder) { - val included = state - .filter { it.state == Filter.TriState.STATE_INCLUDE } - .map { it.id } - - val excluded = state - .filter { it.state == Filter.TriState.STATE_EXCLUDE } - .map { it.id } - - if (included.isNotEmpty()) { - url.addQueryParameter(GENRE_INCLUDE_FILTER_KEY, included.joinToString(",")) - } - - if (excluded.isNotEmpty()) { - url.addQueryParameter(GENRE_EXCLUDE_FILTER_KEY, excluded.joinToString(",")) - } - } -} - -class GenreValue(name: String, val id: String) : Filter.TriState(name) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt deleted file mode 100644 index 4b6e49e2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Helpers.kt +++ /dev/null @@ -1,67 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull - -class DeepLink( - val mangaSlug: String, - val chapterSlug: String?, -) - -fun parseDeepLink(query: String, baseHost: String): DeepLink? { - val url = query.toHttpUrlOrNull() ?: return null - if (url.host != baseHost) return null - - val pathSegments = url.pathSegments - if (pathSegments.getOrNull(0) != SERIES_PATH_SEGMENT) return null - - val mangaSlug = pathSegments.getOrNull(1) - ?.takeIf { it.isNotBlank() } - ?: return null - - val chapterSlug = pathSegments.getOrNull(2) - ?.takeIf { it.isNotBlank() } - - return DeepLink(mangaSlug, chapterSlug) -} - -fun extractMangaSlug(rawUrl: String): String { - val pathSegments = rawUrl.substringBefore('#').trim('/').split('/').filter { it.isNotBlank() } - - return when { - pathSegments.size >= 2 && pathSegments[0] == SERIES_PATH_SEGMENT -> pathSegments[1] - pathSegments.isNotEmpty() -> pathSegments[0] - else -> throw Exception("Invalid manga url") - } -} - -fun extractChapterSlugs(rawUrl: String): Pair? { - val pathSegments = rawUrl.substringBefore('#').trim('/').split('/').filter { it.isNotBlank() } - - return when { - pathSegments.size >= 3 && pathSegments[0] == SERIES_PATH_SEGMENT -> pathSegments[1] to pathSegments[2] - pathSegments.size >= 2 -> pathSegments[pathSegments.size - 2] to pathSegments.last() - else -> null - } -} - -fun buildSlugSearchTerms(slug: String): List { - val withSpaces = slug.replace('-', ' ') - val withoutApostrophe = withSpaces.replace("'", " ") - - return listOf( - withSpaces, - withoutApostrophe, - collapseSpaces(withoutApostrophe), - slug, - ).map { it.trim() } - .filter { it.isNotBlank() } - .distinct() -} - -private fun collapseSpaces(text: String): String { - var value = text - while (value.contains(" ")) { - value = value.replace(" ", " ") - } - return value -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt deleted file mode 100644 index 3cde0ce5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/Mapper.kt +++ /dev/null @@ -1,116 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import org.jsoup.Jsoup -import java.text.SimpleDateFormat - -fun PostSummaryDto.toSMangaSummary(): SManga { - val cleanTitle = postTitle.trim() - if (cleanTitle.isEmpty()) throw Exception(MISSING_TITLE_MESSAGE) - - return SManga.create().apply { - url = "$slug#$id" - title = cleanTitle - thumbnail_url = featuredImage - genre = genres.joinToString { it.name } - status = mapStatus(seriesStatus) - } -} - -fun PostDto.toSMangaDetailsModel(): SManga { - val cleanTitle = postTitle.trim() - if (cleanTitle.isEmpty()) throw Exception(MISSING_TITLE_MESSAGE) - - return SManga.create().apply { - url = "$slug#$id" - title = cleanTitle - description = buildDescription(this@toSMangaDetailsModel) - author = this@toSMangaDetailsModel.author?.trim()?.takeUnless { it.isEmpty() } - artist = this@toSMangaDetailsModel.artist?.trim()?.takeUnless { it.isEmpty() } - genre = buildGenre(seriesType, genres) - status = mapStatus(seriesStatus) - thumbnail_url = featuredImage - initialized = true - } -} - -fun PostChapterDto.toSChapterModel(mangaSlug: String, dateFormat: SimpleDateFormat): SChapter { - val rawNumber = number.toString().trim('"').trim() - val fallbackNumber = slug.substringAfter("chapter-", "") - val chapterNumberText = if (rawNumber.isNotEmpty()) rawNumber else fallbackNumber - val chapterTitle = title?.trim().orEmpty() - - return SChapter.create().apply { - url = "$mangaSlug/$slug#$id" - name = buildString { - if (isAccessible == false || isLocked == true) { - append("🔒 ") - } - - append("Chapter") - if (chapterNumberText.isNotEmpty()) { - append(' ') - append(chapterNumberText) - } - - if (chapterTitle.isNotEmpty()) { - append(" - ") - append(chapterTitle) - } - } - chapterNumberText.toFloatOrNull()?.let { chapter_number = it } - date_upload = dateFormat.tryParse(createdAt) - } -} - -private fun buildDescription(post: PostDto): String? { - val synopsis = post.postContent - ?.takeIf { it.isNotBlank() } - ?.replace("
", "\n") - ?.replace("
", "\n") - ?.replace("
", "\n") - ?.let { Jsoup.parse(it).text() } - ?.trim() - ?.takeIf { it.isNotEmpty() } - - val alternativeTitles = post.alternativeTitles - ?.lineSequence() - ?.map { it.trim() } - ?.filter { it.isNotEmpty() } - ?.toList() - ?.takeIf { it.isNotEmpty() } - ?.joinToString("\n") - - return buildString { - synopsis?.let { append(it) } - alternativeTitles?.let { - if (isNotEmpty()) append("\n\n") - append("Alternative titles:\n") - append(it) - } - }.takeIf { it.isNotBlank() } -} - -private fun buildGenre(seriesType: String?, genres: List): String? { - val values = mutableListOf() - - when (seriesType?.uppercase()) { - "MANGA" -> values += "Manga" - "MANHUA" -> values += "Manhua" - "MANHWA" -> values += "Manhwa" - } - - genres.mapTo(values) { it.name } - - val out = values.distinct().joinToString() - return out.takeIf { it.isNotBlank() } -} - -private fun mapStatus(status: String?): Int = when (status) { - "ONGOING", "COMING_SOON", "MASS_RELEASED" -> SManga.ONGOING - "COMPLETED" -> SManga.COMPLETED - "CANCELLED", "DROPPED" -> SManga.CANCELLED - else -> SManga.UNKNOWN -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt deleted file mode 100644 index 188e56c7..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/en/vortexscan/VortexScan.kt +++ /dev/null @@ -1,290 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.en.vortexscan - -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("VORTEXSCANS", "Vortex Scans", "en") -class VortexScans : - HttpSource(), - ConfigurableSource { - - private lateinit var summaryOn: String - private lateinit var summaryOff: String - override val name = "Vortex Scans" - override val baseUrl = "https://vortexscans.org" - private val apiUrl = "https://api.vortexscans.org" - override val lang = "en" - override val supportsLatest = true - - private val preferences by getPreferencesLazy() - - private val baseHost = baseUrl.toHttpUrl().host - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) - private val showLockedChapters - get() = preferences.getBoolean(SHOW_LOCKED_CHAPTERS_PREF_KEY, SHOW_LOCKED_CHAPTERS_DEFAULT) - - override fun headersBuilder() = super.headersBuilder() - .set("Referer", "$baseUrl/") - - private val apiHeaders: Headers by lazy { - headersBuilder() - .set("Accept", "application/json, text/plain, */*") - .set("Origin", baseUrl) - .build() - } - - // ========================= Popular ========================= - - override fun popularMangaRequest(page: Int): Request = queryRequest( - page = page, - query = "", - orderBy = POPULAR_ORDER_BY, - orderDirection = ORDER_DESC, - ) - - override fun popularMangaParse(response: Response): MangasPage = searchMangaParse(response) - - // ========================= Latest ========================= - - override fun latestUpdatesRequest(page: Int): Request = queryRequest( - page = page, - query = "", - orderBy = LATEST_ORDER_BY, - orderDirection = ORDER_DESC, - ) - - override fun latestUpdatesParse(response: Response): MangasPage = searchMangaParse(response) - - // ========================= Search ========================= - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (page == 1) { - val deepLink = parseDeepLink(query, baseHost) - if (deepLink != null) { - val summary = findPostBySlug(deepLink.mangaSlug) - ?: throw Exception("Series not found") - - val manga = summary.toSMangaSummary() - - return fetchMangaDetails(manga) - .map { MangasPage(listOf(it), false) } - } - } - - return super.fetchSearchManga(page, query, filters) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = queryRequest( - page = page, - query = query.trim(), - filters = filters.filterIsInstance(), - ) - - override fun searchMangaParse(response: Response): MangasPage { - val payload = response.parseAs() - val currentPage = response.request.url.queryParameter("page")?.toIntOrNull() ?: 1 - val hasNextPage = payload.totalCount > (currentPage * PER_PAGE) - - val mangas = payload.posts.map { it.toSMangaSummary() } - return MangasPage(mangas, hasNextPage) - } - - // ========================= Filters ========================= - - override fun getFilterList() = FilterList( - StatusFilter("Status", STATUS_FILTER_KEY, STATUS_OPTIONS), - TypeFilter("Type", TYPE_FILTER_KEY, TYPE_OPTIONS), - SortFilter("Sort by", SORT_FILTER_KEY, SORT_OPTIONS), - SortDirectionFilter("Sort direction", SORT_DIRECTION_FILTER_KEY, SORT_DIRECTION_OPTIONS), - GenreFilter("Genres", GENRE_OPTIONS), - ) - - // ========================= Details ========================= - - override fun getMangaUrl(manga: SManga): String { - val slug = extractMangaSlug(manga.url) - return "$baseUrl/$SERIES_PATH_SEGMENT/$slug" - } - - override fun mangaDetailsRequest(manga: SManga): Request = postRequest(resolvePostId(manga)) - - override fun mangaDetailsParse(response: Response): SManga { - val payload = response.parseAs() - return payload.post.toSMangaDetailsModel() - } - - // ========================= Chapters ========================= - - override fun getChapterUrl(chapter: SChapter): String { - val slugs = extractChapterSlugs(chapter.url) - ?: return "$baseUrl/$SERIES_PATH_SEGMENT/${chapter.url.substringBefore('#').trim('/')}" - - return "$baseUrl/$SERIES_PATH_SEGMENT/${slugs.first}/${slugs.second}" - } - - override fun chapterListRequest(manga: SManga): Request = postRequest(resolvePostId(manga)) - - override fun chapterListParse(response: Response): List { - val payload = response.parseAs() - val mangaSlug = payload.post.slug - - return payload.post.chapters - .filter { chapter -> - showLockedChapters || (chapter.isAccessible != false && chapter.isLocked != true) - } - .map { it.toSChapterModel(mangaSlug, dateFormat) } - } - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_LOCKED_CHAPTERS_PREF_KEY - title = "Show inaccessible chapters" - summaryOn = "Locked/inaccessible chapters will be shown in the chapter list." - summaryOff = "Locked/inaccessible chapters will be hidden from the chapter list." - setDefaultValue(SHOW_LOCKED_CHAPTERS_DEFAULT) - }.also(screen::addPreference) - } - - // ========================= Pages ========================= - - override fun pageListRequest(chapter: SChapter): Request = chapterRequest(resolveChapterId(chapter)) - - override fun pageListParse(response: Response): List { - val payload = response.parseAs() - val chapter = payload.chapter - - if (chapter.isAccessible == false || chapter.isLocked == true) { - throw Exception(LOCKED_CHAPTER_MESSAGE) - } - - val images = chapter.images.sortedBy { it.order ?: Int.MAX_VALUE } - if (images.isEmpty()) { - throw Exception("No pages found") - } - - return images.mapIndexed { index, image -> - Page(index = index, imageUrl = image.url) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - private fun queryRequest( - page: Int, - query: String, - orderBy: String? = null, - orderDirection: String? = null, - filters: List = emptyList(), - ): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_QUERY_PATH) - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", PER_PAGE.toString()) - addQueryParameter("searchTerm", query) - - filters.forEach { it.addUrlParameter(this) } - - if (!orderBy.isNullOrBlank()) { - setQueryParameter(SORT_FILTER_KEY, orderBy) - } - - if (!orderDirection.isNullOrBlank()) { - setQueryParameter(SORT_DIRECTION_FILTER_KEY, orderDirection) - } - }.build() - - return GET(url, apiHeaders) - } - - private fun postRequest(postId: Int): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_POST_PATH) - addQueryParameter("postId", postId.toString()) - }.build() - - return GET(url, apiHeaders) - } - - private fun chapterRequest(chapterId: Int): Request { - val url = apiUrl.toHttpUrl().newBuilder().apply { - encodedPath(API_CHAPTER_PATH) - addQueryParameter("chapterId", chapterId.toString()) - }.build() - - return GET(url, apiHeaders) - } - - private fun resolvePostId(manga: SManga): Int { - manga.url.substringAfter('#', "").toIntOrNull()?.let { return it } - - val slug = extractMangaSlug(manga.url) - val summary = findPostBySlug(slug) - return summary?.id ?: throw Exception("Unable to resolve series id") - } - - private fun resolveChapterId(chapter: SChapter): Int { - chapter.url.substringAfter('#', "").toIntOrNull()?.let { return it } - - val slugs = extractChapterSlugs(chapter.url) - ?: throw Exception("Unable to resolve chapter id") - - val summary = findPostBySlug(slugs.first) - ?: throw Exception("Unable to resolve series id") - - return try { - client.newCall(postRequest(summary.id)).execute().use { response -> - if (!response.isSuccessful) { - throw Exception("Unable to resolve chapter id") - } - - val payload = response.parseAs() - val chapterDto = payload.post.chapters.firstOrNull { it.slug == slugs.second } - ?: throw Exception("Unable to resolve chapter id") - - chapterDto.id - } - } catch (_: Exception) { - throw Exception("Unable to resolve chapter id") - } - } - - private fun findPostBySlug(slug: String): PostSummaryDto? { - val normalizedSlug = slug.trim().trim('/').lowercase() - val terms = buildSlugSearchTerms(normalizedSlug) - - for (term in terms) { - try { - client.newCall(queryRequest(page = 1, query = term)).execute().use { response -> - if (response.isSuccessful) { - val payload = response.parseAs() - val post = payload.posts.firstOrNull { it.slug.equals(normalizedSlug, ignoreCase = true) } - if (post != null) { - return post - } - } - } - } catch (_: Exception) { - } - } - - return null - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt deleted file mode 100644 index 6ad673c8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/Filters.kt +++ /dev/null @@ -1,99 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.mangaoni - -import eu.kanade.tachiyomi.source.model.Filter - -open class UriPartFilter(displayName: String, private val vals: Array>) : Filter.Select(displayName, vals.map { it.first }.toTypedArray()) { - fun toUriPart() = vals[state].second -} - -class StatusFilter : - UriPartFilter( - "Estado", - arrayOf( - Pair("Estado", "false"), - Pair("En desarrollo", "1"), - Pair("Completo", "0"), - ), - ) - -class TypeFilter : - UriPartFilter( - "Tipo", - arrayOf( - Pair("Todo", "false"), - Pair("Mangas", "0"), - Pair("Manhwas", "1"), - Pair("One Shot", "2"), - Pair("Manhuas", "3"), - Pair("Novelas", "4"), - ), - ) - -class GenreFilter : - UriPartFilter( - "Géneros", - arrayOf( - Pair("Todos", "false"), - Pair("Comedia", "1"), - Pair("Drama", "2"), - Pair("Acción", "3"), - Pair("Escolar", "4"), - Pair("Romance", "5"), - Pair("Ecchi", "6"), - Pair("Aventura", "7"), - Pair("ShÅnen", "8"), - Pair("ShÅjo", "9"), - Pair("Deportes", "10"), - Pair("Psicológico", "11"), - Pair("Fantasía", "12"), - Pair("Mecha", "13"), - Pair("Gore", "14"), - Pair("Yaoi", "15"), - Pair("Yuri", "16"), - Pair("Misterio", "17"), - Pair("Sobrenatural", "18"), - Pair("Seinen", "19"), - Pair("Ficción", "20"), - Pair("Harem", "21"), - Pair("Webtoon", "25"), - Pair("Histórico", "27"), - Pair("Músical", "30"), - Pair("Ciencia ficción", "31"), - Pair("ShÅjo-ai", "32"), - Pair("Josei", "33"), - Pair("Magia", "34"), - Pair("Artes Marciales", "35"), - Pair("Horror", "36"), - Pair("Demonios", "37"), - Pair("Supervivencia", "38"), - Pair("Recuentos de la vida", "39"), - Pair("ShÅnen ai", "40"), - Pair("Militar", "41"), - Pair("Eroge", "42"), - Pair("Isekai", "43"), - ), - ) - -class AdultContentFilter : - UriPartFilter( - "Contenido +18", - arrayOf( - Pair("Mostrar todo", "false"), - Pair("Mostrar solo +18", "1"), - Pair("No mostrar +18", "0"), - ), - ) - -class SortBy : - Filter.Sort( - "Ordenar por", - arrayOf("Visitas", "Recientes", "Alfabético"), - Selection(0, false), - ) { - fun toUriPart() = when (state?.index) { - 0 -> "visitas" - 1 -> "id" - 2 -> "nombre" - else -> "visitas" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt deleted file mode 100644 index 874747b2..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/mangaoni/MangaOni.kt +++ /dev/null @@ -1,226 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.mangaoni - -import android.content.SharedPreferences -import android.net.Uri -import android.util.Base64 -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.tryParse -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import java.nio.charset.Charset -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("MANGAONI", "MangaOni", "es") -class MangaOni : - HttpSource(), - ConfigurableSource { - - override val name = "MangaOni" - - override val id: Long = 2202687009511923782 - - override val baseUrl = "https://manga-oni.com" - - override val lang = "es" - - override val supportsLatest = true - - private val preferences: SharedPreferences by getPreferencesLazy() - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) - - override fun popularMangaRequest(page: Int) = GET( - url = "$baseUrl/directorio?genero=false&estado=false&filtro=visitas&tipo=false&adulto=${if (hideNSFWContent()) "0" else "false"}&orden=desc&p=$page", - headers = headers, - ) - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val mangas = document.select("#article-div a").map { element -> - SManga.create().apply { - setUrlWithoutDomain(element.attr("abs:href")) - thumbnail_url = element.select("img").attr("abs:data-src") - title = element.select("div:eq(1)").text() - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/recientes?p=$page", headers) - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - - val mangas = document.select("div._1bJU3").map { element -> - SManga.create().apply { - thumbnail_url = element.select("img").attr("abs:data-src") - element.selectFirst("a[data-test=latest-update-name]")?.also { - title = it.text() - setUrlWithoutDomain(it.attr("abs:href")) - } - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val uri = Uri.parse("$baseUrl/${if (query.isNotBlank()) "buscar" else "directorio"}").buildUpon() - - if (query.isNotBlank()) { - uri.appendQueryParameter("q", query) - } else { - val adultContent = filters.firstInstanceOrNull()?.toUriPart() - ?: if (hideNSFWContent()) "0" else "false" - uri.appendQueryParameter("adulto", adultContent) - - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("estado", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("tipo", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { uri.appendQueryParameter("genero", it.toUriPart()) } - filters.firstInstanceOrNull()?.let { - uri.appendQueryParameter("filtro", it.toUriPart()) - uri.appendQueryParameter("orden", if (it.state?.ascending == true) "asc" else "desc") - } - } - uri.appendQueryParameter("p", page.toString()) - return GET(uri.toString(), headers) - } - - override fun searchMangaParse(response: Response): MangasPage { - if (!response.isSuccessful) throw Exception("Búsqueda fallida ${response.code}") - - val document = response.asJsoup() - - val mangas = if (document.location().startsWith("$baseUrl/directorio")) { - document.select("#article-div a").map { element -> - SManga.create().apply { - setUrlWithoutDomain(element.attr("abs:href")) - thumbnail_url = element.select("img").attr("abs:data-src") - title = element.select("div:eq(1)").text() - } - } - } else { - document.select("#article-div > div").map { element -> - SManga.create().apply { - thumbnail_url = element.select("img").attr("abs:src") - title = element.select("a").firstOrNull { it.text().isNotBlank() }?.text() - ?: throw Exception("Title not found") - element.selectFirst("a")?.also { - setUrlWithoutDomain(it.attr("abs:href")) - } - } - } - } - - val hasNextPage = document.selectFirst("ul.pagination a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun mangaDetailsParse(response: Response): SManga { - val document = response.asJsoup() - - return SManga.create().apply { - title = document.selectFirst("h1")?.text() ?: throw Exception("Title not found") - thumbnail_url = document.select("img[src*=cover]").attr("abs:src") - description = document.select("div#sinopsis").lastOrNull()?.ownText() - author = document.select("div#info-i").text().let { - if (it.contains("Autor", true)) { - it.substringAfter("Autor:").substringBefore("Fecha:").trim() - } else { - "N/A" - } - } - artist = author - genre = document.select("div#categ a").joinToString(", ") { it.text() } - status = when (document.selectFirst("strong:contains(Estado) + span")?.text()) { - "En desarrollo" -> SManga.ONGOING - "Finalizado" -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - } - } - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - - return document.select("div#c_list a").map { element -> - SChapter.create().apply { - name = element.text() - setUrlWithoutDomain(element.attr("abs:href")) - chapter_number = element.select("span").attr("data-num").toFloatOrNull() ?: -1f - date_upload = dateFormat.tryParse(element.select("span").attr("datetime")) - } - } - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - - val encoded = document.selectFirst("script:containsData(unicap)") - ?.data()?.substringAfter("'")?.substringBefore("'") - ?: throw Exception("unicap not found") - val drop = encoded.length % 4 - val decoded = Base64.decode(encoded.dropLast(drop), Base64.DEFAULT).toString(Charset.defaultCharset()) - val path = decoded.substringBefore("||") - - return decoded.substringAfter("[").substringBefore("]").split(",").mapIndexed { i, file -> - Page(i, imageUrl = path + file.removeSurrounding("\"")) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - override fun getFilterList(): FilterList { - val filterList = mutableListOf( - Filter.Header("NOTA: Se ignoran si se usa el buscador"), - Filter.Separator(), - SortBy(), - StatusFilter(), - TypeFilter(), - GenreFilter(), - ) - - if (!hideNSFWContent()) { - filterList.add(AdultContentFilter()) - } - return FilterList(filterList) - } - - override fun setupPreferenceScreen(screen: androidx.preference.PreferenceScreen) { - val contentPref = android.preference.CheckBoxPreference(screen.context).apply { - key = CONTENT_PREF - title = CONTENT_PREF_TITLE - summary = CONTENT_PREF_SUMMARY - setDefaultValue(CONTENT_PREF_DEFAULT_VALUE) - } - } - - private fun hideNSFWContent(): Boolean = preferences.getBoolean(CONTENT_PREF, CONTENT_PREF_DEFAULT_VALUE) - - companion object { - private const val CONTENT_PREF = "showNSFWContent" - private const val CONTENT_PREF_TITLE = "Ocultar contenido +18" - private const val CONTENT_PREF_SUMMARY = "Ocultar el contenido erótico en mangas populares y filtros, no funciona en los mangas recientes ni búsquedas textuales." - private const val CONTENT_PREF_DEFAULT_VALUE = false - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt deleted file mode 100644 index beef1a42..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/Dto.kt +++ /dev/null @@ -1,170 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.olympusscan - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import java.text.ParseException -import java.text.SimpleDateFormat - -@Serializable -class RankingDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class PayloadSeriesDto(val data: PayloadSeriesDataDto) - -@Serializable -class PayloadSeriesDataDto( - val series: SeriesDto, -) - -@Serializable -class SeriesDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class PayloadMangaDto(val data: List) - -@Serializable -class MangaDto( - val id: Int, - val name: String, - val slug: String, - private val cover: String? = null, - val type: String? = null, - private val summary: String? = null, - private val status: MangaStatusDto? = null, - private val genres: List? = null, -) { - fun toSManga() = SManga.create().apply { - title = name - url = id.toString() - thumbnail_url = cover - } - - fun toSMangaDetails() = toSManga().apply { - description = summary - status = parseStatus() - genre = genres?.joinToString { it.name.trim() } - } - - private fun parseStatus(): Int { - val status = this.status ?: return SManga.UNKNOWN - return when (status.id) { - 1 -> SManga.ONGOING - 3 -> SManga.ON_HIATUS - 4 -> SManga.COMPLETED - 5 -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } -} - -@Serializable -class NewChaptersDto( - val data: List, - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} - -@Serializable -class LatestMangaDto( - val id: Int, - private val name: String, - val slug: String, - private val cover: String? = null, - val type: String? = null, -) { - fun toSManga() = SManga.create().apply { - title = name - url = id.toString() - thumbnail_url = cover - } -} - -@Serializable -class MangaDetailDto( - var data: MangaDto, -) - -@Serializable -class PayloadChapterDto(var data: List, val meta: MetaDto) - -@Serializable -class ChapterDto( - private val id: Int, - private val name: String, - @SerialName("published_at") private val date: String, -) { - fun toSChapter(mangaId: String, dateFormat: SimpleDateFormat) = SChapter.create().apply { - name = "Capitulo ${this@ChapterDto.name}" - url = "$mangaId/$id" - date_upload = try { - dateFormat.parse(date)!!.time - } catch (e: ParseException) { - 0L - } - } -} - -@Serializable -class MetaDto(val total: Int) - -@Serializable -class PayloadPagesDto(val chapter: PageDto) - -@Serializable -class PageDto(val pages: List) - -@Serializable -class MangaStatusDto( - val id: Int, -) - -@Serializable -class GenresStatusesDto( - val genres: List, - val statuses: List, -) - -@Serializable -class FilterDto( - val id: Int, - val name: String, -) - -@Serializable -class BookmarksWrapperDto( - private val data: List = emptyList(), - val meta: BookmarksMetaDto, -) { - fun getBookmarks() = data.filter { it.type == "comic" && it.id != null && it.slug != null } -} - -@Serializable -class BookmarkDto( - val id: Int?, - val slug: String?, - val type: String?, -) - -@Serializable -class BookmarksMetaDto( - @SerialName("current_page") private val currentPage: Int, - @SerialName("last_page") private val lastPage: Int, -) { - fun hasNextPage() = currentPage < lastPage -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt deleted file mode 100644 index 823932cd..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/olympusscan/OlympusScan.kt +++ /dev/null @@ -1,338 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.olympusscan - -import android.content.SharedPreferences -import android.widget.Toast -import androidx.preference.EditTextPreference -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import org.koitharu.kotatsu.parsers.util.rateLimit -import keiyoushi.utils.getPreferences -import keiyoushi.utils.parseAs -import keiyoushi.utils.toJsonString -import kotlinx.serialization.SerializationException -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.koitharu.kotatsu.parsers.TachiyomiSource -import rx.Observable -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone -import kotlin.time.Duration.Companion.seconds - -@TachiyomiSource("OLYMPUSSCAN", "Olympus Scanlation", "es") -class OlympusScan : - HttpSource(), - ConfigurableSource { - private val fetchedDomainUrlHost by lazy { fetchedDomainUrl.toHttpUrl().host } - private val apiBaseUrlHost by lazy { apiBaseUrl.toHttpUrl().host } - - override val versionId = 3 - private val isCi = System.getenv("CI") == "true" - - override val baseUrl: String get() = when { - isCi -> defaultBaseUrl - else -> preferences.prefBaseUrl - } - - private val defaultBaseUrl: String = "https://olympusxyz.com" - - private val fetchedDomainUrl: String by lazy { - if (!preferences.fetchDomainPref()) return@lazy preferences.prefBaseUrl - try { - val initClient = network.client - val headers = super.headersBuilder().build() - val document = initClient.newCall(GET("https://olympus.pages.dev", headers)).execute().asJsoup() - val domain = document.selectFirst("meta[property=og:url]")?.attr("content") - ?: return@lazy preferences.prefBaseUrl - val host = initClient.newCall(GET(domain, headers)).execute().request.url.host - val newDomain = "https://$host" - preferences.prefBaseUrl = newDomain - newDomain - } catch (_: Exception) { - preferences.prefBaseUrl - } - } - - private val apiBaseUrl by lazy { - fetchedDomainUrl.replace("https://", "https://panel.") - } - - override val lang: String = "es" - override val name: String = "Olympus Scanlation" - - override val supportsLatest: Boolean = true - - private val preferences: SharedPreferences = getPreferences { - this.getString(DEFAULT_BASE_URL_PREF, null).let { domain -> - if (domain != defaultBaseUrl) { - this.edit() - .putString(BASE_URL_PREF, defaultBaseUrl) - .putString(DEFAULT_BASE_URL_PREF, defaultBaseUrl) - .apply() - } - } - } - - override val client by lazy { - val client = network.client.newBuilder() -// .rateLimit(1, 2.seconds) { it.host == fetchedDomainUrlHost } -// .rateLimit(2, 1.seconds) { it.host == apiBaseUrlHost } - .build() - - return@lazy client - } - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - @Volatile - private var seriesList: List = emptyList() - - @Volatile - private var lastFetchTime: Long = 0L - - @Synchronized - private fun fetchSeriesList() { - val now = System.currentTimeMillis() - - if (seriesList.isNotEmpty() && (now - lastFetchTime) < CACHE_DURATION_MS) { - return - } - - val result = client.newCall(GET("$baseUrl/api/series/list")).execute() - if (!result.isSuccessful) { - throw Exception("Failed to fetch series list: HTTP ${result.code}") - } - - val series = result.parseAs() - - val comics = series.data.asSequence() - .filter { it.type == "comic" } - .toList() - - seriesList = comics - lastFetchTime = now - - val newSlugMap = comics.associate { it.id to it.slug } - - preferences.slugMap += newSlugMap - } - - override fun fetchPopularManga(page: Int): Observable { - fetchSeriesList() - return super.fetchPopularManga(page) - } - - override fun popularMangaRequest(page: Int): Request { - val apiUrl = "$baseUrl/api/rankings?page=$page&period=total_ranking".toHttpUrl().newBuilder() - .build() - return GET(apiUrl, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val result = response.parseAs() - val slugMap = preferences.slugMap.toMutableMap() - val mangaList = result.data - .filter { it.type == "comic" } - .map { - slugMap[it.id] = it.slug - it.toSManga() - } - preferences.slugMap = slugMap - return MangasPage(mangaList, hasNextPage = result.hasNextPage()) - } - - override fun fetchLatestUpdates(page: Int): Observable { - fetchSeriesList() - return super.fetchLatestUpdates(page) - } - - override fun latestUpdatesRequest(page: Int): Request { - val apiUrl = "$baseUrl/api/new-chapters?page=$page".toHttpUrl().newBuilder() - .build() - return GET(apiUrl, headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val result = response.parseAs() - val slugMap = preferences.slugMap.toMutableMap() - val mangaList = result.data.filter { it.type == "comic" } - .map { - slugMap[it.id] = it.slug - it.toSManga() - } - preferences.slugMap = slugMap - return MangasPage(mangaList, result.hasNextPage()) - } - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - fetchSeriesList() - return Observable.just(parseSearchManga(page, query)) - } - - private fun parseSearchManga(page: Int, query: String): MangasPage { - val filteredList = seriesList.filter { it.name.contains(query, ignoreCase = true) } - val paginatedList = filteredList.drop((page - 1) * 20).take(20) - val hasNextPage = page * 20 < filteredList.size - return MangasPage(paginatedList.map { it.toSManga() }, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList) = throw Exception("La búsqueda no es compatible.") - - override fun searchMangaParse(response: Response) = throw Exception("La búsqueda no es compatible.") - - override fun getMangaUrl(manga: SManga): String { - val slug = preferences.slugMap[manga.url.toInt()]!! - return "$baseUrl/series/comic-$slug" - } - - override fun fetchMangaDetails(manga: SManga): Observable { - fetchSeriesList() - return super.fetchMangaDetails(manga) - } - - override fun mangaDetailsRequest(manga: SManga): Request { - val slug = preferences.slugMap[manga.url.toInt()]!! - - val apiUrl = "$baseUrl/api/series/$slug?type=comic" - return GET(url = apiUrl, headers = headers) - } - - override fun mangaDetailsParse(response: Response): SManga { - val result = response.parseAs() - return result.data.toSMangaDetails() - } - - override fun getChapterUrl(chapter: SChapter): String { - val mangaId = chapter.url.substringBefore("/") - val chapterId = chapter.url.substringAfter("/") - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - return "$baseUrl/capitulo/$chapterId/comic-$mangaSlug" - } - - override fun fetchChapterList(manga: SManga): Observable> { - fetchSeriesList() - return super.fetchChapterList(manga) - } - - override fun chapterListRequest(manga: SManga): Request { - val mangaId = manga.url - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - - return paginatedChapterListRequest(mangaSlug, mangaId, 1) - } - - private fun paginatedChapterListRequest(mangaSlug: String, mangaId: String, page: Int): Request = GET( - url = "$apiBaseUrl/api/series/$mangaSlug/chapters?page=$page&direction=desc&type=comic#$mangaId", - headers = headers, - ) - - override fun chapterListParse(response: Response): List { - val mangaId = response.request.url.fragment ?: "" - val slug = response.request.url.toString() - .substringAfter("/series/") - .substringBefore("/chapters") - - val data = response.parseAs() - var resultSize = data.data.size - var page = 2 - while (data.meta.total > resultSize) { - val newRequest = paginatedChapterListRequest(slug, mangaId, page) - val newResponse = client.newCall(newRequest).execute() - val newData = newResponse.parseAs() - data.data += newData.data - resultSize += newData.data.size - page += 1 - } - return data.data.map { it.toSChapter(mangaId, dateFormat) }.sortedBy { it.chapter_number } - } - - override fun pageListRequest(chapter: SChapter): Request { - val mangaId = chapter.url.substringBefore("/") - val chapterId = chapter.url.substringAfter("/") - val mangaSlug = preferences.slugMap[mangaId.toInt()]!! - - return GET("$baseUrl/api/capitulo/comic-$mangaSlug/$chapterId") - } - - override fun pageListParse(response: Response): List = response.parseAs().chapter.pages.mapIndexed { i, img -> - Page(i, imageUrl = img) - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = FETCH_DOMAIN_PREF - title = "Buscar dominio automáticamente" - summary = "Intenta buscar el dominio automáticamente al abrir la fuente." - setDefaultValue(FETCH_DOMAIN_PREF_DEFAULT) - }.also { screen.addPreference(it) } - - EditTextPreference(screen.context).apply { - key = BASE_URL_PREF - title = "Editar URL de la fuente" - summary = "Para uso temporal, si la extensión se actualiza se perderá el cambio." - dialogTitle = "Editar URL de la fuente" - setDefaultValue(defaultBaseUrl) - }.also { screen.addPreference(it) } - } - - private var cachedBaseUrl: String? = null - private var SharedPreferences.prefBaseUrl: String - get() { - if (cachedBaseUrl == null) { - cachedBaseUrl = getString(BASE_URL_PREF, defaultBaseUrl)!! - } - return cachedBaseUrl!! - } - set(value) { - cachedBaseUrl = value - edit().putString(BASE_URL_PREF, value).apply() - } - - private fun SharedPreferences.fetchDomainPref() = getBoolean(FETCH_DOMAIN_PREF, FETCH_DOMAIN_PREF_DEFAULT) - - private var slugMapCache: Map? = null - private var SharedPreferences.slugMap: Map - get() { - slugMapCache?.let { return it } - val json = getString(SLUG_MAP, "{}")!! - slugMapCache = try { - json.parseAs>() - } catch (_: SerializationException) { - emptyMap() - } - return slugMapCache!! - } - set(map) { - slugMapCache = map - edit().putString(SLUG_MAP, map.toJsonString()).apply() - } - - companion object { - private const val BASE_URL_PREF = "overrideBaseUrl" - private const val DEFAULT_BASE_URL_PREF = "defaultBaseUrl" - - private const val FETCH_DOMAIN_PREF = "fetchDomain" - private const val FETCH_DOMAIN_PREF_DEFAULT = true - - private const val SLUG_MAP = "slugMap" - - private const val CACHE_DURATION_MS = 60 * 60 * 1000L // 1 hour - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt deleted file mode 100644 index 96fef581..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/es/submanhwa/SubManhwa.kt +++ /dev/null @@ -1,140 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.es.submanhwa - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.tryParse -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.TachiyomiSource -import java.text.SimpleDateFormat -import java.util.Locale - -@TachiyomiSource("SUBMANHWA", "Submanhwa", "es") -class Submanhwa : HttpSource() { - - override val name = "Submanhwa" - override val baseUrl = "https://submanhwa.com" - override val lang = "es" - override val supportsLatest = true - - private val dateFormat = SimpleDateFormat("dd MMM. yyyy", Locale.ENGLISH) - - override fun headersBuilder() = super.headersBuilder() - .add("Accept-Language", "es-PE,es;q=0.9,en-US;q=0.8,en;q=0.7") - .add("Referer", "$baseUrl/") - - override fun popularMangaRequest(page: Int): Request = GET( - baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("filterList") - addQueryParameter("page", page.toString()) - addQueryParameter("sortBy", "views") - addQueryParameter("asc", "false") - }.build(), - headers, - ) - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(".series-card").map { element -> - SManga.create().apply { - title = element.selectFirst(".series-title")!!.text() - setUrlWithoutDomain(element.selectFirst("a")!!.absUrl("href")) - thumbnail_url = element.selectFirst("img")!!.absUrl("src") - } - } - val hasNextPage = document.selectFirst("li a[rel=next]") != null - - return MangasPage(mangas, hasNextPage) - } - - override fun latestUpdatesRequest(page: Int): Request = GET(baseUrl, headers) - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select("div[class^=manga-item]").map { element -> - SManga.create().apply { - title = element.selectFirst("h3[class^=manga-title] a")!!.text() - setUrlWithoutDomain(element.selectFirst("a")!!.absUrl("href")) - thumbnail_url = element.selectFirst("img")!!.absUrl("src") - } - } - - return MangasPage(mangas, false) - } - - override fun searchMangaRequest( - page: Int, - query: String, - filters: FilterList, - ): Request = GET( - baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("filterList") - addQueryParameter("page", page.toString()) - addQueryParameter("sortBy", "views") - addQueryParameter("asc", "false") - addQueryParameter("alpha", query) - }.build(), - headers, - ) - - override fun searchMangaParse(response: Response): MangasPage = popularMangaParse(response) - - override fun mangaDetailsParse(response: Response): SManga { - val document = response.asJsoup() - return SManga.create().apply { - title = document.selectFirst(".manga-title-centered")!!.text() - thumbnail_url = document.selectFirst("img")?.absUrl("src") - description = document.selectFirst("h5:contains(Resumen) + p")?.text() - - val box = document.selectFirst(".main-content > .boxed-modern") - - status = when (box?.selectFirst(".detail-label:contains(Estado) + .detail-value span")?.text()?.lowercase()) { - "completa" -> SManga.COMPLETED - "en curso" -> SManga.ONGOING - else -> SManga.UNKNOWN - } - - author = box?.selectFirst(".detail-label:contains(Autor) + .detail-value a")?.text() - artist = box?.selectFirst(".detail-label:contains(Artist) + .detail-value a")?.text() - genre = box?.select(".detail-label:contains(Categor) + .detail-value a")?.joinToString { it.text() } - } - } - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - return document.select(".chapters-grid [class^=chapter-card]").map { element -> - SChapter.create().apply { - val a = element.selectFirst("a.chapter-link")!! - name = a.text() - setUrlWithoutDomain(a.absUrl("href")) - - val date = element.selectFirst("span:has(i.glyphicon-time)")?.text() - ?: element.selectFirst(".chapter-preview-meta > span")?.text() - - date_upload = dateFormat.tryParse(date) - } - } - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - return document.select("#all img").mapIndexed { idx, img -> - Page(idx, imageUrl = img.imgAttr()) - } - } - - private fun Element.imgAttr(): String = when { - hasAttr("data-src") -> attr("abs:data-src") - else -> attr("abs:src") - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt deleted file mode 100644 index 1ec2ad2c..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Keyoapp.kt +++ /dev/null @@ -1,391 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers - -import android.content.SharedPreferences -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.lib.i18n.Intl -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import keiyoushi.utils.tryParse -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.Locale - -abstract class Keyoapp : - HttpSource(), - ConfigurableSource { - - protected val preferences: SharedPreferences by getPreferencesLazy() - - override val supportsLatest = true - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) - - protected val intl = Intl( - language = lang, - baseLanguage = "en", - availableLanguages = setOf("ar", "en", "fr"), - classLoader = this::class.java.classLoader!!, - ) - - // ============================== Popular ============================== - - override fun popularMangaRequest(page: Int): Request = GET(baseUrl, headers) - - open val popularMangaTitleSelector = listOf( - "Popular", - "Popularie", - "Trending", - ) - - open fun popularMangaSelector(): String = selector( - "div:contains(%s) + div .group.overflow-hidden.grid", - popularMangaTitleSelector, - ) - - open fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - thumbnail_url = element.getImageUrl("*[style*=background-image]") - element.selectFirst("a[href]")!!.run { - title = attr("title") - setUrlWithoutDomain(attr("abs:href")) - } - } - - open fun popularMangaNextPageSelector(): String? = null - - override fun popularMangaParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - val mangas = document.select(popularMangaSelector()).map { popularMangaFromElement(it) } - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) } != null - return MangasPage(mangas, hasNextPage) - } - - // ============================== Latest =============================== - - override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/latest/", headers) - - open fun latestUpdatesSelector(): String = "div.grid > div.group" - - open fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element) - - open fun latestUpdatesNextPageSelector(): String? = null - - override fun latestUpdatesParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - val mangas = document.select(latestUpdatesSelector()).map { latestUpdatesFromElement(it) } - val hasNextPage = latestUpdatesNextPageSelector()?.let { document.selectFirst(it) } != null - return MangasPage(mangas, hasNextPage) - } - - // ============================== Search =============================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("series") - addPathSegment("") - if (query.isNotBlank()) { - addQueryParameter("q", query) - } - filters.firstInstanceOrNull()?.also { filter -> - filter.state - .filter { it.state } - .forEach { genre -> - addQueryParameter("genre", genre.id) - } - } - }.build() - - return GET(url, headers) - } - - open fun searchMangaSelector() = "#searched_series_page > button" - - open fun searchMangaFromElement(element: Element): SManga = popularMangaFromElement(element) - - open fun searchMangaNextPageSelector(): String? = null - - override fun searchMangaParse(response: Response): MangasPage { - runCatching { fetchGenres() } - val document = response.asJsoup() - - val query = response.request.url.queryParameter("q") ?: "" - val genres = response.request.url.queryParameterValues("genre").filterNotNull() - - val mangaList = document.select(searchMangaSelector()) - .filter { it.attr("title").contains(query, true) } - .filter { entry -> - val entryGenres = runCatching { - entry.attr("tags").parseAs>() - }.getOrDefault(emptyList()) - genres.all { genre -> entryGenres.any { it.equals(genre, true) } } - } - .map(::searchMangaFromElement) - - return MangasPage(mangaList, false) - } - - // ============================== Filters ============================== - - /** - * Automatically fetched genres from the source to be used in the filters. - */ - private var genresList: List = emptyList() - - /** - * Inner variable to control the genre fetching failed state. - */ - private var fetchGenresFailed: Boolean = false - - /** - * Inner variable to control how much tries the genres request was called. - */ - private var fetchGenresAttempts: Int = 0 - - class Genre(name: String, val id: String = name) : Filter.CheckBox(name) - - protected class GenreList(title: String, genres: List) : Filter.Group(title, genres) - - override fun getFilterList(): FilterList = if (genresList.isNotEmpty()) { - FilterList( - GenreList("Genres", genresList), - ) - } else { - FilterList( - Filter.Header("Press 'Reset' to attempt to show the genres"), - ) - } - - /** - * Fetch the genres from the source to be used in the filters. - */ - protected open fun fetchGenres() { - if (fetchGenresAttempts <= 3 && (genresList.isEmpty() || fetchGenresFailed)) { - val genres = runCatching { - client.newCall(genresRequest()).execute() - .use { parseGenres(it.asJsoup()) } - } - - fetchGenresFailed = genres.isFailure - genresList = genres.getOrNull().orEmpty() - fetchGenresAttempts++ - } - } - - protected open fun genresRequest(): Request = GET("$baseUrl/series/", headers) - - /** - * Get the genres from the search page document. - * - * @param document The search page document - */ - protected open fun parseGenres(document: Document): List = document.select("#series_tags_page > button") - .map { btn -> - Genre(btn.text(), btn.attr("tag")) - } - - // ============================== Details ============================== - protected open val descriptionSelector: String = "div:containsOwn(Synopsis) ~ div" - protected open val statusSelector: String = "div:has(span:containsOwn(Status)) ~ div" - protected open val authorSelector: String = "div:has(span:containsOwn(Author)) ~ div" - protected open val artistSelector: String = "div:has(span:containsOwn(Artist)) ~ div" - protected open val genreSelector: String = "div.grid:has(>h1) > div > a:not([title='Status'])" - - protected open val typeSelector: String = "div:has(span:containsOwn(Type)) ~ div" - protected open val dateSelector: String = ".text-xs" - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - open fun mangaDetailsParse(document: Document): SManga = SManga.create().apply { - title = document.selectFirst("div.grid > h1")!!.text() - thumbnail_url = document.getImageUrl("div[class*=photoURL]") - description = document.selectFirst(descriptionSelector)?.text() - status = document.selectFirst(statusSelector).parseStatus() - author = document.selectFirst(authorSelector)?.text() - artist = document.selectFirst(artistSelector)?.text() - genre = buildList { - document.selectFirst(typeSelector)?.text()?.replaceFirstChar { - if (it.isLowerCase()) { - it.titlecase( - Locale.ENGLISH, - ) - } else { - it.toString() - } - }?.let(::add) - document.select(genreSelector).forEach { add(it.text()) } - }.joinToString() - } - - protected fun Element?.parseStatus(): Int = when (this?.text()?.lowercase()) { - "ongoing" -> SManga.ONGOING - "dropped" -> SManga.CANCELLED - "paused" -> SManga.ON_HIATUS - "completed" -> SManga.COMPLETED - else -> SManga.UNKNOWN - } - - // ============================= Chapters ============================== - protected open val paidChapterSelector: String = "img[alt~=Coin]" - - override fun chapterListParse(response: Response): List = response.asJsoup().select(chapterListSelector()).map { chapterFromElement(it) }.reversed() - - open fun chapterListSelector(): String { - if (!preferences.showPaidChapters) { - return "#chapters > a:not(:has(.text-sm span:matches(Upcoming))):not(:has($paidChapterSelector))" - } - return "#chapters > a:not(:has(.text-sm span:matches(Upcoming)))" - } - - open fun chapterFromElement(element: Element): SChapter = SChapter.create().apply { - setUrlWithoutDomain(element.selectFirst("a[href]")!!.attr("abs:href")) - name = element.selectFirst(".text-sm")!!.text() - element.selectFirst(dateSelector)?.run { - date_upload = text().trim().parseDate() - } - if (element.select(paidChapterSelector).isNotEmpty()) { - name = "🔒 $name" - } - } - - // =============================== Pages =============================== - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - open fun pageListParse(document: Document): List { - val cdnUrl = getCdnUrl(document) - document.select("#pages > img") - .map { it.attr("uid") } - .filter { it.isNotEmpty() } - .also { if (it.isNotEmpty() && cdnUrl == null) throw Exception(intl["chapter_page_url_not_found"]) } - .mapIndexed { index, img -> - Page(index, url = document.location(), imageUrl = "$cdnUrl/$img") - } - .takeIf { it.isNotEmpty() } - ?.also { return it } - - // Fallback, old method - return document.select("#pages > img") - .map { it.imgAttr() } - .filter { it.contains(oldImgCdnRegex) } - .mapIndexed { index, img -> - Page(index, url = document.location(), imageUrl = img) - } - } - - protected open fun getCdnUrl(document: Document): String? = document.select("script") - .firstOrNull { CDN_HOST_REGEX.containsMatchIn(it.html()) } - ?.let { - val cdnHost = CDN_HOST_REGEX.find(it.html()) - ?.groups?.get(1)?.value - ?.replace(CDN_CLEAN_REGEX, "") - "https://$cdnHost/uploads" - } - - private val oldImgCdnRegex = Regex("""^(https?:)?//cdn\d*\.keyoapp\.com""") - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ============================= Utilities ============================= - - // From mangathemesia - private fun Element.imgAttr(): String = when { - hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") - hasAttr("data-src") -> attr("abs:data-src") - else -> attr("abs:src") - } - - protected open fun Element.getImageUrl(selector: String): String? = this.selectFirst(selector)?.let { element -> - IMG_REGEX.find(element.attr("style"))?.groups?.get(1)?.value - ?.toHttpUrlOrNull()?.let { - it.newBuilder() - .setQueryParameter("w", "480") // Keyoapp returns the dynamic size of the thumbnail to any size - .build() - .toString() - } - } - - private fun String.parseDate(): Long = if (this.contains("ago")) { - this.parseRelativeDate() - } else { - dateFormat.tryParse(this) - } - - private fun String.parseRelativeDate(): Long { - val now = Calendar.getInstance().apply { - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - } - - val relativeDate = this.split(" ").firstOrNull() - ?.replace("one", "1") - ?.replace("a", "1") - ?.toIntOrNull() - ?: return 0L - - when { - "second" in this -> now.add(Calendar.SECOND, -relativeDate) - - // parse: 30 seconds ago - "minute" in this -> now.add(Calendar.MINUTE, -relativeDate) - - // parses: "42 minutes ago" - "hour" in this -> now.add(Calendar.HOUR, -relativeDate) - - // parses: "1 hour ago" and "2 hours ago" - "day" in this -> now.add(Calendar.DAY_OF_YEAR, -relativeDate) - - // parses: "2 days ago" - "week" in this -> now.add(Calendar.WEEK_OF_YEAR, -relativeDate) - - // parses: "2 weeks ago" - "month" in this -> now.add(Calendar.MONTH, -relativeDate) - - // parses: "2 months ago" - "year" in this -> now.add(Calendar.YEAR, -relativeDate) // parse: "2 years ago" - } - return now.timeInMillis - } - - private fun selector(selector: String, contains: List): String = contains.joinToString { selector.replace("%s", it) } - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_PAID_CHAPTERS_PREF - title = intl["pref_show_paid_chapter_title"] - setDefaultValue(SHOW_PAID_CHAPTERS_DEFAULT) - }.also(screen::addPreference) - } - - protected val SharedPreferences.showPaidChapters: Boolean - get() = getBoolean(SHOW_PAID_CHAPTERS_PREF, SHOW_PAID_CHAPTERS_DEFAULT) - - companion object { - private const val SHOW_PAID_CHAPTERS_PREF = "pref_show_paid_chap" - private const val SHOW_PAID_CHAPTERS_DEFAULT = false - val CDN_HOST_REGEX = """realUrl\s*=\s*`[^`]+//([^/]+)""".toRegex() - val CDN_CLEAN_REGEX = """\$\{[^}]*\}""".toRegex() - val IMG_REGEX = """url\(['"]?([^(['")])]+)""".toRegex() - } -} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt deleted file mode 100644 index a7941e41..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/Madara.kt +++ /dev/null @@ -1,1179 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers - -import android.util.Base64 -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.lib.cryptoaes.CryptoAES -import keiyoushi.lib.i18n.Intl -import keiyoushi.utils.decodeHex -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.jsonArray -import kotlinx.serialization.json.jsonObject -import kotlinx.serialization.json.jsonPrimitive -import okhttp3.FormBody -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import rx.Observable -import uy.kohesive.injekt.injectLazy -import java.text.ParseException -import java.text.SimpleDateFormat -import java.util.Calendar -import java.util.Locale - -abstract class Madara( - override val name: String, - override val baseUrl: String, - final override val lang: String, - protected val dateFormat: SimpleDateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US), -) : HttpSource() { - - override val supportsLatest = true - - override val client = network.cloudflareClient - - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - protected val xhrHeaders by lazy { - headersBuilder() - .set("X-Requested-With", "XMLHttpRequest") - .build() - } - - protected open val json: Json by injectLazy() - - protected val intl = Intl( - language = lang, - baseLanguage = "en", - availableLanguages = setOf("en", "pt-BR", "es"), - classLoader = this::class.java.classLoader!!, - ) - - /** - * If enabled, will attempt to remove non-manga items in popular and latest. - * The filter will not be used in search as the theme doesn't set the CSS class. - * Can be disabled if the source incorrectly sets the entry types. - */ - protected open val filterNonMangaItems = true - - /** - * The CSS selector used to filter manga items in popular and latest - * if `filterNonMangaItems` is set to `true`. Can be override if needed. - * If the flag is set to `false`, it will be empty by default. - */ - protected open val mangaEntrySelector: String by lazy { - if (filterNonMangaItems) ".manga" else "" - } - - /** - * Automatically fetched genres from the source to be used in the filters. - */ - protected open var genresList: List = emptyList() - - /** - * Whether genres have been fetched - */ - private var genresFetched: Boolean = false - - /** - * Inner variable to control how much tries the genres request was called. - */ - private var fetchGenresAttempts: Int = 0 - - /** - * Disable it if you don't want the genres to be fetched. - */ - protected open val fetchGenres: Boolean = true - - /** - * The path used in the URL for the manga pages. Can be - * changed if needed as some sites modify it to other words. - */ - protected open val mangaSubString = "manga" - - /** - * enable if the site use "madara_load_more" to load manga on the site - * Typically has "load More" instead of next/previous page - * - * with LoadMoreStrategy.AutoDetect it tries to detect if site uses `madara_load_more` - */ - protected open val useLoadMoreRequest = LoadMoreStrategy.AutoDetect - - enum class LoadMoreStrategy { - AutoDetect, - Always, - Never, - } - - /** - * internal variable to save if site uses load_more or not - */ - private var loadMoreRequestDetected = LoadMoreDetection.Pending - - private enum class LoadMoreDetection { - Pending, - True, - False, - } - - protected fun detectLoadMore(document: Document) { - if (useLoadMoreRequest == LoadMoreStrategy.AutoDetect && - loadMoreRequestDetected == LoadMoreDetection.Pending - ) { - loadMoreRequestDetected = when (document.selectFirst("nav.navigation-ajax") != null) { - true -> LoadMoreDetection.True - false -> LoadMoreDetection.False - } - } - } - - protected fun useLoadMoreRequest(): Boolean = when (useLoadMoreRequest) { - LoadMoreStrategy.Always -> true - LoadMoreStrategy.Never -> false - else -> loadMoreRequestDetected == LoadMoreDetection.True - } - - // Popular Manga - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val entries = document.select(popularMangaSelector()) - .map(::popularMangaFromElement) - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) } != null - - detectLoadMore(document) - - return MangasPage(entries, hasNextPage) - } - - // exclude/filter bilibili manga from list - protected open fun popularMangaSelector() = "div.page-item-detail:not(:has(a[href*='bilibilicomics.com']))$mangaEntrySelector , .manga__item" - - open val popularMangaUrlSelector = "div.post-title a" - open val popularMangaUrlSelectorImg = "img" - - protected open fun popularMangaFromElement(element: Element): SManga { - val manga = SManga.create() - - with(element) { - selectFirst(popularMangaUrlSelector)!!.let { - manga.setUrlWithoutDomain(it.attr("abs:href")) - manga.title = it.ownText() - } - - selectFirst(popularMangaUrlSelectorImg)?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - - return manga - } - - override fun popularMangaRequest(page: Int): Request = if (useLoadMoreRequest()) { - loadMoreRequest(page, popular = true) - } else { - GET("$baseUrl/$mangaSubString/${searchPage(page)}?m_orderby=views", headers) - } - - protected open fun popularMangaNextPageSelector(): String? = if (useLoadMoreRequest()) { - "body:not(:has(.no-posts))" - } else { - "div.nav-previous, nav.navigation-ajax, a.nextpostslink" - } - - // Related Manga - protected open fun relatedMangaSelector() = ".related-reading-wrap" - - override fun relatedMangaListParse(response: Response): List { - val document = response.asJsoup() - return document.select(relatedMangaSelector()) - .mapNotNull { manga -> - SManga.create().apply { - manga.selectFirst(".widget-title a")?.let { - setUrlWithoutDomain(it.attr("abs:href")) - title = it.ownText() - } ?: return@mapNotNull null - manga.selectFirst(".widget-thumbnail img")?.let { - thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - } - } - - // Latest Updates - - protected open fun latestUpdatesSelector() = popularMangaSelector() - - protected open fun latestUpdatesFromElement(element: Element): SManga { - // Even if it's different from the popular manga's list, the relevant classes are the same - return popularMangaFromElement(element) - } - - override fun latestUpdatesRequest(page: Int): Request = if (useLoadMoreRequest()) { - loadMoreRequest(page, popular = false) - } else { - GET("$baseUrl/$mangaSubString/${searchPage(page)}?m_orderby=latest", headers) - } - - protected open fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector() - - override fun latestUpdatesParse(response: Response): MangasPage { - val mp = popularMangaParse(response) - val mangas = mp.mangas.distinctBy { it.url } - return MangasPage(mangas, mp.hasNextPage) - } - - // load more - protected fun loadMoreRequest(page: Int, popular: Boolean): Request { - val formBody = FormBody.Builder().apply { - add("action", "madara_load_more") - add("page", (page - 1).toString()) - add("template", "madara-core/content/content-archive") - add("vars[orderby]", "meta_value_num") - add("vars[paged]", "1") - add("vars[post_type]", "wp-manga") - add("vars[post_status]", "publish") - add("vars[meta_key]", if (popular) "_wp_manga_views" else "_latest_update") - add("vars[order]", "desc") - add("vars[sidebar]", "right") - add("vars[manga_archives_item_layout]", "big_thumbnail") - }.build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, formBody) - } - - // Search Manga - - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (query.startsWith("https://")) { - val url = query.toHttpUrl() - if (url.host != baseUrl.toHttpUrl().host) { - throw Exception("Unsupported url") - } - if (url.pathSegments.size < 2) { - throw Exception("Unsupported url") - } - val slug = url.pathSegments[1] - return fetchSearchManga(page, "$URL_SEARCH_PREFIX$slug", filters) - } - if (query.startsWith(URL_SEARCH_PREFIX)) { - val mangaUrl = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment(mangaSubString) - addPathSegment(query.substringAfter(URL_SEARCH_PREFIX)) - addPathSegment("") // add trailing slash - }.build() - return client.newCall(GET(mangaUrl, headers)) - .asObservableSuccess().map { response -> - val manga = mangaDetailsParse(response).apply { - setUrlWithoutDomain(mangaUrl.toString()) - initialized = true - } - - MangasPage(listOf(manga), false) - } - } - - return super.fetchSearchManga(page, query, filters) - } - - protected open fun searchPage(page: Int): String = if (page == 1) { - "" - } else { - "page/$page/" - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = if (useLoadMoreRequest()) { - searchLoadMoreRequest(page, query, filters) - } else { - searchRequest(page, query, filters) - } - - protected open fun searchRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/${searchPage(page)}".toHttpUrl().newBuilder() - url.addQueryParameter("s", query) - url.addQueryParameter("post_type", "wp-manga") - filters.forEach { filter -> - when (filter) { - is AuthorFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("author", filter.state) - } - } - - is ArtistFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("artist", filter.state) - } - } - - is YearFilter -> { - if (filter.state.isNotBlank()) { - url.addQueryParameter("release", filter.state) - } - } - - is StatusFilter -> { - filter.state.forEach { - if (it.state) { - url.addQueryParameter("status[]", it.id) - } - } - } - - is OrderByFilter -> { - if (filter.state != 0) { - url.addQueryParameter("m_orderby", filter.toUriPart()) - } - } - - is AdultContentFilter -> { - url.addQueryParameter("adult", filter.toUriPart()) - } - - is GenreConditionFilter -> { - url.addQueryParameter("op", filter.toUriPart()) - } - - is GenreList -> { - filter.state - .filter { it.state } - .let { list -> - if (list.isNotEmpty()) { - list.forEach { genre -> url.addQueryParameter("genre[]", genre.id) } - } - } - } - - else -> {} - } - } - return GET(url.build(), headers) - } - - protected open fun searchLoadMoreRequest(page: Int, query: String, filters: FilterList): Request { - val formBody = FormBody.Builder().apply { - add("action", "madara_load_more") - add("page", (page - 1).toString()) - add("template", "madara-core/content/content-search") - add("vars[paged]", "1") - add("vars[template]", "archive") - add("vars[sidebar]", "right") - add("vars[post_type]", "wp-manga") - add("vars[post_status]", "publish") - add("vars[manga_archives_item_layout]", "big_thumbnail") - - if (filterNonMangaItems) { - add("vars[meta_query][0][key]", "_wp_manga_chapter_type") - add("vars[meta_query][0][value]", "manga") - } - - add("vars[s]", query) - - var metaQueryIdx = if (filterNonMangaItems) 1 else 0 - var taxQueryIdx = 0 - val genres = filters.filterIsInstance().firstOrNull()?.state - ?.filter { it.state } - ?.map { it.id } - .orEmpty() - - filters.forEach { filter -> - when (filter) { - is AuthorFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-author") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is ArtistFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-artist") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is YearFilter -> { - if (filter.state.isNotBlank()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-release") - add("vars[tax_query][$taxQueryIdx][field]", "name") - add("vars[tax_query][$taxQueryIdx][terms]", filter.state) - - taxQueryIdx++ - } - } - - is StatusFilter -> { - val statuses = filter.state - .filter { it.state } - .map { it.id } - - if (statuses.isNotEmpty()) { - add("vars[meta_query][$metaQueryIdx][key]", "_wp_manga_status") - - statuses.forEachIndexed { i, slug -> - add("vars[meta_query][$metaQueryIdx][value][$i]", slug) - } - - metaQueryIdx++ - } - } - - is OrderByFilter -> { - if (filter.state != 0) { - when (filter.toUriPart()) { - "latest" -> { - add("vars[orderby]", "meta_value_num") - add("vars[order]", "DESC") - add("vars[meta_key]", "_latest_update") - } - - "alphabet" -> { - add("vars[orderby]", "post_title") - add("vars[order]", "ASC") - } - - "rating" -> { - add("vars[orderby][query_average_reviews]", "DESC") - add("vars[orderby][query_total_reviews]", "DESC") - } - - "trending" -> { - add("vars[orderby]", "meta_value_num") - add("vars[meta_key]", "_wp_manga_week_views_value") - add("vars[order]", "DESC") - } - - "views" -> { - add("vars[orderby]", "meta_value_num") - add("vars[meta_key]", "_wp_manga_views") - add("vars[order]", "DESC") - } - - else -> { - add("vars[orderby]", "date") - add("vars[order]", "DESC") - } - } - } - } - - is AdultContentFilter -> { - if (filter.state != 0) { - add("vars[meta_query][$metaQueryIdx][key]", "manga_adult_content") - add( - "vars[meta_query][$metaQueryIdx][compare]", - if (filter.state == 1) "not exists" else "exists", - ) - - metaQueryIdx++ - } - } - - is GenreConditionFilter -> { - if (filter.state == 1 && genres.isNotEmpty()) { - add("vars[tax_query][$taxQueryIdx][operation]", "AND") - } - } - - is GenreList -> { - if (genres.isNotEmpty()) { - add("vars[tax_query][$taxQueryIdx][taxonomy]", "wp-manga-genre") - add("vars[tax_query][$taxQueryIdx][field]", "slug") - - genres.forEachIndexed { i, slug -> - add("vars[tax_query][$taxQueryIdx][terms][$i]", slug) - } - - taxQueryIdx++ - } - } - - else -> {} - } - } - }.build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, formBody) - } - - protected open val statusFilterOptions: Map = - mapOf( - intl["status_filter_completed"] to "end", - intl["status_filter_ongoing"] to "on-going", - intl["status_filter_canceled"] to "canceled", - intl["status_filter_on_hold"] to "on-hold", - ) - - protected open val orderByFilterOptions: Map = mapOf( - intl["order_by_filter_relevance"] to "", - intl["order_by_filter_latest"] to "latest", - intl["order_by_filter_az"] to "alphabet", - intl["order_by_filter_rating"] to "rating", - intl["order_by_filter_trending"] to "trending", - intl["order_by_filter_views"] to "views", - intl["order_by_filter_new"] to "new-manga", - ) - - protected open val genreConditionFilterOptions: Map = - mapOf( - intl["genre_condition_filter_or"] to "", - intl["genre_condition_filter_and"] to "1", - ) - - protected open val adultContentFilterOptions: Map = - mapOf( - intl["adult_content_filter_all"] to "", - intl["adult_content_filter_none"] to "0", - intl["adult_content_filter_only"] to "1", - ) - - open class UriPartFilter(displayName: String, private val vals: Array>, state: Int = 0) : Filter.Select(displayName, vals.map { it.first }.toTypedArray(), state) { - fun toUriPart() = vals[state].second - } - - open class Tag(name: String, val id: String) : Filter.CheckBox(name) - - protected class AuthorFilter(title: String) : Filter.Text(title) - protected class ArtistFilter(title: String) : Filter.Text(title) - protected class YearFilter(title: String) : Filter.Text(title) - protected class StatusFilter(title: String, status: List) : Filter.Group(title, status) - - protected class OrderByFilter(title: String, options: List>, state: Int = 0) : UriPartFilter(title, options.toTypedArray(), state) - - protected class GenreConditionFilter(title: String, options: List>) : - UriPartFilter( - title, - options.toTypedArray(), - ) - - protected class AdultContentFilter(title: String, options: List>) : - UriPartFilter( - title, - options.toTypedArray(), - ) - - protected class GenreList(title: String, genres: List) : Filter.Group(title, genres.map { GenreCheckBox(it.name, it.id) }) - class GenreCheckBox(name: String, val id: String = name) : Filter.CheckBox(name) - class Genre(val name: String, val id: String = name) - - override fun getFilterList(): FilterList { - launchIO { fetchGenres() } - - val filters = mutableListOf( - AuthorFilter(intl["author_filter_title"]), - ArtistFilter(intl["artist_filter_title"]), - YearFilter(intl["year_filter_title"]), - StatusFilter( - title = intl["status_filter_title"], - status = statusFilterOptions.map { Tag(it.key, it.value) }, - ), - OrderByFilter( - title = intl["order_by_filter_title"], - options = orderByFilterOptions.toList(), - state = 0, - ), - AdultContentFilter( - title = intl["adult_content_filter_title"], - options = adultContentFilterOptions.toList(), - ), - ) - - if (genresList.isNotEmpty()) { - filters += listOf( - Filter.Separator(), - Filter.Header(intl["genre_filter_header"]), - GenreConditionFilter( - title = intl["genre_condition_filter_title"], - options = genreConditionFilterOptions.toList(), - ), - GenreList( - title = intl["genre_filter_title"], - genres = genresList, - ), - ) - } else if (fetchGenres) { - filters += listOf( - Filter.Separator(), - Filter.Header(intl["genre_missing_warning"]), - ) - } - - return FilterList(filters) - } - - override fun searchMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - - val entries = document.select(searchMangaSelector()) - .map(::searchMangaFromElement) - val hasNextPage = searchMangaNextPageSelector()?.let { document.selectFirst(it) } != null - - detectLoadMore(document) - - return MangasPage(entries, hasNextPage) - } - - protected open fun searchMangaSelector() = "div.c-tabs-item__content , .manga__item" - - protected open val searchMangaUrlSelector = "div.post-title a" - - protected open fun searchMangaFromElement(element: Element): SManga { - val manga = SManga.create() - - with(element) { - selectFirst(searchMangaUrlSelector)!!.let { - manga.setUrlWithoutDomain(it.attr("abs:href")) - manga.title = it.ownText() - } - selectFirst("img")?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it), true) - } - } - - return manga - } - - protected open fun searchMangaNextPageSelector() = popularMangaNextPageSelector() - - // Manga Details Parse - - protected val completedStatusList: Array = arrayOf( - "Completed", - "Completo", - "Completado", - "Concluído", - "Concluido", - "Finalizado", - "Achevé", - "Terminé", - "Hoàn Thành", - "مكتملة", - "مكتمل", - "已完结", - "Tamamlandı", - "Äã hoàn thành", - "Завершено", - "Tamamlanan", - "Complété", - ) - - protected val ongoingStatusList: Array = arrayOf( - "OnGoing", "ПродолжаетÑÑ", "Updating", "Em Lançamento", "Em lançamento", "Em andamento", - "Em Andamento", "En cours", "En Cours", "En cours de publication", "Ativo", "Lançando", "Äang Tiến Hành", "Còn Nữa", "Devam Ediyor", - "Devam ediyor", "In Corso", "In Arrivo", "مستمرة", "مستمر", "En Curso", "En curso", "Emision", - "Curso", "En marcha", "Publicandose", "Publicándose", "En emision", "连载中", "Em Lançamento", "Devam Ediyo", - "Äang làm", "Em postagem", "Devam Eden", "Em progresso", "Em curso", "Atualizações Semanais", - ) - - protected val hiatusStatusList: Array = arrayOf( - "On Hold", - "Pausado", - "En espera", - "Durduruldu", - "Beklemede", - "Äang chá»", - "متوقÙ", - "En Pause", - "Заморожено", - "En attente", - ) - - protected val canceledStatusList: Array = arrayOf( - "Canceled", - "Cancelado", - "Ä°ptal Edildi", - "Güncel", - "Äã hủy", - "ملغي", - "Abandonné", - "Заброшено", - "Annulé", - ) - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - protected open fun mangaDetailsParse(document: Document): SManga { - val manga = SManga.create() - with(document) { - manga.title = selectFirst(mangaDetailsSelectorTitle)!!.ownText() - select(mangaDetailsSelectorAuthor).eachText().filter { - it.notUpdating() - }.joinToString().takeIf { it.isNotBlank() }?.let { - manga.author = it - } - select(mangaDetailsSelectorArtist).eachText().filter { - it.notUpdating() - }.joinToString().takeIf { it.isNotBlank() }?.let { - manga.artist = it - } - select(mangaDetailsSelectorDescription).let { - if (it.select("p").text().isNotEmpty()) { - manga.description = it.select("p").joinToString(separator = "\n\n") { p -> - p.text().replace("
", "\n") - } - } else { - manga.description = it.text() - } - } - selectFirst(mangaDetailsSelectorThumbnail)?.let { - manga.thumbnail_url = processThumbnail(imageFromElement(it)) - } - select(mangaDetailsSelectorStatus).last()?.let { - manga.status = with(it.text().filter { ch -> ch.isLetterOrDigit() || ch.isWhitespace() }.trim()) { - when { - containsIn(completedStatusList) -> SManga.COMPLETED - containsIn(ongoingStatusList) -> SManga.ONGOING - containsIn(hiatusStatusList) -> SManga.ON_HIATUS - containsIn(canceledStatusList) -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - } - val genres = select(mangaDetailsSelectorGenre) - .mapTo(ArrayList()) { element -> element.text() } - - if (mangaDetailsSelectorTag.isNotEmpty()) { - select(mangaDetailsSelectorTag).forEach { element -> - if ( - element.text().length <= 25 && - element.text().contains("read", true).not() && - element.text().contains(name, true).not() && - element.text().contains(name.replace(" ", ""), true).not() && - element.text().contains(manga.title, true).not() && - element.text().contains(altName, true).not() - ) { - genres.add(element.text()) - } - } - } - - // add manga/manhwa/manhua thinggy to genre - document.selectFirst(seriesTypeSelector)?.ownText()?.let { - if (it.isEmpty().not() && it.notUpdating() && it != "-") { - genres.add(it) - } - } - - manga.genre = genres.distinctBy(String::lowercase).joinToString() - - // add alternative name to manga description - document.selectFirst(altNameSelector)?.ownText()?.let { - if (it.isBlank().not() && it.notUpdating()) { - manga.description = when { - manga.description.isNullOrBlank() -> "$altName $it" - else -> "${manga.description}\n\n$altName $it" - } - } - } - } - - return manga - } - - // Manga Details Selector - open val mangaDetailsSelectorTitle = "div.post-title h3, div.post-title h1, #manga-title > h1" - open val mangaDetailsSelectorAuthor = "div.author-content > a, div.manga-authors > a" - open val mangaDetailsSelectorArtist = "div.artist-content > a" - open val mangaDetailsSelectorStatus = "div.summary-content, div.summary-heading:contains(Status) + div" - open val mangaDetailsSelectorDescription = "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt" - open val mangaDetailsSelectorThumbnail = "div.summary_image img" - open val mangaDetailsSelectorGenre = "div.genres-content a" - open val mangaDetailsSelectorTag = "div.tags-content a" - - open val seriesTypeSelector = ".post-content_item:contains(Type) .summary-content" - open val altNameSelector = ".post-content_item:contains(Alt) .summary-content" - open val altName = intl["alt_names_heading"] - open val updatingRegex = "Updating|Atualizando".toRegex(RegexOption.IGNORE_CASE) - - fun String.notUpdating(): Boolean = this.contains(updatingRegex).not() - - private fun String.containsIn(array: Array): Boolean = this.lowercase() in array.map { it.lowercase() } - - protected open fun imageFromElement(element: Element): String? = when { - element.hasAttr("data-src") -> element.attr("abs:data-src") - element.hasAttr("data-lazy-src") -> element.attr("abs:data-lazy-src") - element.hasAttr("srcset") -> element.attr("abs:srcset").getSrcSetImage() - element.hasAttr("data-cfsrc") -> element.attr("abs:data-cfsrc") - element.hasAttr("data-manga-src") -> element.attr("abs:data-manga-src") - else -> element.attr("abs:src") - } - - /** - * Get the best image quality available from srcset - */ - protected open fun String.getSrcSetImage(): String? = this.split(" ") - .filter(URL_REGEX::matches) - .maxOfOrNull(String::toString) - - /** - * Apply any additional processing to the thumbnail URL if needed. - */ - protected open fun processThumbnail(url: String?, fromSearch: Boolean = false): String? = url - - /** - * Set it to true if the source uses the new AJAX endpoint to - * fetch the manga chapters instead of the old admin-ajax.php one. - */ - protected open val useNewChapterEndpoint: Boolean = false - - /** - * Internal attribute to control if it should always use the - * new chapter endpoint after a first check if useNewChapterEndpoint is - * set to false. Using a separate variable to still allow the other - * one to be overridable manually in each source. - */ - private var oldChapterEndpointDisabled: Boolean = false - - protected open fun oldXhrChaptersRequest(mangaId: String): Request { - val form = FormBody.Builder() - .add("action", "manga_get_chapters") - .add("manga", mangaId) - .build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", xhrHeaders, form) - } - - protected open fun xhrChaptersRequest(mangaUrl: String): Request = POST("$mangaUrl/ajax/chapters", xhrHeaders) - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - - launchIO { countViews(document) } - - val chaptersWrapper = document.select("div[id^=manga-chapters-holder]") - - var chapterElements = document.select(chapterListSelector()) - - if (chapterElements.isEmpty() && !chaptersWrapper.isNullOrEmpty()) { - val mangaUrl = document.location().removeSuffix("/") - val mangaId = chaptersWrapper.attr("data-id") - - var xhrRequest = if (useNewChapterEndpoint || oldChapterEndpointDisabled) { - xhrChaptersRequest(mangaUrl) - } else { - oldXhrChaptersRequest(mangaId) - } - var xhrResponse = client.newCall(xhrRequest).execute() - - // Newer Madara versions throws HTTP 400 when using the old endpoint. - if (!useNewChapterEndpoint && xhrResponse.code == 400) { - xhrResponse.close() - // Set it to true so following calls will be made directly to the new endpoint. - oldChapterEndpointDisabled = true - - xhrRequest = xhrChaptersRequest(mangaUrl) - xhrResponse = client.newCall(xhrRequest).execute() - } - - chapterElements = xhrResponse.asJsoup().select(chapterListSelector()) - xhrResponse.close() - } - - return chapterElements.map(::chapterFromElement) - } - - protected open fun chapterListSelector() = "li.wp-manga-chapter" - - protected open fun chapterDateSelector() = "span.chapter-release-date" - - open val chapterUrlSelector = "a" - - // can cause some issue for some site. blocked by cloudflare when opening the chapter pages - open val chapterUrlSuffix = "?style=list" - - protected open fun chapterFromElement(element: Element): SChapter { - val chapter = SChapter.create() - - with(element) { - selectFirst(chapterUrlSelector)!!.let { urlElement -> - chapter.url = urlElement.attr("abs:href").let { - it.substringBefore("?style=paged") + if (!it.endsWith(chapterUrlSuffix)) chapterUrlSuffix else "" - } - chapter.name = urlElement.text() - } - // Dates can be part of a "new" graphic or plain text - // Added "title" alternative - chapter.date_upload = selectFirst("img:not(.thumb)")?.attr("alt")?.let { parseRelativeDate(it) } - ?: selectFirst("span a")?.attr("title")?.let { parseRelativeDate(it) } - ?: parseChapterDate(selectFirst(chapterDateSelector())?.text()) - } - - return chapter - } - - open fun parseChapterDate(date: String?): Long { - date ?: return 0 - - fun SimpleDateFormat.tryParse(string: String): Long = try { - parse(string)?.time ?: 0 - } catch (_: ParseException) { - 0 - } - - return when { - // Handle 'yesterday' and 'today', using midnight - WordSet("yesterday", "يوم واحد").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("today").startsWith(date) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("يومين").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -2) // day before yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("ago", "atrás", "önce", "قبل", "trÆ°á»›c").endsWith(date) -> { - parseRelativeDate(date) - } - - WordSet("hace", "năm", "tháng", "tuần", "ngày", "giá»", "phút", "giây").startsWith(date) -> { - parseRelativeDate(date) - } - - // Handle "jour" with a number before it - date.contains(Regex("""\b\d+ jour""")) -> { - parseRelativeDate(date) - } - - date.contains(Regex("""\d(st|nd|rd|th)""")) -> { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - date.split(" ").map { - if (it.contains(Regex("""\d\D\D"""))) { - it.replace(Regex("""\D"""), "") - } else { - it - } - } - .let { dateFormat.tryParse(it.joinToString(" ")) } - } - - else -> dateFormat.tryParse(date) - } - } - - // Parses dates in this form: - // 21 horas ago - protected open fun parseRelativeDate(date: String): Long { - val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 - val cal = Calendar.getInstance() - - return when { - WordSet("hari", "gün", "jour", "día", "dia", "day", "วัน", "ngày", "giorni", "أيام", "天").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("jam", "saat", "heure", "hora", "hour", "ชั่วโมง", "giá»", "ore", "ساعة", "å°æ—¶").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("menit", "dakika", "min", "minute", "minuto", "นาที", "دقائق", "phút").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("detik", "segundo", "second", "วินาที", "giây").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("week", "semana", "tuần").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - WordSet("month", "mes", "tháng").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "año", "năm").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0 - } - } - - override fun pageListRequest(chapter: SChapter): Request { - if (chapter.url.startsWith("http")) { - return GET(chapter.url, headers) - } - return super.pageListRequest(chapter) - } - - open val pageListParseSelector = "div.page-break, li.blocks-gallery-item, .reading-content .text-left:not(:has(.blocks-gallery-item)) img" - - open val chapterProtectorSelector = "#chapter-protector-data" - open val chapterProtectorPasswordPrefix = "wpmangaprotectornonce='" - open val chapterProtectorDataPrefix = "chapter_data='" - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - protected open fun pageListParse(document: Document): List { - launchIO { countViews(document) } - - val chapterProtector = document.selectFirst(chapterProtectorSelector) - ?: return document.select(pageListParseSelector).mapIndexed { index, element -> - val imageUrl = element.selectFirst("img")?.let { imageFromElement(it) } - Page(index, document.location(), imageUrl) - } - val chapterProtectorHtml = chapterProtector.attr("src") - .takeIf { it.startsWith("data:text/javascript;base64,") } - ?.substringAfter("data:text/javascript;base64,") - ?.let { Base64.decode(it, Base64.DEFAULT).toString(Charsets.UTF_8) } - ?: chapterProtector.html() - val password = chapterProtectorHtml - .substringAfter(chapterProtectorPasswordPrefix) - .substringBefore("';") - val chapterData = json.parseToJsonElement( - chapterProtectorHtml - .substringAfter(chapterProtectorDataPrefix) - .substringBefore("';") - .replace("\\/", "/"), - ).jsonObject - - val unsaltedCiphertext = Base64.decode(chapterData["ct"]!!.jsonPrimitive.content, Base64.DEFAULT) - val salt = chapterData["s"]!!.jsonPrimitive.content.decodeHex() - val ciphertext = salted + salt + unsaltedCiphertext - - val rawImgArray = CryptoAES.decrypt(Base64.encodeToString(ciphertext, Base64.DEFAULT), password) - val imgArrayString = json.parseToJsonElement(rawImgArray).jsonPrimitive.content - val imgArray = json.parseToJsonElement(imgArrayString).jsonArray - - return imgArray.mapIndexed { idx, it -> - Page(idx, document.location(), it.jsonPrimitive.content) - } - } - - override fun imageRequest(page: Page): Request = GET(page.imageUrl!!, headers.newBuilder().set("Referer", page.url).build()) - - override fun imageUrlParse(response: Response) = throw UnsupportedOperationException() - - /** - * Set it to false if you want to disable the extension reporting the view count - * back to the source website through admin-ajax.php. - */ - protected open val sendViewCount: Boolean = true - - protected open fun countViewsRequest(document: Document): Request? { - val wpMangaData = document.selectFirst("script#wp-manga-js-extra") - ?.data() ?: return null - - val wpMangaInfo = wpMangaData - .substringAfter("var manga = ") - .substringBeforeLast(";") - - val wpManga = json.parseToJsonElement(wpMangaInfo).jsonObject - - if (wpManga["enable_manga_view"]?.jsonPrimitive?.content == "1") { - val formBuilder = FormBody.Builder() - .add("action", "manga_views") - .add("manga", wpManga["manga_id"]!!.jsonPrimitive.content) - - if (wpManga["chapter_slug"] != null) { - formBuilder.add("chapter", wpManga["chapter_slug"]!!.jsonPrimitive.content) - } - - val formBody = formBuilder.build() - - val newHeaders = headersBuilder() - .set("Referer", document.location()) - .build() - - return POST("$baseUrl/wp-admin/admin-ajax.php", newHeaders, formBody) - } - - return null - } - - /** - * Send the view count request to the Madara endpoint. - * - * @param document The response document with the wp-manga data - */ - protected fun countViews(document: Document) { - if (!sendViewCount) { - return - } - - try { - val request = countViewsRequest(document) ?: return - client.newCall(request).execute().close() - } catch (_: Exception) { } - } - - /** - * Fetch the genres from the source to be used in the filters. - */ - protected fun fetchGenres() { - if (fetchGenres && fetchGenresAttempts < 3 && !genresFetched) { - try { - client.newCall(genresRequest()).execute() - .use { parseGenres(it.asJsoup()) } - .also { - genresFetched = true - } - .takeIf { it.isNotEmpty() } - ?.also { - genresList = it - } - } catch (_: Exception) { - } finally { - fetchGenresAttempts++ - } - } - } - - /** - * The request to the search page (or another one) that have the genres list. - */ - protected open fun genresRequest(): Request = GET("$baseUrl/?s=genre&post_type=wp-manga", headers) - - /** - * Get the genres from the search page document. - * - * @param document The search page document - */ - protected open fun parseGenres(document: Document): List = document.selectFirst("div.checkbox-group") - ?.select("div.checkbox") - .orEmpty() - .map { li -> - Genre( - li.selectFirst("label")!!.text(), - li.selectFirst("input[type=checkbox]")!!.`val`(), - ) - } - - protected val salted = "Salted__".toByteArray(Charsets.UTF_8) - - private val scope = CoroutineScope(Dispatchers.IO) - - protected fun launchIO(block: () -> Unit) = scope.launch { block() } - - companion object { - const val URL_SEARCH_PREFIX = "slug:" - val URL_REGEX = """^(https?://[^\s/$.?#].[^\s]*)${'$'}""".toRegex() - } -} - -class WordSet(private vararg val words: String) { - fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) } - fun startsWith(dateString: String): Boolean = words.any { dateString.startsWith(it, ignoreCase = true) } - fun endsWith(dateString: String): Boolean = words.any { dateString.endsWith(it, ignoreCase = true) } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt deleted file mode 100644 index d5fbc0e8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwa.kt +++ /dev/null @@ -1,160 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response - -abstract class EZManhwa( - override val name: String, - override val baseUrl: String, - val apiUrl: String, - final override val lang: String = "en", -) : HttpSource(), - ConfigurableSource { - - override val supportsLatest = true - - private val preferences by getPreferencesLazy() - - override fun headersBuilder() = super.headersBuilder() - .set("Accept", "application/json, text/plain, */*") - .set("Referer", "$baseUrl/") - - override fun getMangaUrl(manga: SManga) = "$baseUrl/series/${manga.url}" - - // Chapter URLs are stored as: series/{seriesSlug}/chapters/{chapterSlug} - // The website drops "chapters/" from the path, so we strip it here. - override fun getChapterUrl(chapter: SChapter) = "$baseUrl/${chapter.url.replace("/chapters/", "/")}" - - // ── Browse ─────────────────────────────────────────────────────────────── - - override fun popularMangaRequest(page: Int) = GET("$apiUrl/series?page=$page&perPage=20&sort=popular", headers) - - override fun latestUpdatesRequest(page: Int) = GET("$apiUrl/series?page=$page&perPage=20&sort=latest", headers) - - override fun popularMangaParse(response: Response) = parseSeriesList(response) - override fun latestUpdatesParse(response: Response) = parseSeriesList(response) - - private fun parseSeriesList(response: Response): MangasPage { - val dto = response.parseAs() - val mangas = dto.data.mapNotNull { if (it.type != "NOVEL") it.toSManga() else null } - return MangasPage(mangas, dto.currentPage < dto.totalPages) - } - - // ── Search ─────────────────────────────────────────────────────────────── - - // Base implementation sends filters only during browse. - // Override if the source's search endpoint behaviour differs. - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val isSearch = query.isNotBlank() - val endpoint = if (isSearch) "$apiUrl/series/search" else "$apiUrl/series" - val url = endpoint.toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "20") - if (isSearch) { - addQueryParameter("q", query) - } else { - var sortAdded = false - for (filter in filters) { - when (filter) { - is EZManhwaSortFilter -> { - addQueryParameter("sort", filter.value) - sortAdded = true - } - is EZManhwaStatusFilter -> if (filter.value.isNotBlank()) addQueryParameter("status", filter.value) - is EZManhwaTypeFilter -> if (filter.value.isNotBlank()) addQueryParameter("type", filter.value) - else -> {} - } - } - if (!sortAdded) addQueryParameter("sort", "latest") - } - }.build() - return GET(url, headers) - } - - override fun searchMangaParse(response: Response) = parseSeriesList(response) - - // ── Details ────────────────────────────────────────────────────────────── - - override fun mangaDetailsRequest(manga: SManga) = GET("$apiUrl/series/${manga.url}", headers) - - override fun mangaDetailsParse(response: Response) = response.parseAs().toSManga().apply { initialized = true } - - // ── Chapters ───────────────────────────────────────────────────────────── - - override fun chapterListRequest(manga: SManga) = GET("$apiUrl/series/${manga.url}/chapters?page=1&perPage=100&sort=desc", headers) - - override fun chapterListParse(response: Response): List { - // pathSegments for /api/v1/series/{slug}/chapters: [api, v1, series, slug, chapters] - val seriesSlug = response.request.url.pathSegments[3] - val initialData = response.parseAs() - val chapters = mutableListOf() - - fun parsePage(dto: EZManhwaChapterListDto) { - dto.data.mapNotNullTo(chapters) { - if (shouldShowChapter(it)) it.toSChapter(seriesSlug) else null - } - } - - parsePage(initialData) - var curr = initialData.currentPage - while (curr < initialData.totalPages) { - curr++ - val nextUrl = response.request.url.newBuilder() - .setQueryParameter("page", curr.toString()).build() - parsePage( - client.newCall(GET(nextUrl, headers)).execute() - .parseAs(), - ) - } - return chapters - } - - open fun shouldShowChapter(chapter: EZManhwaChapterDto): Boolean = chapter.requiresPurchase != true || - preferences.getBoolean(SHOW_LOCKED_CHAPTER_PREF_KEY, false) - - // ── Pages ──────────────────────────────────────────────────────────────── - - override fun pageListRequest(chapter: SChapter) = GET("$apiUrl/${chapter.url}", headers) - - override fun pageListParse(response: Response): List { - val data = response.parseAs() - if (data.requiresPurchase == true) { - throw Exception( - "Chapter requires purchase (${data.totalImages} pages). " + - "Log in via webview and purchase to read.", - ) - } - return data.images?.mapIndexed { i, img -> Page(i, imageUrl = img.url) } - ?: throw Exception("No images found. Chapter may be locked or require login via webview.") - } - - override fun imageUrlParse(response: Response) = throw UnsupportedOperationException("Not used") - - override fun getFilterList() = FilterList(EZManhwaSortFilter(), EZManhwaStatusFilter(), EZManhwaTypeFilter()) - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = SHOW_LOCKED_CHAPTER_PREF_KEY - title = "Show locked chapters" - summary = "Show chapters requiring coins. Note: They only load if owned/logged in via webview." - setDefaultValue(false) - }.also(screen::addPreference) - } - - companion object { - const val SHOW_LOCKED_CHAPTER_PREF_KEY = "pref_show_locked_chapters" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt deleted file mode 100644 index 4a4d0e6e..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaDto.kt +++ /dev/null @@ -1,124 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import keiyoushi.utils.tryParse -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable -import org.jsoup.Jsoup -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.TimeZone - -internal val EZMANHWA_DATE_FORMAT by lazy { - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH).apply { - timeZone = TimeZone.getTimeZone("UTC") - } -} - -internal val CHAPTER_PREFIX_REGEX = Regex("^(?i)(chapter|ch\\.?|episode|ep\\.?)\\s*.*") - -@Serializable -data class EZManhwaSeriesListDto( - val data: List, - val totalPages: Int, - @SerialName("current") val currentPage: Int, -) - -// Used for both list (partial) and details (full) responses. -// Fields absent in the list response deserialize as null and are -// only populated after mangaDetailsParse sets initialized = true. -@Serializable -data class EZManhwaSeriesDto( - val slug: String, - val title: String, - val cover: String? = null, - val type: String? = null, - val status: String? = null, - val alternativeTitles: String? = null, - val description: String? = null, - val author: String? = null, - val artist: String? = null, - val genres: List? = null, -) { - fun toSManga() = SManga.create().apply { - url = slug - title = this@EZManhwaSeriesDto.title - thumbnail_url = cover - author = this@EZManhwaSeriesDto.author?.trim()?.takeIf { it.isNotBlank() } - artist = this@EZManhwaSeriesDto.artist?.trim()?.takeIf { it.isNotBlank() } - description = buildString { - this@EZManhwaSeriesDto.description?.let { append(Jsoup.parse(it).text()) } - if (!alternativeTitles.isNullOrBlank()) { - append("\n\nAlternative Titles: $alternativeTitles") - } - }.trim().takeIf { it.isNotBlank() } - genre = genres?.joinToString { it.name } - status = when (this@EZManhwaSeriesDto.status) { - "ONGOING", "MASS_RELEASED" -> SManga.ONGOING - "COMPLETED" -> SManga.COMPLETED - "DROPPED" -> SManga.CANCELLED - "HIATUS" -> SManga.ON_HIATUS - else -> SManga.UNKNOWN - } - } -} - -@Serializable -data class EZManhwaGenreDto(val name: String) - -@Serializable -data class EZManhwaChapterListDto( - val data: List, - val totalPages: Int = 1, - @SerialName("current") val currentPage: Int = 1, -) - -@Serializable -data class EZManhwaChapterDto( - val slug: String, - val number: Double? = null, - val title: String? = null, - val requiresPurchase: Boolean? = null, - val createdAt: String? = null, -) { - fun toSChapter(seriesSlug: String) = SChapter.create().apply { - // Unified URL format: series/{seriesSlug}/chapters/{chapterSlug} - // pageListRequest uses: $apiUrl/${chapter.url} - // getChapterUrl uses: $baseUrl/${chapter.url.replace("/chapters/", "/")} - url = "series/$seriesSlug/chapters/$slug" - val prefix = if (requiresPurchase == true) "\uD83D\uDD12 " else "" - val numStr = number?.let { - if (it % 1.0 == 0.0) it.toInt().toString() else it.toString() - } ?: "" - - val parsedTitle = title?.trim()?.takeIf { it.isNotBlank() } - - val chapterName = if (numStr.isBlank()) { - parsedTitle ?: "Chapter" - } else { - when { - parsedTitle == null || parsedTitle == numStr -> "Chapter $numStr" - isChapterTitle(parsedTitle, numStr) -> parsedTitle - parsedTitle.startsWith("-") || parsedTitle.startsWith(":") -> "Chapter $numStr $parsedTitle" - else -> "Chapter $numStr - $parsedTitle" - } - } - - name = prefix + chapterName - chapter_number = number?.toFloat() ?: -1f - date_upload = EZMANHWA_DATE_FORMAT.tryParse(createdAt) - } - - private fun isChapterTitle(title: String, numStr: String): Boolean = title.contains(numStr) && CHAPTER_PREFIX_REGEX.matches(title) -} - -@Serializable -data class EZManhwaPageListDto( - val images: List? = null, - val requiresPurchase: Boolean? = null, - val totalImages: Int? = null, -) - -@Serializable -data class EZManhwaImageDto(val url: String) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt deleted file mode 100644 index 027a93c8..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/ezmanhwa/EZManhwaFilters.kt +++ /dev/null @@ -1,27 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.ezmanhwa - -import eu.kanade.tachiyomi.source.model.Filter - -class EZManhwaSortFilter : Filter.Select("Sort", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("latest", "popular", "newest", "alphabetical") - val DISPLAY = arrayOf("Latest", "Popular", "Newest", "Alphabetical") - } -} - -class EZManhwaStatusFilter : Filter.Select("Status", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("", "ONGOING", "COMPLETED", "HIATUS", "DROPPED") - val DISPLAY = arrayOf("All", "Ongoing", "Completed", "Hiatus", "Dropped") - } -} - -class EZManhwaTypeFilter : Filter.Select("Type", DISPLAY) { - val value get() = VALUES[state] - companion object { - private val VALUES = arrayOf("", "MANGA", "MANHWA", "MANHUA") - val DISPLAY = arrayOf("All", "Manga", "Manhwa", "Manhua") - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt deleted file mode 100644 index 1d685b12..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdults.kt +++ /dev/null @@ -1,950 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import android.content.SharedPreferences -import android.util.Log -import androidx.preference.PreferenceScreen -import androidx.preference.SwitchPreferenceCompat -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.network.POST -import eu.kanade.tachiyomi.network.asObservableSuccess -import eu.kanade.tachiyomi.source.ConfigurableSource -import eu.kanade.tachiyomi.source.model.Filter -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.model.UpdateStrategy -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.getPreferencesLazy -import keiyoushi.utils.parseAs -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.Dispatchers -import kotlinx.coroutines.launch -import kotlinx.serialization.SerializationException -import kotlinx.serialization.json.JsonObject -import okhttp3.FormBody -import okhttp3.HttpUrl -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements -import rx.Observable -import java.text.SimpleDateFormat -import kotlin.collections.iterator - -abstract class GalleryAdults( - override val name: String, - override val baseUrl: String, - override val lang: String = "all", - protected open val mangaLang: String = LANGUAGE_MULTI, - protected val simpleDateFormat: SimpleDateFormat? = null, -) : HttpSource(), - ConfigurableSource { - - protected open val xhrHeaders = headers.newBuilder() - .add("X-Requested-With", "XMLHttpRequest") - .build() - - /* Preferences */ - protected val preferences: SharedPreferences by getPreferencesLazy() - - protected open val useShortTitlePreference = true - - override fun setupPreferenceScreen(screen: PreferenceScreen) { - SwitchPreferenceCompat(screen.context).apply { - key = PREF_SHORT_TITLE - title = "Display Short Titles" - setDefaultValue(false) - }.also(screen::addPreference) - } - - protected val SharedPreferences.shortTitle - get() = getBoolean(PREF_SHORT_TITLE, false) - - /* List detail */ - protected class SMangaDto( - val title: String, - val url: String, - val thumbnail: String?, - val lang: String, - ) - - protected open fun Element.mangaTitle(selector: String = ".caption"): String? = mangaFullTitle(selector).let { - if (preferences.shortTitle) it?.shortenTitle() else it - } - - protected open fun Element.mangaFullTitle(selector: String) = selectFirst(selector)?.text() - - protected open fun String.shortenTitle() = this.replace(shortenTitleRegex, "").trim() - - protected open val shortenTitleRegex = Regex("""(\[[^]]*]|[({][^)}]*[)}])""") - - protected open fun Element.mangaUrl() = selectFirst(".inner_thumb a")?.attr("abs:href") - - protected open fun Element.mangaThumbnail() = selectFirst(".inner_thumb img")?.imgAttr() - - // Overwrite this to filter other languages' manga from search result. - // Default to [mangaLang] won't filter anything - protected open fun Element.mangaLang() = mangaLang - - protected open fun HttpUrl.Builder.addPageUri(page: Int): HttpUrl.Builder { - val url = toString() - if (!url.endsWith('/') && !url.contains('?')) { - addPathSegment("") // trailing slash (/) - } - addQueryParameter("page", page.toString()) - return this - } - - /* Popular */ - override fun popularMangaRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - if (mangaLang.isNotBlank()) addPathSegments("language/$mangaLang") - if (supportsLatest) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(popularMangaSelector()).map { element -> - popularMangaFromElement(element) - } - val hasNextPage = popularMangaNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - return MangasPage(mangas, hasNextPage) - } - - protected open fun popularMangaSelector() = "div.thumb" - - protected open fun popularMangaFromElement(element: Element): SManga = SManga.create().apply { - title = element.mangaTitle()!! - setUrlWithoutDomain(element.mangaUrl()!!) - thumbnail_url = element.mangaThumbnail() - } - - protected open fun popularMangaNextPageSelector(): String? = ".pagination li.active + li:not(.disabled)" - - /* Latest */ - override fun latestUpdatesRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - if (mangaLang.isNotBlank()) addPathSegments("language/$mangaLang") - addPageUri(page) - } - return GET(url.build(), headers) - } - - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - val mangas = document.select(latestUpdatesSelector()).map { element -> - latestUpdatesFromElement(element) - } - val hasNextPage = latestUpdatesNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - return MangasPage(mangas, hasNextPage) - } - - protected open fun latestUpdatesSelector() = popularMangaSelector() - - protected open fun latestUpdatesFromElement(element: Element) = popularMangaFromElement(element) - - protected open fun latestUpdatesNextPageSelector(): String? = popularMangaNextPageSelector() - - /* Search */ - override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable { - if (query.startsWith("https://")) { - val url = query.toHttpUrl() - if (url.host != baseUrl.toHttpUrl().host) { - throw Exception("Unsupported url") - } - val id = url.pathSegments.getOrNull(1) - ?: throw Exception("Unsupported url") - return fetchSearchManga(page, "$PREFIX_ID_SEARCH$id", filters) - } - - val randomEntryFilter = filters.firstInstanceOrNull() - - return when { - randomEntryFilter?.state == true -> { - client.newCall(randomEntryRequest()) - .asObservableSuccess() - .map { response -> randomEntryParse(response) } - } - - query.startsWith(PREFIX_ID_SEARCH) -> { - val id = query.removePrefix(PREFIX_ID_SEARCH) - client.newCall(searchMangaByIdRequest(id)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, id) } - } - - query.toIntOrNull() != null -> { - client.newCall(searchMangaByIdRequest(query)) - .asObservableSuccess() - .map { response -> searchMangaByIdParse(response, query) } - } - - else -> { - client.newCall(searchMangaRequest(page, query, filters)) - .asObservableSuccess() - .map { response -> searchMangaParse(response) } - } - } - } - - protected open fun randomEntryRequest(): Request = GET("$baseUrl/random/", headers) - - protected open fun randomEntryParse(response: Response): MangasPage { - val document = response.asJsoup() - - val url = response.request.url.toString() - val id = url.removeSuffix("/").substringAfterLast('/') - return MangasPage( - listOf( - SManga.create().apply { - title = document.mangaTitle("h1")!! - setUrlWithoutDomain("$baseUrl/$idPrefixUri/$id/") - thumbnail_url = document.getCover() - }, - ), - false, - ) - } - - /** - * Manga URL: $baseUrl/$idPrefixUri// - */ - protected open val idPrefixUri = "gallery" - - protected open fun searchMangaByIdRequest(id: String): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment(idPrefixUri) - addPathSegments("$id/") - } - return GET(url.build(), headers) - } - - protected open fun searchMangaByIdParse(response: Response, id: String): MangasPage { - val details = mangaDetailsParse(response.asJsoup()) - details.url = "/$idPrefixUri/$id/" - return MangasPage(listOf(details), false) - } - - protected open val useIntermediateSearch: Boolean = false - protected open val supportAdvancedSearch: Boolean = false - protected open val supportSpeechless: Boolean = false - protected open val useBasicSearch: Boolean - get() = !useIntermediateSearch - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - val favoriteFilter = filters.firstInstanceOrNull() - - // Speechless - val speechlessFilter = filters.firstInstanceOrNull() - - // Advanced search - val advancedSearchFilters = filters.filterIsInstance() - - return when { - favoriteFilter?.state == true -> - favoriteFilterSearchRequest(page, query, filters) - - supportSpeechless && speechlessFilter?.state == true -> - speechlessFilterSearchRequest(page, query, filters) - - supportAdvancedSearch && advancedSearchFilters.any { it.state.isNotBlank() } -> - advancedSearchRequest(page, query, filters) - - selectedGenres.size == 1 && query.isBlank() -> - tagBrowsingSearchRequest(page, query, filters) - - useIntermediateSearch -> - intermediateSearchRequest(page, query, filters) - - useBasicSearch && (selectedGenres.size > 1 || query.isNotBlank()) -> - basicSearchRequest(page, query, filters) - - sortOrderFilter?.state == 1 -> - latestUpdatesRequest(page) - - else -> - popularMangaRequest(page) - } - } - - protected open val basicSearchKey = "q" - - /** - * Basic Search: support query string with multiple-genres filter by adding genres to query string. - */ - protected open fun basicSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegments("search/") - addEncodedQueryParameter(basicSearchKey, buildQueryString(selectedGenres.map { it.name }, query)) - if (sortOrderFilter?.state == 0) addQueryParameter("sort", "popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - protected open val intermediateSearchKey = "key" - - /** - * This supports filter query search with languages, categories (manga, doujinshi...) - * with additional sort orders. - */ - protected open fun intermediateSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Intermediate search - val categoryFilters = filters.firstInstanceOrNull() - - // Only for query string or multiple tags - val url = "$baseUrl/search/".toHttpUrl().newBuilder().apply { - getSortOrderURIs().forEachIndexed { index, pair -> - addQueryParameter(pair.second, toBinary(sortOrderFilter?.state == index)) - } - categoryFilters?.state?.forEach { - addQueryParameter(it.uri, toBinary(it.state)) - } - getLanguageURIs().forEach { pair -> - addQueryParameter( - pair.second, - toBinary(mangaLang == pair.first || mangaLang == LANGUAGE_MULTI), - ) - } - addEncodedQueryParameter(intermediateSearchKey, buildQueryString(selectedGenres.map { it.name }, query)) - addPageUri(page) - } - return GET(url.build(), headers) - } - - protected open val advancedSearchKey = "key" - protected open val advancedSearchUri = "advsearch" - - /** - * Advanced Search normally won't support search for string but allow to include/exclude specific - * tags/artists/groups/parodies/characters - */ - protected open fun advancedSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Intermediate search - val categoryFilters = filters.firstInstanceOrNull() - // Advanced search - val advancedSearchFilters = filters.filterIsInstance() - - val url = "$baseUrl/$advancedSearchUri/".toHttpUrl().newBuilder().apply { - getSortOrderURIs().forEachIndexed { index, pair -> - addQueryParameter(pair.second, toBinary(sortOrderFilter?.state == index)) - } - categoryFilters?.state?.forEach { - addQueryParameter(it.uri, toBinary(it.state)) - } - getLanguageURIs().forEach { pair -> - addQueryParameter( - pair.second, - toBinary( - mangaLang == pair.first || - mangaLang == LANGUAGE_MULTI, - ), - ) - } - - // Build this query string: +tag:"bat+man"+-tag:"cat"+artist:"Joe"... - // +tag must be encoded into %2Btag while the rest are not needed to encode - val keys = emptyList().toMutableList() - keys.addAll(selectedGenres.map { "%2Btag:\"${it.name}\"" }) - advancedSearchFilters.forEach { filter -> - val key = when (filter) { - is TagsFilter -> "tag" - is ParodiesFilter -> "parody" - is ArtistsFilter -> "artist" - is CharactersFilter -> "character" - is GroupsFilter -> "group" - else -> null - } - if (key != null) { - keys.addAll( - filter.state.trim() - .replace(regexSpaceNotAfterComma, "+") - .replace(" ", "") - .split(',') - .mapNotNull { - val match = regexExcludeTerm.find(it) - match?.groupValues?.let { groups -> - "${if (groups[1].isNotBlank()) "-" else "%2B"}$key:\"${groups[2]}\"" - } - }, - ) - } - } - addEncodedQueryParameter(advancedSearchKey, keys.joinToString("+")) - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Convert space( ) typed in search-box into plus(+) in URL. Then: - * - uses plus(+) to search for exact match - * - use comma(, ) for separate terms, as AND condition. - * Plus(+) after comma(, ) doesn't have any effect. - */ - protected open fun buildQueryString(tags: List, query: String): String = (tags + query).filter(String::isNotBlank) - .joinToString(",") { - // any space except after a comma (we're going to replace spaces only between words) - it.trim() - .replace(regexSpaceNotAfterComma, "+") - .replace(" ", "") - } - - protected open fun tagBrowsingSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - val genresFilter = filters.firstInstanceOrNull() - val selectedGenres = genresFilter?.state?.filter { it.state } ?: emptyList() - - // Browsing single tag's catalog - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("tag") - addPathSegment(selectedGenres.single().uri) - if (sortOrderFilter?.state == 0) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Browsing speechless titles. Some sites exclude speechless titles from normal search and - * allow browsing separately. - */ - protected open fun speechlessFilterSearchRequest(page: Int, query: String, filters: FilterList): Request { - // Basic search - val sortOrderFilter = filters.firstInstanceOrNull() - - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegment("language") - addPathSegment(LANGUAGE_SPEECHLESS) - if (sortOrderFilter?.state == 0) addPathSegment("popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Browsing user's personal favorites saved on site. This requires login in view WebView. - */ - protected open fun favoriteFilterSearchRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/$favoritePath".toHttpUrl().newBuilder() - return POST( - url.build().toString(), - xhrHeaders, - FormBody.Builder() - .add("page", page.toString()) - .build(), - ) - } - - protected open val favoritePath = "user/fav_pags.php" - - protected open fun loginRequired(document: Document, url: String): Boolean = ( - url.contains("/login/") && - document.select("input[value=Login]").isNotEmpty() - ) - - override fun searchMangaParse(response: Response): MangasPage { - val document = response.asJsoup() - if (loginRequired(document, response.request.url.toString())) { - throw Exception("Log in via WebView to view favorites") - } else { - val hasNextPage = searchMangaNextPageSelector()?.let { document.selectFirst(it) != null } ?: false - val mangas = document.select(searchMangaSelector()) - .searchMangaFromElements(hasNextPage) - return MangasPage(mangas, hasNextPage) - } - } - - protected open fun Elements.searchMangaFromElements(hasNextPage: Boolean): List = mapNotNull { - SMangaDto( - title = it.mangaTitle() ?: return@mapNotNull null, - url = it.mangaUrl() ?: return@mapNotNull null, - thumbnail = it.mangaThumbnail(), - lang = it.mangaLang(), - ) - } - .let { unfiltered -> - val results = unfiltered.filter { mangaLang.isBlank() || it.lang == mangaLang } - // return at least 1 title if all mangas in current page is of other languages - if (results.isEmpty() && hasNextPage) { - unfiltered.firstOrNull()?.let(::listOf) ?: emptyList() - } else { - results - } - } - .map { - SManga.create().apply { - title = it.title - setUrlWithoutDomain(it.url) - thumbnail_url = it.thumbnail - } - } - - protected open fun searchMangaSelector() = popularMangaSelector() - - protected open fun searchMangaNextPageSelector(): String? = popularMangaNextPageSelector() - - /* Related titles */ - protected open fun relatedMangaSelector() = popularMangaSelector() - - override fun relatedMangaListParse(response: Response): List = response.asJsoup() - .select(relatedMangaSelector()) - .searchMangaFromElements(hasNextPage = false) - - /* Details */ - protected open val mangaDetailInfoSelector = ".gallery_top" - - override fun mangaDetailsParse(response: Response): SManga = mangaDetailsParse(response.asJsoup()) - - protected open fun mangaDetailsParse(document: Document): SManga = document.selectFirst(mangaDetailInfoSelector)!!.run { - SManga.create().apply { - update_strategy = UpdateStrategy.ONLY_FETCH_ONCE - status = SManga.COMPLETED - mangaTitle("h1")?.let { title = it } - thumbnail_url = getCover() - genre = getInfo("Tags") - author = getInfo("Artists") - description = getDescription(document) - } - } - - protected open fun Element.getCover() = selectFirst(".cover img")?.imgAttr() - - protected val regexTag = Regex("Tags?") - - /** - * Parsing document to extract info related to [tag]. - */ - protected abstract fun Element.getInfo(tag: String): String - - protected open fun Element.getDescription(document: Document? = null): String = ( - listOf("Parodies", "Characters", "Languages", "Categories", "Category") - .mapNotNull { tag -> - getInfo(tag) - .takeIf { it.isNotBlank() } - ?.let { "$tag: $it" } - } + - listOfNotNull( - getInfoPages(document), - getInfoAlternativeTitle(), - getInfoFullTitle(), - ) - ) - .joinToString("\n\n") - - protected open fun Element.getInfoPages(document: Document? = null): String? = document?.inputIdValueOf(totalPagesSelector) - ?.takeIf { it.isNotBlank() } - ?.let { "Pages: $it" } - - protected open fun Element.getInfoAlternativeTitle(): String? = selectFirst("h1 + h2, .subtitle")?.ownText() - .takeIf { !it.isNullOrBlank() } - ?.let { "Alternative title: $it" } - - protected open fun Element.getInfoFullTitle(): String? = if (preferences.shortTitle) "Full title: ${mangaFullTitle("h1")}" else null - - protected open fun Element.getTime(): Long = selectFirst(".uploaded") - ?.ownText() - .toDate(simpleDateFormat) - - /* Chapters */ - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - return listOf( - SChapter.create().apply { - name = "Chapter" - scanlator = document.selectFirst(mangaDetailInfoSelector) - ?.getInfo("Groups") - date_upload = document.getTime() - setUrlWithoutDomain(response.request.url.encodedPath) - }, - ) - } - - /* Pages */ - protected open fun Element.inputIdValueOf(string: String): String = select("input[id=$string]").attr("value") - - protected open val pagesRequest = "inc/thumbs_loader.php" - protected open val galleryIdSelector = "gallery_id" - protected open val loadIdSelector = "load_id" - protected open val loadDirSelector = "load_dir" - protected open val totalPagesSelector = "load_pages" - protected open val serverSelector = "load_server" - - protected open fun pageRequestForm(document: Document, totalPages: String, loadedPages: Int): FormBody { - val token = document.select("[name=csrf-token]").attr("content") - val serverNumber = document.serverNumber() - - return FormBody.Builder() - .add("u_id", document.inputIdValueOf(galleryIdSelector)) - .add("g_id", document.inputIdValueOf(loadIdSelector)) - .add("img_dir", document.inputIdValueOf(loadDirSelector)) - .add("visible_pages", loadedPages.toString()) - .add("total_pages", totalPages) - .add("type", "2") // 1 would be "more", 2 is "all remaining" - .apply { - if (token.isNotBlank()) add("_token", token) - if (serverNumber != null) add("server", serverNumber) - } - .build() - } - - protected open val thumbnailSelector = ".gallery_thumb" - - protected open fun Element.getServer(): String { - val domain = baseUrl.toHttpUrl().host - return serverNumber() - ?.let { "m$it.$domain" } - ?: getCover()!!.toHttpUrl().host - } - - protected open fun Element.serverNumber(): String? = inputIdValueOf(serverSelector) - .takeIf { it.isNotBlank() } - - protected open fun Element.parseJson(): String? = selectFirst("script:containsData(parseJSON)")?.data() - ?.substringAfter("$.parseJSON('") - ?.substringBefore("');")?.trim() - - /** - * Page URL: $baseUrl/$pageUri// - */ - protected open val pageUri = "g" - - override fun pageListParse(response: Response): List = pageListParse(response.asJsoup()) - - protected open fun pageListParse(document: Document): List { - val json = document.parseJson() - - if (json != null) { - val loadDir = document.inputIdValueOf(loadDirSelector) - val loadId = document.inputIdValueOf(loadIdSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val server = document.getServer() - val imagesUri = "https://$server/$loadDir/$loadId" - - try { - val pages = mutableListOf() - val images = json.parseAs() - - // JSON string in this form: {"1":"j,1100,1148","2":"j,728,689",... - for (image in images) { - val ext = image.value.toString().replace("\"", "").split(",")[0] - val imageExt = when (ext) { - "p" -> "png" - "b" -> "bmp" - "g" -> "gif" - "w" -> "webp" - else -> "jpg" - } - val idx = image.key.toInt() - pages.add( - Page( - index = idx, - imageUrl = "$imagesUri/${image.key}.$imageExt", - url = "$pageUrl/$idx/", - ), - ) - } - return pages - } catch (_: SerializationException) { - Log.e("GalleryAdults", "Failed to decode JSON") - return this.pageListParseAlternative(document) - } - } else { - return this.pageListParseAlternative(document) - } - } - - /** - * Overwrite this to force extension not blindly converting thumbnails to full image - * by simply removing the trailing "t" from file name. Instead, it will open each page, - * one by one, then parsing for actual image's URL. - * This will be much slower but guaranteed work. - * - * This only apply if site doesn't provide 'parseJSON'. - */ - protected open val parsingImagePageByPage: Boolean = false - - /** - * Either: - * - Load all thumbnails then convert thumbnails to full images. - * - Or request then parse for a list of manga's page's URL, - * which will then request one by one to parse for page's image's URL using [imageUrlParse]. - */ - protected open fun pageListParseAlternative(document: Document): List { - val totalPages = document.inputIdValueOf(totalPagesSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val pages = document.select("$thumbnailSelector a") - .mapNotNull { - if (parsingImagePageByPage) { - it.absUrl("href") - } else { - it.selectFirst("img")?.imgAttr() ?: return@mapNotNull null - } - } - .toMutableList() - - if (totalPages.isNotBlank() && totalPages.toInt() > pages.size) { - val form = pageRequestForm(document, totalPages, pages.size) - - val morePages = client.newCall(POST("$baseUrl/$pagesRequest", xhrHeaders, form)) - .execute().use { it -> - it.asJsoup() - .select("a") - .mapNotNull { - if (parsingImagePageByPage) { - it.absUrl("href") - } else { - it.selectFirst("img")?.imgAttr() ?: return@mapNotNull null - } - } - } - if (morePages.isNotEmpty()) { - pages.addAll(morePages) - } else { - return pageListParseDummy(document) - } - } - - return pages.mapIndexed { idx, url -> - if (parsingImagePageByPage) { - Page(idx, url) - } else { - Page( - index = idx, - imageUrl = url.thumbnailToFull(), - url = "$pageUrl/$idx/", - ) - } - } - } - - /** - * Generate all images using `totalPages`. Supposedly they are sequential. - * Use in case any extension doesn't know how to request for "All thumbnails" - */ - protected open fun pageListParseDummy(document: Document): List { - val loadDir = document.inputIdValueOf(loadDirSelector) - val loadId = document.inputIdValueOf(loadIdSelector) - val galleryId = document.inputIdValueOf(galleryIdSelector) - val pageUrl = "$baseUrl/$pageUri/$galleryId" - - val server = document.getServer() - val imagesUri = "https://$server/$loadDir/$loadId" - - val images = document.select("$thumbnailSelector img") - val thumbUrls = images.map { it.imgAttr() }.toMutableList() - - val totalPages = document.inputIdValueOf(totalPagesSelector) - - if (totalPages.isNotBlank() && totalPages.toInt() > thumbUrls.size) { - val imagesExt = images.first()?.imgAttr()!! - .substringAfterLast('.') - - thumbUrls.addAll( - listOf((images.size + 1)..totalPages.toInt()).flatten().map { - "$imagesUri/${it}t.$imagesExt" - }, - ) - } - return thumbUrls.mapIndexed { idx, url -> - Page( - index = idx, - imageUrl = url.thumbnailToFull(), - url = "$pageUrl/$idx/", - ) - } - } - - override fun imageUrlParse(response: Response): String = imageUrlParse(response.asJsoup()) - - protected open fun imageUrlParse(document: Document): String = document.selectFirst("img#gimg, img#fimg")?.imgAttr() ?: "" - - /* Filters */ - private val scope = CoroutineScope(Dispatchers.IO) - private fun launchIO(block: () -> Unit) = scope.launch { block() } - private var tagsFetched = false - private var tagsFetchAttempt = 0 - - /** - * List of tags in pairs - */ - protected var genres: MutableMap = mutableMapOf() - - protected open fun tagsRequest(page: Int): Request { - val url = baseUrl.toHttpUrl().newBuilder().apply { - addPathSegments("tags/popular") - addPageUri(page) - } - return GET(url.build(), headers) - } - - /** - * Parsing [document] to return a list of tags in pairs. - */ - protected open fun tagsParser(document: Document): List = document.select("a.tag_btn") - .mapNotNull { - Genre( - it.select(".list_tag, .tag_name").text(), - it.attr("href") - .removeSuffix("/").substringAfterLast('/'), - ) - } - - protected open fun requestTags() { - if (!tagsFetched && tagsFetchAttempt < 3) { - launchIO { - val tags = mutableListOf() - try { - (1..5).forEach { page -> - runCatching { - client.newCall(tagsRequest(page)) - .execute().use { it.asJsoup().let(::tagsParser) } - }.onSuccess { parsed -> - tags.addAll(parsed) - } - } - - tags.sortedWith(compareBy { it.name }) - .forEach { genres[it.name] = it.uri } - - tagsFetched = true - } finally { - tagsFetchAttempt++ - } - } - } - } - - override fun getFilterList(): FilterList { - requestTags() - val filters = emptyList>().toMutableList() - if (useIntermediateSearch) { - filters.add(Filter.Header("HINT: Separate search term with comma (,)")) - } - - filters.add(SortOrderFilter(getSortOrderURIs())) - - if (genres.isEmpty()) { - filters.add(Filter.Header("Press 'reset' to attempt to load tags")) - } else { - filters.add(GenresFilter(genres)) - } - - if (useIntermediateSearch || supportAdvancedSearch) { - filters.addAll( - listOf( - Filter.Separator(), - CategoryFilters(getCategoryURIs()), - ), - ) - } - - if (supportAdvancedSearch) { - filters.addAll( - listOf( - Filter.Separator(), - Filter.Header("Advanced filters will ignore query search. Separate terms by comma (,) and precede term with minus (-) to exclude."), - TagsFilter(), - ParodiesFilter(), - ArtistsFilter(), - CharactersFilter(), - GroupsFilter(), - ), - ) - } - - filters.add(Filter.Separator()) - - if (supportSpeechless) { - filters.add(SpeechlessFilter()) - } - filters.add(FavoriteFilter()) - - filters.add(RandomEntryFilter()) - - return FilterList(filters) - } - - protected open fun getSortOrderURIs() = listOf( - Pair("Popular", "pp"), - Pair("Latest", "lt"), - ) + if (useIntermediateSearch || supportAdvancedSearch) { - listOf( - Pair("Downloads", "dl"), - Pair("Top Rated", "tr"), - ) - } else { - emptyList() - } - - protected open fun getCategoryURIs() = listOf( - SearchFlagFilter("Manga", "m"), - SearchFlagFilter("Doujinshi", "d"), - SearchFlagFilter("Western", "w"), - SearchFlagFilter("Image Set", "i"), - SearchFlagFilter("Artist CG", "a"), - SearchFlagFilter("Game CG", "g"), - ) - - protected open fun getLanguageURIs() = listOf( - Pair(LANGUAGE_ENGLISH, "en"), - Pair(LANGUAGE_JAPANESE, "jp"), - Pair(LANGUAGE_SPANISH, "es"), - Pair(LANGUAGE_FRENCH, "fr"), - Pair(LANGUAGE_KOREAN, "kr"), - Pair(LANGUAGE_GERMAN, "de"), - Pair(LANGUAGE_RUSSIAN, "ru"), - ) - - companion object { - const val PREFIX_ID_SEARCH = "id:" - - private const val PREF_SHORT_TITLE = "pref_short_title" - - // references to be used in factory - const val LANGUAGE_MULTI = "" - const val LANGUAGE_ENGLISH = "english" - const val LANGUAGE_JAPANESE = "japanese" - const val LANGUAGE_CHINESE = "chinese" - const val LANGUAGE_KOREAN = "korean" - const val LANGUAGE_SPANISH = "spanish" - const val LANGUAGE_FRENCH = "french" - const val LANGUAGE_GERMAN = "german" - const val LANGUAGE_RUSSIAN = "russian" - const val LANGUAGE_SPEECHLESS = "speechless" - const val LANGUAGE_TRANSLATED = "translated" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt deleted file mode 100644 index f4a0c0c3..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsFilters.kt +++ /dev/null @@ -1,31 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import eu.kanade.tachiyomi.source.model.Filter - -class Genre(name: String, val uri: String) : Filter.CheckBox(name) -class GenresFilter(genres: Map) : - Filter.Group( - "Tags", - genres.map { Genre(it.key, it.value) }, - ) - -class SortOrderFilter(sortOrderURIs: List>) : Filter.Select("Sort By", sortOrderURIs.map { it.first }.toTypedArray()) - -class FavoriteFilter : Filter.CheckBox("Show favorites only (login via WebView)", false) - -class RandomEntryFilter : Filter.CheckBox("Random manga", false) - -// Speechless -class SpeechlessFilter : Filter.CheckBox("Show speechless items only", false) - -// Intermediate search -class SearchFlagFilter(name: String, val uri: String, state: Boolean = true) : Filter.CheckBox(name, state) -class CategoryFilters(flags: List) : Filter.Group("Categories", flags) - -// Advance search -abstract class AdvancedTextFilter(name: String) : Filter.Text(name) -class TagsFilter : AdvancedTextFilter("Tags") -class ParodiesFilter : AdvancedTextFilter("Parodies") -class ArtistsFilter : AdvancedTextFilter("Artists") -class CharactersFilter : AdvancedTextFilter("Characters") -class GroupsFilter : AdvancedTextFilter("Groups") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt deleted file mode 100644 index 04c21f5a..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/parsers/galleryadults/GalleryAdultsUtils.kt +++ /dev/null @@ -1,148 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.parsers.galleryadults - -import keiyoushi.utils.tryParse -import org.jsoup.nodes.Element -import java.text.SimpleDateFormat -import java.util.Calendar - -// any space except after a comma (we're going to replace spaces only between words) -val regexSpaceNotAfterComma = Regex("""(? absUrl("data-cfsrc") - hasAttr("data-src") -> absUrl("data-src") - hasAttr("data-lazy-src") -> absUrl("data-lazy-src") - hasAttr("srcset") -> absUrl("srcset").substringBefore(" ") - else -> absUrl("src") -} - -fun Element.cleanTag(): String = text().cleanTag() -fun String.cleanTag(): String = replace(regexTagCountNumber, "").trim() - -// convert thumbnail URLs to full image URLs -fun String.thumbnailToFull(): String { - val ext = substringAfterLast(".") - return replace("t.$ext", ".$ext") -} - -fun String?.toDate(simpleDateFormat: SimpleDateFormat?): Long { - this ?: return 0L - - return if (simpleDateFormat != null) { - if (contains(regexDateSuffix)) { - // Clean date (e.g. 5th December 2019 to 5 December 2019) before parsing it - split(" ").map { - if (it.contains(regexDate)) { - it.replace(regexNotNumber, "") - } else { - it - } - } - .let { simpleDateFormat.tryParse(it.joinToString(" ")) } - } else { - simpleDateFormat.tryParse(this) - } - } else { - parseDate(this) - } -} - -private fun parseDate(date: String?): Long { - date ?: return 0L - - return when { - // Handle 'yesterday' and 'today', using midnight - WordSet("yesterday", "يوم واحد").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -1) // yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("today", "just now").startsWith(date) -> { - Calendar.getInstance().apply { - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("يومين").startsWith(date) -> { - Calendar.getInstance().apply { - add(Calendar.DAY_OF_MONTH, -2) // day before yesterday - set(Calendar.HOUR_OF_DAY, 0) - set(Calendar.MINUTE, 0) - set(Calendar.SECOND, 0) - set(Calendar.MILLISECOND, 0) - }.timeInMillis - } - - WordSet("ago", "atrás", "önce", "قبل").endsWith(date) -> { - parseRelativeDate(date) - } - - WordSet("hace").startsWith(date) -> { - parseRelativeDate(date) - } - - else -> 0L - } -} - -// Parses dates in this form: 21 hours ago OR "2 days ago (Updated 19 hours ago)" -private fun parseRelativeDate(date: String): Long { - val number = regexRelativeDateTime.find(date)?.value?.toIntOrNull() - ?: date.split(" ").firstOrNull() - ?.replace("one", "1") - ?.replace("a", "1") - ?.toIntOrNull() - ?: return 0L - val now = Calendar.getInstance() - - // Sort by order - return when { - WordSet("detik", "segundo", "second", "วินาที").anyWordIn(date) -> - now.apply { add(Calendar.SECOND, -number) }.timeInMillis - - WordSet("menit", "dakika", "min", "minute", "minuto", "นาที", "دقائق").anyWordIn(date) -> - now.apply { add(Calendar.MINUTE, -number) }.timeInMillis - - WordSet("jam", "saat", "heure", "hora", "hour", "ชั่วโมง", "giá»", "ore", "ساعة", "å°æ—¶").anyWordIn(date) -> - now.apply { add(Calendar.HOUR, -number) }.timeInMillis - - WordSet("hari", "gün", "jour", "día", "dia", "day", "วัน", "ngày", "giorni", "أيام", "天").anyWordIn(date) -> - now.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - - WordSet("week", "semana").anyWordIn(date) -> - now.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - - WordSet("month", "mes").anyWordIn(date) -> - now.apply { add(Calendar.MONTH, -number) }.timeInMillis - - WordSet("year", "año").anyWordIn(date) -> - now.apply { add(Calendar.YEAR, -number) }.timeInMillis - - else -> 0L - } -} - -class WordSet(private vararg val words: String) { - fun anyWordIn(dateString: String): Boolean = words.any { dateString.contains(it, ignoreCase = true) } - fun startsWith(dateString: String): Boolean = words.any { dateString.startsWith(it, ignoreCase = true) } - fun endsWith(dateString: String): Boolean = words.any { dateString.endsWith(it, ignoreCase = true) } -} - -fun toBinary(boolean: Boolean) = if (boolean) "1" else "0" \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt deleted file mode 100644 index f0078e74..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Astratoons.kt +++ /dev/null @@ -1,213 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.network.GET -import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.source.model.Page -import eu.kanade.tachiyomi.source.model.SChapter -import eu.kanade.tachiyomi.source.model.SManga -import eu.kanade.tachiyomi.source.online.HttpSource -import eu.kanade.tachiyomi.util.asJsoup -import keiyoushi.utils.firstInstanceOrNull -import keiyoushi.utils.parseAs -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.Request -import okhttp3.Response -import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.TachiyomiSource -import okhttp3.OkHttpClient - -@TachiyomiSource("ASTRATOONS", "Astratoons", "pt") -class Astratoons : HttpSource() { - - override val name = "Astratoons" - - override val baseUrl = "https://new.astratoons.com" - - override val lang = "pt-BR" - - override val supportsLatest = true - - override val client: OkHttpClient = network.client.newBuilder() - .connectionPool( - okhttp3.ConnectionPool(5,5,java.util.concurrent.TimeUnit.MINUTES) - ) - .build() - - override val versionId: Int = 2 - override fun headersBuilder() = super.headersBuilder() - .add("Referer", "$baseUrl/") - - // ======================== Popular ========================== - - override fun popularMangaRequest(page: Int): Request { - - val url = "$baseUrl/api/comics" - .toHttpUrl() - .newBuilder() - .addQueryParameter("sortBy", "views") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "40") - .build() - - return GET(url, headers) - } - - override fun popularMangaParse(response: Response): MangasPage { - - val dto = response.parseAs() - - return MangasPage( - dto.data.map { - it.toSManga(baseUrl) - }, - dto.currentPage < dto.lastPage - ) - } - - // ======================== Latest ========================== - - override fun latestUpdatesRequest(page: Int): Request { - val url = "$baseUrl/api/comics" - .toHttpUrl() - .newBuilder() - .addQueryParameter("sortBy", "updated_at") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "40") - .build() - - return GET(url, headers) - } - override fun latestUpdatesParse(response: Response): MangasPage { - val dto = response.parseAs() - val mangas = dto.data.map { it.toSManga(baseUrl) } - return MangasPage(mangas, dto.currentPage < dto.lastPage) - } - - // ======================== Search ========================== - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = "$baseUrl/api/comics".toHttpUrl().newBuilder() - .addQueryParameter("page", page.toString()) - - if (query.isNotEmpty()) { - url.addQueryParameter("search", query) - } - - filters.firstInstanceOrNull()?.let { - url.addQueryParameter("sortBy", it.toQuery()) - } - - filters.firstInstanceOrNull()?.toQuery() - ?.takeIf { it.isNotEmpty() } - ?.let { url.addQueryParameter("status", it) } - - filters.firstInstanceOrNull()?.state - ?.filter { it.state } - ?.forEach { url.addQueryParameter("types[]", it.value) } - - filters.firstInstanceOrNull()?.state - ?.filter { it.state } - ?.forEach { url.addQueryParameter("tags[]", it.value) } - - return GET(url.build(), headers) - } - - override fun searchMangaParse(response: Response) = latestUpdatesParse(response) - - // ======================== Details ========================= - - override fun mangaDetailsParse(response: Response) = SManga.create().apply { - val document = response.asJsoup() - title = document.selectFirst("h1")?.text() ?: "Unknown" - thumbnail_url = document.selectFirst("img[class*=object-cover]")?.absUrl("src") - description = document.selectFirst("div.space-y-4 > p")?.text() - ?: document.selectFirst("div:has(>h1) + div")?.text() - genre = document.select("h3:contains(Tags) + div a").joinToString { it.text() } - author = document.selectFirst("span:contains(Autor) > span")?.text() - artist = document.selectFirst("span:contains(Artista) > span")?.text() - - val statusText = document.selectFirst("h3:contains(Informações) + div span.capitalize")?.text() - status = when (statusText?.lowercase()) { - "em andamento", "em dia" -> SManga.ONGOING - "completo" -> SManga.COMPLETED - "hiato" -> SManga.ON_HIATUS - "cancelado", "dropado" -> SManga.CANCELLED - else -> SManga.UNKNOWN - } - } - - // ======================== Chapter ========================= - - override fun chapterListRequest(manga: SManga): Request = mangaDetailsRequest(manga) - - override fun chapterListParse(response: Response): List { - val document = response.asJsoup() - val html = document.html() - val mangaId = MANGA_ID.find(html)?.groupValues?.get(1) - ?: throw Exception("Could not find manga id") - - var page = 1 - var hasMore = true - val chapters = mutableListOf() - - while (hasMore) { - val url = "$baseUrl/api/comics/$mangaId/chapters".toHttpUrl().newBuilder() - .addQueryParameter("search", "") - .addQueryParameter("order", "desc") - .addQueryParameter("page", page.toString()) - .addQueryParameter("perPage", "100") - .build() - - val res = client.newCall(GET(url, headers)).execute() - val dto = res.parseAs() - - val fragment = Jsoup.parseBodyFragment(dto.html, baseUrl) - chapters += fragment.select("a").map { element -> - SChapter.create().apply { - name = element.selectFirst(".text-lg")?.text() ?: "Chapter" - setUrlWithoutDomain(element.absUrl("href")) - } - } - - hasMore = dto.hasMore - page++ - } - - return chapters - } - - // ======================== Pages =========================== - - override fun imageRequest(page: Page): Request { - val imageHeaders = headersBuilder() - .set("Referer", page.url) - .build() - - return GET(page.imageUrl!!, imageHeaders) - } - - override fun pageListParse(response: Response): List { - val document = response.asJsoup() - return document.select("#reader-container img[src], #reader-container canvas[data-src]") - .mapIndexed { index, element -> - val imageUrl = element.absUrl("src").ifEmpty { element.absUrl("data-src") } - Page(index, document.location(), imageUrl) - } - } - - override fun imageUrlParse(response: Response): String = throw UnsupportedOperationException() - - // ======================== Filters ========================= - - override fun getFilterList() = FilterList( - SortFilter(), - StatusFilter(), - TypeFilter(), - TagFilter(), - ) - - companion object { - val MANGA_ID = """comicId:\s*(\d+)""".toRegex() - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt deleted file mode 100644 index 2dcfdf18..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Dto.kt +++ /dev/null @@ -1,31 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.source.model.SManga -import kotlinx.serialization.SerialName -import kotlinx.serialization.Serializable - -@Serializable -class ChapterListDto( - val hasMore: Boolean, - val html: String, -) - -@Serializable -class ComicsResponseDto( - val data: List, - @SerialName("current_page") val currentPage: Int, - @SerialName("last_page") val lastPage: Int, -) - -@Serializable -class ComicDto( - @SerialName("title") private val title: String, - @SerialName("slug") private val slug: String, - @SerialName("cover_image") private val coverImage: String, -) { - fun toSManga(baseUrl: String) = SManga.create().apply { - this.title = this@ComicDto.title - this.thumbnail_url = "$baseUrl/storage/$coverImage" - this.url = "/comics/$slug" - } -} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt deleted file mode 100644 index 295abf64..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/tachiyomi/pt/astratoons/Filters.kt +++ /dev/null @@ -1,80 +0,0 @@ -package org.koitharu.kotatsu.parsers.site.tachiyomi.pt.astratoons - -import eu.kanade.tachiyomi.source.model.Filter - -internal class CheckBox(name: String, val value: String) : Filter.CheckBox(name) - -internal class SortFilter : Filter.Select("Ordenar por", arrayOf("Mais Recente", "Título", "Nº de Capítulos")) { - fun toQuery() = when (state) { - 0 -> "updated_at" - 1 -> "title" - 2 -> "chapters_count" - else -> "updated_at" - } -} - -internal class StatusFilter : Filter.Select("Status", arrayOf("Todo Status", "Em dia", "Em andamento", "Completo", "Cancelado", "Hiato", "Dropado")) { - fun toQuery() = when (state) { - 1 -> "em dia" - 2 -> "em andamento" - 3 -> "completo" - 4 -> "cancelado" - 5 -> "hiato" - 6 -> "dropado" - else -> "" - } -} - -internal class TypeFilter : - Filter.Group( - "Tipos", - listOf( - CheckBox("Manga", "Manga"), - CheckBox("Manhwa", "Manhwa"), - CheckBox("Manhua", "Manhua"), - CheckBox("Webtoon", "Webtoon"), - ), - ) - -internal class TagFilter : - Filter.Group( - "Gêneros", - listOf( - CheckBox("Ação", "acao"), - CheckBox("Artes Marciais", "artes-marciais"), - CheckBox("Aventura", "aventura"), - CheckBox("Comédia", "comedia"), - CheckBox("Crime", "crime"), - CheckBox("Cultivo", "cultivo"), - CheckBox("Dança", "danca"), - CheckBox("Drama", "drama"), - CheckBox("Ecchi", "ecchi"), - CheckBox("Esporte", "esporte"), - CheckBox("Fantasia", "fantasia"), - CheckBox("Harém", "harem"), - CheckBox("Histórico", "historico"), - CheckBox("Isekai", "isekai"), - CheckBox("Iskai", "iskai"), - CheckBox("Magia", "magia"), - CheckBox("Mistério", "misterio"), - CheckBox("Monstros", "monstros"), - CheckBox("Murim", "murim"), - CheckBox("Reencarnação", "reencarnacao"), - CheckBox("Regressão", "regressao"), - CheckBox("Romance", "romance"), - CheckBox("Sci-fi", "sci-fi"), - CheckBox("Seinen", "seinen"), - CheckBox("Shounen", "shounen"), - CheckBox("Silence", "silence"), - CheckBox("Slice of Life", "slice-of-life"), - CheckBox("Sobrenatural", "sobrenatural"), - CheckBox("Super poderes", "super-poderes"), - CheckBox("Suspense", "suspense"), - CheckBox("Tragédia", "tragedia"), - CheckBox("Troca de gênero", "troca-de-genero"), - CheckBox("Viagem no tempo", "viagem-no-tempo"), - CheckBox("Vida Cotidiana", "vida-cotidiana"), - CheckBox("Vida Escolar", "vida-escolar"), - CheckBox("Zumbis", "zumbis"), - ), - ) \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt deleted file mode 100644 index 46a98888..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/Context.kt +++ /dev/null @@ -1,7 +0,0 @@ -package org.koitharu.kotatsu.parsers.util - -import android.app.Application -import uy.kohesive.injekt.Injekt -import uy.kohesive.injekt.api.get - -val applicationContext: Application get() = Injekt.get() \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt deleted file mode 100644 index ddacdcb5..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/MangaParserExt.kt +++ /dev/null @@ -1,23 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.util - -import org.koitharu.kotatsu.parsers.model.MangaListFilter - -/** - * Extension properties to help with smart-casting across modules. - * These return a local copy of the property value. - */ -public val MangaListFilter.queryOrNull: String? - get() = query - -public val MangaListFilter.authorOrNull: String? - get() = author - -public inline fun T?.ifNotNull(block: (T) -> Unit) { - if (this != null) block(this) -} - -public fun String?.nullIfEmpty(): String? = if (isNullOrEmpty()) null else this - - diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt b/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt deleted file mode 100644 index 08962d96..00000000 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/util/WebViewHelper.kt +++ /dev/null @@ -1,20 +0,0 @@ -@file:Suppress("warnings", "unused") - -package org.koitharu.kotatsu.parsers.util - -import org.koitharu.kotatsu.parsers.MangaLoaderContext - -public class WebViewHelper( - private val context: MangaLoaderContext, -) { - - public suspend fun getLocalStorageValue(domain: String, key: String): String? { - return context.evaluateJs("https://$domain/", "window.localStorage.getItem(\"$key\")") - } - - public suspend fun getUrlValue(url: String, value: String): String? { - return context.evaluateJs(url, "new URLSearchParams(window.location.search).get('$value');") - } -} - - diff --git a/src/main/kotlin/tsuki/parsers/ChanParser.kt b/src/main/kotlin/tsuki/parsers/ChanParser.kt new file mode 100644 index 00000000..488bfee0 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/ChanParser.kt @@ -0,0 +1,305 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.core.AbstractMangaParser +import tsuki.exception.AuthRequiredException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.host +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.removeSurrounding +import tsuki.util.requireElementById +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode + +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.internal.StringUtil +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import kotlin.text.isBlank + +internal abstract class ChanParser( + context: MangaLoaderContext, + source: MangaParserSource, +) : AbstractMangaParser(context, source), MangaParserAuthProvider { + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val authUrl: String + get() = "https://${domain}" + + override suspend fun isAuthorized(): Boolean = context.cookieJar.getCookies(domain).any { it.name == "dle_user_id" } + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { + val domain = domain + val doc = webClient.httpGet(buildUrl(offset, order, filter)).parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("content") + ?: doc.parseFailed("Cannot find root") + return root.select("div.content_row").mapNotNull { row -> + val a = row.selectFirst("div.manga_row1")?.selectFirst("h2")?.selectFirst("a") + ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = row.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = row.selectFirst("div.manga_images")?.selectFirst("img") + ?.absUrl("src").orEmpty(), + tags = row.selectFirst("div.genre")?.select("a")?.mapToSet { + MangaTag( + title = it.text().toTagName(), + key = it.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + }.orEmpty(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast(" + val a = tr.selectFirst("a") + val href = a?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null + MangaChapter( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + title = tr.selectFirst("a")?.textOrNull(), + number = i + 1f, + volume = 0, + url = href, + scanlator = null, + branch = null, + uploadDate = dateFormat.parseSafe(tr.selectFirst("div.date")?.text()), + source = source, + ) + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + val pos = data.indexOf("\"fullimg") + if (pos == -1) { + continue + } + val json = data.substring(pos).substringAfter('[').substringBefore(';') + .substringBeforeLast(']') + val domain = domain + return json.split(",").mapNotNull { + it.trim() + .removeSurrounding('"', '\'') + .toRelativeUrl(domain) + .takeUnless(String::isBlank) + }.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } + + private suspend fun fetchAvailableTags(): Set { + val domain = domain + val doc = webClient.httpGet("https://$domain/mostfavorites&sort=manga").parseHtml() + val root = doc.body().selectFirst("div.main_fon")?.getElementById("side") + ?.select("ul")?.last() ?: doc.parseFailed("Cannot find root") + return root.select("li.sidetag").mapToSet { li -> + val a = li.children().lastOrNull() ?: li.parseFailed("a is null") + MangaTag( + title = a.text().toTagName(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://${domain}").parseHtml().body() + val root = doc.requireElementById("top_user") + val a = root.getElementsByAttributeValueContaining("href", "/user/").firstOrNull() + ?: throw AuthRequiredException(source) + return a.attr("href").removeSuffix('/').substringAfterLast('/') + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.replace("/manga/", "/related/").toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("right") + return root.select("div.related").mapNotNull { div -> + val info = div.selectFirst(".related_info") ?: return@mapNotNull null + val a = info.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val title = a.text().parseTitle() + val author = info.getElementsByAttributeValueStarting( + "href", + "/mangaka", + ).firstOrNull()?.text() + Manga( + id = generateUid(a.attrAsRelativeUrlAnyHost("href")), + url = href, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + altTitles = setOfNotNull(title.second), + title = title.first, + authors = setOfNotNull(author), + coverUrl = div.selectFirst("img")?.absUrl("src").orEmpty(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + protected open fun buildUrl( + offset: Int, + order: SortOrder, + filter: MangaListFilter, + ): HttpUrl { + val builder = urlBuilder() + builder.addQueryParameter("offset", offset.toString()) + when { + !filter.query.isNullOrEmpty() -> { + builder.addQueryParameter("do", "search") + builder.addQueryParameter("subaction", "search") + builder.addQueryParameter("search_start", ((offset / 40) + 1).toString()) + builder.addQueryParameter("full_search", "0") + builder.addQueryParameter("result_from", (offset + 1).toString()) + builder.addQueryParameter("result_num", "40") + builder.addQueryParameter("story", filter.query) + builder.addQueryParameter("need_sort_date", "false") + } + + else -> { + if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { + builder.addPathSegment("tags") + val joiner = StringUtil.StringJoiner("+") + filter.tags.forEach { joiner.add(it.key) } + filter.tagsExclude.forEach { joiner.add("-"); joiner.append(it.key) } + builder.addPathSegment(joiner.complete()) + builder.addQueryParameter( + "n", + when (order) { + SortOrder.RATING, + SortOrder.POPULARITY, + -> "favdesc" + + SortOrder.ALPHABETICAL -> "abcasc" + else -> "" // SortOrder.NEWEST + }, + ) + } else { + when (order) { + SortOrder.POPULARITY -> builder.addPathSegment("mostviews") + SortOrder.ALPHABETICAL -> builder.addPathSegment("catalog") + SortOrder.RATING -> builder.addPathSegment("mostfavorites") + else -> { // SortOrder.NEWEST + builder.addPathSegment("manga") + builder.addPathSegment("new") + } + } + } + } + } + return builder.build() + } + + private fun String.toTagName() = replace('_', ' ').toTitleCase() + + private fun String.parseTitle(): Pair { + var depth = 0 + for (i in indices.reversed()) { + val c = this[i] + if (c == '(') { + depth-- + if (depth == 0 && (i + 2) < lastIndex && i > 0) { + return substring(i + 1, lastIndex).trim() to substring(0, i).trim().nullIfEmpty() + } + } else if (c == ')') { + depth++ + } + } + return this to null + } + + private fun Element.attrAsRelativeUrlAnyHost(attributeKey: String): String { + val attr = attr(attributeKey) + if (attr.isEmpty() || attr.startsWith("data:")) { + throw IllegalArgumentException("Wrong url $attr") + } + if (attr.startsWith('/')) { + return attr + } + val host = attr.toHttpUrl().host + return attr.substringAfter(host) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt similarity index 84% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt rename to src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt index cabf4992..151065aa 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/DoujinDesuParser.kt +++ b/src/main/kotlin/tsuki/parsers/DoujinDesuParser.kt @@ -1,373 +1,410 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.net.URLDecoder -import java.time.Instant -import java.util.* -import kotlin.time.Duration.Companion.milliseconds - -internal abstract class DoujinDesuParser( - context: MangaLoaderContext, - source: MangaParserSource -): PagedMangaParser(context, source, pageSize = 18) { - - protected abstract val defaultTypes: String - protected abstract val availableContentTypes: Set - - override val configKeyDomain = ConfigKey.Domain("doujin.desu.xxx") - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - @Volatile - private var genresCache: Set? = null - private val genresMutex = Mutex() - - @get:Synchronized - private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 - } - - override val defaultSortOrder: SortOrder = SortOrder.UPDATED - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, SortOrder.NEWEST, SortOrder.ALPHABETICAL, SortOrder.POPULARITY - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isAuthorSearchSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = availableContentTypes, - ) - - private suspend fun getOrFetchGenres(): Set { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - genresCache = fetchAvailableTags() - genresCache!! - } - } - } - - private suspend fun fetchAvailableTags(): Set { - val url = "/api/terms?taxonomy=genre".toAbsoluteUrl(domain) - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val array = JSONArray(decrypted) - return (0 until array.length()).mapTo(mutableSetOf()) { i -> - val obj = array.getJSONObject(i) - MangaTag(key = obj.getString("slug"), title = obj.getString("name"), source = source) - } - } - - override fun getRequestHeaders() = Headers.Builder() - .add("X-Requested-With", "XMLHttpRequest") - .add("Referer", "https://$domain/") - .add("X-App-Secret", "dfdf72051dbfdc7d76889ebd31324e74") - .build() - - private suspend fun executeWithCloudflareRetry(url: String, headers: Headers): okhttp3.Response { - var response = webClient.httpGet(url, headers) - if (response.code == 403) { - val bodyString = response.body?.string().orEmpty() - if ("Unauthorized" in bodyString || "Access denied" in bodyString) { - response.close() - refreshCloudflare() - response = webClient.httpGet(url, headers) - } - } - if (response.code in 500..599) { - response.close() - kotlinx.coroutines.delay(1000L.milliseconds) - response = webClient.httpGet(url, headers) - } - return response - } - - private suspend fun refreshCloudflare() { - webClient.httpGet("https://$domain/", getRequestHeaders()).close() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val limit = pageSize - val offset = (page - 1) * limit - - val url = urlBuilder().apply { - addPathSegment("api") - addPathSegment("manga") - addQueryParameter("limit", limit.toString()) - addQueryParameter("offset", offset.toString()) - - filter.query?.let { addQueryParameter("search", it) } - - val sortParam = when (order) { - SortOrder.POPULARITY -> "rating" - SortOrder.ALPHABETICAL -> "title_asc" - SortOrder.NEWEST -> "newest" - SortOrder.UPDATED -> "latest_chapter" - else -> "latest_chapter" - } - addQueryParameter("sort", sortParam) - - filter.types.oneOrThrowIfMany()?.let { - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.DOUJINSHI -> "doujinshi" - else -> null - }?.let { type -> addQueryParameter("type", type) } - } - - filter.states.oneOrThrowIfMany()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> null - }?.let { status -> addQueryParameter("status", status) } - } - - if (filter.tags.isNotEmpty()) { - addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) - } - }.build() - - val jsonResponse = executeWithCloudflareRetry(url.toString(), getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val array = JSONArray(decrypted) - - return (0 until array.length()).map { i -> - val obj = array.getJSONObject(i) - val slug = obj.getString("slug") - val href = "/manga/$slug" - Manga( - id = generateUid(href), - title = obj.getString("title"), - altTitles = emptySet(), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - rating = obj.optDouble("rating", 0.0).toFloat() / 10f, - contentRating = ContentRating.ADULT, - coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> - if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) - }, - tags = emptySet(), - state = null, - authors = emptySet(), - largeCoverUrl = null, - description = null, - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val slug = manga.url.removePrefix("/manga/").removeSuffix("/") - val url = "/api/manga/$slug".toAbsoluteUrl(domain) - - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val obj = JSONObject(decrypted) - - val state = when (obj.optString("status")) { - "completed" -> MangaState.FINISHED - "ongoing" -> MangaState.ONGOING - else -> null - } - - val author = obj.optString("author").takeIf { it.isNotEmpty() && it != "null" } - - val tags = mutableSetOf() - val genresArray = obj.optJSONArray("manga_genres") - if (genresArray != null) { - for (i in 0 until genresArray.length()) { - val genreObj = genresArray.getJSONObject(i).getJSONObject("genres") - tags.add( - MangaTag( - key = genreObj.getString("slug"), - title = genreObj.getString("name"), - source = source, - ) - ) - } - } - - val chaptersArray = obj.getJSONArray("chapters") - val chapters = ArrayList(chaptersArray.length()) - for (i in 0 until chaptersArray.length()) { - val chapObj = chaptersArray.getJSONObject(i) - val chNum = chapObj.optDouble("chapter_number", 0.0).toFloat() - val chTitle = if (chNum == chNum.toLong().toFloat()) { - "Chapter ${chNum.toLong()}" - } else { - "Chapter $chNum" - } - chapters.add( - MangaChapter( - id = generateUid("/reader/${chapObj.getString("id")}"), - title = chTitle, - number = chNum, - volume = 0, - url = "/reader/${chapObj.getString("id")}", - scanlator = null, - uploadDate = runCatching { Instant.parse(chapObj.optString("created_at")).toEpochMilli() }.getOrDefault(0L), - branch = null, - source = source, - ) - ) - } - - val rawDesc = obj.optString("description").takeIf { it.isNotEmpty() && it != "null" } - val cleanDesc = rawDesc?.let { html -> - val document = org.jsoup.Jsoup.parseBodyFragment(html) - val fullText = document.text() - val cutoffRegex = Regex("""download\s+(batch|volume)""", RegexOption.IGNORE_CASE) - val cutoffIndex = cutoffRegex.find(fullText)?.range?.first - val textBeforeDownload = if (cutoffIndex != null) fullText.substring(0, cutoffIndex) else fullText - textBeforeDownload - .replace(Regex("""^Sinopsis:\s*""", RegexOption.IGNORE_CASE), "") - .replace(Regex("""https?://\S+"""), "") - .replace(Regex("""\[url=.*?]|\[/url]"""), "") - .trim() - } - - val coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> - if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) - } - - val fullManga = manga.copy( - authors = setOfNotNull(author), - description = cleanDesc, - state = state, - rating = obj.optDouble("rating", 0.0).toFloat() / 10f, - tags = tags, - coverUrl = coverUrl ?: manga.coverUrl, - chapters = chapters.reversed() - ) - - detailsCache[manga.url] = fullManga - return fullManga - } - - override suspend fun getPages(chapter: MangaChapter): List { - val chId = chapter.url.removePrefix("/reader/").removeSuffix("/") - val url = "/api/chapters/$chId".toAbsoluteUrl(domain) - - val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() - val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) - val obj = JSONObject(decrypted) - - val pagesList = mutableListOf() - val contentUrls = obj.optJSONArray("content_urls") - if (contentUrls != null && contentUrls.length() > 0) { - for (i in 0 until contentUrls.length()) { - pagesList.add( - MangaPage( - id = generateUid(contentUrls.getString(i)), - url = transformPageUrl(contentUrls.getString(i)), - preview = null, - source = source, - ) - ) - } - } else { - val signedUrlsStr = obj.optString("signed_content_urls") - if (signedUrlsStr.isNotEmpty()) { - val signedUrls = JSONArray(signedUrlsStr) - for (i in 0 until signedUrls.length()) { - pagesList.add( - MangaPage( - id = generateUid(signedUrls.getString(i)), - url = transformPageUrl(signedUrls.getString(i)), - preview = null, - source = source, - ) - ) - } - } - } - - return pagesList - } - - private fun toSigned32(x: Long): Int { - var y = x and 0xFFFFFFFFL - if (y >= 0x80000000L) y -= 0x100000000L - return y.toInt() - } - - private fun wH(e: Int): String { - val t = "doujindesu-scrapers-cannot-read-this-super-secret-salt-2026-v2_$e" - var a = 0 - for (ch in t) { - a = (a shl 5) - a + ch.code - a = toSigned32(a.toLong()) - } - var l = if (a != 0) kotlin.math.abs(a).toLong() else 123456789L - return buildString { - repeat(32) { - l = (l * 1664525L + 1013904223L) % 4294967296L - append((33 + (l % 93)).toInt().toChar()) - } - } - } - - private fun lU(): List { - val now = System.currentTimeMillis() - val t = now / 3600000L - return listOf(wH(t.toInt()), wH((t - 1).toInt()), wH((t + 1).toInt())) - } - - private fun yre(encryptedHex: String, key: String): String { - val bytes = encryptedHex.chunked(2).mapNotNull { it.toIntOrNull(16) } - var d = 42 - val keyLen = key.length - return buildString { - for ((idx, byteVal) in bytes.withIndex()) { - val keyChar = key[idx % keyLen].code - val k = byteVal xor keyChar xor (idx * 13) xor d - append((k and 0xFF).toChar()) - d = (d + byteVal) % 256 - } - } - } - - private fun decrypt(encryptedHex: String): String { - for (key in lU()) { - try { - val decoded = yre(encryptedHex, key) - return URLDecoder.decode(decoded, "UTF-8") - } catch (_: Exception) { - } - } - throw RuntimeException("Decryption failed") - } - - private fun transformPageUrl(page: String): String = when { - page.contains("/uploads/") && !page.contains("/storage/uploads/") -> - page.replace("/uploads/", "/storage/uploads/") - page.contains("/upload/") && !page.contains("/storage/upload/") -> - page.replace("/upload/", "/storage/upload/") - else -> page - } -} +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlBuilder + +import okhttp3.Headers +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.net.URLDecoder +import java.time.Instant +import java.util.EnumSet +import kotlin.math.abs +import kotlin.text.iterator +import kotlin.time.Duration.Companion.milliseconds +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +// Only for the source id/DoujinDesu? + +internal abstract class DoujinDesuParser( + context: MangaLoaderContext, + source: MangaParserSource +): PagedMangaParser(context, source, pageSize = 18) { + + override val configKeyDomain = ConfigKey.Domain("doujin.desu.xxx") + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + @Volatile + private var genresCache: Set? = null + private val genresMutex = Mutex() + + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.NEWEST_ASC + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.DOUJINSHI, + ContentType.MANHWA + ), + ) + + private suspend fun getOrFetchGenres(): Set { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + genresCache = fetchAvailableTags() + genresCache!! + } + } + } + + private suspend fun fetchAvailableTags(): Set { + val url = "/api/terms?taxonomy=genre".toAbsoluteUrl(domain) + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val array = JSONArray(decrypted) + return (0 until array.length()).mapTo(mutableSetOf()) { i -> + val obj = array.getJSONObject(i) + MangaTag(key = obj.getString("slug"), title = obj.getString("name"), source = source) + } + } + + override fun getRequestHeaders() = Headers.Builder() + .add("X-Requested-With", "XMLHttpRequest") + .add("Referer", "https://$domain/") + .add("X-App-Secret", "dfdf72051dbfdc7d76889ebd31324e74") + .build() + + private suspend fun executeWithCloudflareRetry(url: String, headers: Headers): Response { + var response = webClient.httpGet(url, headers) + if (response.code == 403) { + val bodyString = response.body.string() + if ("Unauthorized" in bodyString || "Access denied" in bodyString) { + response.close() + refreshCloudflare() + response = webClient.httpGet(url, headers) + } + } + if (response.code in 500..599) { + response.close() + delay(1000L.milliseconds) + response = webClient.httpGet(url, headers) + } + return response + } + + private suspend fun refreshCloudflare() { + webClient.httpGet("https://$domain/", getRequestHeaders()).close() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val limit = pageSize + val offset = (page - 1) * limit + + val url = urlBuilder().apply { + addPathSegment("api") + addPathSegment("manga") + addQueryParameter("limit", limit.toString()) + addQueryParameter("offset", offset.toString()) + + filter.query?.let { addQueryParameter("search", it) } + + val sortParam = when (order) { + SortOrder.POPULARITY -> "rating" + SortOrder.ALPHABETICAL -> "title_asc" + SortOrder.NEWEST -> "newest" + SortOrder.NEWEST_ASC -> "oldest" + SortOrder.UPDATED -> "latest_chapter" + else -> "latest_chapter" + } + addQueryParameter("sort", sortParam) + + filter.types.oneOrThrowIfMany()?.let { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.DOUJINSHI -> "doujinshi" + else -> null + }?.let { type -> addQueryParameter("type", type) } + } + + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> null + }?.let { status -> addQueryParameter("status", status) } + } + + if (filter.tags.isNotEmpty()) { + addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) + } + }.build() + + val jsonResponse = executeWithCloudflareRetry(url.toString(), getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val array = JSONArray(decrypted) + + return (0 until array.length()).map { i -> + val obj = array.getJSONObject(i) + val slug = obj.getString("slug") + val href = "/manga/$slug" + Manga( + id = generateUid(href), + title = obj.getString("title"), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = obj.optDouble("rating", 0.0).toFloat() / 10f, + contentRating = ContentRating.ADULT, + coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> + if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) + }, + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val slug = manga.url.removePrefix("/manga/").removeSuffix("/") + val url = "/api/manga/$slug".toAbsoluteUrl(domain) + + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val obj = JSONObject(decrypted) + + val state = when (obj.optString("status")) { + "completed" -> MangaState.FINISHED + "ongoing" -> MangaState.ONGOING + else -> null + } + + val author = obj.optString("author").takeIf { it.isNotEmpty() && it != "null" } + + val tags = mutableSetOf() + val genresArray = obj.optJSONArray("manga_genres") + if (genresArray != null) { + for (i in 0 until genresArray.length()) { + val genreObj = genresArray.getJSONObject(i).getJSONObject("genres") + tags.add( + MangaTag( + key = genreObj.getString("slug"), + title = genreObj.getString("name"), + source = source, + ) + ) + } + } + + val chaptersArray = obj.getJSONArray("chapters") + val chapters = ArrayList(chaptersArray.length()) + for (i in 0 until chaptersArray.length()) { + val chapObj = chaptersArray.getJSONObject(i) + val chNum = chapObj.optDouble("chapter_number", 0.0).toFloat() + val chTitle = if (chNum == chNum.toLong().toFloat()) { + "Chapter ${chNum.toLong()}" + } else { + "Chapter $chNum" + } + chapters.add( + MangaChapter( + id = generateUid("/reader/${chapObj.getString("id")}"), + title = chTitle, + number = chNum, + volume = 0, + url = "/reader/${chapObj.getString("id")}", + scanlator = null, + uploadDate = runCatching { Instant.parse(chapObj.optString("created_at")).toEpochMilli() }.getOrDefault(0L), + branch = null, + source = source, + ) + ) + } + + val rawDesc = obj.optString("description").takeIf { it.isNotEmpty() && it != "null" } + val cleanDesc = rawDesc?.let { html -> + val document = Jsoup.parseBodyFragment(html) + val fullText = document.text() + val cutoffRegex = Regex("""download\s+(batch|volume)""", RegexOption.IGNORE_CASE) + val cutoffIndex = cutoffRegex.find(fullText)?.range?.first + val textBeforeDownload = if (cutoffIndex != null) fullText.substring(0, cutoffIndex) else fullText + textBeforeDownload + .replace(Regex("""^Sinopsis:\s*""", RegexOption.IGNORE_CASE), "") + .replace(Regex("""https?://\S+"""), "") + .replace(Regex("""\[url=.*?]|\[/url]"""), "") + .trim() + } + + val coverUrl = obj.optString("cover_url").takeIf { it.isNotEmpty() }?.let { cover -> + if (cover.startsWith("http")) cover else cover.toAbsoluteUrl(domain) + } + + val result = manga.copy( + authors = setOfNotNull(author), + description = cleanDesc, + state = state, + rating = obj.optDouble("rating", 0.0).toFloat() / 10f, + tags = tags, + coverUrl = coverUrl ?: manga.coverUrl, + chapters = chapters.sortedBy { it.number } + ) + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + override suspend fun getPages(chapter: MangaChapter): List { + val chId = chapter.url.removePrefix("/reader/").removeSuffix("/") + val url = "/api/chapters/$chId".toAbsoluteUrl(domain) + + val jsonResponse = executeWithCloudflareRetry(url, getRequestHeaders()).parseJson() + val decrypted = decrypt(jsonResponse.getString("_enc_resp_")) + val obj = JSONObject(decrypted) + + val pagesList = mutableListOf() + val contentUrls = obj.optJSONArray("content_urls") + if (contentUrls != null && contentUrls.length() > 0) { + for (i in 0 until contentUrls.length()) { + pagesList.add( + MangaPage( + id = generateUid(contentUrls.getString(i)), + url = transformPageUrl(contentUrls.getString(i)), + preview = null, + source = source, + ) + ) + } + } else { + val signedUrlsStr = obj.optString("signed_content_urls") + if (signedUrlsStr.isNotEmpty()) { + val signedUrls = JSONArray(signedUrlsStr) + for (i in 0 until signedUrls.length()) { + pagesList.add( + MangaPage( + id = generateUid(signedUrls.getString(i)), + url = transformPageUrl(signedUrls.getString(i)), + preview = null, + source = source, + ) + ) + } + } + } + + return pagesList + } + + private fun toSigned32(x: Long): Int { + var y = x and 0xFFFFFFFFL + if (y >= 0x80000000L) y -= 0x100000000L + return y.toInt() + } + + private fun wH(e: Int): String { + val t = "doujindesu-scrapers-cannot-read-this-super-secret-salt-2026-v2_$e" + var a = 0 + for (ch in t) { + a = (a shl 5) - a + ch.code + a = toSigned32(a.toLong()) + } + var l = if (a != 0) abs(a).toLong() else 123456789L + return buildString { + repeat(32) { + l = (l * 1664525L + 1013904223L) % 4294967296L + append((33 + (l % 93)).toInt().toChar()) + } + } + } + + private fun lU(): List { + val now = System.currentTimeMillis() + val t = now / 3600000L + return listOf(wH(t.toInt()), wH((t - 1).toInt()), wH((t + 1).toInt())) + } + + private fun yre(encryptedHex: String, key: String): String { + val bytes = encryptedHex.chunked(2).mapNotNull { it.toIntOrNull(16) } + var d = 42 + val keyLen = key.length + return buildString { + for ((idx, byteVal) in bytes.withIndex()) { + val keyChar = key[idx % keyLen].code + val k = byteVal xor keyChar xor (idx * 13) xor d + append((k and 0xFF).toChar()) + d = (d + byteVal) % 256 + } + } + } + + private fun decrypt(encryptedHex: String): String { + for (key in lU()) { + try { + val decoded = yre(encryptedHex, key) + return URLDecoder.decode(decoded, "UTF-8") + } catch (_: Exception) { + } + } + throw RuntimeException("Decryption failed") + } + + private fun transformPageUrl(page: String): String = when { + page.contains("/uploads/") && !page.contains("/storage/uploads/") -> + page.replace("/uploads/", "/storage/uploads/") + page.contains("/upload/") && !page.contains("/storage/upload/") -> + page.replace("/upload/", "/storage/upload/") + else -> page + } +} diff --git a/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt new file mode 100644 index 00000000..262320fd --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/EZManhwaParser.kt @@ -0,0 +1,270 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet + +import org.json.JSONObject +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +abstract class EZManhwaParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + api: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + protected open val apiUrl = api + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isAuthorSearchSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.NOVEL, + ), + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val endpoint = if (filter.query.isNullOrBlank()) "$apiUrl/series" else "$apiUrl/series/search" + val url = endpoint.toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + + if (filter.query.isNullOrBlank()) { + val sort = when (order) { + SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "latest" + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popular" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "newest" + SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "alphabetical" + else -> "latest" + } + addQueryParameter("sort", sort) + + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + MangaState.ABANDONED -> "DROPPED" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + }) + } + } else { + addQueryParameter("q", filter.query) + } + }.build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: JSONObject): List { + val data = json.getJSONArray("data") + return data.mapJSON { obj -> + obj.toManga(domain, source, isNsfwSource) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val url = "$apiUrl/series/${manga.url}" + val json = webClient.httpGet(url).parseJson() + val updatedManga = json.toManga(domain, source, isNsfwSource).copy( + id = manga.id, + source = source, + ) + val chapters = fetchChapters(manga.url) + return updatedManga.copy(chapters = chapters) + } + + private suspend fun fetchChapters(seriesSlug: String): List { + val allChapters = mutableListOf() + var page = 1 + var hasMore = true + + while (hasMore) { + val url = "$apiUrl/series/$seriesSlug/chapters?page=$page&perPage=100&sort=desc" + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + val totalPages = json.getInt("totalPages") + val current = json.getInt("current") + + data.mapJSONToSet { chObj -> + chObj.toSChapter(seriesSlug, dateFormat) + }.let { allChapters.addAll(it) } + + hasMore = current < totalPages + page++ + } + return allChapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val url = "$apiUrl/${chapter.url}" + val json = webClient.httpGet(url).parseJson() + + if (json.optBoolean("requiresPurchase", false)) { + throw AuthRequiredException( + source, + RuntimeException("This chapter requires coins. Purchase it via the website to read.") + ) + } + + val images = json.getJSONArray("images") + return (0 until images.length()).map { i -> + val imgUrl = images.getJSONObject(i).getString("url") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + fun JSONObject.toManga(domain: String, source: MangaSource, isNsfw: Boolean): Manga { + val slug = getString("slug") + val title = getString("title") + val cover = optString("cover", "").takeIf { it.isNotBlank() && it != "null" } + val status = optString("status", "").takeIf { it.isNotBlank() && it != "null" } + val altTitles = optString("alternativeTitles", "").takeIf { it.isNotBlank() && it != "null" } + val description = optString("description", "").takeIf { it.isNotBlank() && it != "null" } + val author = optString("author", "").takeIf { it.isNotBlank() && it != "null" } + val artist = optString("artist", "").takeIf { it.isNotBlank() && it != "null" } + val genresArray = optJSONArray("genres") + val genres = genresArray?.mapJSON { it.getString("name") } ?: emptyList() + + val state = when (status?.uppercase(Locale.ROOT)) { + "ONGOING", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "DROPPED" -> MangaState.ABANDONED + "HIATUS" -> MangaState.PAUSED + else -> null + } + + val altSet = altTitles?.split(",") + ?.map { it.trim() } + ?.filter { it.isNotBlank() && it != "null" } + ?.toSet() ?: emptySet() + val authors = setOfNotNull(author, artist).filter { it.isNotBlank() && it != "null" }.toSet() + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "/series/$slug".toAbsoluteUrl(domain), + title = title, + altTitles = altSet, + coverUrl = cover, + description = description, + authors = authors, + tags = genres.map { MangaTag(key = it, title = it, source = source) }.toSet(), + state = state, + rating = RATING_UNKNOWN, + source = source, + contentRating = if (isNsfw) ContentRating.ADULT else null, + ) + } + + private fun JSONObject.toSChapter(seriesSlug: String, dateFormat: SimpleDateFormat): MangaChapter { + val slug = getString("slug") + val number = optDouble("number", -1.0).takeIf { it != -1.0 }?.toFloat() ?: -1f + val title = optString("title", "").takeIf { it.isNotBlank() && it != "null" } + val createdAt = optString("createdAt", "").takeIf { it.isNotBlank() && it != "null" } + + val prefix = if (optBoolean("requiresPurchase", false)) "🔒 " else "" + val numStr = if (number % 1.0 == 0.0) number.toInt().toString() else number.toString() + val chapterName = when { + title == null || title == numStr -> "Chapter $numStr" + title.matches(Regex("^(?i)(chapter|ch\\.?|episode|ep\\.?)\\s*.*")) -> title + title.startsWith("-") || title.startsWith(":") -> "Chapter $numStr $title" + else -> "Chapter $numStr - $title" + } + + return MangaChapter( + id = generateUid(slug), + url = "series/$seriesSlug/chapters/$slug", + title = prefix + chapterName, + number = number, + volume = 0, + uploadDate = dateFormat.parseSafe(createdAt), + scanlator = null, + branch = null, + source = source, + ) + } + + private fun SimpleDateFormat.parseSafe(dateStr: String?): Long { + return dateStr?.let { runCatching { parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt similarity index 82% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt rename to src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt index bff94429..7514ac1d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryAdultsParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryAdultsParser.kt @@ -1,4 +1,37 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.ErrorMessages + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import androidx.collection.ArraySet import kotlinx.coroutines.async @@ -6,13 +39,14 @@ import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* +import java.util.EnumSet +import java.util.Locale + +/* +Default: One chapter per manga + +Override getDetails chapters section if source contains more than one chapter. + */ internal abstract class GalleryAdultsParser( context: MangaLoaderContext, @@ -55,22 +89,16 @@ internal abstract class GalleryAdultsParser( ), ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { val url = buildString { append("https://") append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?q=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) append("&") } - else -> { val tag = filter.tags.oneOrThrowIfMany() val lang = filter.locale @@ -124,9 +152,7 @@ internal abstract class GalleryAdultsParser( ) } } - - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. + private suspend fun fetchAvailableTags(): Set { return coroutineScope { (1..3).map { page -> @@ -221,4 +247,4 @@ internal abstract class GalleryAdultsParser( .trim() protected open fun Locale.toLanguagePath() = getDisplayLanguage(Locale.ENGLISH).lowercase() -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt b/src/main/kotlin/tsuki/parsers/GalleryParser.kt similarity index 75% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt rename to src/main/kotlin/tsuki/parsers/GalleryParser.kt index a2d0eff4..d2957e4b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/GalleryParser.kt +++ b/src/main/kotlin/tsuki/parsers/GalleryParser.kt @@ -1,12 +1,37 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.attrOrNull +import tsuki.util.attrOrThrow +import tsuki.util.extractChapterNumber +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectFirstOrThrow +import tsuki.util.urlBuilder -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet internal abstract class GalleryParser( context: MangaLoaderContext, @@ -33,10 +58,9 @@ internal abstract class GalleryParser( override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions() override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query val url = urlBuilder().apply { when { - !query.isNullOrEmpty() -> addQueryParameter("search", query) + !filter.query.isNullOrEmpty() -> addQueryParameter("search", filter.query) filter.tags.isNotEmpty() -> addPathSegments(filter.tags.first().key) order == SortOrder.POPULARITY -> addPathSegment("hot") } @@ -75,18 +99,20 @@ internal abstract class GalleryParser( } } + override suspend fun getDetails(manga: Manga): Manga { val content = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val description = content.selectFirst("div.article-info")?.text()?.trim() ?: "" val df = SimpleDateFormat("HH:mm dd-MM-yyyy") val time = content.selectFirst("div.article-info > small")?.text()?.trim() val chapters = content.selectFirst("nav.pagination")?.select("a.pagination-link") - ?.mapChapters { index, element -> + ?.map { element -> val relUrl = element.attrAsRelativeUrl("href") + val chapterText = element.ownText() MangaChapter( id = generateUid(relUrl), - title = null, - number = index + 1f, + title = chapterText, + number = chapterText.extractChapterNumber(), volume = 0, url = relUrl, scanlator = null, @@ -94,7 +120,9 @@ internal abstract class GalleryParser( branch = null, source = source, ) - }.orEmpty() + } + ?.sortedBy { it.number } + .orEmpty() return manga.copy(chapters = chapters, description = description) } @@ -121,4 +149,4 @@ internal abstract class GalleryParser( ) }.toSet() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/parsers/GroupleParser.kt b/src/main/kotlin/tsuki/parsers/GroupleParser.kt new file mode 100644 index 00000000..cf14bd4e --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/GroupleParser.kt @@ -0,0 +1,616 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.exception.ParseException +import tsuki.network.CommonHeaders + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.YEAR_MAX +import tsuki.model.YEAR_MIN + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.concatUrl +import tsuki.util.findGroupValue +import tsuki.util.flatMapChapters +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.headersContentLength +import tsuki.util.ifZero +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.mapToSet +import tsuki.util.metaValue +import tsuki.util.nullIfEmpty +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.runCatchingCancellable +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectFirstParent +import tsuki.util.substringBetween +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.upBy +import tsuki.util.urlBuilder +import tsuki.util.urlEncoded + +import androidx.collection.MutableScatterMap +import androidx.collection.ScatterMap +import kotlinx.coroutines.flow.channelFlow +import kotlinx.coroutines.flow.firstOrNull +import kotlinx.coroutines.launch +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import okio.IOException +import org.json.JSONArray +import org.jsoup.nodes.Element +import java.net.HttpURLConnection +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +private const val PAGE_SIZE = 70 +private const val NSFW_ALERT = "ÑекÑуальные Ñцены" +private const val NOTHING_FOUND = "Ðичего не найдено" +private const val MIN_IMAGE_SIZE = 1024L +private const val HEADER_ACCEPT = "Accept" +private const val RELATED_TITLE = "СвÑзанные произведениÑ" +private const val COPYRIGHT_ALERT = "Запрещена Ð¿ÑƒÐ±Ð»Ð¸ÐºÐ°Ñ†Ð¸Ñ Ð¿Ñ€Ð¾Ð¸Ð·Ð²ÐµÐ´ÐµÐ½Ð¸Ñ" +private const val NO_CHAPTERS = "Ð’ Ñтой манге еще нет ни одной главы" + +internal abstract class GroupleParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val siteId: Int, +) : AbstractMangaParser(context, source), MangaParserAuthProvider, Interceptor { + + @Volatile + private var cachedPagesServer: String? = null + + override val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (X11; U; UNICOS lcLinux; en-US) Gecko/20140730 (KHTML, like Gecko, Safari/419.3) Arora/0.8.0", + ) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(false) + private val tagsIndex = suspendLazy(initializer = ::fetchTagsMap) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add(CommonHeaders.REFERER, "https://$domain/") + .add(CommonHeaders.USER_AGENT, config[userAgentKey]) + .add(CommonHeaders.ACCEPT_LANGUAGE, "ru,en-US;q=0.7,en;q=0.3") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.ADDED, + ) + + override val authUrl: String + get() { + val targetUri = "https://${domain}/".urlEncoded() + return "https://3.grouple.co/internal/auth/sso?siteId=$siteId&=targetUri=$targetUri" + } + + override suspend fun isAuthorized(): Boolean = hasAuthCookie() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.FINISHED, MangaState.ABANDONED, MangaState.UPCOMING), + ) + + override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List { + val domain = domain + val root = if (filter.isEmpty()) { + webClient.httpGet( + "https://$domain/list?sortType=${ + getSortKey(order) + }&offset=${offset upBy PAGE_SIZE}", + ).parseHtml().body().let { doc -> (doc.getElementById("mangaBox") ?: doc.getElementById("mangaResults")) } + } else { + advancedSearch(offset, order, filter).parseHtml() + } + checkNotNull(root) { "Root not found" } + val tiles = root.selectFirst("div.tiles.row") + if (tiles == null) { + if (root.getElementsContainingOwnText(NOTHING_FOUND).isNotEmpty()) { + return emptyList() + } + root.parseFailed("No tiles found") + } + return tiles.select("div.tile").mapNotNull(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + val doc = response.parseHtml() + val root = doc.body().requireElementById("mangaBox") + val dateFormat = SimpleDateFormat("dd.MM.yy", Locale.US) + val coverImg = root.selectFirst("img.cr-hero-poster__img") + val translations = if (config[splitTranslationsKey]) { + root.selectFirst("div.translator-selection") + ?.select(".translator-selection-item") + ?.associate { + it.id().removePrefix("tr-").toLong() to it.selectFirst(".translator-selection-name")?.textOrNull() + } + } else { + null + } + val newSource = source + val chaptersList = root.getElementById("chapters-list") + var isRestricted = false + if (chaptersList == null && root.getElementsContainingOwnText(NO_CHAPTERS).isEmpty()) { + if (root.getElementsContainingOwnText(COPYRIGHT_ALERT).isNotEmpty()) { + isRestricted = true + } else { + root.parseFailed("No chapters found") + } + } + val hashRegex = Regex("window.user_hash\\s*=\\s*\'([^\']+)\'") + val userHash = doc.select("script").firstNotNullOfOrNull { it.html().findGroupValue(hashRegex) } + val hasNsfwAlert = root.select(".alert-warning").any { it.ownText().contains(NSFW_ALERT) } + return manga.copy( + source = newSource, + title = doc.metaValue("name") ?: manga.title, + altTitles = root.select(".cr-hero-names__alt > span > span") + .filterNot { it.hasClass("cr-hero-names__alt-separator") } + .flatMapTo(linkedSetOf()) { element -> + element.text().split('/').mapNotNull { it.trim().nullIfEmpty() } + } + manga.altTitles, + publicUrl = response.request.url.toString(), + description = root.selectFirst("div.cr-description__content")?.html(), + largeCoverUrl = coverImg?.attrAsAbsoluteUrlOrNull("src"), + coverUrl = manga.coverUrl + ?: coverImg?.attrAsAbsoluteUrlOrNull("src"), + tags = root.select("a[href*='/list/genre/']") + .mapTo(manga.tags.toMutableSet()) { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").removeSuffix('/').substringAfterLast('/'), + source = source, + ) + }, + state = if (isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = manga.authors, + contentRating = (if (hasNsfwAlert) ContentRating.SUGGESTIVE else ContentRating.SAFE) + .coerceAtLeast(manga.contentRating ?: ContentRating.SAFE), + chapters = chaptersList?.select("a.chapter-link") + ?.flatMapChapters(reversed = true) { a -> + val tr = a.selectFirstParent("tr") ?: return@flatMapChapters emptyList() + val href = a.attrAsRelativeUrl("href") + val number = tr.attr("data-num").toFloatOrNull()?.div(10f) ?: 0f + val volume = tr.attr("data-vol").toIntOrNull() ?: 0 + if (translations.isNullOrEmpty() || a.attr("data-translations").isEmpty()) { + var translators = "" + val translatorElement = a.attr("title") + if (translatorElement.isNotBlank()) { + translators = translatorElement.replace("(Переводчик),", "&").removeSuffix(" (Переводчик)") + } + listOf( + MangaChapter( + id = generateUid(href), + title = a.text().removePrefix(manga.title).trim().nullIfEmpty(), + number = number, + volume = volume, + url = href.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(tr.selectFirst("td.date")?.text()), + scanlator = translators, + source = newSource, + branch = null, + ), + ) + } else { + val translationData = JSONArray(a.attr("data-translations")) + translationData.mapJSON { jo -> + val personId = jo.getLong("personId") + val link = href.withQueryParam("tran", personId.toString()) + MangaChapter( + id = generateUid(link), + title = a.text().removePrefix(manga.title).trim(), + number = number, + volume = volume, + url = link.withQueryParam("d", userHash), + uploadDate = dateFormat.parseSafe(jo.getStringOrNull("dateCreated")), + scanlator = null, + source = newSource, + branch = translations[personId], + ) + } + } + }.orEmpty(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List { + if (chapter.source != source && chapter.source is MangaParserSource) { // handle redirects between websites + return context.newParserInstance(chapter.source).getPages(chapter) + } + val url = chapter.url.toAbsoluteUrl(domain).toHttpUrl().newBuilder().setQueryParameter("mtr", "1").build() + val doc = webClient.httpGet(url).parseHtml() + val scripts = doc.select("script") + for (script in scripts) { + val data = script.html() + var pos = data.indexOf("rm_h.readerDoInit(") + if (pos != -1) { + parsePagesV2(data, pos)?.let { return it } + } + pos = data.indexOf("rm_h.readerInit( 0,") + if (pos != -1) { + parsePagesV1(data, pos)?.let { return it } + } + pos = data.indexOf(".readerInit(") + if (pos != -1) { + parsePagesV3(data, pos).let { return it } + } + } + doc.parseFailed("Pages list not found at ${chapter.url}") + } + + override suspend fun getPageUrl(page: MangaPage): String { + val parts = page.url.split('|') + if (parts.size < 2) { + return page.url + } + val path = parts.last() + // fast path + cachedPagesServer?.let { host -> + val url = concatUrl("https://$host/", path) + if (tryHead(url)) { + return url + } else { + cachedPagesServer = null + } + } + // slow path + val candidates = HashSet((parts.size - 1) * 2) + for (i in 0 until parts.size - 1) { + val server = parts[i].trim().ifEmpty { "https://$domain/" } + candidates.add(concatUrl(server, path)) + candidates.add(concatUrl(server, path.substringBeforeLast('?'))) + } + val resolvedUrl = channelFlow { + for (url in candidates) { + launch { + if (tryHead(url)) { + send(url) + } + } + } + }.firstOrNull() ?: throw ParseException("No reachable page URL", page.url) + cachedPagesServer = resolvedUrl.toHttpUrlOrNull()?.host + return resolvedUrl + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://${domain}/list/genres/sort_name").parseHtml() + val root = doc.body().getElementById("mangaBox")?.selectFirst("div.leftContent")?.selectFirst("table.table") + ?: doc.parseFailed("Cannot find root") + return root.select("a.element-link").mapToSet { a -> + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/'), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://grouple.co/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else res + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (!request.header(HEADER_ACCEPT).isNullOrEmpty()) { + return chain.proceed(request).checkIfAuthRequired() + } + val ext = request.url.pathSegments.lastOrNull()?.substringAfterLast('.', "")?.lowercase(Locale.ROOT) + return if (ext == "jpg" || ext == "jpeg" || ext == "png" || ext == "webp") { + chain.proceed( + request.newBuilder().header(HEADER_ACCEPT, "image/webp,image/png;q=0.9,image/jpeg,*/*;q=0.8").build(), + ) + } else { + chain.proceed(request).checkIfAuthRequired() + } + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(splitTranslationsKey) + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("mangaBox").select("h4").first { it.ownText() == RELATED_TITLE } + .nextElementSibling() ?: doc.parseFailed("Cannot find root") + return root.select("div.tile").mapNotNull(::parseManga) + } + + private fun getSortKey(sortOrder: SortOrder) = when (sortOrder) { + SortOrder.ALPHABETICAL -> "name" + SortOrder.POPULARITY -> "rate" + SortOrder.UPDATED -> "updated" + SortOrder.ADDED, + SortOrder.NEWEST, + -> "created" + + SortOrder.RATING -> "votes" + else -> "rate" + } + + private suspend fun advancedSearch(offset: Int, order: SortOrder, filter: MangaListFilter): Response { + val tagsMap = if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty()) { + tagsIndex.get() + } else { + null + } + val url = urlBuilder() + .addPathSegment("search") + .addPathSegment("advancedResults") + url.addQueryParameter("q", filter.query) + url.addQueryParameter("offset", offset.toString()) + filter.tags.forEach { tag -> + val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "in") + } + filter.tagsExclude.forEach { tag -> + val tagId = requireNotNull(requireNotNull(tagsMap)[tag.title.lowercase()]) { "Tag ${tag.title} not found" } + url.addQueryParameter(tagId, "ex") + } + url.addQueryParameter( + "years", + buildString { + append(filter.yearFrom.ifZero { YEAR_MIN }) + append(',') + append(filter.yearTo.ifZero { YEAR_MAX }) + }, + ) + url.addQueryParameter( + "sortType", + when (order) { + SortOrder.RATING -> "USER_RATING" + SortOrder.ALPHABETICAL -> "NAME" + SortOrder.ADDED -> "YEAR" + SortOrder.POPULARITY -> "POPULARITY" + SortOrder.NEWEST -> "DATE_CREATE" + SortOrder.UPDATED -> "DATE_UPDATE" + else -> "RATING" + }, + ) + filter.states.forEach { state -> + when (state) { + MangaState.FINISHED -> "s_completed" + MangaState.ABANDONED -> "s_abandoned_popular" + MangaState.UPCOMING -> "s_wait_upload" + else -> null + }?.let { + url.addQueryParameter(it, "in") + } + } + + return webClient.httpGet(url.build()) + } + + private suspend fun tryHead(url: String): Boolean = runCatchingCancellable { + webClient.httpHead(url).use { response -> + response.isSuccessful && !response.isPumpkin() && response.headersContentLength() >= MIN_IMAGE_SIZE + } + }.getOrDefault(false) + + private fun Response.isPumpkin(): Boolean = request.url.host == "upload.wikimedia.org" + + private fun parseManga(node: Element): Manga? { + val imgDiv = node.selectFirst("div.img") ?: return null + val descDiv = node.selectFirst("div.desc") ?: return null + if (descDiv.selectFirst("i.fa-user") != null || descDiv.selectFirst("i.fa-external-link") != null) { + return null // skip author + } + val href = imgDiv.selectFirst("a")?.attrAsAbsoluteUrlOrNull("href") ?: return null + val title = descDiv.selectFirst("h3")?.selectFirst("a")?.text() ?: return null + val tileInfo = descDiv.selectFirst("div.tile-info") + val relUrl = href.toRelativeUrl(domain) + if (relUrl.contains("://")) { + return null + } + val author = tileInfo?.selectFirst("a.person-link")?.text() + return Manga( + id = generateUid(relUrl), + url = relUrl, + publicUrl = href, + title = title, + altTitles = setOfNotNull(descDiv.selectFirst("h5")?.textOrNull()), + coverUrl = imgDiv.selectFirst("img.lazy")?.attrAsAbsoluteUrlOrNull("data-original")?.replace("_p.", "."), + rating = node.selectFirst(".compact-rate")?.attr("title")?.toFloatOrNull()?.div(5f) + ?: RATING_UNKNOWN, + authors = setOfNotNull(author), + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + tags = tileInfo?.select("a.element-link")?.mapToSet { + MangaTag( + title = it.text().toTitleCase(), + key = it.attr("href").substringAfterLast('/'), + source = source, + ) + }.orEmpty(), + state = when { + node.selectFirst("div.tags")?.selectFirst("span.mangaCompleted") != null -> MangaState.FINISHED + + else -> null + }, + source = source, + ) + } + + private fun parsePagesV1(data: String, pos: Int): List? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(1) + val servers = ja.getJSONArray(3).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV2(data: String, pos: Int): List? { + val json = data.substring(pos).substringAfter('(').substringBefore('\n').substringBeforeLast(')') + if (json.isEmpty()) { + return null + } + val ja = JSONArray("[$json]") + val pages = ja.getJSONArray(0) + val servers = ja.getJSONArray(2).mapJSON { it.getString("path") } + val serversStr = servers.joinToString("|") + return (0 until pages.length()).map { i -> + val page = pages.getJSONArray(i) + val primaryServer = page.getString(0) + val url = page.getString(2) + val fullSrc = if ("$primaryServer|$serversStr|$url".contains("one-way.work")) { + // domain that does not need a token + "$primaryServer|$serversStr|${url}".substringBefore("?") + } else { + "$primaryServer|$serversStr|$url" + } + MangaPage( + id = generateUid(url), + url = fullSrc, + preview = null, + source = source, + ) + } + } + + private fun parsePagesV3(data: String, pos: Int): List { + val json = JSONArray( + data.substring(pos) + .substringBetween("(", ")") + .removePrefix("chapterInfo,") + .substringBeforeLast(','), + ) + return (0 until json.length()).map { i -> + val ja = json.getJSONArray(i) + val server = ja.getString(0).ifEmpty { "https://$domain" } + val url = ja.getString(2) + val fullUrl = concatUrl(server, url) + MangaPage( + id = generateUid(url), + url = if (fullUrl.contains("one-way.work")) { + // domain that does not need a token + fullUrl.substringBefore("?") + } else { + fullUrl + }, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTagsMap(): ScatterMap { + val url = "https://$domain/search/advanced" + val properties = + webClient.httpGet(url).parseHtml().body().selectFirst("form.search-form")?.select("div.form-group") + ?.find { it.selectFirst("li.property") != null } + ?.select("li.property") + ?: throw ParseException("Genres filter element not found", url) + val result = MutableScatterMap(properties.size) + properties.forEach { li -> + val name = li.text().lowercase() + val id = li.selectFirstOrThrow("input").id() + result[name] = id + } + return result + } + + private fun String.withQueryParam(name: String, value: String?): String { + if (value == null) return this + return toAbsoluteUrl(domain) + .toHttpUrl() + .newBuilder() + .setQueryParameter(name, value) + .build() + .toString() + .toRelativeUrl(domain) + } + + @Throws(IOException::class) + private fun Response.checkIfAuthRequired(): Response { + val lastPathSegment = request.url.pathSegments.lastOrNull() + if (lastPathSegment == "login") { + closeQuietly() + throw AuthRequiredException(source) + } + if (code == HttpURLConnection.HTTP_NOT_FOUND) { + if (!hasAuthCookie()) { + closeQuietly() + throw AuthRequiredException(source) + } else { + return newBuilder().code(HttpURLConnection.HTTP_OK).build() + } + } + return this + } + + private fun hasAuthCookie() = context.cookieJar.getCookies(domain).any { it.name == "gwt" } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt similarity index 76% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt rename to src/main/kotlin/tsuki/parsers/HeanCmsParser.kt index 3e72bf24..d4b4e8ea 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HeanCmsParser.kt +++ b/src/main/kotlin/tsuki/parsers/HeanCmsParser.kt @@ -1,15 +1,43 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.json.unescapeJson +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.extractChapterNumber import org.json.JSONArray import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale internal abstract class HeanCmsParser( context: MangaLoaderContext, @@ -55,14 +83,13 @@ internal abstract class HeanCmsParser( protected open val paramsUpdated = "latest" override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query val url = buildString { append("https://") append(apiPath) append("/query?query_string=") - query?.let { - append(query.urlEncoded()) + filter.query?.let { + append(filter.query.urlEncoded()) } append("&series_type=Comic&perPage=$pageSize") @@ -154,23 +181,23 @@ internal abstract class HeanCmsParser( val response = webClient.httpGet(url).parseJson() val data = response.getJSONArray("data").asTypedList() val dateFormat = SimpleDateFormat(datePattern, Locale.ENGLISH) - return manga.copy( - chapters = data.mapChapters(reversed = true) { i, it -> - val chapterUrl = - "/series/${it.getJSONObject("series").getString("series_slug")}/${it.getString("chapter_slug")}" - MangaChapter( - id = generateUid(it.getLong("id")), - title = it.getString("chapter_name"), - number = i + 1f, - volume = 0, - url = chapterUrl, - scanlator = null, - uploadDate = dateFormat.parseSafe(it.getString("created_at").substringBefore("T")), - branch = null, - source = source, - ) - }, - ) + + val chapters = data.map { ch -> + val chapterUrl = "/series/${ch.getJSONObject("series").getString("series_slug")}/${ch.getString("chapter_slug")}" + val name = ch.getString("chapter_name") + MangaChapter( + id = generateUid(ch.getLong("id")), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = dateFormat.parseSafe(ch.getString("created_at").substringBefore("T")), + branch = null, + source = source, + ) + }.sortedBy { it.number } + return manga.copy(chapters = chapters) } protected open val selectPages = ".flex > img:not([alt])" @@ -210,4 +237,4 @@ internal abstract class HeanCmsParser( } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt b/src/main/kotlin/tsuki/parsers/HiperParser.kt similarity index 80% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt rename to src/main/kotlin/tsuki/parsers/HiperParser.kt index 4b3b2a74..06f9e7cd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/HiperParser.kt +++ b/src/main/kotlin/tsuki/parsers/HiperParser.kt @@ -1,380 +1,423 @@ -package org.koitharu.kotatsu.parsers.parsers - -import okhttp3.Headers -import okhttp3.Request -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -abstract class HiperParser( - context: MangaLoaderContext, - source: MangaParserSource, - protected val domainName: String, -) : PagedMangaParser(context, source, pageSize = 30) { - - protected open val availableContentTypes: Set = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA) - - protected open val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED) - - protected open val availableContentRating: Set = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) - - protected open val mangaPath = "manga" - - override val configKeyDomain = ConfigKey.Domain(domainName) - override val availableSortOrders: Set = EnumSet.of( - SortOrder.POPULARITY, - SortOrder.UPDATED, - SortOrder.NEWEST, - SortOrder.NEWEST_ASC, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - SortOrder.RELEVANCE - ) - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - ) - - val client: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request() - val newRequest = request.newBuilder() - .header("Referer", "https://$domainName/") - .header("Origin", "https://$domainName") - .build() - var response = chain.proceed(newRequest) - if (response.code == 401) { - response.close() - val acceptHeaders = Headers.Builder() - .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - .build() - val homeRequest = Request.Builder() - .url("https://$domainName/") - .headers(acceptHeaders) - .build() - chain.proceed(homeRequest).close() - response = chain.proceed(request) - } - response - } - .build() - OkHttpWebClient(httpClient, source) - } - - @Volatile - private var genresListCache: List? = null - - override suspend fun getFilterOptions(): MangaListFilterOptions { - if (genresListCache == null) { - try { - val input = "%7B%220%22%3A%7B%22json%22%3Anull%2C%22meta%22%3A%7B%22values%22%3A%5B%22undefined%22%5D%7D%7D%7D" - val res = client.httpGet(apiUrl("search.genres", input), getRequestHeaders()) - val body = res.body?.string() - if (body != null) { - val root = JSONArray(body) - val jsonArray = root.optJSONObject(0) - ?.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") - if (jsonArray != null) { - val tags = mutableListOf() - for (i in 0 until jsonArray.length()) { - val obj = jsonArray.optJSONObject(i) ?: continue - val name = obj.optString("name", "") - if (name.isNotEmpty()) tags.add(MangaTag(name, name, source)) - } - if (tags.isNotEmpty()) genresListCache = tags - } - } - } catch (_: Exception) {} - } - return MangaListFilterOptions( - availableTags = genresListCache?.toSet() ?: emptySet(), - availableContentTypes = availableContentTypes, - availableStates = availableStates, - availableContentRating = availableContentRating, - ) - } - - private fun apiUrl(trpcPath: String, input: String) = "https://$domainName/api/trpc/$trpcPath?batch=1&input=${input.urlEncoded()}" - - private fun searchPayload(query: String, sort: String, limit: Int, offset: Int, filter: MangaListFilter): String { - val genres = filter.tags.map { it.key } - val types = filter.types.map { it.name.lowercase() } - val statusStr = filter.states.firstOrNull()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "cancelled" - else -> null - } - } - val contentRatingStr = filter.contentRating.firstOrNull()?.let { - when (it) { - ContentRating.SAFE -> "safe" - ContentRating.SUGGESTIVE -> "suggestive" - ContentRating.ADULT -> "pornographic" - } - } - return JSONObject().apply { - put("0", JSONObject().apply { - put("json", JSONObject().apply { - put("q", query) - put("sort", sort) - put("filters", JSONObject().apply { - if (genres.isNotEmpty()) put("genres", JSONArray(genres)) else put("genres", JSONObject.NULL) - if (types.isNotEmpty()) put("type", types.first()) else put("type", JSONObject.NULL) - if (statusStr != null) put("status", statusStr) else put("status", JSONObject.NULL) - if (contentRatingStr != null) put("contentRating", contentRatingStr) else put("contentRating", JSONObject.NULL) - put("author", JSONObject.NULL) - put("artist", JSONObject.NULL) - put("year", JSONObject.NULL) - }) - put("limit", limit) - put("offset", offset) - put("maxRating", "pornographic") - }) - put("meta", JSONObject().apply { - put("values", JSONObject().apply { - if (genres.isEmpty()) put("filters.genres", JSONArray().put("undefined")) - if (types.isEmpty()) put("filters.type", JSONArray().put("undefined")) - if (statusStr == null) put("filters.status", JSONArray().put("undefined")) - if (contentRatingStr == null) put("filters.contentRating", JSONArray().put("undefined")) - put("filters.author", JSONArray().put("undefined")) - put("filters.artist", JSONArray().put("undefined")) - put("filters.year", JSONArray().put("undefined")) - }) - }) - }) - }.toString() - } - - private fun detailsPayload(slug: String) = JSONObject().apply { - put("0", JSONObject().apply { - put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { put("slug", slug) }) - }) - }.toString() - - private fun chaptersPayload(mangaId: Long) = JSONObject().apply { - put("0", JSONObject().apply { - put("json", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { - put("seriesId", mangaId) - put("sort", "best") - put("page", 1) - put("limit", 20) - }) - put("meta", JSONObject().apply { - put("values", JSONObject().apply { put("chapterId", JSONArray().put("undefined")) }) - }) - }) - put("2", JSONObject().apply { - put("json", JSONObject().apply { put("seriesId", mangaId) }) - }) - }.toString() - - private fun pagesPayload(slug: String, number: Float) = JSONObject().apply { - put("0", JSONObject().apply { - put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) - }) - put("1", JSONObject().apply { - put("json", JSONObject().apply { put("slug", slug) }) - }) - put("2", JSONObject().apply { - put("json", JSONObject().apply { - put("seriesSlug", slug) - put("chapterNumber", number.toDouble()) - }) - }) - put("3", JSONObject().apply { - put("json", JSONObject().apply { put("position", "footer_bottom") }) - }) - }.toString() - - private fun parseLastItemJsonArray(body: String): JSONArray? = try { - val root = JSONArray(body) - root.optJSONObject(root.length() - 1) - ?.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") - } catch (_: Exception) { null } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val q = filter.query?.trim().orEmpty() - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "recent" - SortOrder.NEWEST -> "newest" - SortOrder.NEWEST_ASC -> "oldest" - SortOrder.RATING -> "score" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "relevance" - else -> "newest" - } - val input = searchPayload(q, sort, pageSize, (page - 1) * pageSize, filter) - val res = client.httpGet(apiUrl("search.query", input), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val root = JSONArray(body) - val first = root.optJSONObject(0) ?: return emptyList() - val hits = first.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONObject("json") - ?.optJSONArray("hits") ?: return emptyList() - return (0 until hits.length()).mapNotNull { i -> - hits.optJSONObject(i)?.let { obj -> - val id = obj.optLong("id", -1).takeIf { it > 0 } ?: return@mapNotNull null - val slug = obj.optString("slug", "").takeIf { it.isNotEmpty() } ?: return@mapNotNull null - val title = obj.optString("title", "") - val cover = obj.optString("coverUrl", "") - Manga( - id = generateUid("/$mangaPath/$slug#$id"), - url = "/$mangaPath/$slug#$id", - publicUrl = "https://$domainName/$mangaPath/$slug#$id", - title = title, coverUrl = cover, - altTitles = emptySet(), authors = emptySet(), tags = emptySet(), - rating = RATING_UNKNOWN, state = null, - contentRating = ContentRating.ADULT, source = source - ) - } - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#") - .takeIf { it.isNotBlank() } ?: return manga - val res = client.httpGet(apiUrl("auth.me,series.bySlugWithGenres", detailsPayload(slug)), getRequestHeaders()) - val body = res.body?.string() ?: return manga - val detailJson = try { - val arr = JSONArray(body) - arr.getJSONObject(arr.length() - 1) - .optJSONObject("result")?.optJSONObject("data")?.optJSONObject("json") - } catch (_: Exception) { null } ?: return manga - - val title = detailJson.optString("title", manga.title) - val desc = detailJson.optString("synopsis", manga.description) - val cover = detailJson.optString("coverUrl", manga.coverUrl) - val authors = detailJson.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - val genres = detailJson.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).map { arr.getString(it) } - } ?: emptyList() - val status = detailJson.optString("status", null) - val chapters = loadChapters(manga) - - return manga.copy( - title = title, description = desc, coverUrl = cover, - authors = authors.toSet(), - tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), - state = when (status?.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "cancelled" -> MangaState.ABANDONED - else -> null - }, - chapters = chapters, - ) - } - - private suspend fun loadChapters(manga: Manga): List { - val mangaId = manga.url.substringAfterLast("#").toLongOrNull() ?: return emptyList() - val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#").ifBlank { return emptyList() } - val res = client.httpGet(apiUrl("auth.me,comments.list,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val chaptersArray = parseLastItemJsonArray(body) ?: return emptyList() - return (0 until chaptersArray.length()).mapNotNull { i -> - chaptersArray.optJSONObject(i)?.let { obj -> - val number = obj.optDouble("number", 0.0).toFloat() - val createdAt = obj.optString("createdAt", null) - val uploadDate = createdAt?.let { dateFormat.parse(it)?.time } ?: 0L - MangaChapter( - id = generateUid("/$mangaPath/$slug/$number"), - url = "/$mangaPath/$slug/$number", - title = "Chapter ${number.formatForTitle()}", - number = number, volume = 0, uploadDate = uploadDate, - scanlator = null, branch = null, source = source - ) - } - }.sortedBy { it.number } - } - - // triyng first tRPC API if it fails then chapter html scraping - override suspend fun getPages(chapter: MangaChapter): List { - val path = chapter.url.removePrefix("/$mangaPath/") - val slug = path.substringBefore("/") - val num = path.substringAfter("/").toFloatOrNull() ?: return emptyList() - - val input = pagesPayload(slug, num) - val res = client.httpGet(apiUrl("auth.me,series.bySlug,reader.chapterPages", input), getRequestHeaders()) - val body = res.body?.string() ?: return emptyList() - val apiPages = parsePages(body) - if (!apiPages.isNullOrEmpty()) return apiPages - - val chapterUrl = "https://$domainName/$mangaPath/$slug/$num" - val doc = client.httpGet(chapterUrl).parseHtml() - - // search in json results image URLs - val script = doc.select("script").find { it.data().contains("\"webpUrl\"") } - ?: doc.select("script").find { it.data().contains("\"avifUrl\"") } - if (script != null) { - val arrayRegex = Regex("""\[\s*\{.*?\"(?:webp|avif)Url\".*?\}\s*\]""", RegexOption.DOT_MATCHES_ALL) - val match = arrayRegex.find(script.data())?.value - if (match != null) { - return try { - val jsonArray = JSONArray(match) - (0 until jsonArray.length()).mapNotNull { i -> - val page = jsonArray.optJSONObject(i) ?: return@mapNotNull null - val url = page.optString("webpUrl", "").ifEmpty { - page.optString("avifUrl", "") - }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } catch (_: Exception) { emptyList() } - } - } - - return emptyList() - } - - private fun parsePages(body: String): List? { - val elements = try { JSONArray(body) } catch (_: Exception) { return null } - for (i in 0 until elements.length()) { - if (elements.optJSONObject(i)?.has("error") == true) return null - } - val last = elements.optJSONObject(elements.length() - 1) ?: return null - val items = last.optJSONObject("result") - ?.optJSONObject("data") - ?.optJSONArray("json") ?: return null - return (0 until items.length()).mapNotNull { i -> - val page = items.optJSONObject(i) ?: return@mapNotNull null - val url = page.optString("webpUrl", "").ifEmpty { - page.optString("avifUrl", "") - }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage) = page.url - - private fun Float.formatForTitle() = if (this == toLong().toFloat()) toLong().toString() else toString() - - companion object { - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) - } -} +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import okhttp3.Headers +import okhttp3.Request +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +abstract class HiperParser( + context: MangaLoaderContext, + source: MangaParserSource, + protected val domainName: String, +) : PagedMangaParser(context, source, pageSize = 30) { + + protected open val availableContentTypes: Set = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA) + protected open val availableStates: Set = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED) + protected open val availableContentRating: Set = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + protected open val mangaPath = "manga" + + override val configKeyDomain = ConfigKey.Domain(domainName) + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL + ) + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + val client: OkHttpWebClient by lazy { + val httpClient = context.httpClient.newBuilder() + .addInterceptor { chain -> + val request = chain.request() + val newRequest = request.newBuilder() + .header("Referer", "https://$domainName/") + .header("Origin", "https://$domainName") + .build() + var response = chain.proceed(newRequest) + if (response.code == 401) { + response.close() + val acceptHeaders = Headers.Builder() + .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + .build() + val homeRequest = Request.Builder() + .url("https://$domainName/") + .headers(acceptHeaders) + .build() + chain.proceed(homeRequest).close() + response = chain.proceed(request) + } + response + } + .build() + OkHttpWebClient(httpClient, source) + } + + @Volatile + private var genresListCache: List? = null + + override suspend fun getFilterOptions(): MangaListFilterOptions { + if (genresListCache == null) { + try { + val input = "%7B%220%22%3A%7B%22json%22%3Anull%2C%22meta%22%3A%7B%22values%22%3A%5B%22undefined%22%5D%7D%7D%7D" + val res = client.httpGet(apiUrl("search.genres", input), getRequestHeaders()) + val body = res.body?.string() + if (body != null) { + val root = JSONArray(body) + val jsonArray = root.optJSONObject(0) + ?.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") + if (jsonArray != null) { + val tags = mutableListOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.optJSONObject(i) ?: continue + val name = obj.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name, name, source)) + } + if (tags.isNotEmpty()) genresListCache = tags + } + } + } catch (_: Exception) {} + } + return MangaListFilterOptions( + availableTags = genresListCache?.toSet() ?: emptySet(), + availableContentTypes = availableContentTypes, + availableStates = availableStates, + availableContentRating = availableContentRating, + ) + } + + private fun apiUrl(trpcPath: String, input: String) = "https://$domainName/api/trpc/$trpcPath?batch=1&input=${input.urlEncoded()}" + + private fun searchPayload(query: String, sort: String, limit: Int, offset: Int, filter: MangaListFilter): String { + val genres = filter.tags.map { it.key } + val typeValue = filter.types.firstOrNull()?.name?.lowercase() + val statusValue = filter.states.firstOrNull()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> null + } + } + val ratingValue = filter.contentRating.firstOrNull()?.let { + when (it) { + ContentRating.SAFE -> "safe" + ContentRating.SUGGESTIVE -> "suggestive" + ContentRating.ADULT -> "pornographic" + } + } + + return JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject().apply { + put("q", query) + put("sort", sort) + put("filters", JSONObject().apply { + if (genres.isEmpty()) put("genres", JSONObject.NULL) + else put("genres", JSONArray(genres)) + put("type", typeValue) + put("status", statusValue) + put("contentRating", ratingValue) + put("author", JSONObject.NULL) + put("artist", JSONObject.NULL) + put("year", JSONObject.NULL) + }) + put("limit", limit) + put("offset", offset) + put("maxRating", "pornographic") + }) + put("meta", JSONObject().apply { + put("values", JSONObject().apply { + if (genres.isEmpty()) put("filters.genres", JSONArray().put("undefined")) + if (typeValue == null) put("filters.type", JSONArray().put("undefined")) + if (statusValue == null) put("filters.status", JSONArray().put("undefined")) + if (ratingValue == null) put("filters.contentRating", JSONArray().put("undefined")) + put("filters.author", JSONArray().put("undefined")) + put("filters.artist", JSONArray().put("undefined")) + put("filters.year", JSONArray().put("undefined")) + }) + }) + }) + }.toString() + } + + private fun detailsPayload(slug: String) = JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject.NULL) + put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { put("slug", slug) }) + }) + }.toString() + + private fun chaptersPayload(mangaId: Long) = JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { + put("seriesId", mangaId) + put("chapterId", JSONObject.NULL) + put("sort", "best") + put("page", 1) + put("limit", 9999) + }) + put("meta", JSONObject().apply { + put("values", JSONObject().apply { put("chapterId", JSONArray().put("undefined")) }) + }) + }) + put("2", JSONObject().apply { + put("json", JSONObject().apply { put("seriesId", mangaId) }) + }) + }.toString() + + private fun pagesPayload(slug: String, number: Float) = JSONObject().apply { + put("0", JSONObject().apply { + put("json", JSONObject.NULL) + put("meta", JSONObject().apply { put("values", JSONArray().put("undefined")) }) + }) + put("1", JSONObject().apply { + put("json", JSONObject().apply { put("slug", slug) }) + }) + put("2", JSONObject().apply { + put("json", JSONObject().apply { + put("seriesSlug", slug) + put("chapterNumber", number.toDouble()) + }) + }) + put("3", JSONObject().apply { + put("json", JSONObject().apply { put("position", "footer_bottom") }) + }) + }.toString() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val q = filter.query?.trim().orEmpty() + val sort = when (order) { + SortOrder.RELEVANCE -> "relevance" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "score" + SortOrder.UPDATED -> "recent" + SortOrder.NEWEST -> "newest" + SortOrder.NEWEST_ASC -> "oldest" + SortOrder.ALPHABETICAL -> "alphabetical" + else -> "relevance" + } + val input = searchPayload(q, sort, pageSize, (page - 1) * pageSize, filter) + val res = client.httpGet(apiUrl("search.query", input), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val root = JSONArray(body) + val first = root.optJSONObject(0) ?: return emptyList() + val hits = first.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONObject("json") + ?.optJSONArray("hits") ?: return emptyList() + return (0 until hits.length()).mapNotNull { i -> + hits.optJSONObject(i)?.let { obj -> + val id = obj.optLong("id", -1).takeIf { it > 0 } ?: return@mapNotNull null + val slug = obj.optString("slug", "").takeIf { it.isNotEmpty() } ?: return@mapNotNull null + val title = obj.optString("title", "") + val cover = obj.optString("coverUrl", "") + Manga( + id = generateUid("/$mangaPath/$slug#$id"), + url = "/$mangaPath/$slug#$id", + publicUrl = "https://$domainName/$mangaPath/$slug#$id", + title = title, + coverUrl = cover, + altTitles = emptySet(), + authors = emptySet(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = ContentRating.ADULT, + source = source + ) + } + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#") + .takeIf { it.isNotBlank() } ?: return manga + val res = client.httpGet(apiUrl("auth.me,series.bySlugWithGenres", detailsPayload(slug)), getRequestHeaders()) + val body = res.body?.string() ?: return manga + val detailJson = try { + val arr = JSONArray(body) + arr.getJSONObject(arr.length() - 1) + .optJSONObject("result")?.optJSONObject("data")?.optJSONObject("json") + } catch (_: Exception) { null } ?: return manga + + val title = detailJson.optString("title", manga.title) + val desc = detailJson.optString("synopsis", manga.description) + val cover = detailJson.optString("coverUrl", manga.coverUrl) + val authors = detailJson.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val genres = detailJson.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val status = detailJson.optString("status", null) + val chapters = loadChapters(manga) + + return manga.copy( + title = title, description = desc, coverUrl = cover, + authors = authors.toSet(), + tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = when (status?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + }, + chapters = chapters, + ) + } + + private suspend fun loadChapters(manga: Manga): List { + val mangaId = manga.url.substringAfterLast("#").toLongOrNull() ?: return emptyList() + val slug = manga.url.substringAfterLast("$mangaPath/").substringBefore("#").ifBlank { return emptyList() } + val res = client.httpGet(apiUrl("auth.me,series.chapters", chaptersPayload(mangaId)), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val chaptersArray = parseLastItemJsonArray(body) ?: return emptyList() + return (0 until chaptersArray.length()).mapNotNull { i -> + chaptersArray.optJSONObject(i)?.let { obj -> + val number = obj.optDouble("number", 0.0).toFloat() + val title = obj.optString("title", null) + val createdAt = obj.optString("createdAt", null) + val uploadDate = createdAt?.let { dateFormat.parse(it)?.time } ?: 0L + val displayTitle = when { + !title.isNullOrBlank() && title != "null" && !Regex("""\d+""").containsMatchIn(title) -> + "Chapter ${number.formatForTitle()} $title" + else -> "Chapter ${number.formatForTitle()}" + } + MangaChapter( + id = generateUid("/$mangaPath/$slug/$number"), + url = "/$mangaPath/$slug/$number", + title = displayTitle, + number = number, + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source + ) + } + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val path = chapter.url.removePrefix("/$mangaPath/") + val slug = path.substringBefore("/") + val num = path.substringAfter("/").toFloatOrNull() ?: return emptyList() + + val input = pagesPayload(slug, num) + val res = client.httpGet(apiUrl("auth.me,series.bySlug,reader.chapterPages", input), getRequestHeaders()) + val body = res.body?.string() ?: return emptyList() + val apiPages = parsePages(body) + if (!apiPages.isNullOrEmpty()) return apiPages + + val chapterUrl = "https://$domainName/$mangaPath/$slug/$num" + val doc = client.httpGet(chapterUrl).parseHtml() + + val script = doc.select("script").find { it.data().contains("\"webpUrl\"") } + ?: doc.select("script").find { it.data().contains("\"avifUrl\"") } + if (script != null) { + val arrayRegex = Regex("""\[\s*\{.*?\"(?:webp|avif)Url\".*?\}\s*\]""", RegexOption.DOT_MATCHES_ALL) + val match = arrayRegex.find(script.data())?.value + if (match != null) { + return try { + val jsonArray = JSONArray(match) + (0 until jsonArray.length()).mapNotNull { i -> + val page = jsonArray.optJSONObject(i) ?: return@mapNotNull null + val url = page.optString("webpUrl", "").ifEmpty { + page.optString("avifUrl", "") + }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } catch (_: Exception) { emptyList() } + } + } + + return emptyList() + } + + private fun parsePages(body: String): List? { + val elements = try { JSONArray(body) } catch (_: Exception) { return null } + for (i in 0 until elements.length()) { + if (elements.optJSONObject(i)?.has("error") == true) return null + } + val last = elements.optJSONObject(elements.length() - 1) ?: return null + val items = last.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") ?: return null + return (0 until items.length()).mapNotNull { i -> + val page = items.optJSONObject(i) ?: return@mapNotNull null + val url = page.optString("webpUrl", "").ifEmpty { + page.optString("avifUrl", "") + }.takeIf { it.isNotEmpty() } ?: return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + private fun parseLastItemJsonArray(body: String): JSONArray? = try { + val root = JSONArray(body) + root.optJSONObject(root.length() - 1) + ?.optJSONObject("result") + ?.optJSONObject("data") + ?.optJSONArray("json") + } catch (_: Exception) { null } + + override suspend fun getPageUrl(page: MangaPage) = page.url + + private fun Float.formatForTitle() = if (this == toLong().toFloat()) toLong().toString() else toString() + + companion object { + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) + } +} diff --git a/src/main/kotlin/tsuki/parsers/LibSocialParser.kt b/src/main/kotlin/tsuki/parsers/LibSocialParser.kt new file mode 100644 index 00000000..d197bf81 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/LibSocialParser.kt @@ -0,0 +1,510 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.network.UserAgents + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.LinkResolver +import tsuki.util.SCHEME_HTTPS +import tsuki.util.WebViewHelper +import tsuki.util.concatUrl +import tsuki.util.formatSimple +import tsuki.util.generateUid +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNull +import tsuki.util.json.mapJSONNotNullToSet +import tsuki.util.json.unescapeJson +import tsuki.util.nl2br +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.removeSurrounding +import tsuki.util.suspendlazy.suspendLazy + +import androidx.collection.ArraySet +import androidx.collection.IntObjectMap +import androidx.collection.IntSet +import androidx.collection.MutableIntSet +import androidx.collection.MutableScatterMap +import androidx.collection.ScatterMap +import androidx.collection.intObjectMapOf +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.runBlocking +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONException +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +internal abstract class LibSocialParser( + context: MangaLoaderContext, + source: MangaParserSource, + protected val siteId: Int, + siteDomains: String, +) : PagedMangaParser(context, source, pageSize = 60), MangaParserAuthProvider { + + protected val apiHost = "api.cdnlibs.org" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.CHROME_MOBILE) + + override val authUrl: String + get() = "https://$domain/ru/front/auth" + + override suspend fun isAuthorized(): Boolean { + val token = getAuthData()?.optJSONObject("token")?.getStringOrNull("access_token") + return !token.isNullOrEmpty() + } + + override suspend fun getUsername(): String = getAuthData() + ?.getJSONObject("auth") + ?.getString("username") + ?: throw AuthRequiredException(source) + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val configKeyDomain = ConfigKey.Domain(siteDomains) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.PAUSED, + MangaState.UPCOMING, + ), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val token = runBlocking { getAuthData() }?.optJSONObject("token")?.getStringOrNull("access_token") + val requestBuilder = chain.request().newBuilder() + if (!token.isNullOrEmpty()) { + requestBuilder.header("Authorization", "Bearer $token") + } + requestBuilder.header("Site-Id", siteId.toString()) + return chain.proceed(requestBuilder.build()) + } + + private val statesMap = intObjectMapOf( + 1, MangaState.ONGOING, + 2, MangaState.FINISHED, + 3, MangaState.UPCOMING, + 4, MangaState.PAUSED, + 5, MangaState.ABANDONED, + ) + private val imageServers = suspendLazy(initializer = ::fetchServers) + private val splitTranslationsKey = ConfigKey.SplitByTranslations(true) + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + null to null, + SERVER_MAIN to "Первый", + SERVER_SECONDARY to "Второй", + SERVER_COMPRESS to "СжатиÑ", + SERVER_DOWNLOAD to "Загрузки", + SERVER_CROP to "Обрезки", + ), + defaultValue = null, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val urlBuilder = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addQueryParameter("site_id[]", siteId.toString()) + .addQueryParameter("fields[]", "rate") + .addQueryParameter("fields[]", "rate_avg") + .addQueryParameter("page", page.toString()) + for (state in filter.states) { + urlBuilder.addQueryParameter("status[]", statesMap.keyOf(state).toString()) + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("${tag.typeKey()}[]", tag.key.drop(1)) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("${tag.typeKey()}_exclude[]", tag.key.drop(1)) + } + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("q", filter.query) + } + urlBuilder.addQueryParameter( + "sort_by", + when (order) { + SortOrder.UPDATED -> "last_chapter_at" + SortOrder.POPULARITY -> "views" + SortOrder.RATING -> "rate_avg" + SortOrder.NEWEST -> "created_at" + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + -> "rus_name" + + else -> null + }, + ) + urlBuilder.addQueryParameter( + "sort_type", + when (order) { + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL_DESC, + -> "desc" + + SortOrder.ALPHABETICAL -> "asc" + else -> null + }, + ) + val json = webClient.httpGet(urlBuilder.build()).parseJson() + val data = json.getJSONArray("data") + return data.mapJSON(::parseManga) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val chaptersDeferred = async { fetchChapters(manga) } + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addQueryParameter("fields[]", "summary") + .addQueryParameter("fields[]", "genres") + .addQueryParameter("fields[]", "tags") + .addQueryParameter("fields[]", "authors") + .addQueryParameter("fields[]", "close_view") + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + val genres = json.getJSONArray("genres").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "g" + jo.getInt("id"), source = source) + } + val tags = json.getJSONArray("tags").mapJSON { jo -> + MangaTag(title = jo.getString("name"), key = "t" + jo.getInt("id"), source = source) + } + val authors = json.getJSONArray("authors").mapJSONNotNullToSet { + it.getStringOrNull("name") + } + val chapters = chaptersDeferred.await() + val isRestricted = json.getIntOrDefault("close_view", 0) > 0 + manga.copy( + title = json.getStringOrNull("rus_name") ?: manga.title, + altTitles = setOfNotNull(json.getStringOrNull("name")), + tags = tagsSetOf(tags, genres), + state = if (chapters.isEmpty() && isRestricted) { + MangaState.RESTRICTED + } else { + manga.state + }, + authors = authors, + contentRating = json.optJSONObject("ageRestriction")?.let { + val id = it.getIntOrDefault("id", -1) + if (id >= 4) ContentRating.SUGGESTIVE else sourceContentRating + } ?: manga.contentRating, + description = json.parseLibSocialSummary()?.nl2br(), + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List = coroutineScope { + val pages = async { + webClient.httpGet( + concatUrl("https://$apiHost/api/manga/", chapter.url), + ).parseJson().getJSONObject("data") + } + val servers = imageServers.get() + val json = pages.await() + val primaryServer = getPrimaryImageServer(servers) + json.getJSONArray("pages").mapJSON { jo -> + val url = jo.getString("url") + MangaPage( + id = generateUid(jo.getLong("id")), + url = concatUrl(primaryServer, url), + preview = servers[SERVER_COMPRESS]?.let { concatUrl(it, url) }, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set = coroutineScope { + val tags = async { fetchTags("tags") } + val genres = async { fetchTags("genres") } + tagsSetOf(tags.await(), genres.await()) + } + + override suspend fun getRelatedManga(seed: Manga): List { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(seed.url) + .addPathSegment("similar") + .build(), + ).parseJson().getJSONArray("data") + return json.mapJSON { jo -> + parseManga(jo.getJSONObject("media")) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + return resolver.resolveManga(this, link.pathSegments.lastOrNull() ?: return null) + } + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(splitTranslationsKey) + keys.add(preferredServerKey) + } + + private fun parseManga(jo: JSONObject): Manga { + val cover = jo.getJSONObject("cover") + return Manga( + id = generateUid(jo.getLong("id")), + title = jo.getString("rus_name").ifEmpty { jo.getString("name") }, + altTitles = setOfNotNull(jo.getString("name")), + url = jo.getString("slug_url"), + publicUrl = "https://$domain/ru/manga/" + jo.getString("slug_url"), + rating = jo.optJSONObject("rating") + ?.getFloatOrDefault("average", RATING_UNKNOWN * 10f)?.div(10f) ?: RATING_UNKNOWN, + contentRating = sourceContentRating, + coverUrl = cover.getString("thumbnail"), + tags = setOf(), + state = statesMap[jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1], + authors = emptySet(), + largeCoverUrl = cover.getString("default"), + source = source, + ) + } + + private fun getPrimaryImageServer(servers: ScatterMap): String { + val preferred = config[preferredServerKey] + if (preferred != null) { + servers[preferred]?.let { return it } + } + return checkNotNull(servers[SERVER_MAIN] ?: servers[SERVER_DOWNLOAD] ?: servers[SERVER_SECONDARY]) { + "No available images servers" + } + } + + private suspend fun fetchChapters(manga: Manga): List { + val url = HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("manga") + .addPathSegment(manga.url) + .addPathSegment("chapters") + .build() + val json = webClient.httpGet(url).parseJson().getJSONArray("data") + val builder = ChaptersListBuilder(json.length()) + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US) + val useBranching = config[splitTranslationsKey] + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val volume = jo.getIntOrDefault("volume", 0) + val number = jo.getFloatOrDefault("number", 0f) + val numberString = number.formatSimple() + val name = jo.getStringOrNull("name") + val branches = jo.getJSONArray("branches") + for (j in 0 until branches.length()) { + val bjo = branches.getJSONObject(j) + val isRestricted = bjo.optJSONObject("restricted_view")?.let { + !it.getBooleanOrDefault("is_open", true) + } ?: false + if (isRestricted) { + continue + } + val id = bjo.getLong("id") + val branchId = bjo.getLongOrDefault("branch_id", 0L) + val team = bjo.getJSONArray("teams").optJSONObject(0)?.getStringOrNull("name") + builder += MangaChapter( + id = generateUid(id), + title = name, + number = number, + volume = volume, + url = buildString { + append(manga.url) + append("/chapter?number=") + append(numberString) + append("&volume=") + append(volume) + if (branchId != 0L) { + append("&branch_id=") + append(branchId) + } + }, + scanlator = team, + uploadDate = dateFormat.parseSafe(bjo.getStringOrNull("created_at")), + branch = if (useBranching) team else null, + source = source, + ) + } + } + return builder.toList() + } + + private suspend fun fetchTags(type: String): List { + val data = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api").addPathSegment(type).build(), + ).parseJson().getJSONArray("data") + val prefix = type.first().toString() + return data.mapJSONNotNull { jo -> + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + return@mapJSONNotNull null + } + MangaTag( + title = jo.getString("name"), + key = prefix + jo.getInt("id"), + source = source, + ) + } + } + + private suspend fun fetchServers(): ScatterMap { + val json = webClient.httpGet( + HttpUrl.Builder() + .scheme(SCHEME_HTTPS) + .host(apiHost) + .addPathSegment("api") + .addPathSegment("constants") + .addQueryParameter("fields[]", "imageServers") + .build(), + ).parseJson().getJSONObject("data").getJSONArray("imageServers") + val result = MutableScatterMap() + for (i in 0 until json.length()) { + val jo = json.getJSONObject(i) + val sites = jo.getJSONArray("site_ids").toIntSet() + if (siteId !in sites) { + continue + } + result[jo.getString("id")] = jo.getString("url") + } + return result + } + + private fun IntObjectMap.keyOf(value: V): Int { + forEach { k, v -> + if (v == value) { + return k + } + } + throw NoSuchElementException("No key associated with value $value") + } + + private fun JSONArray.toIntSet(): IntSet { + val result = MutableIntSet(length()) + for (i in 0 until length()) { + result.add(getInt(i)) + } + return result + } + + private fun MangaTag.typeKey() = when (key.firstOrNull()) { + 'g' -> "genres" + 't' -> "tags" + else -> throw IllegalArgumentException("Tag $key($title) is of unknown type") + } + + private fun tagsSetOf(tags: Collection, genres: Collection): Set { + val result = ArraySet(tags.size + genres.size) + val names = HashSet(tags.size + genres.size) + genres.forEach { x -> if (names.add(x.title)) result.add(x) } + tags.forEach { x -> if (names.add(x.title)) result.add(x) } + return result + } + + private suspend fun getAuthData(): JSONObject? { + val raw = WebViewHelper(context).getLocalStorageValue(domain, "auth") ?: return null + return JSONObject(raw.unescapeJson().removeSurrounding('"')) + } + + protected companion object { + + const val SERVER_MAIN = "main" + const val SERVER_SECONDARY = "secondary" + const val SERVER_COMPRESS = "compress" + const val SERVER_DOWNLOAD = "download" + const val SERVER_CROP = "crop" + } +} + +internal fun JSONObject.parseLibSocialSummary(): String? = when (val value = opt("summary")) { + null, + JSONObject.NULL, + -> null + + is String -> value + is JSONObject -> value.parseLibSocialRichText() + else -> throw JSONException("Unsupported summary value: ${value.javaClass.name}") +} + +private fun JSONObject.parseLibSocialRichText(): String { + return when (val type = getString("type")) { + "text" -> getString("text") + "hardBreak" -> "\n" + "paragraph", + "heading", + -> parseLibSocialChildren(separator = "") + + "doc", + "blockquote", + "bulletList", + "orderedList", + "listItem", + -> parseLibSocialChildren(separator = "\n") + + else -> throw JSONException("Unsupported summary node type: $type") + } +} + +private fun JSONObject.parseLibSocialChildren(separator: String): String { + val children = optJSONArray("content") ?: return "" + return children.mapJSON { child -> child.parseLibSocialRichText() }.joinToString(separator) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt similarity index 85% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt rename to src/main/kotlin/tsuki/parsers/LikeMangaParser.kt index 5ff71ada..127b408e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LikeMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/LikeMangaParser.kt @@ -1,4 +1,40 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.concatUrl +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.selectLast import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll @@ -7,14 +43,11 @@ import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.jsoup.Jsoup import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale +import java.util.Calendar internal abstract class LikeMangaParser( context: MangaLoaderContext, @@ -52,17 +85,16 @@ internal abstract class LikeMangaParser( ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query val url = buildString { append("https://") append(domain) append("/?act=search") - query?.let { + filter.query?.let { append("&f") append("[keyword]".urlEncoded()) append("=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&f") @@ -144,7 +176,7 @@ internal abstract class LikeMangaParser( val maxPageChapterSelect = doc.getElementById("nav_list_chapter_id_detail")?.select("a:not(.next)") var maxPageChapter = 1 if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { + maxPageChapterSelect.forEach { val i = it.text().toInt() if (i > maxPageChapter) { maxPageChapter = i @@ -179,7 +211,7 @@ internal abstract class LikeMangaParser( result } } - }.reversed(), + }.sortedBy { it.number } ) } @@ -273,4 +305,4 @@ internal abstract class LikeMangaParser( else -> dateFormat.parseSafe(date) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt b/src/main/kotlin/tsuki/parsers/LilianaParser.kt similarity index 86% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt rename to src/main/kotlin/tsuki/parsers/LilianaParser.kt index 5db210ed..6f2eb03a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/LilianaParser.kt +++ b/src/main/kotlin/tsuki/parsers/LilianaParser.kt @@ -1,16 +1,43 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.mapChapters +import tsuki.util.parseJson +import tsuki.util.selectOrThrow import androidx.collection.scatterSetOf import kotlinx.coroutines.coroutineScope import org.jsoup.Jsoup import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getBooleanOrDefault -import java.util.* + +import java.util.EnumSet internal abstract class LilianaParser( context: MangaLoaderContext, @@ -174,7 +201,7 @@ internal abstract class LilianaParser( in abandoned -> MangaState.ABANDONED else -> null }, - chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = true) { i, element -> + chapters = doc.selectOrThrow("ul > li.chapter").mapChapters(reversed = false) { i, element -> val href = element.selectFirstOrThrow("a").attrAsRelativeUrl("href") MangaChapter( id = generateUid(href), @@ -188,7 +215,7 @@ internal abstract class LilianaParser( branch = null, source = source, ) - }, + }.sortedBy { it.number }, ) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt b/src/main/kotlin/tsuki/parsers/MadaraParser.kt similarity index 77% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt rename to src/main/kotlin/tsuki/parsers/MadaraParser.kt index 482ac579..9b5f2d88 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadaraParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadaraParser.kt @@ -1,4 +1,51 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.MangaParserAuthProvider +import tsuki.exception.AuthRequiredException +import tsuki.exception.ParseException + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.selectOrThrow +import tsuki.util.CryptoAES +import tsuki.util.getCookies +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.selectLast +import tsuki.util.toMutableMap +import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.mapChapters import androidx.collection.scatterSetOf import kotlinx.coroutines.async @@ -7,18 +54,16 @@ import org.json.JSONObject import org.jsoup.HttpStatusException import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.AuthRequiredException -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.net.HttpURLConnection import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.* +import java.util.Base64 +import java.util.Calendar +import java.util.EnumSet + +/** + * Todo: make ContentType filter. + */ internal abstract class MadaraParser( context: MangaLoaderContext, @@ -34,10 +79,60 @@ internal abstract class MadaraParser( keys.add(userAgentKey) } - // Change these values only if the site does not support manga listings via ajax + /** + * AJAX listing mode. + * false (default) – uses POST requests to wp-admin/admin-ajax.php + * true – builds normal GET URLs (?s=, ?page=, /manga-genre/…) and parses the HTML directly. + */ protected open val withoutAjax = false protected open val authorSearchSupported = false + /** Date format pattern used to parse chapter release dates (SimpleDateFormat). */ + protected open val datePattern = "MMMM d, yyyy" + + /** + * Genre fetching: + * listUrl – the page path (relative to domain) that contains the genre list. + * The parser fetches https://domain/ to scrape available genres. + * + * tagPrefix – the URL segment that immediately precedes the genre identifier in filter links. + * Used to extract the genre key from a link like /manga-genre/action/ -> "action". + */ + protected open val listUrl = "manga/" + protected open val tagPrefix = "manga-genre/" + + /** + * Chapter list handling: + * stylePage – suffix appended to chapter URLs to force all images to be shown in one long list. + * postReq – whether to use the old AJAX endpoint for chapter loading. + * true -> POST admin-ajax.php with action=manga_get_chapters&manga= + * false -> POST /manga//ajax/chapters/ with empty body + * postDataReq – form data used when postReq = true. + */ + protected open val stylePage = "?style=list" + protected open val postReq = false + protected open val postDataReq = "action=manga_get_chapters&manga=" + + /** Page extraction selectors */ + protected open val selectBodyPage = "div.main-col-inner div.reading-content" + protected open val selectPage = "div.page-break, div.page-box" + protected open val selectRequiredLogin = ".content-blocked, .login-required" + + /** Details page selectors */ + protected open val selectDesc = "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" + protected open val selectGenre = "div.genres-content a" + protected open val selectTestAsync = "div.listing-chapters_wrap" + protected open val selectState = + "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + + "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + + "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(СтатуÑ)," + + "div.post-content_item:contains(状æ€), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" + protected open val selectAlt = ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" + + /** Chapter list selectors */ + protected open val selectDate = "span.chapter-release-date i" + protected open val selectChapter = "li.wp-manga-chapter" + override val availableSortOrders: Set = setupAvailableSortOrders() private fun setupAvailableSortOrders(): Set { @@ -110,11 +205,6 @@ internal abstract class MadaraParser( } } - protected open val tagPrefix = "manga-genre/" - protected open val datePattern = "MMMM d, yyyy" - protected open val stylePage = "?style=list" - protected open val postReq = false - init { paginator.firstPage = 0 searchPaginator.firstPage = 0 @@ -246,100 +336,95 @@ internal abstract class MadaraParser( "in arrivo", ) - // can be changed to retrieve tags see getTags - protected open val listUrl = "manga/" - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { if (withoutAjax) { val pages = page + 1 - val url = buildString { - append("https://") - append(domain) - - if (pages > 1) { - append("/page/") - append(pages.toString()) - } - append("/?s=") - append(filter.query?.urlEncoded() ?: "") - - append("&post_type=wp-manga") + fun sortParam(): String = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "new-manga" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "" + else -> "" + } - // Known bug: in some cases, if there are no manga with the associated tags, the source returns the full list of manga - if (filter.tags.isNotEmpty()) { - filter.tags.forEach { - append("&genre[]=") - append(it.key) + val url = if (filter.tags.isNotEmpty()) { + val genreSlug = filter.tags.first().key + buildString { + append("https://") + append(domain) + append("/manga-genre/$genreSlug/") + if (pages > 1) { + append("page/") + append(pages) + append("/") } + append("?m_orderby=") + append(sortParam()) } + } else { + buildString { + append("https://") + append(domain) - filter.states.forEach { - append("&status[]=") - when (it) { - MangaState.ONGOING -> append("on-going") - MangaState.FINISHED -> append("end") - MangaState.ABANDONED -> append("canceled") - MangaState.PAUSED -> append("on-hold") - MangaState.UPCOMING -> append("upcoming") - else -> throw IllegalArgumentException("$it not supported") + if (pages > 1) { + append("/page/") + append(pages.toString()) } - } + append("/?s=") + append(filter.query?.urlEncoded() ?: "") + + append("&post_type=wp-manga") - filter.contentRating.oneOrThrowIfMany()?.let { - append("&adult=") - append( + filter.states.forEach { + append("&status[]=") when (it) { - ContentRating.SAFE -> "0" - ContentRating.ADULT -> "1" - else -> "" - }, - ) - } + MangaState.ONGOING -> append("on-going") + MangaState.FINISHED -> append("end") + MangaState.ABANDONED -> append("canceled") + MangaState.PAUSED -> append("on-hold") + MangaState.UPCOMING -> append("upcoming") + else -> throw IllegalArgumentException("$it not supported") + } + } - if (filter.year != 0) { - append("&release=") - append(filter.year.toString()) - } + filter.contentRating.oneOrThrowIfMany()?.let { + append("&adult=") + append( + when (it) { + ContentRating.SAFE -> "0" + ContentRating.ADULT -> "1" + else -> "" + }, + ) + } - filter.author?.takeIf { it.isNotEmpty() }?.let { - append("&author=") - // should be like "minamida-usuke" - append(it.lowercase().replace(" ", "-")) - } + if (filter.year != 0) { + append("&release=") + append(filter.year.toString()) + } - // Support artist - //filter.artist?.let { - // append("&artist=") - // append(filter.artist) - //} - - - append("&m_orderby=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.NEWEST -> append("new-manga") - SortOrder.ALPHABETICAL -> append("alphabet") - SortOrder.RATING -> append("rating") - SortOrder.RELEVANCE -> {} - else -> {} + filter.author?.takeIf { it.isNotEmpty() }?.let { + append("&author=") + append(it.lowercase().replace(" ", "-")) + } + + append("&m_orderby=") + append(sortParam()) } } + val html = try { webClient.httpGet(url).parseHtml() } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", url) - } + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() + else throw ParseException("Can't fetch data from source!", url) } return parseMangaList(html) } else { - val payload = createRequestTemplate() - payload["page"] = page.toString() filter.query?.takeIf { it.isNotEmpty() }?.let { @@ -377,14 +462,6 @@ internal abstract class MadaraParser( payload["vars[tax_query][3][operator]"] = "IN" } - // Support artist - // filter.artist.let { - // payload["vars[tax_query][4][taxonomy]"] = "wp-manga-artist" - // payload["vars[tax_query][4][field]"] = "name" - // payload["vars[tax_query][4][terms][0]"] = filter.artist - // payload["vars[tax_query][4][operator]"] = "IN" - //} - if (filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || filter.year != 0) { payload["vars[tax_query][relation]"] = "AND" } @@ -395,103 +472,80 @@ internal abstract class MadaraParser( payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "desc" } - SortOrder.POPULARITY_ASC -> { payload["vars[meta_key]"] = "_wp_manga_views" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "asc" } - SortOrder.UPDATED -> { payload["vars[meta_key]"] = "_latest_update" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "desc" } - SortOrder.UPDATED_ASC -> { payload["vars[meta_key]"] = "_latest_update" payload["vars[orderby]"] = "meta_value_num" payload["vars[order]"] = "asc" } - SortOrder.NEWEST -> { payload["vars[orderby]"] = "date" payload["vars[order]"] = "desc" } - SortOrder.NEWEST_ASC -> { payload["vars[orderby]"] = "date" payload["vars[order]"] = "asc" } - SortOrder.ALPHABETICAL -> { payload["vars[orderby]"] = "post_title" payload["vars[order]"] = "asc" } - SortOrder.ALPHABETICAL_DESC -> { payload["vars[orderby]"] = "post_title" payload["vars[order]"] = "desc" } - SortOrder.RATING -> { payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - payload["vars[orderby][query_avarage_reviews]"] = "DESC" payload["vars[orderby][query_total_reviews]"] = "DESC" } - SortOrder.RATING_ASC -> { payload["vars[meta_query][0][query_avarage_reviews][key]"] = "_manga_avarage_reviews" payload["vars[meta_query][0][query_total_reviews][key]"] = "_manga_total_votes" - payload["vars[orderby][query_avarage_reviews]"] = "ASC" payload["vars[orderby][query_total_reviews]"] = "ASC" } - - SortOrder.RELEVANCE -> { - payload["vars[orderby]"] = "" - } - + SortOrder.RELEVANCE -> payload["vars[orderby]"] = "" else -> payload["vars[orderby]"] = "" } filter.states.forEach { payload["vars[meta_query][0][0][key]"] = "_wp_manga_status" payload["vars[meta_query][0][0][compare]"] = "IN" - payload["vars[meta_query][0][0][value][]"] = - when (it) { - MangaState.ONGOING -> "on-going" - MangaState.FINISHED -> "end" - MangaState.ABANDONED -> "canceled" - MangaState.PAUSED -> "on-hold" - MangaState.UPCOMING -> "upcoming" - else -> throw IllegalArgumentException("$it not supported") - } + payload["vars[meta_query][0][0][value][]"] = when (it) { + MangaState.ONGOING -> "on-going" + MangaState.FINISHED -> "end" + MangaState.ABANDONED -> "canceled" + MangaState.PAUSED -> "on-hold" + MangaState.UPCOMING -> "upcoming" + else -> throw IllegalArgumentException("$it not supported") + } } filter.contentRating.oneOrThrowIfMany()?.let { payload["vars[meta_query][0][1][key]"] = "manga_adult_content" - payload["vars[meta_query][0][1][value]"] = - when (it) { - ContentRating.SAFE -> "" - ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" - else -> "" - } + payload["vars[meta_query][0][1][value]"] = when (it) { + ContentRating.SAFE -> "" + ContentRating.ADULT -> "a%3A1%3A%7Bi%3A0%3Bs%3A3%3A%22yes%22%3B%7D" + else -> "" + } } val html = try { - webClient.httpPost( - "https://$domain/wp-admin/admin-ajax.php", - payload, - ).parseHtml() + webClient.httpPost("https://$domain/wp-admin/admin-ajax.php", payload).parseHtml() } catch (e: HttpStatusException) { - if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) { - return emptyList() - } else { - throw ParseException("Can't fetch data from source!", domain) - } + if (e.statusCode == HttpURLConnection.HTTP_INTERNAL_ERROR) return emptyList() + else throw ParseException("Can't fetch data from source!", domain) } return parseMangaList(html) } @@ -574,18 +628,6 @@ internal abstract class MadaraParser( } } - protected open val selectDesc = - "div.description-summary div.summary__content, div.summary_content div.post-content_item > h5 + div, div.summary_content div.manga-excerpt, div.post-content div.manga-summary, div.post-content div.desc, div.c-page__content div.summary__content" - protected open val selectGenre = "div.genres-content a" - protected open val selectTestAsync = "div.listing-chapters_wrap" - protected open val selectState = - "div.post-content_item:contains(Status), div.post-content_item:contains(Statut), " + - "div.post-content_item:contains(État), div.post-content_item:contains(حالة العمل), div.post-content_item:contains(Estado), div.post-content_item:contains(สถานะ)," + - "div.post-content_item:contains(Stato), div.post-content_item:contains(Durum), div.post-content_item:contains(Statüsü), div.post-content_item:contains(СтатуÑ)," + - "div.post-content_item:contains(状æ€), div.post-content_item:contains(الحالة), div.post-content_item:contains(Tình trạng)" - protected open val selectAlt = - ".post-content_item:contains(Alt) .summary-content, .post-content_item:contains(Nomes alternativos: ) .summary-content" - protected open suspend fun createMangaTag(a: Element): MangaTag? { return MangaTag( key = a.attr("href").removeSuffix("/").substringAfterLast('/'), @@ -594,6 +636,8 @@ internal abstract class MadaraParser( ) } + protected open fun transformChapterName(element: Element, name: String): String = name + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val fullUrl = manga.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() @@ -639,28 +683,22 @@ internal abstract class MadaraParser( ) } - - protected open val selectDate = "span.chapter-release-date i" - protected open val selectChapter = "li.wp-manga-chapter" - protected open suspend fun getChapters(manga: Manga, doc: Document): List { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.body().select(selectChapter).mapChapters(reversed = true) { _, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() + val baseName = a.selectFirst("p")?.text() ?: a.ownText() + val name = transformChapterName(li, baseName) MangaChapter( id = generateUid(href), title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, url = link, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), source = source, scanlator = null, branch = null, @@ -668,8 +706,6 @@ internal abstract class MadaraParser( } } - protected open val postDataReq = "action=manga_get_chapters&manga=" - protected open suspend fun loadChapters(mangaUrl: String, document: Document): List { val doc = if (postReq) { val mangaId = document.select("div#manga-chapters-holder").attr("data-id") @@ -681,23 +717,21 @@ internal abstract class MadaraParser( webClient.httpPost(url, emptyMap()).parseHtml() } val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.select(selectChapter).mapChapters(reversed = true) { _, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val link = href + stylePage val dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() - val name = a.selectFirst("p")?.text() ?: a.ownText() + val baseName = a.selectFirst("p")?.text() ?: a.ownText() + val name = transformChapterName(li, baseName) MangaChapter( id = generateUid(href), url = link, title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, branch = null, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), scanlator = null, source = source, ) @@ -727,10 +761,6 @@ internal abstract class MadaraParser( } } - protected open val selectBodyPage = "div.main-col-inner div.reading-content" - protected open val selectPage = "div.page-break, div.page-box" - protected open val selectRequiredLogin = ".content-blocked, .login-required" - override suspend fun getPages(chapter: MangaChapter): List { val fullUrl = chapter.url.toAbsoluteUrl(domain) val doc = webClient.httpGet(fullUrl).parseHtml() @@ -893,4 +923,4 @@ internal abstract class MadaraParser( return chunked(2).map { it.toInt(16).toByte() }.toByteArray() } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt similarity index 88% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt rename to src/main/kotlin/tsuki/parsers/MadthemeParser.kt index deb6ba44..6838cd81 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MadthemeParser.kt +++ b/src/main/kotlin/tsuki/parsers/MadthemeParser.kt @@ -1,20 +1,53 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapChapters +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import okhttp3.Headers +import okhttp3.Interceptor import okhttp3.Response import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.util.* import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.* +import java.util.Calendar +import java.util.EnumSet internal abstract class MadthemeParser( context: MangaLoaderContext, @@ -74,7 +107,7 @@ internal abstract class MadthemeParser( .add("Referer", "https://$domain/") .build() - override fun intercept(chain: okhttp3.Interceptor.Chain): Response { + override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() val url = request.url val fragment = url.fragment @@ -103,7 +136,6 @@ internal abstract class MadthemeParser( } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query val url = buildString { append("https://") append(domain) @@ -113,9 +145,9 @@ internal abstract class MadthemeParser( append("?page=") append(page.toString()) - query?.let { + filter.query?.let { append("&q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&sort=") @@ -244,7 +276,7 @@ internal abstract class MadthemeParser( val slug = doc.selectFirstOrThrow("script:containsData(bookSlug)").data().substringAfter("bookSlug = \"") .substringBefore("\";") val docChapter = webClient.httpGet("https://$domain/api/manga/$slug/chapters?source=detail").parseHtml() - return docChapter.select(selectChapter).mapChapters(reversed = true) { i, li -> + return docChapter.select(selectChapter).mapChapters(reversed = false) { i, li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") val dateText = li.selectFirst(selectDate)?.text() @@ -262,7 +294,7 @@ internal abstract class MadthemeParser( scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } protected open val selectPage = "div#chapter-images img" @@ -413,4 +445,4 @@ internal abstract class MadthemeParser( private const val IMAGE_FALLBACK_HOST = "sb.mbcdn.xyz" } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt similarity index 82% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt rename to src/main/kotlin/tsuki/parsers/Manga18Parser.kt index 31033fd4..91d6efb2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/Manga18Parser.kt +++ b/src/main/kotlin/tsuki/parsers/Manga18Parser.kt @@ -1,248 +1,279 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.Headers -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -internal abstract class Manga18Parser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - ) - - @Volatile - private var genresCache: Set? = null - private val genresMutex = Mutex() - - private suspend fun getOrFetchGenres(): Set { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - genresCache = fetchAvailableTags() - genresCache!! - } - } - } - - private suspend fun fetchAvailableTags(): Set { - val doc = webClient.httpGet("https://$domain/$listUrl/").parseHtml() - return doc.select("div.grid_cate li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val href = a.attr("href").removeSuffix('/').substringAfterLast('/') - MangaTag( - key = href, - title = a.text(), - source = source, - ) - } - } - - override fun getRequestHeaders() = Headers.Builder() - .add("Referer", "https://$domain/") - .build() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - ) - - init { - paginator.firstPage = 1 - searchPaginator.firstPage = 1 - } - - protected open val ongoing: Set = setOf("On Going") - protected open val finished: Set = setOf("Completed") - - protected open val listUrl = "list-manga/" - protected open val tagUrl = "manga-list/" - protected open val datePattern = "dd-MM-yyyy" - - protected open val selectDesc = "div.detail_reviewContent" - protected open val selectState = "div.item:contains(Status) div.info_value" - protected open val selectAlt = "div.item:contains(Other name) div.info_value" - protected open val selectTag = "div.info_value > a[href*=/manga-list/]" - protected open val selectAuthor = "div.info_label:contains(author) + div.info_value, div.info_label:contains(autor) + div.info_value" - protected open val selectArtist = "div.info_label:contains(artist) + div.info_value" - protected open val selectDate = "div.item p" - protected open val selectChapter = "div.chapter_box li" - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim().orEmpty() - val tag = filter.tags.firstOrNull() - - val url = buildString(120) { - append("https://") - append(domain) - append('/') - - when { - tag != null && query.isNotEmpty() -> { - throw IllegalArgumentException("Search is not supported with tags") - } - tag != null -> { - append(tagUrl) - append(tag.key) - append('/') - append(page) - } - query.isNotEmpty() -> { - append(listUrl) - append(page) - append("?search=") - append(query.urlEncoded()) - append("&order_by=latest") - } - else -> { - append(listUrl) - append(page) - } - } - if (query.isEmpty()) { - append("?order_by=") - when (order) { - SortOrder.POPULARITY -> append("views") - SortOrder.UPDATED -> append("latest") - SortOrder.ALPHABETICAL -> append("name") - else -> append("latest") - } - } - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - protected open fun parseMangaList(doc: Document): List { - return doc.select("div.story_item").map { div -> - val a = div.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val title = div.selectFirst("div.mg_info") - ?.selectFirst("div.mg_name a") - ?.text() - ?: a.attr("title") - val cover = div.selectFirst("img")?.src() - - Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - coverUrl = cover, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - tags = emptySet(), - authors = emptySet(), - state = null, - source = source, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val info = doc.body().selectFirstOrThrow("div.detail_listInfo") - val chaptersDeferred = async { getChapters(doc) } - val desc = doc.selectFirst(selectDesc)?.html()?.nullIfEmpty() - val state = info.selectFirst(selectState)?.text()?.let { - when (it) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - else -> null - } - } - val alt = info.selectFirst(selectAlt)?.textOrNull()?.takeUnless { it == "Updating" } - val author = info.selectFirst(selectAuthor)?.textOrNull()?.takeUnless { it == "Updating" } - val artist = info.selectFirst(selectArtist)?.textOrNull()?.takeUnless { it == "Updating" } - val tags = doc.body().select(selectTag).mapToSet { a -> - val key = a.attr("href").removeSuffix('/').substringAfterLast('/') - MangaTag( - key = key, - title = a.text().toTitleCase(), - source = source, - ) - } - manga.copy( - tags = tags, - description = desc.nullIfEmpty(), - altTitles = setOfNotNull(alt), - authors = setOfNotNull(author, artist), - state = state, - chapters = chaptersDeferred.await(), - ) - } - - protected open suspend fun getChapters(doc: Document): List { - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> - val a = li.selectFirstOrThrow("a") - val href = a.attrAsRelativeUrl("href") - val dateText = li.selectFirst(selectDate)?.text() - MangaChapter( - id = generateUid(href), - title = a.textOrNull(), - number = i + 1f, - volume = 0, - url = href, - uploadDate = dateFormat.parseSafe(dateText), - source = source, - scanlator = null, - branch = null, - ) - } - } - - override suspend fun getPages(chapter: MangaChapter): List { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - val script = doc.selectFirstOrThrow("script:containsData(slides_p_path)") - val encoded = script.data() - .substringAfter('[') - .substringBefore(",]") - .replace("\"", "") - .split(",") - - return encoded.map { encodedUrl -> - val decoded = context.decodeBase64(encodedUrl).toString(Charsets.UTF_8) - val absoluteUrl = decoded.toAbsoluteUrl(domain) - MangaPage( - id = generateUid(absoluteUrl), - url = absoluteUrl, - preview = null, - source = source, - ) - } - } -} +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.extractChapterNumber + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet + +internal abstract class Manga18Parser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + ) + + @Volatile + private var genresCache: Set? = null + private val genresMutex = Mutex() + + private suspend fun getOrFetchGenres(): Set { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + genresCache = fetchAvailableTags() + genresCache!! + } + } + } + + private suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl/").parseHtml() + return doc.select("div.grid_cate li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix('/').substringAfterLast('/') + MangaTag( + key = href, + title = a.text(), + source = source, + ) + } + } + + override fun getRequestHeaders() = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + protected open val ongoing: Set = setOf("On Going") + protected open val finished: Set = setOf("Completed") + + protected open val listUrl = "list-manga/" + protected open val tagUrl = "manga-list/" + protected open val datePattern = "dd-MM-yyyy" + + protected open val selectDesc = "div.detail_reviewContent" + protected open val selectState = "div.item:contains(Status) div.info_value" + protected open val selectAlt = "div.item:contains(Other name) div.info_value" + protected open val selectTag = "div.info_value > a[href*=/manga-list/]" + protected open val selectAuthor = "div.info_label:contains(author) + div.info_value, div.info_label:contains(autor) + div.info_value" + protected open val selectArtist = "div.info_label:contains(artist) + div.info_value" + protected open val selectDate = "div.item p" + protected open val selectChapter = "div.chapter_box li" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val tag = filter.tags.firstOrNull() + + val url = buildString { + append("https://") + append(domain) + append('/') + + when { + tag != null && !filter.query.isNullOrBlank() -> + throw IllegalArgumentException("Search is not supported with tags") + + tag != null -> { + append(tagUrl) + append(tag.key) + append('/') + append(page) + } + + !filter.query.isNullOrBlank() -> { + append(listUrl) + append(page) + append("?search=") + append(filter.query.orEmpty().urlEncoded()) + append("&order_by=latest") + } + + else -> { + append(listUrl) + append(page) + } + } + + if (filter.query.isNullOrBlank()) { + append("?order_by=") + when (order) { + SortOrder.POPULARITY -> append("views") + SortOrder.UPDATED -> append("latest") + SortOrder.ALPHABETICAL -> append("name") + else -> append("latest") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + protected open fun parseMangaList(doc: Document): List { + return doc.select("div.story_item").map { div -> + val a = div.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val title = div.selectFirst("div.mg_info") + ?.selectFirst("div.mg_name a") + ?.text() + ?: a.attr("title") + val cover = div.selectFirst("img")?.src() + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = cover, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val info = doc.body().selectFirstOrThrow("div.detail_listInfo") + val chaptersDeferred = async { getChapters(doc) } + val desc = doc.selectFirst(selectDesc)?.html()?.nullIfEmpty() + val state = info.selectFirst(selectState)?.text()?.let { + when (it) { + in ongoing -> MangaState.ONGOING + in finished -> MangaState.FINISHED + else -> null + } + } + val alt = info.selectFirst(selectAlt)?.textOrNull()?.takeUnless { it == "Updating" } + val author = info.selectFirst(selectAuthor)?.textOrNull()?.takeUnless { it == "Updating" } + val artist = info.selectFirst(selectArtist)?.textOrNull()?.takeUnless { it == "Updating" } + val tags = doc.body().select(selectTag).mapToSet { a -> + val key = a.attr("href").removeSuffix('/').substringAfterLast('/') + MangaTag( + key = key, + title = a.text().toTitleCase(), + source = source, + ) + } + manga.copy( + tags = tags, + description = desc.nullIfEmpty(), + altTitles = setOfNotNull(alt), + authors = setOfNotNull(author, artist), + state = state, + chapters = chaptersDeferred.await(), + ) + } + + protected open suspend fun getChapters(doc: Document): List { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val name = a.textOrNull() ?: "" + val dateText = li.selectFirst(selectDate)?.text() + MangaChapter( + id = generateUid(href), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = href, + uploadDate = dateFormat.parseSafe(dateText), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val script = doc.selectFirstOrThrow("script:containsData(slides_p_path)") + val encoded = script.data() + .substringAfter('[') + .substringBefore(",]") + .replace("\"", "") + .split(",") + + return encoded.map { encodedUrl -> + val decoded = context.decodeBase64(encodedUrl).toString(Charsets.UTF_8) + val absoluteUrl = decoded.toAbsoluteUrl(domain) + MangaPage( + id = generateUid(absoluteUrl), + url = absoluteUrl, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt similarity index 87% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt rename to src/main/kotlin/tsuki/parsers/MangaReaderParser.kt index 438354da..24f19b9b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaReaderParser.kt +++ b/src/main/kotlin/tsuki/parsers/MangaReaderParser.kt @@ -1,417 +1,454 @@ -package org.koitharu.kotatsu.parsers.parsers - -import androidx.collection.ArrayMap -import kotlinx.coroutines.runBlocking -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import okhttp3.* -import okhttp3.HttpUrl.Companion.toHttpUrl -import okhttp3.internal.closeQuietly -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.Base64 -import java.util.EnumSet - -internal abstract class MangaReaderParser( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int, - searchPageSize: Int, -) : PagedMangaParser(context, source, pageSize, searchPageSize), Interceptor { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set - get() = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.NEWEST, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isSearchSupported = true, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrCreateTagMap().values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ContentType.NOVEL, - ), - ) - - protected open val listUrl = "/manga" - protected open val datePattern = "MMMM d, yyyy" - protected open val isNetShieldProtected = false - - protected var tagCache: ArrayMap? = null - protected val mutex = Mutex() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query - val url = buildString { - append("https://") - append(domain) - - when { - - !query.isNullOrEmpty() -> { - append("/page/") - append(page.toString()) - append("/?s=") - append(query.urlEncoded()) - } - - else -> { - append(listUrl) - - append("/?order=") - append( - when (order) { - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - SortOrder.NEWEST -> "latest" - SortOrder.POPULARITY -> "popular" - SortOrder.UPDATED -> "update" - else -> "" - }, - ) - - filter.tags.forEach { - append("&") - append("genre[]".urlEncoded()) - append("=") - append(it.key) - } - - filter.tagsExclude.forEach { - append("&") - append("genre[]".urlEncoded()) - append("=-") - append(it.key) - } - - if (filter.states.isNotEmpty()) { - filter.states.oneOrThrowIfMany()?.let { - append("&status=") - when (it) { - MangaState.ONGOING -> append("ongoing") - MangaState.FINISHED -> append("completed") - MangaState.PAUSED -> append("hiatus") - else -> append("") - } - } - } - - filter.types.oneOrThrowIfMany()?.let { - append("&type=") - append( - when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - }, - ) - } - - append("&page=") - append(page.toString()) - } - } - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - - protected open val selectMangaList = ".postbody .listupd .bs .bsx" - protected open val selectMangaListImg = "img.ts-post-image" - protected open val selectMangaListTitle = "div.tt" - - protected open fun parseMangaList(docs: Document): List { - return docs.select(selectMangaList).mapNotNull { - val a = it.selectFirst("a") ?: return@mapNotNull null - val relativeUrl = a.attrAsRelativeUrl("href") - val rating = it.selectFirst(".numscore")?.text() - ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - title = it.selectFirst(selectMangaListTitle)?.text() ?: a.attr("title"), - altTitles = emptySet(), - publicUrl = a.attrAsAbsoluteUrl("href"), - rating = rating, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - coverUrl = it.selectFirst(selectMangaListImg)?.src(), - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - } - - protected open val selectChapter = "#chapterlist > ul > li" - override suspend fun getDetails(manga: Manga): Manga { - val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> - val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapChapters null - MangaChapter( - id = generateUid(url), - title = element.selectFirst(".chapternum")?.textOrNull(), - url = url, - number = index + 1f, - volume = 0, - scanlator = null, - uploadDate = dateFormat.parseSafe(element.selectFirst(".chapterdate")?.text()), - branch = null, - source = source, - ) - } - return parseInfo(docs, manga, chapters) - } - - protected open val detailsDescriptionSelector = "div.entry-content" - - open suspend fun parseInfo(docs: Document, manga: Manga, chapters: List): Manga { - /// set if is table - val tableMode = - docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr") - ?: docs.selectFirst("div.seriestucon") - - val tagMap = getOrCreateTagMap() - - val selectTag = if (tableMode != null) { - docs.select(".seriestugenre > a") - } else { - docs.select(".wd-full .mgen > a") - } - - val tags = selectTag.mapNotNullToSet { tagMap[it.text()] } - - val stateSelect = if (tableMode != null) { - tableMode.selectFirst(".infotable td:contains(Status)") - ?: tableMode.selectFirst(".infotable td:contains(Statut)") - ?: tableMode.selectFirst(".infotable td:contains(حالة العمل)") - ?: tableMode.selectFirst(".infotable td:contains(الحالة)") - ?: tableMode.selectFirst(".infotable td:contains(Estado)") - ?: tableMode.selectFirst(".infotable td:contains(สถานะ)") - ?: tableMode.selectFirst(".infotable td:contains(Stato )") - ?: tableMode.selectFirst(".infotable td:contains(Durum)") - ?: tableMode.selectFirst(".infotable td:contains(Statüsü)") - - } else { - docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)") - ?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)") - ?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )") - ?: docs.selectFirst(".tsinfo div:contains(Durum)") ?: docs.selectFirst(".tsinfo div:contains(Statüsü)") - ?: docs.selectFirst(".tsinfo div:contains(Statü)") ?: docs.selectFirst(".tsinfo div:contains(الحالة)") - } - - val state = if (tableMode != null) { - stateSelect?.lastElementSibling() - - } else { - stateSelect?.lastElementChild() - } - - val mangaState = state?.let { - when (it.text()) { - - "مستمرة", "En curso", "En Curso", "Ongoing", "OnGoing", "On going", "Ativo", "En Cours", "En cours", "En cours \uD83D\uDFE2", - "En cours de publication", "Äang tiến hành", "Em lançamento", "em lançamento", "Em Lançamento", "Онгоінг", "Publishing", - "Devam Ediyor", "Em Andamento", "In Corso", "Güncel", "Berjalan", "ПродолжаетÑÑ", "Updating", "Lançando", "In Arrivo", "Emision", - "En emision", "مستمر", "Curso", "En marcha", "Publicandose", "Publicando", "连载中", "Devam ediyor", "Devam Etmekte", - -> MangaState.ONGOING - - "Completed", "Completo", "Complété", "Fini", "Achevé", "Terminé", "Terminé âš«", "Tamamlandı", "Äã hoàn thành", "Hoàn Thành", - "مكتملة", "Завершено", "Finished", "Finalizado", "Completata", "One-Shot", "Bitti", "Tamat", "Completado", "Concluído", - "Concluido", "已完结", "BitmiÅŸ", - -> MangaState.FINISHED - - "Canceled", "Cancelled", "Cancelado", "cancellato", "Cancelados", "Dropped", "Discontinued", "abandonné", "Abandonné", - -> MangaState.ABANDONED - - "Hiatus", "On Hold", "Pausado", "En espera", "En pause", "En Pause", "En attente", - -> MangaState.PAUSED - - else -> null - } - } - - val author = tableMode?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.textOrNull() - ?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.textOrNull() - - val nsfw = docs.selectFirst(".restrictcontainer") != null - || docs.selectFirst(".info-right .alr") != null - || docs.selectFirst(".postbody .alr") != null - - val title = docs.selectFirst("h1.entry-title")?.text() - ?: docs.selectFirst("h1")?.text() - ?: docs.selectFirst(".entry-title")?.text() - ?: docs.selectFirst(".seriestucontent h1")?.text() - ?: docs.selectFirst(".postbody h1")?.text() - ?: docs.selectFirst("title")?.text()?.substringBefore(" - ")?.trim() - ?: manga.title - - return manga.copy( - title = title, - description = docs.selectFirst(detailsDescriptionSelector)?.text(), - state = mangaState, - authors = setOfNotNull(author), - contentRating = if (manga.contentRating == ContentRating.ADULT || nsfw) { - ContentRating.ADULT - } else { - ContentRating.SAFE - }, - tags = tags, - chapters = chapters, - ) - } - - override suspend fun getRelatedManga(seed: Manga): List { - return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) - } - - protected open val encodedSrc = false - protected open val selectScript = "div.wrapper script" - protected open val selectPage = "div#readerarea img" - - protected open val selectTestScript = "script:containsData(ts_reader)" - override suspend fun getPages(chapter: MangaChapter): List { - val chapterUrl = chapter.url.toAbsoluteUrl(domain) - val docs = webClient.httpGet(chapterUrl).parseHtml() - - val test = docs.select(selectTestScript) - if (test.isEmpty() and !encodedSrc) { - return docs.select(selectPage).map { img -> - val url = img.requireSrc().toRelativeUrl(domain) - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source, - ) - } - } else { - val images = if (encodedSrc) { - val script = docs.select(selectScript) - var decode = "" - for (i in script) { - if (i.attr("src").startsWith("data:text/javascript;base64,")) { - decode = Base64.getDecoder().decode(i.attr("src").replace("data:text/javascript;base64,", "")) - .decodeToString() - if (decode.startsWith("ts_reader.run")) { - break - } - } - } - JSONObject(decode.substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - - } else { - val script = docs.selectFirstOrThrow(selectTestScript) - JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) - .getJSONArray("sources") - .getJSONObject(0) - .getJSONArray("images") - } - - val pages = ArrayList(images.length()) - for (i in 0 until images.length()) { - pages.add( - MangaPage( - id = generateUid(images.getString(i)), - url = images.getString(i), - preview = null, - source = source, - ), - ) - } - return pages - } - } - - protected open suspend fun getOrCreateTagMap(): Map = mutex.withLock { - tagCache?.let { return@withLock it } - val url = listUrl.toAbsoluteUrl(domain) - val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") - val tagMap = ArrayMap(tagElements.size) - for (el in tagElements) { - tagMap[el.text()] = MangaTag( - title = el.textOrNull()?.toTitleCase(sourceLocale) ?: continue, - key = el.selectFirst("input")?.attrOrNull("value") ?: continue, - source = source, - ) - } - tagCache = tagMap - return@withLock tagMap - } - - override fun intercept(chain: Interceptor.Chain): Response { - val response = chain.proceed(chain.request()) - if (!isNetShieldProtected) { - return response - } - val contentType = response.mimeType - if ( - contentType?.endsWith("/html") != false && - context.cookieJar.getCookies(domain).none { it.name.contains("NetShield") } - ) { - val cookie = runBlocking { response.copy().parseHtml().getNetShieldCookie() } ?: return response - context.cookieJar.insertCookie(domain, cookie) - return chain.proceed(response.request.newBuilder().build()).also { - response.closeQuietly() - } - } - return response - } - - @Suppress("DEPRECATION") - private suspend fun Document.getNetShieldCookie(): Cookie? = runCatchingCancellable { - val script = select("script").firstNotNullOfOrNull { s -> - s.html().takeIf { x -> x.contains("slowAES.decrypt") } - } ?: return@runCatchingCancellable null - val min = webClient.httpGet("https://$domain/min.js").parseRaw() - val res = context.evaluateJs(min + "\n\n" + script.replace(Regex("document.cookie\\s*=\\s*"), "return ")) - res?.let { - Cookie.parse(baseUri().toHttpUrl(), it) - } - }.getOrNull() - - override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { - val mangaSlug = link.pathSegments[1].nullIfEmpty() ?: return null - val url = "/manga/$mangaSlug/" - return resolver.resolveManga( - this, url, generateUid(url), - ) - } -} +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.LinkResolver +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.attrOrNull +import tsuki.util.copy +import tsuki.util.getCookies +import tsuki.util.insertCookie +import tsuki.util.mimeType +import tsuki.util.parseRaw +import tsuki.util.runCatchingCancellable +import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber + +import androidx.collection.ArrayMap +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Cookie +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.json.JSONObject +import org.jsoup.nodes.Document + +import java.text.SimpleDateFormat +import java.util.Base64 +import java.util.EnumSet + +internal abstract class MangaReaderParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int, + searchPageSize: Int, +) : PagedMangaParser(context, source, pageSize, searchPageSize), Interceptor { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set + get() = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrCreateTagMap().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + + protected open val listUrl = "/manga" + protected open val datePattern = "MMMM d, yyyy" + protected open val isNetShieldProtected = false + + protected var tagCache: ArrayMap? = null + protected val mutex = Mutex() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/page/") + append(page.toString()) + append("/?s=") + append(filter.query.urlEncoded()) + } + else -> { + append(listUrl) + append("/?order=") + append( + when (order) { + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + SortOrder.NEWEST -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.UPDATED -> "update" + else -> "" + }, + ) + filter.tags.forEach { + append("&") + append("genre[]".urlEncoded()) + append("=") + append(it.key) + } + filter.tagsExclude.forEach { + append("&") + append("genre[]".urlEncoded()) + append("=-") + append(it.key) + } + if (filter.states.isNotEmpty()) { + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + when (it) { + MangaState.ONGOING -> append("ongoing") + MangaState.FINISHED -> append("completed") + MangaState.PAUSED -> append("hiatus") + else -> append("") + } + } + } + filter.types.oneOrThrowIfMany()?.let { + append("&type=") + append( + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + }, + ) + } + append("&page=") + append(page.toString()) + } + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + protected open val selectMangaList = ".postbody .listupd .bs .bsx" + protected open val selectMangaListImg = "img.ts-post-image" + protected open val selectMangaListTitle = "div.tt" + + protected open fun parseMangaList(docs: Document): List { + return docs.select(selectMangaList).mapNotNull { + val a = it.selectFirst("a") ?: return@mapNotNull null + val relativeUrl = a.attrAsRelativeUrl("href") + val rating = it.selectFirst(".numscore")?.text() + ?.toFloatOrNull()?.div(10) ?: RATING_UNKNOWN + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = it.selectFirst(selectMangaListTitle)?.text() ?: a.attr("title"), + altTitles = emptySet(), + publicUrl = a.attrAsAbsoluteUrl("href"), + rating = rating, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = it.selectFirst(selectMangaListImg)?.src(), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + protected open val selectChapter = "#chapterlist > ul > li" + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + val chapters = docs.select(selectChapter).mapNotNull { element -> + val url = element.selectFirst("a")?.attrAsRelativeUrlOrNull("href") ?: return@mapNotNull null + val chapterNumEl = element.selectFirst(".chapternum") + val name = chapterNumEl?.textOrNull() ?: "" + MangaChapter( + id = generateUid(url), + title = name, + url = url, + number = name.extractChapterNumber(), + volume = 0, + scanlator = null, + uploadDate = dateFormat.parseSafe(element.selectFirst(".chapterdate")?.text()), + branch = null, + source = source, + ) + }.sortedBy { it.number } + + return parseInfo(docs, manga, chapters) + } + + protected open val detailsDescriptionSelector = "div.entry-content" + + open suspend fun parseInfo(docs: Document, manga: Manga, chapters: List): Manga { + /// set if is table + val tableMode = + docs.selectFirst("div.seriestucontent > div.seriestucontentr") ?: docs.selectFirst("div.seriestucontentr") + ?: docs.selectFirst("div.seriestucon") + + val tagMap = getOrCreateTagMap() + + val selectTag = if (tableMode != null) { + docs.select(".seriestugenre > a") + } else { + docs.select(".wd-full .mgen > a") + } + + val tags = selectTag.mapNotNullToSet { tagMap[it.text()] } + + val stateSelect = if (tableMode != null) { + tableMode.selectFirst(".infotable td:contains(Status)") + ?: tableMode.selectFirst(".infotable td:contains(Statut)") + ?: tableMode.selectFirst(".infotable td:contains(حالة العمل)") + ?: tableMode.selectFirst(".infotable td:contains(الحالة)") + ?: tableMode.selectFirst(".infotable td:contains(Estado)") + ?: tableMode.selectFirst(".infotable td:contains(สถานะ)") + ?: tableMode.selectFirst(".infotable td:contains(Stato )") + ?: tableMode.selectFirst(".infotable td:contains(Durum)") + ?: tableMode.selectFirst(".infotable td:contains(Statüsü)") + + } else { + docs.selectFirst(".tsinfo div:contains(Status)") ?: docs.selectFirst(".tsinfo div:contains(Statut)") + ?: docs.selectFirst(".tsinfo div:contains(حالة العمل)") ?: docs.selectFirst(".tsinfo div:contains(Estado)") + ?: docs.selectFirst(".tsinfo div:contains(สถานะ)") ?: docs.selectFirst(".tsinfo div:contains(Stato )") + ?: docs.selectFirst(".tsinfo div:contains(Durum)") ?: docs.selectFirst(".tsinfo div:contains(Statüsü)") + ?: docs.selectFirst(".tsinfo div:contains(Statü)") ?: docs.selectFirst(".tsinfo div:contains(الحالة)") + } + + val state = if (tableMode != null) { + stateSelect?.lastElementSibling() + + } else { + stateSelect?.lastElementChild() + } + + val mangaState = state?.let { + when (it.text()) { + + "مستمرة", "En curso", "En Curso", "Ongoing", "OnGoing", "On going", "Ativo", "En Cours", "En cours", "En cours \uD83D\uDFE2", + "En cours de publication", "Äang tiến hành", "Em lançamento", "em lançamento", "Em Lançamento", "Онгоінг", "Publishing", + "Devam Ediyor", "Em Andamento", "In Corso", "Güncel", "Berjalan", "ПродолжаетÑÑ", "Updating", "Lançando", "In Arrivo", "Emision", + "En emision", "مستمر", "Curso", "En marcha", "Publicandose", "Publicando", "连载中", "Devam ediyor", "Devam Etmekte", + -> MangaState.ONGOING + + "Completed", "Completo", "Complété", "Fini", "Achevé", "Terminé", "Terminé âš«", "Tamamlandı", "Äã hoàn thành", "Hoàn Thành", + "مكتملة", "Завершено", "Finished", "Finalizado", "Completata", "One-Shot", "Bitti", "Tamat", "Completado", "Concluído", + "Concluido", "已完结", "BitmiÅŸ", + -> MangaState.FINISHED + + "Canceled", "Cancelled", "Cancelado", "cancellato", "Cancelados", "Dropped", "Discontinued", "abandonné", "Abandonné", + -> MangaState.ABANDONED + + "Hiatus", "On Hold", "Pausado", "En espera", "En pause", "En Pause", "En attente", + -> MangaState.PAUSED + + else -> null + } + } + + val author = tableMode?.selectFirst(".infotable td:contains(Author)")?.lastElementSibling()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Author)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Auteur)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Artist)")?.lastElementChild()?.textOrNull() + ?: docs.selectFirst(".tsinfo div:contains(Durum)")?.lastElementChild()?.textOrNull() + + val nsfw = docs.selectFirst(".restrictcontainer") != null + || docs.selectFirst(".info-right .alr") != null + || docs.selectFirst(".postbody .alr") != null + + val title = docs.selectFirst("h1.entry-title")?.text() + ?: docs.selectFirst("h1")?.text() + ?: docs.selectFirst(".entry-title")?.text() + ?: docs.selectFirst(".seriestucontent h1")?.text() + ?: docs.selectFirst(".postbody h1")?.text() + ?: docs.selectFirst("title")?.text()?.substringBefore(" - ")?.trim() + ?: manga.title + + return manga.copy( + title = title, + description = docs.selectFirst(detailsDescriptionSelector)?.text(), + state = mangaState, + authors = setOfNotNull(author), + contentRating = if (manga.contentRating == ContentRating.ADULT || nsfw) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List { + return parseMangaList(webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml()) + } + + protected open val encodedSrc = false + protected open val selectScript = "div.wrapper script" + protected open val selectPage = "div#readerarea img" + + protected open val selectTestScript = "script:containsData(ts_reader)" + override suspend fun getPages(chapter: MangaChapter): List { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val docs = webClient.httpGet(chapterUrl).parseHtml() + + val test = docs.select(selectTestScript) + if (test.isEmpty() and !encodedSrc) { + return docs.select(selectPage).map { img -> + val url = img.requireSrc().toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } else { + val images = if (encodedSrc) { + val script = docs.select(selectScript) + var decode = "" + for (i in script) { + if (i.attr("src").startsWith("data:text/javascript;base64,")) { + decode = Base64.getDecoder().decode(i.attr("src").replace("data:text/javascript;base64,", "")) + .decodeToString() + if (decode.startsWith("ts_reader.run")) { + break + } + } + } + JSONObject(decode.substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + + } else { + val script = docs.selectFirstOrThrow(selectTestScript) + JSONObject(script.data().substringAfter('(').substringBeforeLast(')')) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + } + + val pages = ArrayList(images.length()) + for (i in 0 until images.length()) { + pages.add( + MangaPage( + id = generateUid(images.getString(i)), + url = images.getString(i), + preview = null, + source = source, + ), + ) + } + return pages + } + } + + protected open suspend fun getOrCreateTagMap(): Map = mutex.withLock { + tagCache?.let { return@withLock it } + val url = listUrl.toAbsoluteUrl(domain) + val tagElements = webClient.httpGet(url).parseHtml().select("ul.genrez > li") + val tagMap = ArrayMap(tagElements.size) + for (el in tagElements) { + tagMap[el.text()] = MangaTag( + title = el.textOrNull()?.toTitleCase(sourceLocale) ?: continue, + key = el.selectFirst("input")?.attrOrNull("value") ?: continue, + source = source, + ) + } + tagCache = tagMap + return@withLock tagMap + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (!isNetShieldProtected) { + return response + } + val contentType = response.mimeType + if ( + contentType?.endsWith("/html") != false && + context.cookieJar.getCookies(domain).none { it.name.contains("NetShield") } + ) { + val cookie = runBlocking { response.copy().parseHtml().getNetShieldCookie() } ?: return response + context.cookieJar.insertCookie(domain, cookie) + return chain.proceed(response.request.newBuilder().build()).also { + response.closeQuietly() + } + } + return response + } + + @Suppress("DEPRECATION") + private suspend fun Document.getNetShieldCookie(): Cookie? = runCatchingCancellable { + val script = select("script").firstNotNullOfOrNull { s -> + s.html().takeIf { x -> x.contains("slowAES.decrypt") } + } ?: return@runCatchingCancellable null + val min = webClient.httpGet("https://$domain/min.js").parseRaw() + val res = context.evaluateJs(min + "\n\n" + script.replace(Regex("document.cookie\\s*=\\s*"), "return ")) + res?.let { + Cookie.parse(baseUri().toHttpUrl(), it) + } + }.getOrNull() + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val mangaSlug = link.pathSegments[1].nullIfEmpty() ?: return null + val url = "/manga/$mangaSlug/" + return resolver.resolveManga( + this, url, generateUid(url), + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt similarity index 58% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt rename to src/main/kotlin/tsuki/parsers/MangaThemesia.kt index 86436d32..289ba158 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaThemesia.kt +++ b/src/main/kotlin/tsuki/parsers/MangaThemesia.kt @@ -1,391 +1,486 @@ -package org.koitharu.kotatsu.parsers.parsers - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.jsoup.select.Elements -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -abstract class MangaThemesia( - context: MangaLoaderContext, - source: MangaParserSource, - domain: String, - pageSize: Int = 20, -) : PagedMangaParser(context, source, pageSize) { - - override val configKeyDomain = ConfigKey.Domain(domain) - - override fun onCreateConfig(keys: MutableCollection>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://$domain/") - .build() - - protected open val mangaDirectory = "manga" - protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale.US) - protected open val withoutAjax = false - protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" - protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" - protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num), div.bxcl li:not(a[data-bs-target='#lockedChapterModal'])" - protected open val pageSelector = "div#readerarea img" - - override val availableSortOrders: EnumSet = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.ALPHABETICAL_DESC, - SortOrder.UPDATED, - SortOrder.ADDED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - @Volatile - private var genreCache: Set? = null - private val genreMutex = Mutex() - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.PAUSED, - MangaState.ABANDONED, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHWA, - ContentType.MANHUA, - ContentType.COMICS, - ContentType.NOVEL, - ), - ) - - private suspend fun getOrFetchGenres(): Set { - genreCache?.let { return it } - return genreMutex.withLock { - genreCache ?: fetchGenres().also { genreCache = it } - } - } - - protected open suspend fun fetchGenres(): Set { - val doc = webClient.httpGet("https://$domain/$mangaDirectory/").parseHtml() - val genrez = doc.selectFirst("ul.genrez") ?: return emptySet() - return genrez.select("li").mapNotNull { li -> - val label = li.selectFirst("label")?.text()?.trim() ?: return@mapNotNull null - val input = li.selectFirst("input[type=checkbox]") ?: return@mapNotNull null - val value = input.attr("value").trim() - if (label.isBlank() || value.isBlank()) null - else MangaTag(key = value, title = label, source = source) - }.toSet() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { - val query = filter.query?.trim().orEmpty() - val url = buildString { - append("https://$domain/$mangaDirectory/") - if (query.isNotEmpty()) { - append("?s=${query.urlEncoded()}") - } else { - append("?page=$page") - } - filter.states.firstOrNull()?.let { - append("&status=") - append(when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - MangaState.PAUSED -> "hiatus" - MangaState.ABANDONED -> "dropped" - else -> "" - }) - } - filter.types.firstOrNull()?.let { - append("&type=") - append(when (it) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - ContentType.COMICS -> "comic" - ContentType.NOVEL -> "novel" - else -> "" - }) - } - filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } - filter.tagsExclude.forEach { append("&genre[]=-${it.key.urlEncoded()}") } - append("&order=") - append(when (order) { - SortOrder.UPDATED -> "update" - SortOrder.POPULARITY -> "popular" - SortOrder.ADDED -> "latest" - SortOrder.ALPHABETICAL -> "title" - SortOrder.ALPHABETICAL_DESC -> "titlereverse" - else -> "update" - }) - } - - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - protected open fun parseMangaList(doc: Document): List { - return doc.select(searchSelector).mapNotNull { element -> - parseMangaElement(element) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val title = doc.selectFirst("h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1") - ?.text() ?: manga.title - - val description = doc.selectFirst( - ".desc, .entry-content[itemprop=description], .summary__content, .sinopsis, .entry-content p" - )?.text()?.trim().orEmpty() - - val coverUrl = doc.selectFirst( - ".thumb img, .infomanga img, .summary_image img, .cover img, img.wp-post-image" - )?.imgAttr() ?: manga.coverUrl - - val table = doc.selectFirst("table.infotable") - val hasTable = table != null - - val authors = if (hasTable) { - val authorCell = table.selectFirst("tr:has(td:contains(Author)) td:last-child") - val artistCell = table.selectFirst("tr:has(td:contains(Artist)) td:last-child") - listOfNotNull( - authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, - artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } - ).toSet() - } else { - doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Author) i, .spe span:contains(Author) a") - .map { it.text().trim() } - .filter { it.isNotBlank() } - .toSet() - } - - val tags = doc.select(".mgen a, .gnr a, .seriestugenre a, .genres-content a") - .mapNotNull { a -> - val text = a.text().trim() - if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null - }.toSet() - - val statusText = if (hasTable) { - table.selectFirst("tr:has(td:contains(Status)) td:last-child")?.text() - } else { - doc.select(".imptdt:contains(Status) i, .tsinfo .imptdt:contains(Status) a, .fmed b:contains(Status)+span span") - .text() - .ifEmpty { - doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() - } - } - val state = parseStatus(statusText) - - val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() - ?: doc.selectFirst(".num")?.text()?.toFloatOrNull() - val normalizedRating = if (rating != null && rating > 0) rating / 10f else RATING_UNKNOWN - - val chapters = loadChapters(doc, manga.url) - - return manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - tags = tags, - state = state, - rating = normalizedRating, - chapters = chapters, - ) - } - - protected open fun parseStatus(text: String?): MangaState? { - val status = text?.lowercase() ?: return null - return when (status) { - in ongoing -> MangaState.ONGOING - in finished -> MangaState.FINISHED - in paused -> MangaState.PAUSED - in abandoned -> MangaState.ABANDONED - else -> null - } - } - - protected open suspend fun loadChapters(doc: Document, mangaUrl: String): List { - val chapterElements = doc.select(chapterListSelector) - if (chapterElements.isEmpty() && !withoutAjax) { - return loadChaptersAjax(mangaUrl) - } - return parseChapters(chapterElements) - } - - protected open suspend fun loadChaptersAjax(mangaUrl: String): List { - val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/" - val doc = webClient.httpPost(ajaxUrl, emptyMap()).parseHtml() - return parseChapters(doc.select(chapterListSelector)) - } - - private fun parseChapters(elements: Elements): List { - return elements.mapChapters(reversed = true) { i, element -> - val a = element.selectFirst("a") ?: return@mapChapters null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() - val dateStr = element.selectFirst(".chapterdate")?.text() - val number = extractChapterNumber(name) ?: (i + 1).toFloat() - MangaChapter( - id = generateUid(href), - url = href, - title = name, - number = number, - volume = 0, - uploadDate = parseChapterDate(dateStr), - scanlator = null, - branch = null, - source = source, - ) - } - } - - private fun extractChapterNumber(title: String): Float? { - return Regex("""(?i)(?:chapter|ch\.?)\s*([0-9]+(?:\.[0-9]+)?)""") - .find(title) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - } - - protected open fun parseChapterDate(date: String?): Long { - if (date.isNullOrBlank()) return 0L - val clean = date.trim() - return when { - WordSet(" ago", "atrás", " hace", " назад", " önce", " trÆ°á»›c", "مضت", "قبل").endsWith(clean) -> parseRelativeDate(clean) - WordSet("há ", "منذ", "il y a", "hace", "giá»", "phút").startsWith(clean) -> parseRelativeDate(clean) - WordSet("yesterday", "يوم واحد").startsWith(clean) -> yesterday() - WordSet("today").startsWith(clean) -> today() - WordSet("يومين").startsWith(clean) -> dayBeforeYesterday() - else -> dateFormat.parseSafe(clean) - } - } - - private fun parseRelativeDate(text: String): Long { - val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0L - val cal = Calendar.getInstance() - return when { - WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان").anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis - WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут").anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis - WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giá»", "ore", "ساعة", "å°æ—¶").anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis - WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis - WordSet("week", "semana", "tuần", "أسابيع", "أسبوع").anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis - WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois").anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis - WordSet("year", "año", "năm").anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis - else -> 0L - } - } - - private fun yesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - private fun today() = Calendar.getInstance().apply { set(Calendar.HOUR_OF_DAY, 0); set(Calendar.MINUTE, 0) }.timeInMillis - private fun dayBeforeYesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -2) }.timeInMillis - - override suspend fun getPages(chapter: MangaChapter): List { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - - // json image list - val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) - val jsonMatch = jsonRegex.find(doc.html()) - if (jsonMatch != null) { - val jsonArray = JSONArray(jsonMatch.groupValues[1]) - return (0 until jsonArray.length()).map { i -> - val url = jsonArray.getString(i) - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - // direct - return doc.select(pageSelector).mapNotNull { img -> - val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null - MangaPage(id = generateUid(url), url = url, preview = null, source = source) - } - } - - override suspend fun getRelatedManga(seed: Manga): List { - val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(relatedSelector).mapNotNull { element -> - parseMangaElement(element) - } - } - - protected open fun parseMangaElement(element: Element): Manga? { - val a = element.selectFirst("a") ?: return null - val href = a.attrAsRelativeUrl("href") - val title = a.attr("title").ifBlank { a.text() } - val coverUrl = element.selectFirst("img")?.imgAttr() - return makeManga(href, title, coverUrl) - } - - private fun makeManga(href: String, title: String, coverUrl: String?): Manga { - return Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = title, - altTitles = emptySet(), - authors = emptySet(), - coverUrl = coverUrl, - rating = RATING_UNKNOWN, - tags = emptySet(), - state = null, - contentRating = if (isNsfwSource) ContentRating.ADULT else null, - source = source, - ) - } - - protected fun Element.imgAttr(): String { - for (attr in listOf("data-src", "data-lazy-src", "data-original", "data-cfsrc", "data-image", "src")) { - val value = attr(attr).trim().takeIf { it.isNotEmpty() } ?: continue - return value.toAbsoluteUrl(domain).substringBefore("?") // remove WP resize params - } - val srcset = attr("srcset") - if (srcset.isNotBlank()) { - return srcset.split(",").last().trim().split(" ").first().toAbsoluteUrl(domain) - } - return "" - } - - private fun SimpleDateFormat.parseSafe(date: String): Long = - runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) - - companion object { - val ongoing = setOf( - "ongoing", "on going", "publishing", "updating", "en curso", "ativo", "en cours", - "Ä‘ang tiến hành", "em lançamento", "devam ediyor", "in corso", "güncel", "berjalan", - "продолжаетÑÑ", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة", - ) - val finished = setOf( - "completed", "complete", "finished", "finalizado", "terminé", "tamamlandı", - "đã hoàn thành", "hoàn thành", "مكتملة", "завершено", "completato", "one-shot", - "bitti", "tamat", "concluído", "已完结", "bitmiÅŸ", "achevé", - ) - val paused = setOf( - "hiatus", "on hold", "pausado", "en espera", "en pause", "en attente", - "durduruldu", "beklemede", "Ä‘ang chá»", "متوقÙ", "заморожено", - ) - val abandoned = setOf( - "canceled", "cancelled", "cancelado", "cancellato", "dropped", "discontinued", - "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено", - ) - } -} +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.ContentType +import tsuki.model.WordSet + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded +import tsuki.util.extractChapterNumber +import tsuki.util.mapNotNullToSet + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import org.jsoup.select.Elements +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale + +abstract class MangaThemesia( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 20, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set(CommonHeaders.REFERER, "https://$domain/") + .build() + + protected open val mangaDirectory = "manga" + protected open val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("en")) + protected open val withoutAjax = false + protected open val searchSelector = ".utao .uta .imgu, .listupd .bs .bsx, .listo .bs .bsx" + protected open val relatedSelector = ".related-posts .bsx, .bixbox .bsx, .related-manga .related-reading-wrap" + protected open val chapterListSelector = "div.bxcl li, div.cl li, #chapterlist li, ul li:has(div.chbox):has(div.eph-num)" + protected open val pageSelector = "div#readerarea img" + + override val availableSortOrders: EnumSet = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + @Volatile + private var genreCache: Set? = null + private val genreMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + + private suspend fun getOrFetchGenres(): Set { + genreCache?.let { return it } + return genreMutex.withLock { + genreCache ?: fetchGenres().also { genreCache = it } + } + } + + protected open suspend fun fetchGenres(): Set { + val doc = webClient.httpGet("https://$domain/$mangaDirectory/").parseHtml() + val genrez = doc.selectFirst("ul.genrez") ?: return emptySet() + return genrez.select("li").mapNotNull { li -> + val label = li.selectFirst("label")?.text()?.trim() ?: return@mapNotNull null + val input = li.selectFirst("input[type=checkbox]") ?: return@mapNotNull null + val value = input.attr("value").trim() + if (label.isBlank() || value.isBlank()) null + else MangaTag(key = value, title = label, source = source) + }.toSet() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val url = buildString { + append("https://$domain/$mangaDirectory/") + if (filter.query.isNullOrEmpty()) { + append("?page=$page") + } else { + append("?s=${filter.query.urlEncoded()}") + } + filter.states.firstOrNull()?.let { + append("&status=") + append(it.toQueryParam()) + } + filter.types.firstOrNull()?.let { + append("&type=") + append(it.toQueryParam()) + } + filter.tags.forEach { + append("&genre[]=") + append(it.key.urlEncoded()) + } + append("&order=") + append(order.toQueryParam()) + } + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun MangaState.toQueryParam(): String = when (this) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "dropped" + else -> "" + } + + private fun ContentType.toQueryParam(): String = when (this) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + } + + private fun SortOrder.toQueryParam(): String = when (this) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ADDED -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "update" + } + + protected open fun parseMangaList(doc: Document): List { + return doc.select(searchSelector).mapNotNull { element -> + parseMangaElement(element) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { loadChapters(doc, manga.url) } + + val title = doc.selectFirst( + "h1.entry-title, .ts-breadcrumb li:last-child span, .infomanga h1, .animefull h1" + )?.text() ?: manga.title + + val description = doc.selectFirst( + ".desc, .entry-content[itemprop=description], .summary__content, .sinopsis, .entry-content p" + )?.text()?.trim().orEmpty() + + val coverUrl = doc.selectFirst( + ".thumb img, .infomanga img, .summary_image img, .cover img, img.wp-post-image" + )?.imgAttr() ?: manga.coverUrl + + val table = doc.selectFirst("table.infotable") + val hasTable = table != null + + val authors = if (hasTable) { + val authorCell = table.selectFirst("tr:has(td:contains(Author)) td:last-child") + val artistCell = table.selectFirst("tr:has(td:contains(Artist)) td:last-child") + listOfNotNull( + authorCell?.text().cleanAuthor(), + artistCell?.text().cleanAuthor() + ).toSet() + } else { + doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Author) i, .spe span:contains(Author) a") + .mapNotNull { it.text().cleanAuthor() } + .toSet() + } + + val tags = doc.select(".mgen a, .gnr a, .seriestugenre a, .genres-content a") + .mapNotNullToSet { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + } + + val statusText = if (hasTable) { + table.selectFirst("tr:has(td:contains(Status)) td:last-child")?.text() + } else { + doc.select(".imptdt:contains(Status) i, .tsinfo .imptdt:contains(Status) a, .fmed b:contains(Status)+span span") + .text() + .ifEmpty { + doc.selectFirst("div.post-content_item:contains(Status) div.summary-content")?.text() + } + } + val altTitles = parseAltTitles(doc, table, hasTable) + + val state = parseStatus(statusText) + + val rating = doc.selectFirst(".num[itemprop=ratingValue]")?.attr("content")?.toFloatOrNull() + ?: doc.selectFirst(".num")?.text()?.toFloatOrNull() + val normalizedRating = if (rating != null && rating > 0) rating / 10f else RATING_UNKNOWN + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + tags = tags, + state = state, + rating = normalizedRating, + chapters = chapters, + altTitles = altTitles + ) + } + + protected open fun parseAltTitles(doc: Document, table: Element?, hasTable: Boolean): Set { + return buildSet { + if (hasTable && table != null) { + table.select("tr").forEach { row -> + val label = row.selectFirst("td")?.text()?.trim().orEmpty() + if ( + label.contains("Alternative", ignoreCase = true) || + label.contains("Alt.", ignoreCase = true) || + label.contains("Alt ", ignoreCase = true) + ) { + val cell = row.select("td").lastOrNull() + val text = cell?.text() + if (!text.isNullOrBlank()) { + addAll(splitAltTitles(text)) + } + } + } + } else { + val selectors = listOf( + ".tsinfo .imptdt:contains(Alt) i", + ".tsinfo .imptdt:contains(Alternative) i", + ".spe span:contains(Alt) a", + ".spe span:contains(Alternative) a", + ".alternative", + ".alter", + ".post-content_item:contains(Alt) .summary-content", + ) + doc.select(selectors.joinToString(", ")).forEach { element -> + val text = element.text() + if (text.isNotBlank()) { + addAll(splitAltTitles(text)) + } + } + } + } + } + + protected fun splitAltTitles(text: String?): Set { + return text.orEmpty() + .split(',', ';', '/', '•', '\n') + .map { it.trim() } + .filter { it.isNotBlank() && !it.equals("n/a", true) } + .toSet() + } + + protected open fun parseStatus(text: String?): MangaState? { + val status = text?.lowercase() ?: return null + return when { + ongoingWordSet.anyWordIn(status) -> MangaState.ONGOING + finishedWordSet.anyWordIn(status) -> MangaState.FINISHED + pausedWordSet.anyWordIn(status) -> MangaState.PAUSED + abandonedWordSet.anyWordIn(status) -> MangaState.ABANDONED + else -> null + } + } + + protected open suspend fun loadChapters(doc: Document, mangaUrl: String): List { + val chapterElements = doc.select(chapterListSelector) + if (chapterElements.isEmpty() && !withoutAjax) { + return loadChaptersAjax(mangaUrl) + } + return parseChapters(chapterElements) + } + + protected open suspend fun loadChaptersAjax(mangaUrl: String): List { + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/" + val doc = webClient.httpPost(ajaxUrl, emptyMap()).parseHtml() + return parseChapters(doc.select(chapterListSelector)) + } + + private fun parseChapters(elements: Elements): List { + return elements.mapNotNull { element -> + val a = element.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + if (href.isBlank()) return@mapNotNull null + val name = a.selectFirst(".chapternum")?.text() ?: a.ownText() + val dateStr = element.selectFirst(".chapterdate")?.text() + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = parseChapterDate(dateStr), + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + } + + protected open fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val clean = date.trim() + return when { + agoWordSet.endsWith(clean) -> parseRelativeDate(clean) + fromWordSet.startsWith(clean) -> parseRelativeDate(clean) + yesterdayWordSet.startsWith(clean) -> yesterday() + todayWordSet.startsWith(clean) -> today() + dayBeforeYesterdayWordSet.startsWith(clean) -> dayBeforeYesterday() + else -> dateFormat.parseSafe(clean) + } + } + + private fun parseRelativeDate(text: String): Long { + val number = Regex("""(\d+)""").find(text)?.value?.toIntOrNull() ?: return 0L + val cal = Calendar.getInstance() + return when { + secondsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + minutesWordSet.anyWordIn(text) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + hoursWordSet.anyWordIn(text) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + daysWordSet.anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + weeksWordSet.anyWordIn(text) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number * 7) }.timeInMillis + monthsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + yearsWordSet.anyWordIn(text) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0L + } + } + + private fun yesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + private fun today() = Calendar.getInstance().apply { set(Calendar.HOUR_OF_DAY, 0); set(Calendar.MINUTE, 0) }.timeInMillis + private fun dayBeforeYesterday() = Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -2) }.timeInMillis + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + // json image list + val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + val jsonMatch = jsonRegex.find(doc.html()) + if (jsonMatch != null) { + val jsonArray = JSONArray(jsonMatch.groupValues[1]) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + // direct + return doc.select(pageSelector).mapNotNull { img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List { + val doc = webClient.httpGet(seed.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(relatedSelector).mapNotNull { element -> + parseMangaElement(element) + } + } + + protected open fun parseMangaElement(element: Element): Manga? { + val a = element.selectFirst("a") ?: return null + val href = a.attrAsRelativeUrl("href") + if (href.isBlank()) return null + val title = a.attr("title").ifBlank { a.text() } + val coverUrl = element.selectFirst("img")?.imgAttr() + return makeManga(href, title, coverUrl) + } + + private fun makeManga(href: String, title: String, coverUrl: String?): Manga { + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = coverUrl, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + + protected fun Element.imgAttr(): String { + for (attr in listOf("data-src", "data-lazy-src", "data-original", "data-cfsrc", "data-image", "src")) { + val value = attr(attr).trim().takeIf { it.isNotEmpty() } ?: continue + return value.toAbsoluteUrl(domain).substringBefore("?") // remove WP resize params + } + val srcset = attr("srcset") + if (srcset.isNotBlank()) { + return srcset.split(",").last().trim().split(" ").first().toAbsoluteUrl(domain) + } + return "" + } + + private fun String?.cleanAuthor(): String? = + this?.trim()?.takeIf { it.isNotBlank() && it !in AUTHOR_PLACEHOLDERS } + + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) + + companion object { + private val AUTHOR_PLACEHOLDERS = setOf("n/a", "N/A", "Updating") + + private val ongoingWordSet = WordSet( + "ongoing", "on going", "publishing", "updating", "en curso", "ativo", "en cours", + "Ä‘ang tiến hành", "em lançamento", "devam ediyor", "in corso", "güncel", "berjalan", + "продолжаетÑÑ", "lançando", "in arrivo", "连载中", "devam etmekte", "مستمرة" + ) + private val finishedWordSet = WordSet( + "completed", "complete", "finished", "finalizado", "terminé", "tamamlandı", + "đã hoàn thành", "hoàn thành", "مكتملة", "завершено", "completato", "one-shot", + "bitti", "tamat", "concluído", "已完结", "bitmiÅŸ", "achevé" + ) + private val pausedWordSet = WordSet( + "hiatus", "on hold", "pausado", "en espera", "en pause", "en attente", + "durduruldu", "beklemede", "Ä‘ang chá»", "متوقÙ", "заморожено" + ) + private val abandonedWordSet = WordSet( + "canceled", "cancelled", "cancelado", "cancellato", "dropped", "discontinued", + "abandonné", "iptal edildi", "đã hủy", "ملغي", "заброшено" + ) + + private val agoWordSet = WordSet(" ago", "atrás", " hace", " назад", " önce", " trÆ°á»›c", "مضت", "قبل") + private val fromWordSet = WordSet("há ", "منذ", "il y a", "hace", "giá»", "phút") + private val yesterdayWordSet = WordSet("yesterday", "يوم واحد") + private val todayWordSet = WordSet("today") + private val dayBeforeYesterdayWordSet = WordSet("يومين") + + private val secondsWordSet = WordSet("detik", "segundo", "second", "วินาที", "giây", "ثوان") + private val minutesWordSet = WordSet("menit", "dakika", "min", "minute", "minuto", "mins", "นาที", "دقائق", "phút", "минут") + private val hoursWordSet = WordSet("jam", "saat", "heure", "hora", "hour", "hours", "ชั่วโมง", "giá»", "ore", "ساعة", "å°æ—¶") + private val daysWordSet = WordSet("hari", "gün", "jour", "día", "dia", "day", "days", "días", "วัน", "ngày", "giorni", "أيام", "天", "день") + private val weeksWordSet = WordSet("week", "semana", "tuần", "أسابيع", "أسبوع") + private val monthsWordSet = WordSet("month", "months", "mes", "meses", "tháng", "أشهر", "mois") + private val yearsWordSet = WordSet("year", "año", "năm") + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt similarity index 80% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt rename to src/main/kotlin/tsuki/parsers/MangaboxParser.kt index 7f29cc7a..fd42fb09 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/MangaboxParser.kt +++ b/src/main/kotlin/tsuki/parsers/MangaboxParser.kt @@ -1,55 +1,52 @@ @file:Suppress("DEPRECATION") -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.FlexiblePagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder +import tsuki.model.WordSet +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toTitleCase +import tsuki.util.nullIfEmpty +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.toRelativeUrl +import tsuki.util.extractChapterNumber import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrlOrNull import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.FlexiblePagedMangaParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Exclude -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.AUTHOR -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase import java.text.DateFormat import java.text.SimpleDateFormat import java.util.Calendar import java.util.EnumSet -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* - internal abstract class MangaboxParser( context: MangaLoaderContext, @@ -73,23 +70,23 @@ internal abstract class MangaboxParser( override val searchQueryCapabilities: MangaSearchQueryCapabilities get() = MangaSearchQueryCapabilities( SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class, Exclude::class), + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class, QueryCriteria.Exclude::class), isMultiple = true, ), SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), isMultiple = false, ), SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), isMultiple = true, ), SearchCapability( - field = AUTHOR, - criteriaTypes = setOf(Include::class), + field = SearchableField.AUTHOR, + criteriaTypes = setOf(QueryCriteria.Include::class), isMultiple = false, isExclusive = true, ), @@ -120,8 +117,8 @@ internal abstract class MangaboxParser( protected open val datePattern = "MMM dd,yy" override suspend fun getListPage(query: MangaSearchQuery, page: Int): List { - val titleQuery = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TITLE_NAME } + val titleQuery = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } ?.value ?.toString() ?.trim() @@ -132,8 +129,8 @@ internal abstract class MangaboxParser( return parseSearchResults(doc) } - val tagCriterion = query.criteria.filterIsInstance>() - .firstOrNull { it.field == TAG } + val tagCriterion = query.criteria.filterIsInstance>() + .firstOrNull { it.field == SearchableField.TAG } val tagKey = tagCriterion?.values?.firstOrNull()?.let { value -> when (value) { @@ -283,26 +280,24 @@ internal abstract class MangaboxParser( protected open suspend fun getChapters(doc: Document): List { val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return doc.body().select(selectChapter).mapChapters(reversed = true) { i, li -> + return doc.body().select(selectChapter).map { li -> val a = li.selectFirstOrThrow("a") val href = a.attrAsRelativeUrl("href") + val name = a.text() val dateText = li.select(selectDate).last()?.text() MangaChapter( id = generateUid(href), - title = a.text(), - number = i + 1f, + title = name, + number = name.extractChapterNumber(), volume = 0, url = href, - uploadDate = parseChapterDate( - dateFormat, - dateText, - ), + uploadDate = parseChapterDate(dateFormat, dateText), source = source, scanlator = null, branch = null, ) - } + }.sortedBy { it.number } } protected open val selectPage = "div#vungdoc img, div.container-chapter-reader img" @@ -459,4 +454,4 @@ internal abstract class MangaboxParser( private val backupImageRegex = Regex("""backupImage\s*=\s*\[([^]]+)]""") private val chapterImagesRegex = Regex("""chapterImages\s*=\s*\[([^]]+)]""") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/parsers/NatsuParser.kt b/src/main/kotlin/tsuki/parsers/NatsuParser.kt new file mode 100644 index 00000000..66442a11 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/NatsuParser.kt @@ -0,0 +1,408 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.RATING_UNKNOWN +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaState + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.await +import tsuki.util.parseJsonArray +import tsuki.util.toRelativeUrl +import tsuki.util.urlBuilder + +import okhttp3.Request +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.MultipartBody +import org.json.JSONArray +import org.jsoup.HttpStatusException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import org.jsoup.nodes.Document +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope + +internal abstract class NatsuParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 24, +) : PagedMangaParser(context, source, pageSize, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + override val sourceLocale: Locale = Locale.ENGLISH + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.RELEVANCE, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + @Volatile + private var genreCache: Set? = null + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.NOVEL, + ), + ) + + private suspend fun getOrFetchGenres(): Set { + genreCache?.let { return it } + val tags = fetchAvailableTags() + genreCache = tags + return tags + } + + private var nonce: String? = null + + private suspend fun getNonce(): String { + if (nonce == null) { + val doc = webClient.httpGet( + "https://${domain}/wp-admin/admin-ajax.php?type=search_form&action=get_nonce" + ).parseHtml() + nonce = doc.selectFirst("input[name=search_nonce]")?.attr("value") + ?: throw Exception("Unable to get nonce") + } + return nonce!! + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val formParts = mutableMapOf() + formParts["nonce"] = getNonce() + + formParts["inclusion"] = "OR" + formParts["genre"] = if (filter.tags.isNotEmpty()) + JSONArray(filter.tags.map { it.key }).toString() + else "[]" + + formParts["exclusion"] = "OR" + formParts["genre_exclude"] = if (filter.tagsExclude.isNotEmpty()) + JSONArray(filter.tagsExclude.map { it.key }).toString() + else "[]" + + formParts["page"] = page.toString() + + formParts["author"] = if (!filter.author.isNullOrEmpty()) { + JSONArray(filter.author!!).toString() + } else { + "[]" + } + + formParts["artist"] = "[]" + formParts["project"] = "0" + + formParts["type"] = if (filter.types.isNotEmpty()) + JSONArray(filter.types.mapNotNull { type -> + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> null + } + }).toString() + else "[]" + + formParts["status"] = if (filter.states.isNotEmpty()) + JSONArray(filter.states.mapNotNull { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.ABANDONED -> "cancelled" + MangaState.PAUSED -> "on-hiatus" + else -> null + } + }).toString() + else "[]" + + formParts["order"] = "desc" + formParts["orderby"] = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "bookmarked" + else -> "popular" + } + + if (!filter.query.isNullOrEmpty()) { + formParts["query"] = filter.query!! + } + + val doc = httpPost("https://${domain}/wp-admin/admin-ajax.php?action=advanced_search", formParts) + return parseMangaList(doc) + } + + protected open suspend fun parseMangaList(doc: Document): List { + val slugs = doc.select("div > a[href*=/manga/]:has(> img)").mapNotNull { a -> + a.absUrl("href").toHttpUrlOrNull()?.pathSegments?.getOrNull(1) + }.filter { it.isNotEmpty() } + + if (slugs.isEmpty()) return emptyList() + + val url = "https://$domain/wp-json/wp/v2/manga".toHttpUrl().newBuilder().apply { + slugs.forEach { addQueryParameter("slug[]", it) } + addQueryParameter("per_page", "${slugs.size + 1}") + addQueryParameter("_embed", null) + }.build() + + val jsonArray = webClient.httpGet(url).parseJsonArray() + val mangaMap = mutableMapOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getJSONObject("title").getString("rendered") + val cover = obj.optJSONObject("_embedded") + ?.optJSONArray("wp:featuredmedia") + ?.optJSONObject(0) + ?.optString("source_url", null) + mangaMap[slug] = Manga( + id = generateUid("/manga/$slug"), + url = "/manga/$slug", + publicUrl = "https://$domain/manga/$slug/", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + return slugs.mapNotNull { mangaMap[it] } + } + + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val result = try { + coroutineScope { + val slug = manga.url.removePrefix("/manga/").removeSuffix("/") + val id = getMangaIdFromSlug(slug) ?: return@coroutineScope manga + + val chaptersDeferred = async { loadChapters(id.toString(), "https://$domain/manga/$slug/") } + + val url = "https://$domain/wp-json/wp/v2/manga/$id?_embed" + val obj = webClient.httpGet(url).parseJson() + val title = obj.getJSONObject("title").getString("rendered") + val description = obj.getJSONObject("content").getString("rendered") + .let { org.jsoup.Jsoup.parseBodyFragment(it).wholeText().trim() } + val cover = obj.optJSONObject("_embedded") + ?.optJSONArray("wp:featuredmedia") + ?.optJSONObject(0) + ?.optString("source_url", null) + + val embedded = obj.optJSONObject("_embedded") + val terms = embedded?.optJSONArray("wp:term") + + fun getTermNames(taxonomy: String): List { + if (terms == null) return emptyList() + for (i in 0 until terms.length()) { + val termArray = terms.optJSONArray(i) ?: continue + if (termArray.length() > 0 && termArray.getJSONObject(0).optString("taxonomy") == taxonomy) + return (0 until termArray.length()).map { j -> termArray.getJSONObject(j).getString("name") } + } + return emptyList() + } + + val genres = getTermNames("genre") + getTermNames("type") + val statusList = getTermNames("status") + val state = when { + statusList.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING + statusList.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED + statusList.any { it.equals("Cancelled", ignoreCase = true) } -> MangaState.ABANDONED + statusList.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED + else -> null + } + val authors = getTermNames("series-author") + val tags = genres.map { name -> MangaTag(name, name.lowercase().replace(" ", "-"), source) }.toSet() + val altTitles = fetchAltTitlesFromPage("https://$domain/manga/$slug/") + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + description = description, + coverUrl = cover ?: manga.coverUrl, + state = state, + authors = authors.toSet(), + tags = tags, + chapters = chapters, + altTitles = altTitles, + ) + } + } catch (_: Exception) { + manga + } + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private suspend fun getMangaIdFromSlug(slug: String): Int? { + val url = "https://$domain/wp-json/wp/v2/manga?slug=$slug" + val jsonArray = webClient.httpGet(url).parseJsonArray() + return if (jsonArray.length() > 0) jsonArray.getJSONObject(0).getInt("id") else null + } + + private suspend fun fetchAltTitlesFromPage(pageUrl: String): Set { + val doc = webClient.httpGet(pageUrl).parseHtml() + val titleElement = doc.selectFirst("h1[itemprop=name]") + val altText = titleElement?.nextElementSibling()?.text() + return altText?.split(',')?.mapNotNull { it.trim().takeIf(String::isNotBlank) }?.toSet() ?: emptySet() + } + + protected open suspend fun loadChapters(mangaId: String, mangaAbsoluteUrl: String): List { + val headers = Headers.headersOf( + "HX-Request", "true", + "HX-Target", "chapter-list", + "HX-Trigger", "chapter-list", + "HX-Current-URL", mangaAbsoluteUrl, + "Referer", mangaAbsoluteUrl, + ) + return buildList { + for (page in 1..50) { + val url = urlBuilder() + .addPathSegment("wp-admin") + .addPathSegment("admin-ajax.php") + .addQueryParameter("manga_id", mangaId) + .addQueryParameter("page", page.toString()) + .addQueryParameter("action", "chapter_list") + val doc = try { + webClient.httpGet(url.build(), headers).parseHtml() + } catch (e: HttpStatusException) { + if (e.statusCode == 520) break else throw e + } + val elements = doc.select("div#chapter-list > div[data-chapter-number]") + if (elements.isEmpty()) break + elements.mapNotNullTo(this) { el -> + val a = el.selectFirst("a") ?: return@mapNotNullTo null + val href = a.attrAsRelativeUrl("href").takeIf { it.isNotBlank() } ?: return@mapNotNullTo null + val name = el.selectFirst("div.font-medium span")?.text() ?: "" + val number = el.attr("data-chapter-number").toFloatOrNull() ?: -1f + val dateText = el.selectFirst("time")?.attr("datetime") + val uploadDate = dateFormat.parseSafe(dateText) + MangaChapter( + id = generateUid(href), + title = name, + url = href, + number = number, + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + } + }.sortedBy { it.number } + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US) + private fun SimpleDateFormat.parseSafe(date: String?): Long = + date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + + override suspend fun getPages(chapter: MangaChapter): List { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("main .relative section > img").map { img -> + val url = img.attrAsAbsoluteUrl("src").toRelativeUrl(domain) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + protected open suspend fun fetchAvailableTags(): Set { + val jsonArray = webClient.httpGet( + "https://${domain}/wp-json/wp/v2/genre?per_page=100&page=1&orderby=count&order=desc" + ).parseJsonArray() + val tags = mutableSetOf() + for (i in 0 until jsonArray.length()) { + val obj = jsonArray.getJSONObject(i) + val name = obj.optString("name").takeIf { it.isNotBlank() } ?: continue + val slug = obj.optString("slug").takeIf { it.isNotBlank() } ?: continue + tags += MangaTag(name, slug, source) + } + return tags + } + + protected open suspend fun httpPost(url: String, form: Map): Document { + val body = MultipartBody.Builder().setType(MultipartBody.FORM) + form.forEach { (k, v) -> body.addFormDataPart(k, v) } + val request = Request.Builder().url(url).post(body.build()) + .header("User-Agent", config[userAgentKey]) + .header("Referer", "https://${domain}/advanced-search/") + .header("Origin", "https://${domain}") + .build() + val response = context.httpClient.newCall(request).await() + return response.parseHtml() + } +} diff --git a/src/main/kotlin/tsuki/parsers/OriginesParser.kt b/src/main/kotlin/tsuki/parsers/OriginesParser.kt new file mode 100644 index 00000000..407bba74 --- /dev/null +++ b/src/main/kotlin/tsuki/parsers/OriginesParser.kt @@ -0,0 +1,374 @@ +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseFailed +import tsuki.util.removeSuffix +import tsuki.util.textOrNull +import tsuki.util.toTitleCase +import tsuki.util.mapNotNullToSet + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONObject +import org.jsoup.Jsoup +import java.time.LocalDate +import java.time.ZoneId +import java.util.EnumSet +import java.util.Locale + +abstract class OriginesParser( + context: MangaLoaderContext, + source: MangaParserSource, + domain: String, + pageSize: Int = 30, +) : PagedMangaParser(context, source, pageSize) { + + override val configKeyDomain = ConfigKey.Domain(domain) + + /** Path prefix for series */ + protected abstract val mangaPath: String + + /** Legacy paths used by older URLs. */ + protected open val legacyMangaPaths: Set = emptySet() + + /** Origins as label to slug (optional). */ + protected open val origins: List> = emptyList() + + /** Path used to scrape genres (relative to domain). */ + protected open val listUrl = "oeuvre/" + + /** Check madara. */ + protected open val tagPrefix = "manga-genres/" + + /** Hardcoded genres. */ + protected open val genres: List> = emptyList() + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val knownPaths: Set by lazy { legacyMangaPaths + mangaPath } + + private fun String.pathSegments(): List { + val path = if (startsWith("http")) { + toHttpUrlOrNull()?.encodedPath ?: this + } else this + return path.substringBefore('?') + .substringBefore('#') + .split('/') + .filter { it.isNotBlank() && it !in knownPaths } + } + + private fun String.toMangaSlug(): String = pathSegments().firstOrNull() ?: this + + private fun String.toChapterSlug(): String = pathSegments().take(2).joinToString("/") + + private suspend fun getCatalogue( + page: Int, + query: String = "", + genres: String = "", + status: String = "tous", + rating: String = "0", + origin: String = "", + sort: String = "recents", + chapterMin: String = "0", + chapterMax: String = "0", + ): List { + val form = mapOf( + "action" to "madara_child_catalogue", + "s" to query, + "genres" to genres, + "statut" to status, + "note" to rating, + "origine" to origin, + "tri" to sort, + "chmin" to chapterMin, + "chmax" to chapterMax, + "page" to page.toString(), + "auteur" to "", + "artiste" to "", + "annee" to "", + ) + + val response = webClient.httpPost("https://$domain/wp-admin/admin-ajax.php", form) + val json = response.parseJson() + val html = json.getJSONObject("data").optString("html") + if (html.isEmpty()) return emptyList() + + val doc = Jsoup.parseBodyFragment(html) + return doc.select("a.ori-card:has(span.ori-card-title)").map { element -> + val href = element.attr("href").toMangaSlug() + val title = element.selectFirst("span.ori-card-title")!!.text() + val cover = element.selectFirst("img")?.absUrl("src") + Manga( + id = generateUid(href), + url = href, + publicUrl = "https://$domain/$mangaPath/$href/", + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val sort = when (order) { + SortOrder.POPULARITY -> "populaire" + SortOrder.RATING -> "notes" + SortOrder.ALPHABETICAL -> "az" + else -> "recents" + } + + val genreTags = filter.tags.filter { !it.key.startsWith(ORIGIN_PREFIX) } + val originTags = filter.tags.filter { it.key.startsWith(ORIGIN_PREFIX) } + val genreParam = genreTags.joinToString(",") { it.key } + val originParam = originTags.joinToString(",") { it.key.removePrefix(ORIGIN_PREFIX) } + + val statusParam = filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> "en-cours" + MangaState.FINISHED -> "termine" + else -> "tous" + } + } ?: "tous" + + return getCatalogue( + page = page, + query = filter.query.orEmpty(), + genres = genreParam, + status = statusParam, + origin = originParam, + sort = sort, + ) + } + + protected open suspend fun fetchAvailableTags(): Set { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + val body = doc.body() + val root1 = body.selectFirst("header")?.selectFirst("ul.second-menu") + val root2 = body.selectFirst("div.genres_wrap")?.selectFirst("ul.list-unstyled") + if (root1 == null && root2 == null) { + doc.parseFailed("Root not found") + } + val list = root1?.select("li").orEmpty() + root2?.select("li").orEmpty() + val keySet = HashSet(list.size) + return list.mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val href = a.attr("href").removeSuffix('/').substringAfterLast(tagPrefix, "") + if (href.isEmpty() || !keySet.add(href)) { + return@mapNotNullToSet null + } + MangaTag( + key = href, + title = a.ownText().ifEmpty { + a.selectFirst(".menu-image-title")?.textOrNull() + }?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null, + source = source, + ) + } + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val allTags = mutableSetOf() + try { + allTags.addAll(fetchAvailableTags()) + } catch (_: Exception) { + } + genres.forEach { (title, slug) -> + allTags.add(MangaTag(key = slug, title = title, source = source)) + } + origins.forEach { (title, slug) -> + allTags.add(MangaTag(key = "$ORIGIN_PREFIX$slug", title = "Origine: $title", source = source)) + } + + return MangaListFilterOptions( + availableTags = allTags, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + ) + } + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val slug = manga.url.toMangaSlug() + val fullUrl = "https://$domain/$mangaPath/$slug/" + val doc = webClient.httpGet(fullUrl).parseHtml() + + val infos = doc.select("div.ori-sr-infos dt").associate { dt -> + dt.text().lowercase(Locale.FRENCH) to dt.nextElementSibling()?.text().orEmpty() + } + + val title = doc.selectFirst("h1.ori-sr-title")?.text() ?: manga.title + val cover = doc.selectFirst("div.ori-sr-cover img")?.absUrl("src") ?: manga.coverUrl.orEmpty() + + val description = buildString { + doc.select("div.ori-sr-syn-texte p").eachText().forEach { appendLine(it) } + infos["nom alternatif"]?.takeIf { it.isNotBlank() }?.let { + appendLine() + append("Nom alternatif: ", it) + } + }.trim() + + val authors = listOfNotNull( + infos["auteur"]?.takeIf { it.isNotBlank() }, + infos["scénario"]?.takeIf { it.isNotBlank() }, + infos["artiste"]?.takeIf { it.isNotBlank() }, + infos["dessin"]?.takeIf { it.isNotBlank() } + ).distinct().toSet() + + val tags = doc.select("div.ori-sr-genres a.ori-sr-genre").asSequence().map { it.text() } + .plus(infos["type"].orEmpty()) + .filter { it.isNotBlank() } + .map { MangaTag(it, it, source) } + .toSet() + + val state = when (infos["statut"]?.lowercase(Locale.FRENCH)) { + "en cours" -> MangaState.ONGOING + "terminé" -> MangaState.FINISHED + "en pause" -> MangaState.PAUSED + "abandonné", "annulé" -> MangaState.ABANDONED + else -> null + } + + val chaptersDeferred = async { fetchChapters(slug) } + + manga.copy( + url = slug, + publicUrl = fullUrl, + title = title, + altTitles = setOfNotNull(infos["nom alternatif"]?.takeIf { it.isNotBlank() }), + rating = doc.selectFirst("span.total_votes")?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + coverUrl = cover, + description = description, + authors = authors, + tags = tags, + state = state, + chapters = chaptersDeferred.await(), + ) + } + + private suspend fun fetchChapters(slug: String): List { + val doc = webClient.httpPost( + "https://$domain/$mangaPath/$slug/ajax/chapters/", + emptyMap() + ).parseHtml() + return doc.select("div.ori-chl-row").map { element -> + val link = element.selectFirst("a.ori-chl-corps")!! + val href = link.attr("href").toChapterSlug() + val name = element.selectFirst("span.ori-chl-nom")?.text() ?: link.text() + val dateText = element.selectFirst("span.ori-chl-date")?.attr("title") + ?: element.selectFirst("span.ori-chl-date")?.text() + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = parseChapterDate(dateText), + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List { + val fullUrl = "https://$domain/$mangaPath/${chapter.url}/" + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("div.reading-content img.wp-manga-chapter-img").mapIndexed { _, img -> + val url = when { + img.hasAttr("data-src") -> img.absUrl("data-src").trim() + else -> img.absUrl("src").trim() + } + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val regex = Regex("""(\d{1,2})\s+(\p{L}+)\.?(?:\s+(\d{4}))?""") + val match = regex.find(date) ?: return 0L + val (day, month, year) = match.destructured + val monthNumber = monthNumber(month) ?: return 0L + val today = LocalDate.now(ZoneId.of("Europe/Paris")) + return runCatching { + var chapterDate = LocalDate.of( + year.toIntOrNull() ?: today.year, + monthNumber, + day.toInt() + ) + if (year.isEmpty() && chapterDate.isAfter(today)) { + chapterDate = chapterDate.minusYears(1) + } + chapterDate.atStartOfDay(ZoneId.of("Europe/Paris")).toInstant().toEpochMilli() + }.getOrDefault(0L) + } + + private fun monthNumber(month: String): Int? { + val name = month.lowercase(Locale.FRENCH) + return when { + name.startsWith("jan") -> 1 + name.startsWith("fev") || name.startsWith("fév") -> 2 + name.startsWith("mar") -> 3 + name.startsWith("avr") -> 4 + name.startsWith("mai") -> 5 + name.startsWith("juin") -> 6 + name.startsWith("juil") -> 7 + name.startsWith("ao") -> 8 + name.startsWith("sep") -> 9 + name.startsWith("oct") -> 10 + name.startsWith("nov") -> 11 + name.startsWith("dec") || name.startsWith("déc") -> 12 + else -> null + } + } + + private fun okhttp3.Response.parseJson(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + companion object { + private const val ORIGIN_PREFIX = "origin:" + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt similarity index 87% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt rename to src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt index bbd41656..a9058f70 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/parsers/ZeistMangaParser.kt +++ b/src/main/kotlin/tsuki/parsers/ZeistMangaParser.kt @@ -1,4 +1,42 @@ -package org.koitharu.kotatsu.parsers.parsers +package tsuki.parsers + +import tsuki.MangaLoaderContext +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.ErrorMessages + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlDecode +import tsuki.util.extractChapterNumber import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -6,17 +44,8 @@ import org.json.JSONArray import org.json.JSONObject import org.jsoup.Jsoup import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet internal abstract class ZeistMangaParser( context: MangaLoaderContext, @@ -91,14 +120,13 @@ internal abstract class ZeistMangaParser( override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query val url = buildString { append("https://") append(domain) append("/feeds/posts/default/-/") when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append(mangaCategory) append("?alt=json&orderby=published&max-results=") append((maxMangaResults + 1).toString()) @@ -107,7 +135,7 @@ internal abstract class ZeistMangaParser( append("&q=label:") append(mangaCategory) append("+") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } else -> { @@ -281,7 +309,7 @@ internal abstract class ZeistMangaParser( webClient.httpGet(url).parseJson().getJSONObject("feed").getJSONArray("entry").asTypedList() .reversed() val dateFormat = SimpleDateFormat(datePattern, sourceLocale) - return json.mapIndexedNotNull { i, j -> + return json.mapNotNull { j -> val name = j.getJSONObject("title").getString("\$t") val href = j.getJSONArray("link").asTypedList().first { it.getString("rel") == "alternate" } @@ -290,20 +318,20 @@ internal abstract class ZeistMangaParser( val slug = mangaUrl.substringAfterLast('/') val slugChapter = href.substringAfterLast('/') if (slug == slugChapter) { - return@mapIndexedNotNull null + return@mapNotNull null } MangaChapter( id = generateUid(href), url = href, title = name, - number = i + 1f, + number = name.extractChapterNumber(), volume = 0, branch = null, uploadDate = dateFormat.parseSafe(dateText), scanlator = null, source = source, ) - } + }.sortedBy { it.number } } protected open val selectPage = @@ -353,4 +381,4 @@ internal abstract class ZeistMangaParser( } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/all/Kagane.kt b/src/main/kotlin/tsuki/site/all/Kagane.kt new file mode 100644 index 00000000..9d11e5f6 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Kagane.kt @@ -0,0 +1,961 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.CloudFlareHelper + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.urlBuilder + +import okhttp3.Interceptor +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import kotlinx.coroutines.delay +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.net.URI +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import kotlin.time.Duration.Companion.milliseconds +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock + +/** + * Created from: https://github.com/glitch-228 + * Modified from: InvalidDavid + */ + +@MangaSourceParser("KAGANE", "Kagane") +internal class Kagane(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KAGANE, pageSize = 35) { + + override val configKeyDomain = ConfigKey.Domain("kagane.to") + private val apiUrl = "https://$domain" + + private val dataSaverKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "false" to "Normal", + "true" to "Data saving" + ), + defaultValue = "false", + ) + + private val isDataSaver: Boolean + get() = config[dataSaverKey] == "true" + + override fun onCreateConfig(keys: MutableCollection>) { + super.onCreateConfig(keys) + keys.add(dataSaverKey) + } + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + private fun Locale.toKaganeLangCode(): String { + if (language == "pt" && country.equals("br", ignoreCase = true)) return "pt-BR" + if (language == "es" && country == "419") return "es-419" + if (language == "zh" && country.equals("CN", ignoreCase = true)) return "zh-Hans" + if (language == "zh" && country.equals("TW", ignoreCase = true)) return "zh-Hant" + return language + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = genresMutex.withLock { + genresCache ?: fetchGenres().also { genresCache = it } + } + return MangaListFilterOptions( + availableTags = genres, + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS, + ContentType.OTHER, + ), + availableLocales = KAGANE_LANGS, + ) + } + + @Volatile + private var genresCache: Set? = null + private val genresMutex = Mutex() + + private fun fetchGenres(): Set = GENRES + .filter { UUID_REGEX.matches(it.first) } + .map { MangaTag(it.second, it.first, source) } + .toSet() + + private fun parseContentRating(value: String?): ContentRating? { + return when (value?.lowercase(Locale.ROOT)) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "adult", "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val sortParam = when (order) { + SortOrder.RELEVANCE -> "" + SortOrder.POPULARITY -> "total_views,desc" + SortOrder.POPULARITY_ASC -> "total_views,asc" + SortOrder.RATING -> "avg_views,desc" + SortOrder.RATING_ASC -> "avg_views,asc" + SortOrder.UPDATED -> "updated_at,desc" + SortOrder.UPDATED_ASC -> "updated_at,asc" + SortOrder.NEWEST -> "created_at,desc" + SortOrder.NEWEST_ASC -> "created_at,asc" + SortOrder.ALPHABETICAL -> "series_name,asc" + SortOrder.ALPHABETICAL_DESC -> "series_name,desc" + else -> "updated_at,desc" + } + + val urlBuilder = "$apiUrl/api/v2/search/series".toHttpUrl().newBuilder() + .addQueryParameter("page", (page - 1).toString()) + .addQueryParameter("size", pageSize.toString()) + if (sortParam.isNotBlank()) { + urlBuilder.addQueryParameter("sort", sortParam) + } + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("exact", "1") + } + val url = urlBuilder.build().toString() + val jsonBody = JSONObject() + if (!filter.query.isNullOrEmpty()) { + jsonBody.put("title", filter.query) + } + jsonBody.put("source_type", JSONArray().apply { + put("Official") + put("Unofficial") + put("Mixed") + }) + jsonBody.put("content_lang", JSONArray().apply { + val selectedLocale = filter.locale + if (selectedLocale != null) { + put(selectedLocale.toKaganeLangCode()) + } else { + KAGANE_LANGS.forEach(::put) + } + }) + + val genreIds = filter.tags.map { it.key }.filter { UUID_REGEX.matches(it) } + if (genreIds.isNotEmpty()) { + val genresArr = JSONArray() + genreIds.forEach { genresArr.put(it) } + val genresObj = JSONObject() + genresObj.put("values", genresArr) + genresObj.put("match_all", false) + jsonBody.put("genres", genresObj) + } + if (filter.tagsExclude.isNotEmpty()) { + val excludedGenreIds = filter.tagsExclude.map { it.key }.filter { UUID_REGEX.matches(it) } + if (excludedGenreIds.isNotEmpty()) { + val genresObj = jsonBody.optJSONObject("genres") ?: JSONObject().also { + jsonBody.put("genres", it) + } + if (!genresObj.has("values")) { + genresObj.put("values", JSONArray()) + } + genresObj.put("exclude", JSONArray().apply { + excludedGenreIds.forEach(::put) + }) + } + } + jsonBody.put("content_rating", JSONArray().apply { + val ratings = filter.contentRating.ifEmpty { + EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + } + if (ContentRating.SAFE in ratings) put("Safe") + if (ContentRating.SUGGESTIVE in ratings) put("Suggestive") + if (ContentRating.ADULT in ratings) { + put("Erotica") + put("Pornographic") + } + }) + + if (filter.states.isNotEmpty()) { + val statuses = JSONArray() + filter.states.forEach { state -> + when (state) { + MangaState.ONGOING -> statuses.put("Ongoing") + MangaState.FINISHED -> statuses.put("Completed") + MangaState.PAUSED -> statuses.put("Hiatus") + MangaState.ABANDONED -> statuses.put("Abandoned") + else -> Unit + } + } + if (statuses.length() > 0) { + jsonBody.put("upload_status", statuses) + } + } + + if (filter.types.isNotEmpty()) { + val formats = JSONArray() + filter.types.forEach { type -> + when (type) { + ContentType.MANGA -> formats.put("Manga") + ContentType.MANHWA -> formats.put("Manhwa") + ContentType.MANHUA -> formats.put("Manhua") + ContentType.COMICS -> formats.put("Comic") + ContentType.OTHER -> formats.put("Other") + else -> Unit + } + } + if (formats.length() > 0) { + jsonBody.put("format", formats) + } + } + + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val responseBody = executeWithCloudflareCheck( + url = url, + block = { webClient.httpPost(url.toHttpUrl(), jsonBody, headers) }, + parse = { it.parseRaw() }, + ) + + if (responseBody.isCloudflareChallenge()) { + requestCloudflareVerification(url) + } + + val response = try { + JSONObject(responseBody) + } catch (_: Exception) { + throw Exception("Invalid JSON search response: $responseBody") + } + + val content = response.optJSONArray("content") + ?: response.optJSONObject("result")?.optJSONArray("items") + ?: return emptyList() + + return (0 until content.length()).mapNotNull { i -> + val item = content.getJSONObject(i) + val id = item.optString("id").ifBlank { item.optString("series_id") } + if (id.isBlank()) return@mapNotNull null + val name = item.optString("name").ifBlank { item.optString("title") }.ifBlank { return@mapNotNull null } + val src = item.optString("source").ifBlank { item.optString("source_name") } + val title = if (src.isNotEmpty()) "$name [$src]" else name + val coverImageId = item.optString("cover_image_id").ifBlank { item.optString("coverImageId") } + val coverUrl = if (coverImageId.isNotBlank()) { + "$apiUrl/api/v2/image/$coverImageId" + } else { + "$apiUrl/api/v2/series/$id/thumbnail" + } + + Manga( + id = generateUid(id), + url = id, + publicUrl = "https://$domain/series/$id", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = parseContentRating(item.optString("content_rating")), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val seriesId = manga.url + val url = "$apiUrl/api/v2/series/$seriesId" + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val resp = executeWithCloudflareCheck( + url = url, + block = { webClient.httpGet(url, headers) }, + parse = { it }, + ) + val respBody = resp.body?.string() ?: "" + if (!resp.isSuccessful) throw Exception("Details error ${resp.code}: $respBody") + val json = try { + JSONObject(respBody) + } catch (_: Exception) { + throw Exception("Invalid JSON details: $respBody") + } + + val state = when ( + json.optString("publication_status") + .ifBlank { json.optString("upload_status") } + .ifBlank { json.optString("status") } + .uppercase(Locale.ROOT) + ) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED", "ENDED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + "ABANDONED", "CANCELLED", "CANCELED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val genres = json.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).mapNotNull { i -> + when (val item = arr.opt(i)) { + is String -> { + if (UUID_REGEX.matches(item)) { + MangaTag(item, item, source) + } else { + null + } + } + is JSONObject -> { + val key = item.optGenreId() + val name = item.optGenreName() + if (key.isNotBlank() && name.isNotBlank()) { + MangaTag(name, key, source) + } else { + null + } + } + else -> null + } + }.toSet() + } ?: emptySet() + + val authors = linkedSetOf() + json.optJSONArray("authors")?.let { arr -> + for (i in 0 until arr.length()) { + when (val item = arr.opt(i)) { + is String -> item.takeIf { it.isNotBlank() }?.let(authors::add) + is JSONObject -> item.optString("name") + .ifBlank { item.optString("title") } + .takeIf { it.isNotBlank() } + ?.let(authors::add) + } + } + } + if (authors.isEmpty()) { + json.optJSONArray("series_staff")?.let { arr -> + for (i in 0 until arr.length()) { + val staff = arr.optJSONObject(i) ?: continue + val role = staff.optString("role") + if ( + role.contains("author", ignoreCase = true) || + role.contains("story", ignoreCase = true) || + role.contains("artist", ignoreCase = true) || + role.contains("art", ignoreCase = true) + ) { + staff.optString("name").takeIf { it.isNotBlank() }?.let(authors::add) + } + } + } + } + + val altTitles = json.optJSONArray("series_alternate_titles")?.let { arr -> + buildSet { + for (i in 0 until arr.length()) { + val item = arr.optJSONObject(i) ?: continue + item.optString("title").takeIf { it.isNotBlank() }?.let(::add) + } + } + } ?: emptySet() + + val description = buildString { + json.optString("description") + .ifBlank { json.optString("summary") } + .takeIf { it.isNotBlank() } + ?.let { + append(it.trim()) + } + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Associated Name(s):\n") + altTitles.forEach { + append(it) + append('\n') + } + } + }.trim().ifBlank { null } + + val coverUrl = json.optJSONArray("series_covers") + ?.optJSONObject(0) + ?.optString("image_id") + ?.takeIf { it.isNotBlank() } + ?.let { "$apiUrl/api/v2/image/$it" } + ?: manga.coverUrl + + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH) + fun parseChapters(content: JSONArray): List { + val chapters = ArrayList(content.length()) + for (i in 0 until content.length()) { + val ch = content.optJSONObject(i) ?: continue + val chId = ch.optString("book_id") + .ifBlank { ch.optString("id") } + .ifBlank { ch.optString("bookId") } + if (chId.isBlank()) continue + val chapterNo = ch.optString("chapter_no") + val chapterNumber = chapterNo.toChapterNumberOrNull() + ?: ch.optDouble("number", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + val sortNumber = ch.optDouble("sort_no", Double.NaN).takeIf { !it.isNaN() }?.toFloat() + ?: ch.optDouble("number_sort", ch.optDouble("numberSort", Double.NaN)).takeIf { !it.isNaN() }?.toFloat() + val number = when { + sortNumber != null && chapterNumber != null && sortNumber >= chapterNumber -> sortNumber + sortNumber != null && chapterNumber == null -> sortNumber + chapterNumber != null -> chapterNumber + else -> 0f + } + val rawTitle = ch.optString("title").ifBlank { ch.optString("name") }.trim() + val chTitle = rawTitle.ifBlank { + chapterNo.takeIf { it.isNotBlank() }?.let { "Ch.$it" }.orEmpty() + }.ifBlank { "Chapter $number" } + val volume = ch.optString("volume_no") + .ifBlank { ch.optString("volume") } + .toIntOrNull() ?: 0 + val dateStr = ch.optString("published_on") + .ifBlank { ch.optString("release_date") } + .ifBlank { ch.optString("releaseDate") } + .ifBlank { ch.optString("created_at") } + val groups = ch.optJSONArray("groups") + chapters.add( + MangaChapter( + id = generateUid("$seriesId:$chId"), + title = chTitle, + number = number, + volume = volume, + url = "/series/$seriesId/reader/$chId", + uploadDate = dateFormat.parseSafe(dateStr), + source = source, + scanlator = groups?.let { arr -> + buildList { + for (j in 0 until arr.length()) { + arr.optJSONObject(j)?.optString("title")?.takeIf { it.isNotBlank() }?.let(::add) + } + }.joinToString().ifBlank { null } + }, + branch = null, + ), + ) + } + return chapters.sortedWith( + compareBy { it.number <= 0f } + .thenBy { it.number } + .thenBy { it.volume } + .thenBy { it.title.orEmpty() }, + ) + } + + var chapters = parseChapters( + json.optJSONArray("series_books") + ?: json.optJSONArray("seriesBooks") + ?: json.optJSONArray("books") + ?: json.optJSONArray("content") + ?: JSONArray(), + ) + if (chapters.isEmpty()) { + val chaptersUrl = "$apiUrl/api/v2/series/$seriesId/books/list" + val chapterResp = webClient.httpGet(chaptersUrl, headers).parseJson() + chapters = parseChapters( + chapterResp.optJSONArray("series_books") + ?: chapterResp.optJSONArray("seriesBooks") + ?: chapterResp.optJSONArray("content") + ?: JSONArray(), + ) + } + + return manga.copy( + title = json.optString("title").ifBlank { manga.title }, + altTitles = altTitles, + coverUrl = coverUrl, + description = description, + state = state, + authors = authors, + tags = genres, + chapters = chapters, + contentRating = parseContentRating(json.optString("content_rating")) ?: manga.contentRating, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List { + return emptyList() + } + + private var cacheUrl = "https://kstatic.to" + private var accessToken: String = "" + + override suspend fun getPages(chapter: MangaChapter): List { + val uri = URI(chapter.url) + val pathParts = uri.path.split("/").filter { it.isNotEmpty() } + if (pathParts.size < 4) throw Exception("Invalid chapter URL format: ${chapter.url}") + + val chapterId = pathParts.last() + val challenge = getChallengeResponse(chapterId) + accessToken = challenge.optString("access_token").ifBlank { + challenge.optString("accessToken") + }.ifBlank { + throw Exception("Invalid challenge response: missing access token") + } + cacheUrl = challenge.optString("cache_url").ifBlank { + challenge.optString("cacheUrl") + }.ifBlank { + throw Exception("Invalid challenge response: missing cache url") + } + + val pages = parseManifestPages(challenge) + if (pages.isEmpty()) { + throw Exception("Invalid challenge response: missing pages manifest") + } + + return pages.sortedBy { it.pageNumber }.map { page -> + val ext = page.ext?.takeIf { it.isNotBlank() } ?: "jxl" + val imageUrl = "$cacheUrl/api/v2/books/page".toHttpUrl().newBuilder().apply { + if (isDataSaver) addPathSegment("datasaver") + addPathSegment(chapterId) + addPathSegment("${page.pageUuid}.$ext") + addQueryParameter("token", accessToken) + addQueryParameter("is_datasaver", isDataSaver.toString()) + }.build().toString() + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun executeWithCloudflareCheck( + url: String, + block: suspend () -> Response, + parse: (Response) -> T, + ): T { + CloudFlareHelper.getClearanceCookie(context.cookieJar, "https://$domain") + val response = block() + return when (CloudFlareHelper.checkResponseForProtection(response)) { + CloudFlareHelper.PROTECTION_NOT_DETECTED -> parse(response) + CloudFlareHelper.PROTECTION_CAPTCHA -> { + response.close() + delay(CLOUDFLARE_RETRY_DELAY_MS.milliseconds) + val retryResponse = block() + when (CloudFlareHelper.checkResponseForProtection(retryResponse)) { + CloudFlareHelper.PROTECTION_NOT_DETECTED -> parse(retryResponse) + else -> { + retryResponse.close() + requestCloudflareVerification(url) + } + } + } + CloudFlareHelper.PROTECTION_BLOCKED -> { + response.close() + requestCloudflareVerification(url) + } + else -> { + response.close() + requestCloudflareVerification(url) + } + } + } + + private fun requestCloudflareVerification(url: String, cause: Throwable? = null): Nothing { + try { + context.requestBrowserAction(this, "https://$domain/") + } catch (e: UnsupportedOperationException) { + throw ParseException( + "Cloudflare verification required. Open Kagane in WebView and retry.", + url, + cause ?: e, + ) + } + } + + private fun String.isCloudflareChallenge(): Boolean { + return contains("cf-mitigated", ignoreCase = true) + || contains("Just a moment", ignoreCase = true) + || contains("challenges.cloudflare.com", ignoreCase = true) + || contains("/cdn-cgi/challenge-platform/", ignoreCase = true) + || contains("Sorry, you have been blocked", ignoreCase = true) + || contains("cf-error-details", ignoreCase = true) + || contains("cf-chl-bypass", ignoreCase = true) + } + + private data class ManifestPage( + val pageNumber: Int, + val pageUuid: String, + val ext: String?, + ) + + private fun parseManifestPages(challenge: JSONObject): List { + val pagesJson = challenge.optJSONObject("manifest")?.optJSONArray("pages") + ?: challenge.optJSONArray("pages") + ?: JSONArray() + return buildList { + for (i in 0 until pagesJson.length()) { + val page = pagesJson.optJSONObject(i) ?: continue + val pageUuid = page.optString("page_id") + .ifBlank { page.optString("pageId") } + .ifBlank { page.optString("page_uuid") } + .ifBlank { page.optString("pageUuid") } + if (pageUuid.isBlank()) continue + add( + ManifestPage( + pageNumber = page.optInt( + "page_no", + page.optInt("pageNo", page.optInt("page_number", i + 1)), + ), + pageUuid = pageUuid, + ext = page.optString("ext").ifBlank { null }, + ), + ) + } + } + } + + private var integrityToken: String = "" + private var integrityTokenExp: Long = 0L + + private val isIntegrityTokenValid: Boolean + get() = integrityToken.isNotBlank() && System.currentTimeMillis() < integrityTokenExp + + private suspend fun getIntegrityToken(): String { + if (isIntegrityTokenValid) { + return integrityToken + } + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + + val response = executeWithCloudflareCheck( + url = "$apiUrl/api/integrity", + block = { + webClient.httpPost( + urlBuilder().addPathSegments("api/integrity").build(), + JSONObject(), + headers, + ) + }, + parse = { it.parseJson() }, + ) + + val token = response.optString("token") + if (token.isBlank()) { + throw Exception("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = response.optLong("exp", 0L) * 1000L + return integrityToken + } + + private suspend fun getChallengeResponse(chapterId: String): JSONObject { + val integrityToken = getIntegrityToken() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver" + return executeWithCloudflareCheck( + url = challengeUrl, + block = { webClient.httpPost(challengeUrl.toHttpUrl(), JSONObject(), headers) }, + parse = { it.parseJson() }, + ) + } + + private fun String.toChapterNumberOrNull(): Float? = trim() + .replace(',', '.') + .toFloatOrNull() + + private fun getIntegrityTokenBlocking(): String { + if (isIntegrityTokenValid) { + return integrityToken + } + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .build() + val request = Request.Builder() + .url(urlBuilder().addPathSegments("api/integrity").build()) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Integrity token request failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + val token = json.optString("token") + if (token.isBlank()) { + throw IOException("Failed to retrieve integrity token") + } + integrityToken = token + integrityTokenExp = json.optLong("exp", 0L) * 1000L + } + return integrityToken + } + + private fun refreshAccessTokenBlocking(chapterId: String) { + val integrityToken = getIntegrityTokenBlocking() + val headers = getRequestHeaders().newBuilder() + .add("Origin", "https://$domain") + .add("Referer", "https://$domain/") + .add("x-integrity-token", integrityToken) + .build() + val challengeUrl = "$apiUrl/api/v2/books/$chapterId?is_datasaver=$isDataSaver".toHttpUrl() + val request = Request.Builder() + .url(challengeUrl) + .post(JSONObject().toString().toRequestBody("application/json".toMediaType())) + .headers(headers) + .build() + context.httpClient.newCall(request).execute().use { response -> + if (!response.isSuccessful) { + throw IOException("Challenge refresh failed ${response.code}") + } + val json = JSONObject(response.body?.string() ?: "") + accessToken = json.optString("access_token").ifBlank { + json.optString("accessToken") + }.ifBlank { + throw IOException("Invalid challenge response: missing access token") + } + cacheUrl = json.optString("cache_url").ifBlank { + json.optString("cacheUrl") + }.ifBlank { + throw IOException("Invalid challenge response: missing cache url") + } + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + val host = url.host + var requestBuilder = request.newBuilder() + .removeHeader("Content-Encoding") + .removeHeader("cf-connecting-ip") + if (host == domain || host.endsWith(".$domain")) { + requestBuilder = requestBuilder + .header("Origin", "https://$domain") + .header("Referer", "https://$domain/") + } + val newRequest = requestBuilder.build() + + var response = chain.proceed(newRequest) + + if (url.queryParameterNames.contains("token") && + (response.code == 401 || response.code == 403 || response.code == 507) + ) { + response.close() + val segments = url.pathSegments + val chapterId = segments.getOrNull(4) + if (chapterId != null) { + runCatching { refreshAccessTokenBlocking(chapterId) }.onSuccess { + val retryRequest = newRequest.newBuilder() + .url(url.newBuilder().setQueryParameter("token", accessToken).build()) + .build() + response = chain.proceed(retryRequest) + } + } + } + return response + } + + companion object { + private const val CLOUDFLARE_RETRY_DELAY_MS = 6_000L + + private val UUID_REGEX = Regex( + "^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$", + ) + + private val KAGANE_LANGS: Set = setOf( + Locale("af"), + Locale("ar"), + Locale("az"), + Locale("be"), + Locale("bg"), + Locale("bn"), + Locale("ca"), + Locale("cs"), + Locale("cv"), + Locale("da"), + Locale.GERMAN, + Locale("el"), + Locale.ENGLISH, + Locale("eo"), + Locale("es"), + Locale("es","419"), + Locale("et"), + Locale("eu"), + Locale("fa"), + Locale("fi"), + Locale("fil"), + Locale.FRENCH, + Locale("ga"), + Locale("he"), + Locale("hi"), + Locale("hr"), + Locale("hu"), + Locale("id"), + Locale.ITALIAN, + Locale.JAPANESE, + Locale("jv"), + Locale("ka"), + Locale("kk"), + Locale.KOREAN, + Locale("la"), + Locale("lt"), + Locale("mn"), + Locale("ms"), + Locale("my"), + Locale("ne"), + Locale("nl"), + Locale("no"), + Locale("pl"), + Locale("pt"), + Locale("pt", "br"), + Locale("ro"), + Locale("ru"), + Locale("sk"), + Locale("sl"), + Locale("sq"), + Locale("sr"), + Locale("sv"), + Locale("ta"), + Locale("te"), + Locale("th"), + Locale("tr"), + Locale("uk"), + Locale("ur"), + Locale("uz"), + Locale("vi"), + Locale.SIMPLIFIED_CHINESE, + Locale.TRADITIONAL_CHINESE, + ) + + private fun JSONObject.optGenreId(): String = optString("genre_id").ifBlank { optString("id") } + + private fun JSONObject.optGenreName(): String = + optString("genre_name") + .ifBlank { optString("genreName") } + .ifBlank { optString("name") } + .ifBlank { optString("title") } + + private val GENRES = listOf( + "019bef86-56a6-71a1-af75-c1e43f48fba4" to "4-koma", + "019c3a02-5288-777a-b5d8-1bf9f79163ae" to "AI", + "019c1fe4-fabf-7242-a0a8-7866f98f705e" to "Action", + "019c1b6f-b682-7fee-9a9b-86e648b77e9e" to "Adventure", + "019c1fea-1b14-72d5-b2f8-c3194c4e1efe" to "Award Winning", + "019c1fe5-51fe-7f26-aee3-f057ac50f81d" to "Comedy", + "019c3a02-b00d-731d-884c-41460acc81e4" to "Coming of Age", + "019c2043-f726-7f78-9c58-7caf4662bf1f" to "Cooking", + "019c2043-d238-744b-8d5a-21bbc5166bd7" to "Crime", + "019c1fe8-cad1-7842-82a7-db5a87fc0a5a" to "Demons", + "019c1fea-2ddc-7e74-9935-a8b1b7782dd2" to "Doujinshi", + "019c1b6f-3d3b-7528-a3e2-a791cfdcc4f7" to "Drama", + "019c200d-6238-7364-9422-c50bd75e4e0d" to "Ecchi", + "019c1fe9-da2f-7938-a795-2b862b221cb3" to "Fan Colored", + "019c1b6f-51ed-7991-a796-a8ca26e86666" to "Fantasy", + "019c1fe9-9527-7f47-bf28-36ab2d6d4acd" to "Full Color", + "019c1fe7-5189-7619-9094-6e4b85c139ec" to "Gender Bender", + "019c1fe6-6717-77fa-8205-85d8e8a8755c" to "Gore", + "019c1fe8-74eb-73e6-840e-7916f145012e" to "Harem", + "019c3a01-d241-7d44-836d-095b98d4dc1e" to "Hentai", + "019c1fe6-7ea3-7741-9191-5ba94f6e9552" to "Historical", + "019c1b6f-c2e3-7909-93cf-0a48a2f2278f" to "Horror", + "019c1fe6-3ba5-7e76-aa0d-8756c314e2fa" to "Isekai", + "019c1b70-101d-7c9c-a28c-cc2439104443" to "Josei", + "019c1feb-8388-7c89-8c2d-ddfb530065e6" to "LGBTQIA+", + "019c1fe8-2221-7d69-9cd2-b67d64924994" to "Magic", + "019c1fe7-13fa-7019-a5a6-e7bbceb200f2" to "Magical Girls", + "019c1fe8-4c93-7f12-a6f0-e805fe4ade21" to "Martial Arts", + "019c2044-1fd9-7e84-b9c9-cbf20945a664" to "Mecha", + "019c1fe7-2c1e-730c-af33-ca3bef3a161d" to "Medical", + "019c1fe8-e822-7938-a04c-dee0a24f022c" to "Military", + "019c1fe8-3261-7c3d-9103-428f9a54eb41" to "Monsters", + "019c200f-1825-738c-b7c1-67266b9520b1" to "Music", + "019c1b6f-7991-7edb-8c7f-df6876f09a94" to "Mystery", + "019c1fe8-a048-7453-a06d-212678bf777a" to "Office Workers", + "019c1fe9-f0a1-71c6-bbe0-6afc3822647a" to "Official Colored", + "019c22cd-bb85-7541-9db6-7e6ea3394d0b" to "Omegaverse", + "019c1fe9-ae8d-7294-bcc8-9bd4269b94d7" to "Oneshot", + "019c1fe7-380c-72a5-8668-930c5e90ec44" to "Philosophical", + "019c1fe9-4fe3-776a-b11c-a14d76a52e7d" to "Police", + "019c1fe9-0e7c-7782-8265-c29ee6c19e79" to "Post-Apocalyptic", + "019c1ee5-dfa9-7b69-86a8-16831c1d81fe" to "Psychological", + "019c1fe8-5a6d-73bd-835d-16ae059bce6e" to "Reincarnation", + "019c1fe8-8837-78de-a911-f54754c2e713" to "Reverse Harem", + "019beb9a-a36a-718c-b551-d917a0818cc7" to "Romance", + "019c1fe7-db44-77a1-acc2-28c2bbb49ae1" to "School Life", + "019c1fe5-b704-72a6-a676-a8ad87ebcdd8" to "Sci-Fi", + "019c1fea-983f-7279-944e-8fb6e2cccdbc" to "Seinen", + "019c1b6f-fe1c-7eed-b93b-e1bf6dff63fd" to "Shoujo", + "019c1ee5-626f-7284-b5e8-8f3df5c5f76b" to "Shoujo Ai", + "019c1fea-85c7-76a7-996a-b41ce0cb05f8" to "Shounen", + "019c1ee5-4be9-766d-b545-5b87d7cdff96" to "Shounen Ai", + "019c1fe5-84d9-7a67-9240-1f303dc231bc" to "Slice of Life", + "019c2042-2a6f-7bcb-9c29-6098b8426aee" to "Smut", + "019c1fe6-bc40-7aa4-bac6-29a6cc29b559" to "Sports", + "019c1b6f-9c7b-77a0-8b54-e62925422b91" to "Supernatural", + "019c1fe8-b2ab-7e15-a55b-9fb28901cc9d" to "Survival", + "019c1fe6-0228-7052-98a2-fe6df7994d93" to "Thriller", + "019c1fe8-f932-7a19-a502-4ad6a2adfae4" to "Time Travel", + "019c1fe6-211d-7e73-927d-3c54b66a9055" to "Tragedy", + "019c1fe9-78cc-7d2e-be56-cc3f8442ccf0" to "Vampires", + "019c1fe9-21a7-7bbd-9763-10a5855a8e5c" to "Video Games", + "019c1fe9-3609-73d2-b4bf-8420cc15c5b2" to "Villainess", + "019c1fe7-0344-755b-b4df-c019f4cdbc21" to "Wuxia", + "019c1ee5-38f9-7324-9f7d-07a01dbf88a6" to "Yaoi", + "019c1ee5-7e79-7fd2-8fcc-de739858c851" to "Yuri", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt similarity index 85% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt rename to src/main/kotlin/tsuki/site/all/MangaDexParser.kt index fa3d1fc6..12c32b75 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/MangaDexParser.kt +++ b/src/main/kotlin/tsuki/site/all/MangaDexParser.kt @@ -1,6 +1,61 @@ @file:Suppress("DEPRECATION") -package org.koitharu.kotatsu.parsers.site.kotatsu.all +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.FlexibleMangaParser +import tsuki.exception.ParseException +import tsuki.network.UserAgents + +import tsuki.model.ContentRating +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria.Exclude +import tsuki.model.search.QueryCriteria.Include +import tsuki.model.search.QueryCriteria.Match +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.model.search.SearchableField.AUTHOR +import tsuki.model.search.SearchableField.CONTENT_RATING +import tsuki.model.search.SearchableField.CONTENT_TYPE +import tsuki.model.search.SearchableField.DEMOGRAPHIC +import tsuki.model.search.SearchableField.LANGUAGE +import tsuki.model.search.SearchableField.ORIGINAL_LANGUAGE +import tsuki.model.search.SearchableField.PUBLICATION_YEAR +import tsuki.model.search.SearchableField.STATE +import tsuki.model.search.SearchableField.TAG +import tsuki.model.search.SearchableField.TITLE_NAME + +import tsuki.util.ChaptersListBuilder +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toIntUp +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList +import tsuki.util.json.entries +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.async @@ -9,25 +64,9 @@ import kotlinx.coroutines.coroutineScope import okhttp3.HttpUrl import org.json.JSONArray import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.FlexibleMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.* -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField -import org.koitharu.kotatsu.parsers.model.search.SearchableField.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* import java.text.SimpleDateFormat -import java.util.* -import okhttp3.Interceptor -import okhttp3.Response -import org.koitharu.kotatsu.parsers.network.CommonHeaders +import java.util.EnumSet +import java.util.Locale private const val PAGE_SIZE = 20 private const val CHAPTERS_FIRST_PAGE_SIZE = 120 @@ -37,26 +76,12 @@ private const val CHAPTERS_MAX_COUNT = 10_000 // strange api behavior, looks lik private const val LOCALE_FALLBACK = "en" private const val SERVER_DATA = "data" private const val SERVER_DATA_SAVER = "data-saver" -private const val USER_AGENT = "Usagi/1 (Android)" @MangaSourceParser("MANGADEXORG", "MangaDex") internal class MangaDexParser(context: MangaLoaderContext) : FlexibleMangaParser(context, MangaParserSource.MANGADEXORG) { - override val userAgentKey = ConfigKey.UserAgent( - USER_AGENT, - - ) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - val newRequest = request.newBuilder() - .header(CommonHeaders.USER_AGENT, USER_AGENT) - .build() - - return chain.proceed(newRequest) - } - override val configKeyDomain = ConfigKey.Domain("mangadex.org") + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) private val preferredServerKey = ConfigKey.PreferredImageServer( presetValues = mapOf( diff --git a/src/main/kotlin/tsuki/site/all/MangaFire.kt b/src/main/kotlin/tsuki/site/all/MangaFire.kt new file mode 100644 index 00000000..435311b0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/MangaFire.kt @@ -0,0 +1,674 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.MangaParserAuthProvider +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.nullIfEmpty + +import okhttp3.Interceptor +import org.jsoup.Jsoup +import java.util.EnumSet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.supervisorScope +import java.util.concurrent.TimeUnit +import java.util.concurrent.ScheduledThreadPoolExecutor +import java.util.concurrent.Semaphore +import okio.ByteString +import okio.ByteString.Companion.decodeBase64 +import okhttp3.HttpUrl.Builder + +internal class VrfSigner { + fun interceptor() = Interceptor { chain -> + val request = chain.request() + val url = request.url + if (url.encodedPath.startsWith("/api/")) { + val params = url.queryParameterNames + .flatMap { key -> url.queryParameterValues(key).map { key to it } } + .sortedBy { it.first } + val sortedQueryUrl = buildString { + append(url.encodedPath.removePrefix("/api")) + if (params.isNotEmpty()) { + append('?') + var lastKey = "" + var index = 0 + append( + params.joinToString("&") { (key, value) -> + val newKey = if (key.endsWith("[]")) { + if (lastKey != key) index = 0 + lastKey = key + key.replace("[]", "[${index++}]") + } else { + key + } + "$newKey=$value" + }, + ) + } + } + val builder = url.newBuilder().query(null) + params.forEach { (k, v) -> builder.addQueryParameter(k, v) } + builder.addQueryParameter("vrf", sign(sortedQueryUrl)) + val newRequest = request.newBuilder().url(builder.build()).build() + chain.proceed(newRequest) + } else { + chain.proceed(request) + } + } + + private fun sign(path: String): String { + var data = path.toByteArray(Charsets.UTF_8) + for ((table, key, iv) in STAGES) { + data = encryptStage(data, table, key, iv) + } + return ByteString.of(*data).base64Url().trimEnd('=') + } + + private fun encryptStage(data: ByteArray, table: ByteArray, key: ByteArray, iv: Int): ByteArray { + val out = ByteArray(data.size) + var prev = iv + val keySize = key.size + for (i in data.indices) { + prev = table[(data[i].toInt() xor key[i % keySize].toInt() xor prev) and 0xFF].toInt() and 0xFF + out[i] = prev.toByte() + } + return out + } + + companion object { + private fun decodeB64(str: String) = str.decodeBase64()!!.toByteArray() + + private const val TABLE_1 = "yINlmUNho8VYJT+ibTIP+9ESiULpVEtMOoD6U6lRE0R/xwXo/Xp9NrUgC4cw/Lmo33vUyjUE40kUoEWIr/fxfNNcq2s79ShQ5NhNrFnJ4hXPwOu/SuXzIbuTQKGFvfm08E9jvCfqAtoDqvQq3dVWPQFmJjgvkISBeXY3BgANR+yVnjGbcxZ47d6kLNfZPIayTq3/YGySb1KuVZodWp/WGNAO5pfMcpaK53Hhs0allBszaMaxuouOwdxbwgxIw6YunSsXjI05Yi0j9j4eHKfSXR8Ifo/Od+8iamRfCXTyvm7NGRGYdcQ0ywcK/u6RXhrbcCm4t2eCtrDgQVecJGkQ+A==" + private const val KEY_1 = "0Ec58JOY3uBzJK9m3zqIOpdlF7UFiax9DmA=" + private const val TABLE_2 = "IUFltCxD3Oc2cwCgkJffthaOg9cgPUb0LgW6H/VtfcF0kc5F25t+aWj6JH9VOhOaY0rAFdUxlDnl5BLNvwEJvQtP5qcw7vdb/K+chnbwnspSHT8mz5lqwz41TezG0hkO06FTjJZhsyNuFLDpD2ZZxQj/QIRcF90zpmQ7Byu483WsQqUE0C342HL+JXngRB6fRzxRyVTaKu83h7UYTJ0QMt6ixFh6S3F8gqkKwrGTL3jHNBsD45UnifK8+RGtishQV2K3rujLKEkiZxpr2dYcudFW4oFsDKhad3CLBvuyTqsCo4B7mL5IKQ1vXo/MOOvq1I1d8ar9X6Ttu5KF4fZgiA==" + private const val KEY_2 = "AAdjb1iPY8CiDmq9H34tKTBF8a3oDQ==" + private const val TABLE_3 = "NQHlu1/wVO5EmkwQymF810qqY2xG1k2obcas4Z9mCsPEIFl9pRIjFxbJ7ybMHbBckT5Ton85E0FOeHezbh/mjlEYpmpnlXOS8dgrqeq2KfxImTh1YK9y0PeMNhzA1OQzSY9brYOJq/l2QnE/hwOeZIhPixVSKIUlDb5vLcH6RWKxkIEMuP0bDwIqQ71AJJaEaMJL7A6YtyIwoRT+L5v4aZzodN/0+3nOGsfblFjgxSfPzVDjNFeNl5P26+kEC/8AHgdrpAbt3hHz3HrRN1Y6e+JHgF7ncFWnoF0y3THL1S71WgWGCa6KtSzTCCG58n68nTyj2T3Sshk7utqCtMi/ZQ==" + private const val KEY_3 = "DELOJgPsVaCcblDtTGMdHzM=" + + private val STAGES: List> = listOf( + Triple(decodeB64(TABLE_1), decodeB64(KEY_1), 0x5A), + Triple(decodeB64(TABLE_2), decodeB64(KEY_2), 0x35), + Triple(decodeB64(TABLE_3), decodeB64(KEY_3), 0xBA), + ) + } +} + +private class RateLimitInterceptor( + maxRequestsPerSecond: Int = 2 +) : Interceptor { + + private val semaphore = Semaphore(maxRequestsPerSecond) + private val scheduler = ScheduledThreadPoolExecutor(1).apply { + removeOnCancelPolicy = true + } + private val releaseDelayMs = 1000L / maxRequestsPerSecond + + override fun intercept(chain: Interceptor.Chain): okhttp3.Response { + semaphore.acquire() + try { + return chain.proceed(chain.request()) + } finally { + scheduler.schedule({ semaphore.release() }, releaseDelayMs, TimeUnit.MILLISECONDS) + } + } +} + +internal abstract class MangaFireParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val siteLang: String, +) : PagedMangaParser(context, source, 30), Interceptor, MangaParserAuthProvider { + + override val configKeyDomain = ConfigKey.Domain("mangafire.to") + + override val availableSortOrders: Set = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.RELEVANCE, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + private val apiClient by lazy { + val newHttpClient = context.httpClient.newBuilder() +// .addInterceptor(RateLimitInterceptor(2)) future feature? + .addInterceptor(VrfSigner().interceptor()) + .addInterceptor { chain -> + chain.proceed( + chain.request().newBuilder() + .header("Referer", "https://$domain/") + .header("Accept", "application/json") + .build() + ) + } + .build() + OkHttpWebClient(newHttpClient, source) + } + + override fun intercept(chain: Interceptor.Chain) = chain.proceed( + chain.request().newBuilder() + .header("Referer", "https://$domain/") + .build() + ) + + private suspend fun apiGetJsonWithRetry(url: String, maxRetries: Int = 5): org.json.JSONObject { + var lastResponse: org.json.JSONObject? = null + repeat(maxRetries) { + val json = apiClient.httpGet(url).parseJson() + if (json.optString("message").equals("Missing token.", ignoreCase = true)) { + lastResponse = json + return@repeat + } + return json + } + throw ParseException( + "Missing token after $maxRetries retries. Last response: $lastResponse", + url, + ) + } + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearRangeSupported = true, + ) + + companion object { + val GENRE_MAP = mapOf( + "Action" to "1", + "Adult" to "268929", + "Adventure" to "78", + "Avant Garde" to "3", + "Boys Love" to "4", + "Comedy" to "5", + "Crime" to "268921", + "Demons" to "77", + "Drama" to "6", + "Ecchi" to "7", + "Fantasy" to "79", + "Girls Love" to "9", + "Gourmet" to "10", + "Harem" to "11", + "Hentai" to "268930", + "Historical" to "268922", + "Horror" to "530", + "Isekai" to "13", + "Iyashikei" to "531", + "Josei" to "15", + "Kids" to "532", + "Magic" to "539", + "Magical Girls" to "268923", + "Mahou Shoujo" to "533", + "Martial Arts" to "534", + "Mature" to "268931", + "Mecha" to "19", + "Medical" to "268924", + "Military" to "535", + "Music" to "21", + "Mystery" to "22", + "Parody" to "23", + "Philosophical" to "268925", + "Psychological" to "536", + "Reverse Harem" to "25", + "Romance" to "26", + "School" to "73", + "Sci-Fi" to "28", + "Seinen" to "537", + "Shoujo" to "30", + "Shounen" to "31", + "Slice of Life" to "538", + "Smut" to "268932", + "Space" to "33", + "Sports" to "34", + "Super Power" to "75", + "Superhero" to "268926", + "Supernatural" to "76", + "Suspense" to "37", + "Thriller" to "38", + "Tragedy" to "268927", + "Vampire" to "39", + "Wuxia" to "268928" + ) + } + + private val tags by lazy { + GENRE_MAP.entries.map { (title, id) -> + MangaTag(title, id, source) + }.toSet() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.ABANDONED, MangaState.PAUSED, MangaState.UPCOMING, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), + availableDemographics = EnumSet.of( + Demographic.SHOUNEN, + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.JOSEI + ), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE, ContentRating.ADULT) + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List { + val urlBuilder = Builder() + .scheme("https") + .host(domain) + .addPathSegments("api/titles") + .addQueryParameter("page", page.toString()) + .addQueryParameter("limit", "50") + .addQueryParameter("languages[]", siteLang) + + if (!filter.query.isNullOrBlank()) { + urlBuilder.addQueryParameter("keyword", filter.query) + } + if (filter.yearFrom > 0) { + urlBuilder.addQueryParameter("year_from", filter.yearFrom.toString()) + } + if (filter.yearTo > 0) { + urlBuilder.addQueryParameter("year_to", filter.yearTo.toString()) + } + + filter.contentRating.forEach { rating -> + urlBuilder.addQueryParameter("content_rating[]", when (rating) { + ContentRating.SAFE -> "safe" + ContentRating.SUGGESTIVE -> "suggestive" + ContentRating.ADULT -> "pornographic" + }) + } + + filter.types.forEach { type -> + when (type) { + ContentType.MANGA -> urlBuilder.addQueryParameter("types[]", "manga") + ContentType.MANHWA -> urlBuilder.addQueryParameter("types[]", "manhwa") + ContentType.MANHUA -> urlBuilder.addQueryParameter("types[]", "manhua") + ContentType.OTHER -> urlBuilder.addQueryParameter("types[]", "other") + else -> {} + } + } + + filter.demographics.forEach { demo -> + when (demo) { + Demographic.JOSEI -> urlBuilder.addQueryParameter("demographics[]", "268919") + Demographic.SEINEN -> urlBuilder.addQueryParameter("demographics[]", "268920") + Demographic.SHOUJO -> urlBuilder.addQueryParameter("demographics[]", "268917") + Demographic.SHOUNEN -> urlBuilder.addQueryParameter("demographics[]", "268918") + else -> {} + } + } + + filter.tags.forEach { urlBuilder.addQueryParameter("genres_in[]", it.key) } + filter.tagsExclude.forEach { urlBuilder.addQueryParameter("genres_ex[]", it.key) } + + filter.states.forEach { state -> + when (state) { + MangaState.ONGOING -> urlBuilder.addQueryParameter("statuses[]", "releasing") + MangaState.FINISHED -> urlBuilder.addQueryParameter("statuses[]", "finished") + MangaState.ABANDONED -> urlBuilder.addQueryParameter("statuses[]", "discontinued") + MangaState.PAUSED -> urlBuilder.addQueryParameter("statuses[]", "on_hiatus") + MangaState.UPCOMING -> urlBuilder.addQueryParameter("statuses[]", "not_yet_released") + else -> {} + } + } + + val sortParam = if (!filter.query.isNullOrBlank()) { + "relevance" to "desc" + } else when (order) { + SortOrder.UPDATED -> "chapter_updated_at" to "desc" + SortOrder.POPULARITY -> "views_total" to "desc" + SortOrder.RATING -> "score" to "desc" + SortOrder.NEWEST -> "created_at" to "desc" + SortOrder.ALPHABETICAL -> "title" to "asc" + SortOrder.RELEVANCE -> "relevance" to "desc" + SortOrder.POPULARITY_WEEK -> "views_7d" to "desc" + SortOrder.POPULARITY_MONTH -> "views_30d" to "desc" + else -> null + } + sortParam?.let { (field, dir) -> + urlBuilder.addQueryParameter("order[$field]", dir) + } + + val url = urlBuilder.build().toString() + + try { + val response = apiGetJsonWithRetry(url) + val items = response.optJSONArray("items") + ?: throw ParseException("Missing 'items' array in API response", url) + + val mangas = mutableListOf() + for (i in 0 until items.length()) { + try { + val obj = items.getJSONObject(i) + val hid = obj.getString("hid") + val title = obj.getString("title") + val slug = obj.optString("slug").nullIfEmpty() + val poster = obj.optJSONObject("poster") + val cover = poster?.optString("large") + ?: poster?.optString("medium") + ?: poster?.optString("small") + ?: "" + + val urlPath = "/title/$hid${slug?.let { "-$it" } ?: ""}" + + mangas.add( + Manga( + id = generateUid(urlPath), + url = urlPath, + publicUrl = "https://$domain$urlPath", + title = title, + coverUrl = cover, + source = source, + altTitles = emptySet(), + largeCoverUrl = null, + authors = emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + tags = emptySet(), + ) + ) + } catch (e: Exception) { + throw ParseException("Failed to parse manga item at index $i: ${e.message}", url, e) + } + } + return mangas + } catch (e: ParseException) { + throw e + } catch (e: Exception) { + throw ParseException("Failed to load list page: ${e.message}\nSolve captcha in webview and then reload page manually.", url, e) + } + } + + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + val result = try { + coroutineScope { + val hid = extractHid(manga.url) + val detailsJson = apiGetJsonWithRetry("https://$domain/api/titles/$hid") + val data = detailsJson.getJSONObject("data") + val hasVolumes = data.optBoolean("hasVolumes", false) + + val chaptersDeferred = async { fetchChapters(hid, hasVolumes) } + + val title = data.getString("title") + val poster = data.optJSONObject("poster") + val cover = poster?.optString("large") ?: poster?.optString("medium") + ?: poster?.optString("small") + val synopsisHtml = data.optString("synopsisHtml", null) + val status = data.optString("status", null) + val type = data.optString("type", null) + val authorsList = data.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val artistsList = data.optJSONArray("artists")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + }.orEmpty() + val genres = data.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val themes = data.optJSONArray("themes")?.let { arr -> + (0 until arr.length()).map { arr.getJSONObject(it).getString("title") } + } + val altTitlesArray = data.optJSONArray("altTitles")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val rawRating = data.optDouble("rating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + val synopsisText = synopsisHtml?.let { Jsoup.parseBodyFragment(it).text() } ?: "" + + val genreList = buildList { + type?.let { add(it.replaceFirstChar { c -> c.uppercase() }) } + genres?.let { addAll(it) } + themes?.let { addAll(it) } + } + val genreTags = genreList.mapNotNull { name -> + tags.find { it.title == name } + }.toSet() + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + authors = (authorsList + artistsList).toSet(), + description = synopsisText.trim(), + rating = rating, + state = when (status?.lowercase()) { + "releasing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + "discontinued" -> MangaState.ABANDONED + "on_hiatus" -> MangaState.PAUSED + "not_yet_released" -> MangaState.UPCOMING + else -> null + }, + tags = genreTags, + altTitles = altTitlesArray.toSet(), + chapters = chapters, + ) + } + } catch (_: Exception) { + manga + } + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private suspend fun fetchChapters(hid: String, hasVolumes: Boolean): List { + val base = "https://$domain/api/titles/$hid" + val chapters = mutableListOf() + + try { + val firstUrl = "$base/chapters?language=$siteLang&sort=number&order=desc&page=1&limit=200" + val firstJson = apiGetJsonWithRetry(firstUrl) + val firstItems = firstJson.optJSONArray("items") ?: org.json.JSONArray() + val meta = firstJson.optJSONObject("meta") + val lastPage = meta?.optInt("lastPage", 1) ?: 1 + + addChapters(firstItems, hid, chapters) + + if (lastPage > 1) { + supervisorScope { + val deferred = (2..lastPage).map { page -> + async { + try { + val json = apiGetJsonWithRetry( + "$base/chapters?language=$siteLang&sort=number&order=desc&page=$page&limit=200" + ) + json.optJSONArray("items") ?: org.json.JSONArray() + } catch (_: Exception) { + org.json.JSONArray() + } + } + } + deferred.awaitAll().forEach { items -> + addChapters(items, hid, chapters) + } + } + } + } catch (_: Exception) { + } + + if (hasVolumes) { + try { + val volJson = apiGetJsonWithRetry("$base/volumes?language=$siteLang") + val volItems = volJson.optJSONArray("items") ?: org.json.JSONArray() + for (i in 0 until volItems.length()) { + val vol = volItems.getJSONObject(i) + if (vol.optString("language", "") != siteLang) continue + val volId = vol.getInt("id") + val volNumber = vol.getDouble("number").toFloat() + val volName = vol.optString("name", "").takeIf { it.isNotBlank() } + val chapterCount = vol.optInt("chapterCount", 0) + val title = buildString { + append("Vol. ") + append(volNumber.toString().removeSuffix(".0")) + if (volName != null) append(" - $volName") + } + chapters.add( + MangaChapter( + id = generateUid("/title/$hid/vol/$volId"), + title = title, + number = volNumber, + volume = 0, + url = "/title/$hid/vol/$volId", + scanlator = if (chapterCount > 0) "$chapterCount chapters" else "", + uploadDate = 0L, + branch = "Volume", + source = source, + ) + ) + } + } catch (_: Exception) {} + } + + val distinctBranches = chapters.map { it.branch }.distinct() + val useGroups = distinctBranches.size > 1 + return chapters + .map { chapter -> + chapter.copy( + branch = if (useGroups) (chapter.branch ?: "").replaceFirstChar { it.uppercase() } else null + ) + } + .sortedBy { it.number } + } + + private fun addChapters(items: org.json.JSONArray, hid: String, list: MutableList) { + for (i in 0 until items.length()) { + try { + val ch = items.getJSONObject(i) + if (ch.optString("language", "") != siteLang) continue + val id = ch.getInt("id") + val number = ch.getDouble("number").toFloat() + val name = ch.optString("name", null) + val createdAt = ch.optLong("createdAt", 0L) * 1000L + val type = ch.optString("type", "Unknown") + val chapterUrl = "/title/$hid/$id" + val displayName = buildString { + append("Ch. ") + append(number.toString().removeSuffix(".0")) + if (!name.isNullOrBlank()) append(" - $name") + } + list.add( + MangaChapter( + id = generateUid(chapterUrl), + title = displayName, + number = number, + volume = 0, + url = chapterUrl, + scanlator = null, + uploadDate = createdAt, + branch = type, + source = source, + ) + ) + } catch (_: Exception) { + } + } + } + + override suspend fun getPages(chapter: MangaChapter): List = try { + val chapterId = chapter.url.substringAfterLast("/") + val isVolume = chapter.url.contains("/vol/") + val endpoint = if (isVolume) "volumes" else "chapters" + + val response = apiGetJsonWithRetry("https://$domain/api/$endpoint/$chapterId") + val data = response.optJSONObject("data") ?: return emptyList() + val pagesArray = data.optJSONArray("pages") ?: return emptyList() + val pages = ArrayList(pagesArray.length()) + for (i in 0 until pagesArray.length()) { + val pageObj = pagesArray.getJSONObject(i) + val url = pageObj.getString("url") + pages.add( + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + ) + } + pages + } catch (_: Exception) { + emptyList() + } + + override suspend fun getRelatedManga(seed: Manga): List = emptyList() + + override val authUrl: String get() = "https://$domain" + override suspend fun isAuthorized(): Boolean = true + override suspend fun getUsername(): String = "" + + private fun extractHid(url: String): String { + val lastPart = url.removeSuffix("/").substringAfterLast("/") + return when { + lastPart.contains(".") -> lastPart.substringAfterLast(".") + lastPart.contains("-") -> lastPart.substringBefore("-") + else -> lastPart + } + } + + @MangaSourceParser("MANGAFIRE_EN", "MangaFire (English)", "en") + class English(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_EN, "en") + + @MangaSourceParser("MANGAFIRE_ESLA", "MangaFire (Spanish)", "es") + class SpanishLatim(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ESLA, "es-la") + + @MangaSourceParser("MANGAFIRE_ES", "MangaFire (Spanish 2)", "es") + class Spanish(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_ES, "es") + + @MangaSourceParser("MANGAFIRE_FR", "MangaFire (French)", "fr") + class French(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_FR, "fr") + + @MangaSourceParser("MANGAFIRE_JA", "MangaFire (Japanese)", "ja") + class Japanese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_JA, "ja") + + @MangaSourceParser("MANGAFIRE_PTBR", "MangaFire (Portuguese)", "pt") + class PortugueseBR(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PTBR, "pt-br") + + @MangaSourceParser("MANGAFIRE_PT", "MangaFire (Portuguese 2)", "pt") + class Portuguese(context: MangaLoaderContext) : MangaFireParser(context, MangaParserSource.MANGAFIRE_PT, "pt") +} diff --git a/src/main/kotlin/tsuki/site/all/MangaPlus.kt b/src/main/kotlin/tsuki/site/all/MangaPlus.kt new file mode 100644 index 00000000..4440ecc7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/MangaPlus.kt @@ -0,0 +1,816 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import okio.Buffer +import okio.IOException +import java.util.EnumSet +import java.util.UUID +import java.util.concurrent.Semaphore + +// created very small protobuf reader via Okio +@Throws(IOException::class) +private fun Buffer.readProtobufVarint(): Long { + var result = 0L + var shift = 0 + while (true) { + val b = readByte().toInt() and 0xFF + result = result or ((b and 0x7F).toLong() shl shift) + if (b and 0x80 == 0) break + shift += 7 + if (shift > 63) throw IOException("Malformed varint") + } + return result +} + +@Throws(IOException::class) +private fun Buffer.readProtobufString(): String = readByteArray(readProtobufVarint()).toString(Charsets.UTF_8) + +@Throws(IOException::class) +private fun Buffer.readProtobufBytes(): ByteArray = readByteArray(readProtobufVarint()) + +data class MPResponse(val success: SuccessResult?, val error: String?) + +data class SuccessResult( + val titleRankingView: TitleRankingView?, + val webHomeView: WebHomeView?, + val allTitlesView: AllTitlesView?, + val titleDetailView: TitleDetailView?, + val mangaViewer: MangaViewer? +) + +data class TitleRankingView(val rankedTitles: List) +data class RankedTitle(val titles: List) +data class WebHomeView(val groups: List<UpdatedTitleGroup>) +data class UpdatedTitleGroup(val titles: List<UpdatedTitle>) +data class UpdatedTitle(val title: Title?) // can be null + +data class AllTitlesView(val allTitlesGroup: List<AllTitlesGroup>) +data class AllTitlesGroup(val titles: List<Title>) + +data class Title( + val titleId: Int, + val name: String, + val author: String?, + val portraitImageUrl: String, + val language: Int +) + +data class TitleDetailView( + val title: Title, + val overview: String, + val viewingPeriodDescription: String, + val nonAppearanceInfo: String, + val chapterListGroup: List<ChapterListGroup> +) + +data class ChapterListGroup( + val firstChapterList: List<Chapter>, + val lastChapterList: List<Chapter> +) + +data class Chapter( + val chapterId: Int, + val name: String, + val subTitle: String?, + val startTimeStamp: Int +) + +data class MangaViewer( + val pages: List<MangaPlusPage>, + val titleId: Int?, + val viewToken: String? +) + +data class MangaPlusPage(val mangaPage: MPImagePage?) +data class MPImagePage(val imageUrl: String, val encryptionKey: String?) + +// field numbers +private const val FIELD_SUCCESS = 1 +private const val FIELD_ERROR = 2 + +// SuccessResult sub‑fields +private const val FIELD_TITLE_DETAIL_VIEW = 8 +private const val FIELD_MANGA_VIEWER = 10 +private const val FIELD_ALL_TITLES_VIEW = 25 +private const val FIELD_TITLE_RANKING_VIEW= 37 +private const val FIELD_WEB_HOME_VIEW = 38 + +// TitleDetailView fields +private const val FIELD_TITLE = 1 +private const val FIELD_OVERVIEW = 3 +private const val FIELD_VIEWING_PERIOD = 7 +private const val FIELD_NON_APPEARANCE = 8 +private const val FIELD_CHAPTER_LIST_GROUP = 28 + +// Title fields +private const val FIELD_TITLE_ID = 1 +private const val FIELD_TITLE_NAME = 2 +private const val FIELD_AUTHOR = 3 +private const val FIELD_PORTRAIT_IMAGE = 4 +private const val FIELD_LANGUAGE = 7 + +// ChapterListGroup fields +private const val FIELD_FIRST_CHAPTER_LIST = 2 +private const val FIELD_LAST_CHAPTER_LIST = 4 + +// Chapter fields +private const val FIELD_CHAPTER_ID = 2 +private const val FIELD_CHAPTER_NAME = 3 +private const val FIELD_SUBTITLE = 4 +private const val FIELD_START_TIMESTAMP = 6 + +// MangaViewer fields +private const val FIELD_PAGES = 1 +private const val FIELD_TITLE_ID_MV = 9 +private const val FIELD_VIEW_TOKEN = 19 + +// MangaPlusPage fields +private const val FIELD_MANGA_PAGE = 1 + +// MangaPage fields +private const val FIELD_IMAGE_URL = 1 +private const val FIELD_ENCRYPTION_KEY = 5 + +// TitleRankingView fields +private const val FIELD_RANKED_TITLES = 3 +// RankedTitle fields +private const val FIELD_RANKED_TITLE_TITLES = 2 + +// WebHomeView fields +private const val FIELD_GROUPS = 2 +// UpdatedTitleGroup fields +private const val FIELD_GROUP_TITLES = 2 +// UpdatedTitle fields +private const val FIELD_LATEST_CHAPTER = 3 +// LatestChapter fields +private const val FIELD_LATEST_CHAPTER_TITLE = 1 + +// AllTitlesView fields +private const val FIELD_ALL_TITLES_GROUP = 1 +// AllTitlesGroup fields +private const val FIELD_GROUP_TITLES_ALL = 2 + + +private const val one = "low" +private const val two = "high" +private const val three = "super_high" + +@Throws(IOException::class) +private fun Buffer.parseMPResponse(): MPResponse { + var success: SuccessResult? = null + var error: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fieldNumber = (tag shr 3).toInt() + val wireType = (tag and 0x07).toInt() + when (fieldNumber) { + FIELD_SUCCESS -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + success = Buffer().apply { write(bytes) }.parseSuccessResult() + } else skipField(wireType) + } + FIELD_ERROR -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + // parse simple error message (first string body) + val buf = Buffer().apply { write(bytes) } + while (!buf.exhausted()) { + val t = buf.readProtobufVarint() + val fn = (t shr 3).toInt() + val wt = (t and 0x07).toInt() + if (fn == 2 && wt == 2) { + val nested = buf.readProtobufBytes() + val nestedBuf = Buffer().apply { write(nested) } + // Try to extract body field + while (!nestedBuf.exhausted()) { + val nt = nestedBuf.readProtobufVarint() + val nfn = (nt shr 3).toInt() + val nwt = (nt and 0x07).toInt() + if (nfn == 2 && nwt == 2) { + error = nestedBuf.readProtobufString() + break + } else nestedBuf.skipField(nwt) + } + } else buf.skipField(wt) + } + } else skipField(wireType) + } + else -> skipField(wireType) + } + } + return MPResponse(success, error) +} + +@Throws(IOException::class) +private fun Buffer.skipField(wireType: Int) { + when (wireType) { + 0 -> readProtobufVarint() + 1 -> skip(8) + 2 -> skip(readProtobufVarint()) + 5 -> skip(4) + else -> throw IOException("Unknown wire type $wireType") + } +} + +@Throws(IOException::class) +private fun Buffer.parseSuccessResult(): SuccessResult { + var titleDetailView: TitleDetailView? = null + var mangaViewer: MangaViewer? = null + var allTitlesView: AllTitlesView? = null + var titleRankingView: TitleRankingView? = null + var webHomeView: WebHomeView? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fieldNumber = (tag shr 3).toInt() + val wireType = (tag and 0x07).toInt() + when (fieldNumber) { + FIELD_TITLE_DETAIL_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + titleDetailView = Buffer().apply { write(bytes) }.parseTitleDetailView() + } else skipField(wireType) + } + FIELD_MANGA_VIEWER -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + mangaViewer = Buffer().apply { write(bytes) }.parseMangaViewer() + } else skipField(wireType) + } + FIELD_ALL_TITLES_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + allTitlesView = Buffer().apply { write(bytes) }.parseAllTitlesView() + } else skipField(wireType) + } + FIELD_TITLE_RANKING_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + titleRankingView = Buffer().apply { write(bytes) }.parseTitleRankingView() + } else skipField(wireType) + } + FIELD_WEB_HOME_VIEW -> { + if (wireType == 2) { + val bytes = readProtobufBytes() + webHomeView = Buffer().apply { write(bytes) }.parseWebHomeView() + } else skipField(wireType) + } + else -> skipField(wireType) + } + } + return SuccessResult(titleRankingView, webHomeView, allTitlesView, titleDetailView, mangaViewer) +} + +@Throws(IOException::class) +private fun Buffer.parseTitle(): Title { + var titleId = 0 + var name = "" + var author: String? = null + var portraitImageUrl = "" + var language = 0 + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_TITLE_ID -> if (wt == 0) titleId = readProtobufVarint().toInt() + FIELD_TITLE_NAME -> if (wt == 2) name = readProtobufString() + FIELD_AUTHOR -> if (wt == 2) author = readProtobufString() + FIELD_PORTRAIT_IMAGE -> if (wt == 2) portraitImageUrl = readProtobufString() + FIELD_LANGUAGE -> if (wt == 0) language = readProtobufVarint().toInt() + else -> skipField(wt) + } + } + return Title(titleId, name, author, portraitImageUrl, language) +} + +@Throws(IOException::class) +private fun Buffer.parseTitleDetailView(): TitleDetailView { + var title: Title? = null + var overview = "" + var viewingPeriodDescription = "" + var nonAppearanceInfo = "" + val chapterListGroup = mutableListOf<ChapterListGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_TITLE -> if (wt == 2) { + val bytes = readProtobufBytes() + title = Buffer().apply { write(bytes) }.parseTitle() + } + FIELD_OVERVIEW -> if (wt == 2) overview = readProtobufString() + FIELD_VIEWING_PERIOD -> if (wt == 2) viewingPeriodDescription = readProtobufString() + FIELD_NON_APPEARANCE -> if (wt == 2) nonAppearanceInfo = readProtobufString() + FIELD_CHAPTER_LIST_GROUP -> if (wt == 2) { + val bytes = readProtobufBytes() + chapterListGroup += Buffer().apply { write(bytes) }.parseChapterListGroup() + } + else -> skipField(wt) + } + } + return TitleDetailView(title!!, overview, viewingPeriodDescription, nonAppearanceInfo, chapterListGroup) +} + +@Throws(IOException::class) +private fun Buffer.parseChapterListGroup(): ChapterListGroup { + val first = mutableListOf<Chapter>() + val last = mutableListOf<Chapter>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_FIRST_CHAPTER_LIST -> if (wt == 2) { + val bytes = readProtobufBytes() + first += Buffer().apply { write(bytes) }.parseChapter() + } + FIELD_LAST_CHAPTER_LIST -> if (wt == 2) { + val bytes = readProtobufBytes() + last += Buffer().apply { write(bytes) }.parseChapter() + } + else -> skipField(wt) + } + } + return ChapterListGroup(first, last) +} + +@Throws(IOException::class) +private fun Buffer.parseChapter(): Chapter { + var chapterId = 0 + var name = "" + var subTitle: String? = null + var startTimeStamp = 0 + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_CHAPTER_ID -> if (wt == 0) chapterId = readProtobufVarint().toInt() + FIELD_CHAPTER_NAME -> if (wt == 2) name = readProtobufString() + FIELD_SUBTITLE -> if (wt == 2) subTitle = readProtobufString() + FIELD_START_TIMESTAMP -> if (wt == 0) startTimeStamp = readProtobufVarint().toInt() + else -> skipField(wt) + } + } + return Chapter(chapterId, name, subTitle, startTimeStamp) +} + +@Throws(IOException::class) +private fun Buffer.parseMangaViewer(): MangaViewer { + val pages = mutableListOf<MangaPlusPage>() + var titleId: Int? = null + var viewToken: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_PAGES -> if (wt == 2) { + val bytes = readProtobufBytes() + pages += Buffer().apply { write(bytes) }.parseMangaPlusPage() + } + FIELD_TITLE_ID_MV -> if (wt == 0) titleId = readProtobufVarint().toInt() + FIELD_VIEW_TOKEN -> if (wt == 2) viewToken = readProtobufString() + else -> skipField(wt) + } + } + return MangaViewer(pages, titleId, viewToken) +} + +@Throws(IOException::class) +private fun Buffer.parseMangaPlusPage(): MangaPlusPage { + var mangaPage: MPImagePage? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_MANGA_PAGE && wt == 2) { + val bytes = readProtobufBytes() + mangaPage = Buffer().apply { write(bytes) }.parseMPImagePage() + } else skipField(wt) + } + return MangaPlusPage(mangaPage) +} + +@Throws(IOException::class) +private fun Buffer.parseMPImagePage(): MPImagePage { + var imageUrl = "" + var encryptionKey: String? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + when (fn) { + FIELD_IMAGE_URL -> if (wt == 2) imageUrl = readProtobufString() + FIELD_ENCRYPTION_KEY -> if (wt == 2) encryptionKey = readProtobufString() + else -> skipField(wt) + } + } + return MPImagePage(imageUrl, encryptionKey) +} + +@Throws(IOException::class) +private fun Buffer.parseTitleRankingView(): TitleRankingView { + val ranked = mutableListOf<RankedTitle>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_RANKED_TITLES && wt == 2) { + val bytes = readProtobufBytes() + ranked += Buffer().apply { write(bytes) }.parseRankedTitle() + } else skipField(wt) + } + return TitleRankingView(ranked) +} + +@Throws(IOException::class) +private fun Buffer.parseRankedTitle(): RankedTitle { + val titles = mutableListOf<Title>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_RANKED_TITLE_TITLES && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseTitle() + } else skipField(wt) + } + return RankedTitle(titles) +} + +@Throws(IOException::class) +private fun Buffer.parseWebHomeView(): WebHomeView { + val groups = mutableListOf<UpdatedTitleGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUPS && wt == 2) { + val bytes = readProtobufBytes() + groups += Buffer().apply { write(bytes) }.parseUpdatedTitleGroup() + } else skipField(wt) + } + return WebHomeView(groups) +} + +@Throws(IOException::class) +private fun Buffer.parseUpdatedTitleGroup(): UpdatedTitleGroup { + val titles = mutableListOf<UpdatedTitle>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUP_TITLES && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseUpdatedTitle() + } else skipField(wt) + } + return UpdatedTitleGroup(titles) +} + +@Throws(IOException::class) +private fun Buffer.parseUpdatedTitle(): UpdatedTitle { + var title: Title? = null + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_LATEST_CHAPTER && wt == 2) { + val bytes = readProtobufBytes() + val nested = Buffer().apply { write(bytes) } + while (!nested.exhausted()) { + val ntag = nested.readProtobufVarint() + val nfn = (ntag shr 3).toInt() + val nwt = (ntag and 0x07).toInt() + if (nfn == FIELD_LATEST_CHAPTER_TITLE && nwt == 2) { + val titleBytes = nested.readProtobufBytes() + title = Buffer().apply { write(titleBytes) }.parseTitle() + } else nested.skipField(nwt) + } + } else skipField(wt) + } + return UpdatedTitle(title) +} + +@Throws(IOException::class) +private fun Buffer.parseAllTitlesView(): AllTitlesView { + val groups = mutableListOf<AllTitlesGroup>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_ALL_TITLES_GROUP && wt == 2) { + val bytes = readProtobufBytes() + groups += Buffer().apply { write(bytes) }.parseAllTitlesGroup() + } else skipField(wt) + } + return AllTitlesView(groups) +} + +@Throws(IOException::class) +private fun Buffer.parseAllTitlesGroup(): AllTitlesGroup { + val titles = mutableListOf<Title>() + while (!exhausted()) { + val tag = readProtobufVarint() + val fn = (tag shr 3).toInt() + val wt = (tag and 0x07).toInt() + if (fn == FIELD_GROUP_TITLES_ALL && wt == 2) { + val bytes = readProtobufBytes() + titles += Buffer().apply { write(bytes) }.parseTitle() + } else skipField(wt) + } + return AllTitlesGroup(titles) +} + +internal abstract class MangaPlusParser( + context: MangaLoaderContext, + source: MangaParserSource, + private val sourceLang: String, + private val internalLangName: String, + private val internalLangCode: Int +) : SinglePageMangaParser(context, source), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("mangaplus.shueisha.co.jp") + + private val apiUrl = "https://jumpg-webapi.tokyo-cdn.com/api" + private val session = UUID.randomUUID().toString() + private val imageSemaphore = Semaphore(1) + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + one to "Low quality", + two to "High quality", + three to "Super high quality", + ), + defaultValue = two, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, SortOrder.UPDATED, SortOrder.ALPHABETICAL + ) + + override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) + override suspend fun getFilterOptions() = MangaListFilterOptions() + + private suspend fun apiCall(url: String): MPResponse { + val fullUrl = "$apiUrl$url".toHttpUrl().newBuilder().build() + val headers = Headers.headersOf("SESSION-TOKEN", session) + val response = webClient.httpGet(fullUrl, headers) + val body = response.body + return Buffer().apply { write(body.bytes()) }.parseMPResponse() + } + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query + if (!query.isNullOrEmpty()) { + val resp = apiCall("/title_list/allV2") + val titles = resp.success?.allTitlesView?.allTitlesGroup?.flatMap { it.titles } ?: emptyList() + return titles.filterByLang().filter { + it.name.contains(query, true) || it.author?.contains(query, true) ?: false + }.map { it.toManga() } + } + + return when (order) { + SortOrder.POPULARITY -> { + val resp = apiCall("/title_list/rankingV2?lang=$internalLangName&type=hottest&clang=$internalLangName") + resp.success?.titleRankingView?.rankedTitles?.flatMap { it.titles } + .orEmpty().filterByLang().map { it.toManga() } + } + SortOrder.UPDATED -> { + val latestResult = apiCall("/web/web_homeV4?lang=$internalLangName&clang=$internalLangName") + val allTitlesResult = apiCall("/title_list/allV2") + + val latestTitles = latestResult.success?.webHomeView?.groups + ?.flatMap { it.titles } + ?.mapNotNull { it.title } + ?.filterByLang() + .orEmpty() + + val allTitlesGroups = allTitlesResult.success?.allTitlesView?.allTitlesGroup + .orEmpty() + + val titles = latestTitles.mapNotNull { latestTitle -> + allTitlesGroups + .firstOrNull { group -> group.titles.any { it.titleId == latestTitle.titleId } } + ?.titles + ?.firstOrNull { it.language == internalLangCode } + }.distinctBy { it.titleId } + + titles.map { it.toManga() } + } + SortOrder.ALPHABETICAL -> { + val resp = apiCall("/title_list/allV2") + resp.success?.allTitlesView?.allTitlesGroup?.flatMap { it.titles } + .orEmpty().filterByLang().sortedBy { it.name.lowercase() }.map { it.toManga() } + } + else -> emptyList() + } + } + + private fun Title.toManga() = Manga( + id = generateUid(titleId.toString()), + url = titleId.toString(), + publicUrl = "/titles/$titleId".toAbsoluteUrl(domain), + title = name, + coverUrl = portraitImageUrl, + altTitles = emptySet(), + authors = author?.split("/")?.joinToString(", ") { it.trim() }?.let { setOf(it) } ?: emptySet(), + contentRating = null, + rating = RATING_UNKNOWN, + state = null, + source = source, + tags = emptySet(), + ) + + private fun List<Title>.filterByLang() = filter { it.titleId != 0 && it.language == internalLangCode }.distinctBy { it.titleId } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val resp = apiCall("/title_detailV3?title_id=${manga.url}&clang=$internalLangName") + val detail = resp.success?.titleDetailView ?: throw IOException("No detail") + val completed = detail.nonAppearanceInfo.contains(CompletedRegex) || detail.viewingPeriodDescription.contains("latest 0 chapters") + val hiatus = detail.nonAppearanceInfo.contains(Regex("on a hiatus", RegexOption.IGNORE_CASE)) + val result = manga.copy( + title = detail.title.name, + coverUrl = detail.title.portraitImageUrl, + authors = detail.title.author?.split("/")?.joinToString(", ") { it.trim() }?.let { setOf(it) } ?: emptySet(), + description = listOfNotNull(detail.overview.takeIf { it.isNotEmpty() }, detail.viewingPeriodDescription.takeIf { it.isNotEmpty() }).joinToString("\n\n"), + state = when { + completed -> MangaState.FINISHED + hiatus -> MangaState.PAUSED + else -> MangaState.ONGOING + }, + chapters = detail.chapterListGroup + .flatMap { it.firstChapterList + it.lastChapterList } + .filter { it.subTitle != null } + .map { chapter -> + MangaChapter( + id = generateUid(chapter.chapterId.toString()), + url = chapter.chapterId.toString(), + title = chapter.subTitle ?: chapter.name, + number = chapter.name.substringAfter("#").toFloatOrNull() ?: -1f, + volume = 0, + uploadDate = chapter.startTimeStamp * 1000L, + branch = when (sourceLang) { + "PORTUGUESE_BR" -> "Portuguese (Brazil)" + else -> sourceLang.lowercase().toTitleCase() + }, + scanlator = null, + source = source, + ) + } + ) + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val quality = config[preferredServerKey] + val resp = apiCall("/manga_viewer_v3?chapter_id=${chapter.url}&split=yes&img_quality=$quality&clang=$internalLangName") + val viewer = resp.success?.mangaViewer ?: return emptyList() + val viewToken = viewer.viewToken ?: "" + return viewer.pages.mapNotNull { it.mangaPage }.map { mpImage -> + val baseUrl = mpImage.imageUrl + val encryptionKey = mpImage.encryptionKey + val separator = if (baseUrl.contains("?")) "&" else "?" + val fullUrl = buildString { + append(baseUrl) + append(separator).append("vtoken=").append(viewToken) + if (encryptionKey != null) append("#").append(encryptionKey) + } + + MangaPage( + id = generateUid(fullUrl), + url = fullUrl, + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + val host = url.host + + return if (host.startsWith("jumpg-assets")) { + imageSemaphore.acquire() + try { + processImageRequest(chain, request, url) + } finally { + imageSemaphore.release() + } + } else { + processImageRequest(chain, request, url) + } + } + + private fun processImageRequest(chain: Interceptor.Chain, request: okhttp3.Request, url: okhttp3.HttpUrl): Response { + val encryptionKey = url.fragment + val vtoken = url.queryParameter("vtoken") + + val newUrl = url.newBuilder() + .removeAllQueryParameters("vtoken") + .build() + + val newRequest = request.newBuilder() + .url(newUrl) + .apply { + if (vtoken != null) { + header("Plus-Vw-Token", vtoken) + } + } + .build() + + val response = chain.proceed(newRequest) + + if (encryptionKey.isNullOrEmpty()) return response + + val keyStream = encryptionKey.chunked(2).map { it.toInt(16) } + val contentType = response.header("Content-Type") ?: "image/jpeg" + val bytes = response.body.bytes() + val decrypted = bytes.mapIndexed { i, byte -> + (byte.toInt() xor keyStream[i % keyStream.size]).toByte() + }.toByteArray() + + return response.newBuilder() + .body(decrypted.toResponseBody(contentType.toMediaTypeOrNull())) + .build() + } + + companion object { + private val CompletedRegex = Regex("completado|completed?|completo", RegexOption.IGNORE_CASE) + } + + @MangaSourceParser("MANGAPLUSPARSER_EN", "MANGA Plus (English)", "en") + class English(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_EN, "ENGLISH", "eng", 0) + + @MangaSourceParser("MANGAPLUSPARSER_ES", "MANGA Plus (Spanish)", "es") + class Spanish(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ES, "SPANISH", "esp", 1) + + @MangaSourceParser("MANGAPLUSPARSER_FR", "MANGA Plus (French)", "fr") + class French(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_FR, "FRENCH", "fra", 2) + + @MangaSourceParser("MANGAPLUSPARSER_ID", "MANGA Plus (Indonesian)", "id") + class Indonesian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_ID, "INDONESIAN", "ind", 3) + + @MangaSourceParser("MANGAPLUSPARSER_PTBR", "MANGA Plus Portuguese (Brazil)", "pt") + class Portuguese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_PTBR, "PORTUGUESE_BR", "ptb", 4) + + @MangaSourceParser("MANGAPLUSPARSER_RU", "MANGA Plus (Russian)", "ru") + class Russian(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_RU, "RUSSIAN", "rus", 5) + + @MangaSourceParser("MANGAPLUSPARSER_TH", "MANGA Plus (Thai)", "th") + class Thai(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_TH, "THAI", "tha", 6) + + @MangaSourceParser("MANGAPLUSPARSER_VI", "MANGA Plus (Vietnamese)", "vi") + class Vietnamese(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_VI, "VIETNAMESE", "vie", 9) + + @MangaSourceParser("MANGAPLUSPARSER_DE", "MANGA Plus (German)", "de") + class German(context: MangaLoaderContext) : MangaPlusParser(context, MangaParserSource.MANGAPLUSPARSER_DE, "GERMAN", "deu", 7) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt b/src/main/kotlin/tsuki/site/all/Mangaball.kt similarity index 92% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt rename to src/main/kotlin/tsuki/site/all/Mangaball.kt index 34b940db..a6ffb97a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Mangaball.kt +++ b/src/main/kotlin/tsuki/site/all/Mangaball.kt @@ -1,4 +1,38 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.generateUid +import tsuki.util.insertCookies +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.ownTextOrNull +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.Headers import okhttp3.HttpUrl @@ -11,37 +45,6 @@ import org.json.JSONArray import org.json.JSONObject import org.jsoup.HttpStatusException import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.network.WebClient -import org.koitharu.kotatsu.parsers.util.LinkResolver -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.insertCookies -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.ownTextOrNull -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -134,7 +137,9 @@ internal abstract class MangaBallParser( } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } ensureAdultCookie() val doc = client.httpGet(getMangaUrl(manga.url)).parseHtml() @@ -180,7 +185,9 @@ internal abstract class MangaBallParser( chapters = getChapterList(manga.url), contentRating = contentRating, ) - detailsCache[manga.url] = result + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } return result } @@ -548,10 +555,7 @@ internal abstract class MangaBallParser( if (csrfToken.isNullOrEmpty()) { refreshCsrf() if (csrfToken.isNullOrEmpty()) { - refreshCsrf() - } - if (csrfToken.isNullOrEmpty()) { - error("CSRF token not found after two attempts") + error("CSRF token not found") } } return csrfToken!! @@ -582,7 +586,8 @@ internal abstract class MangaBallParser( } } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } @@ -624,7 +629,7 @@ internal abstract class MangaBallParser( companion object { - private const val BASE_URL = "https://$DOMAIN" + private const val BASE_URL = "https://${DOMAIN}" private val GROUP_ID_REGEX = Regex("[a-z0-9]{24}") private val CHAPTER_IMAGES_REGEX = Regex("""const\s+chapterImages\s*=\s*JSON\.parse\(`([^`]+)`\)""") private val ADULT_TAG_TITLES = setOf( diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt similarity index 53% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt rename to src/main/kotlin/tsuki/site/all/Mangadotnet.kt index 1e74f249..0a6b4040 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangadotnet.kt +++ b/src/main/kotlin/tsuki/site/all/Mangadotnet.kt @@ -1,23 +1,47 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone -@MangaSourceParser("MANGADOTNET", "Mangadot.net", "en", ContentType.MANGA) +@MangaSourceParser("MANGADOTNET", "Mangadot.net", type = ContentType.MANGA) internal class Mangadotnet(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANGADOTNET, 20) { override val configKeyDomain = ConfigKey.Domain("mangadot.net") - private val baseUrl = "https://mangadot.net" + private val baseUrl = "https://$domain" private val apiBase = "$baseUrl/api" private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { timeZone = TimeZone.getTimeZone("UTC") @@ -25,31 +49,6 @@ internal class Mangadotnet(context: MangaLoaderContext) : private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) - private val genreNames = setOf( - "Josei", "Seinen", "Shoujo", "Shounen", - "Action", "Adventure", "Comedy", "Drama", "Fantasy", - "Historical", "Horror", "Mecha", "Mystery", "Psychological", - "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", - "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", - "Medical", "Military", "Music", "School Life", "Webtoon", - "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", - "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", - "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", - "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", - "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", - "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", - "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", - "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", - "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", - "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", - "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", - "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", - "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", - "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", - "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", - "Wuxia", "Yaoi", "Yuri", "Zombies" - ) - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { super.onCreateConfig(keys) keys.add(userAgentKey) @@ -84,11 +83,12 @@ internal class Mangadotnet(context: MangaLoaderContext) : } @Suppress("UNCHECKED_CAST") - private suspend fun fetchRscData(url: String, route: String): Map<String, Any?>? { + private suspend fun fetchRscRoute(url: String, route: String): Map<String, Any?> { val flat = webClient.httpGet(url).parseJsonArray() - val decoded = decodeRsc(flat) ?: return null - val routeValue = (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> ?: return null - return routeValue.asMap("data") + val decoded = decodeRsc(flat) + ?: throw ParseException("Failed to decode RSC data", url) + return (decoded as? Map<String, Any?>)?.get(route) as? Map<String, Any?> + ?: throw ParseException("Missing RSC route '$route'", url) } override val filterCapabilities = MangaListFilterCapabilities( @@ -99,63 +99,95 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, // latest - SortOrder.POPULARITY, // most viewed - SortOrder.RATING, // top rated - SortOrder.ALPHABETICAL, // a-z - SortOrder.RELEVANCE, // most tracked + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + SortOrder.RELEVANCE, + SortOrder.ADDED, ) + companion object { + private val GENRES = setOf( + "Action", "Adventure", "Comedy", "Drama", "Fantasy", + "Historical", "Horror", "Mecha", "Mystery", "Psychological", + "Romance", "Sci-Fi", "Slice of Life", "Sports", "Supernatural", "Thriller", "Tragedy", + "Cooking", "Demons", "Ecchi", "Harem", "Isekai", "Magic", "Martial Arts", + "Medical", "Military", "Music", "School Life", "Webtoon", + "Academy", "Acting", "Adult", "Aliens", "Animals", "Anthology", "Apocalypse", + "Avant Garde", "Award Winning", "BDSM", "Boys Love", "Bully", "Business", + "Child Abuse", "Child Neglect", "Comic", "Crime", "crossdressing", "Crossdressing", + "Cultivation", "Delinquents", "Difficult Childhood", "dojinshi", "Doujinshi", + "Erotica", "Female Protagonist", "Femdom", "Fight", "futanari on male", "futunari", + "Gender Bender", "Genderswap", "Ghosts", "Girls Love", "Gore", "Gourmet", + "Gyaru", "Hentai", "Hunters", "Idol", "Incest", "Loli", "Lolicon", "Mafia", + "Magical Girls", "Mahou Shoujo", "Manga", "Manhua", "Manhwa", "Mature", + "Medieval Area", "Monster Girls", "Monsters", "Ninja", "Nobility", + "Office Romance", "Office Worker", "Office Workers", "One Shot", "Otome", + "Overpowered", "Philosophical", "playboy", "Police", "Post-Apocalyptic", + "Reincarnation", "Revenge", "Reverse Harem", "Royalty", "Samurai", "School", + "Seinin", "Shota", "Shotacon", "Shoujo Ai", "Shounen Ai", "Smut", "Superhero", + "Survival", "Suspense", "System", "Time Travel", "Traditional Games", "uncensored", + "Vampires", "Video Games", "Villainess", "Virtual Reality", "War", "Workplace", + "Wuxia", "Yaoi", "Yuri", "Zombies" + ) + } + + private val availableTags = GENRES + .map { MangaTag(it, it, source) } + .toSet() + override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = genreNames.map { name -> MangaTag(name, name, source) }.toSet() return MangaListFilterOptions( - availableTags = tags, + availableTags = availableTags, availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableDemographics = EnumSet.of(Demographic.JOSEI, Demographic.SEINEN, Demographic.SHOUJO, Demographic.SHOUNEN), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER, + ), + availableDemographics = EnumSet.of( + Demographic.JOSEI, + Demographic.SEINEN, + Demographic.SHOUJO, + Demographic.SHOUNEN, + ), + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.GERMAN, + Locale("id"), + Locale("pt", "BR"), + Locale("es"), + Locale("es", "419"), + Locale("th"), + Locale("vi"), + Locale("zh", "Hant"), + ) ) } override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - if (!filter.query.isNullOrBlank() || filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || - filter.states.isNotEmpty() || filter.types.isNotEmpty() || filter.demographics.isNotEmpty()) { - return getSearchPage(page, order, filter) - } - return getBrowsePage(page, order) - } - - private suspend fun getBrowsePage(page: Int, order: SortOrder): List<Manga> { - val sortParam = when (order) { - SortOrder.POPULARITY -> "views" - SortOrder.RATING -> "rating" - SortOrder.ALPHABETICAL -> "alphabetical" - SortOrder.RELEVANCE -> "tracked" - else -> null - } - val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/view-all/latest-updates.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) - if (page > 1) addQueryParameter("page", page.toString()) - sortParam?.let { addQueryParameter("sortBy", it) } - addQueryParameter("_routes", "pages/ViewAllPage") - }.build().toString() - return parseViewAllPage(url) + selectedLanguage = filter.locale?.let { localeToLang(it) } ?: "en" + return getSearchPage(page, order, filter) } private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val adultParam = if (config[showAdultKey]) "both" else "0" - val url = "$baseUrl/search.data".toHttpUrl().newBuilder().apply { - addQueryParameter("adult", adultParam) + val url = "$baseUrl/api/search".toHttpUrl().newBuilder().apply { if (!filter.query.isNullOrBlank()) addQueryParameter("search", filter.query!!) addQueryParameter("page", page.toString()) - addQueryParameter("perPage", "56") - + addQueryParameter("adult", adultParam) + selectedLanguage?.let { addQueryParameter("lang", it) } val (sortBy, sortOrder) = when (order) { SortOrder.UPDATED -> "latest" to "desc" SortOrder.ALPHABETICAL -> "alphabetical" to "asc" SortOrder.POPULARITY -> "views" to "desc" SortOrder.RATING -> "rating" to "desc" - else -> "relevance" to "desc" + SortOrder.RELEVANCE -> "tracked" to "desc" + SortOrder.ADDED -> "chapters" to "desc" + else -> "latest" to "desc" } addQueryParameter("sortBy", sortBy) addQueryParameter("sortOrder", sortOrder) @@ -175,35 +207,33 @@ internal class Mangadotnet(context: MangaLoaderContext) : } filter.demographics.forEach { demographic -> - addQueryParameter("genre", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) + addQueryParameter("genres", demographic.name.lowercase().replaceFirstChar { it.uppercase() }) } - filter.tags.forEach { tag -> addQueryParameter("genre", tag.key) } - filter.tagsExclude.forEach { tag -> addQueryParameter("genre", "-${tag.key}") } + filter.tags.forEach { tag -> + addQueryParameter("genres", tag.key) + } - addQueryParameter("_routes", "pages/SearchPage") + filter.tagsExclude.forEach { tag -> + addQueryParameter("excludeGenres", tag.key) + } }.build().toString() - val dataMap = fetchRscData(url, "pages/SearchPage") ?: return emptyList() - return parseMangaList(dataMap) - } + val json = webClient.httpGet(url).parseJson() + val items = json.optJSONArray("manga_list") + ?: json.optJSONArray("results") + ?: throw ParseException("Missing manga list in search response", url) - private suspend fun parseViewAllPage(url: String): List<Manga> { - val viewAllData = fetchRscData(url, "pages/ViewAllPage") ?: return emptyList() - val mangaListMap = viewAllData.asMap("data") ?: return emptyList() - return parseMangaList(mangaListMap) - } - - private fun parseMangaList(data: Map<String, Any?>): List<Manga> { - val list = data["manga_list"] as? List<*> ?: data["results"] as? List<*> ?: return emptyList() - return list.filterIsInstance<Map<String, Any?>>().map { parseMangaFromList(it) } + return (0 until items.length()).map { i -> + parseMangaFromList(items.getJSONObject(i)) + } } - private fun parseMangaFromList(data: Map<String, Any?>): Manga { - val id = (data["id"] as? Number)?.toInt()?.toString() ?: "" - val title = data["title"] as? String ?: "" - val photo = data["photo"] as? String - val coverUrl = photo?.let { + private fun parseMangaFromList(data: JSONObject): Manga { + val id = (data.optInt("id", data.optInt("manga_id", 0))).takeIf { it > 0 }?.toString() ?: "" + val title = data.optString("title") + val photo = data.optString("photo") + val coverUrl = photo?.takeIf { it.isNotBlank() }?.let { when { it.startsWith("/") -> "$baseUrl$it" it.startsWith("http") -> it @@ -226,19 +256,26 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + private var selectedLanguage: String? = null + private fun localeToLang(locale: Locale): String { + return when (locale.language) { + "pt" if locale.country == "BR" -> "pt-br" + "es" if locale.country == "419" -> "es-la" + "zh" if locale.country == "Hant" -> "zh-hk" + else -> locale.language + } } - @Suppress("UNCHECKED_CAST") - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val url = "$baseUrl/manga/${manga.url}.data?_routes=pages/MangaDetailPage" - val mangaData = fetchRscData(url, "pages/MangaDetailPage") ?: return manga + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") + + val mangaInfo = routeValue + .asMap("data") + ?.asMap("mangaData") + ?.asMap("manga") + ?: throw ParseException("Missing manga data in detail route", url) - val mangaInfo = mangaData.asMap("mangaData")?.asMap("manga") ?: return manga val title = mangaInfo["title"] as? String ?: manga.title val description = mangaInfo["description"] as? String val photo = mangaInfo["photo"] as? String @@ -249,6 +286,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : else -> null } } ?: manga.coverUrl + val bannerImage = mangaInfo["banner_image"] as? String val largeCoverUrl = bannerImage?.let { when { @@ -258,9 +296,19 @@ internal class Mangadotnet(context: MangaLoaderContext) : } } - val genres = (mangaInfo["genres"] as? List<*>)?.filterIsInstance<String>()?.mapNotNull { it.trim().ifBlank { null } }.orEmpty() - val altTitles = (mangaInfo["alt_titles"] as? List<*>)?.filterIsInstance<String>()?.mapNotNull { it.trim().ifBlank { null } }?.toSet().orEmpty() + val genres = (mangaInfo["genres"] as? List<*>) + ?.filterIsInstance<String>() + ?.mapNotNull { it.trim().ifBlank { null } } + .orEmpty() + + val altTitles = (mangaInfo["alt_titles"] as? List<*>) + ?.filterIsInstance<String>() + ?.mapNotNull { it.trim().ifBlank { null } } + ?.toSet() + .orEmpty() + val origin = mangaInfo["country_of_origin"] as? String + val authorRaw = mangaInfo["authors"] as? String val author = parseJsonArrayString(authorRaw)?.joinToString(", ") ?: authorRaw @@ -281,25 +329,25 @@ internal class Mangadotnet(context: MangaLoaderContext) : } else -> null } - val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { MangaTag(key = it, title = it, source = source) } + + val tagSet = genres.mapTo(LinkedHashSet(genres.size)) { + MangaTag(key = it, title = it, source = source) + } when (origin) { "JP" -> tagSet.add(MangaTag("Manga", "Manga", source)) "KR" -> tagSet.add(MangaTag("Manhwa", "Manhwa", source)) "CN" -> tagSet.add(MangaTag("Manhua", "Manhua", source)) } - val tagsArray = mangaInfo["tags"] as? List<Map<String, Any?>> - tagsArray?.flatMap { category -> - (category["tags"] as? List<Map<String, Any?>> ?: emptyList()) - .mapNotNull { it["name"] as? String } - }?.distinct()?.take(20)?.forEach { tagSet.add(MangaTag(it, it, source)) } - val rawRating = (mangaInfo["avg_rating"] as? Number)?.toDouble() ?: -1.0 val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - val chapters = fetchChapters(manga.url) + val chapterLang = selectedLanguage ?: "en" + val chaptersDeferred = async { + fetchChaptersAndVolumes(manga.url, chapterLang) + } - val result = manga.copy( + manga.copy( title = title, coverUrl = coverUrl, largeCoverUrl = largeCoverUrl, @@ -309,11 +357,8 @@ internal class Mangadotnet(context: MangaLoaderContext) : tags = tagSet, state = state, authors = setOfNotNull(author).filterTo(mutableSetOf()) { it.isNotBlank() }, - chapters = chapters, + chapters = chaptersDeferred.await(), ) - - detailsCache[manga.url] = result - return result } private fun parseJsonArrayString(raw: String?): List<String>? { @@ -324,8 +369,20 @@ internal class Mangadotnet(context: MangaLoaderContext) : } catch (_: Exception) { null } } - private suspend fun fetchChapters(mangaId: String): List<MangaChapter> { - val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=en").parseJsonArray() + private suspend fun fetchChaptersAndVolumes(mangaId: String, lang: String): List<MangaChapter> = coroutineScope { + val chaptersDeferred = async { fetchChaptersFromApi(mangaId, lang) } + val volumesDeferred = async { fetchVolumesFromApi(mangaId, lang) } + + val chapters = chaptersDeferred.await() + val volumes = volumesDeferred.await() + val builder = ChaptersListBuilder(chapters.size + volumes.size) + builder.addAll(chapters) + builder.addAll(volumes) + builder.toList() + } + + private suspend fun fetchChaptersFromApi(mangaId: String, lang: String): List<MangaChapter> { + val response = webClient.httpGet("$apiBase/manga/$mangaId/chapters/list?lang=$lang").parseJsonArray() val allChapters = (0 until response.length()).map { response.getJSONObject(it) } val chaptersByTeam = mutableMapOf<String, MutableList<JSONObject>>() @@ -339,7 +396,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : } val allNumbers = allChapters.map { it.optDouble("chapter_number", 0.0).toFloat() }.distinct().sorted() - val builder = ChaptersListBuilder(allChapters.size * chaptersByTeam.size) + val result = mutableListOf<MangaChapter>() for ((team, teamChapters) in chaptersByTeam) { val mapByNum = teamChapters.associateBy { it.optDouble("chapter_number", 0.0).toFloat() } @@ -361,7 +418,7 @@ internal class Mangadotnet(context: MangaLoaderContext) : put("id", chId) put("source", src) }.toString() - builder.add( + result.add( MangaChapter( id = generateUid("$team-$chId"), title = title, @@ -376,45 +433,42 @@ internal class Mangadotnet(context: MangaLoaderContext) : ) } } + return result + } + private suspend fun fetchVolumesFromApi(mangaId: String, lang: String): List<MangaChapter> { val volumesResponse = runCatching { - webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=en").parseJsonArray() - }.getOrNull() - - if (volumesResponse != null) { - val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } - .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } - - for (vol in volumes) { - if (vol.optString("language", "en") != "en") continue - val volId = vol.getInt("id") - val volNum = vol.optDouble("volume_number", 0.0).toFloat() - val volDate = vol.optString("date_added", "").nullIfEmpty() - val volSource = vol.optString("source", "user") - val volGroup = vol.optString("group_name", "").nullIfEmpty() - ?: vol.optString("scanlator_name", "").nullIfEmpty() - ?: "Unknown" - val volumeUrl = JSONObject().apply { - put("id", volId) - put("source", volSource) - }.toString() - builder.add( - MangaChapter( - id = generateUid("vol-$volId"), - title = "Volume ${volNum.toString().removeSuffix(".0")}", - number = volNum, - volume = 0, - url = volumeUrl, - uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = volGroup, - branch = "Volume", - ) - ) - } + webClient.httpGet("$apiBase/manga/$mangaId/volumes?lang=$lang").parseJsonArray() + }.getOrNull() ?: return emptyList() + + val volumes = (0 until volumesResponse.length()).map { volumesResponse.getJSONObject(it) } + .sortedBy { it.optDouble("volume_number", 0.0).toFloat() } + + return volumes.mapNotNull { vol -> + if (vol.optString("language", "en") != lang) return@mapNotNull null + val volId = vol.getInt("id") + val volNum = vol.optDouble("volume_number", 0.0).toFloat() + val volDate = vol.optString("date_added", "").nullIfEmpty() + val volSource = vol.optString("source", "user") + val volGroup = vol.optString("group_name", "").nullIfEmpty() + ?: vol.optString("scanlator_name", "").nullIfEmpty() + ?: "Unknown" + val volumeUrl = JSONObject().apply { + put("id", volId) + put("source", volSource) + }.toString() + MangaChapter( + id = generateUid("vol-$volId"), + title = "Volume ${volNum.toString().removeSuffix(".0")}", + number = volNum, + volume = 0, + url = volumeUrl, + uploadDate = volDate?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = volGroup, + branch = "Volume", + ) } - - return builder.toList() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { @@ -442,17 +496,24 @@ internal class Mangadotnet(context: MangaLoaderContext) : } } - @Suppress("UNCHECKED_CAST") override suspend fun getRelatedManga(seed: Manga): List<Manga> { val url = "$baseUrl/manga/${seed.url}.data?_routes=pages/MangaDetailPage" - val data = fetchRscData(url, "pages/MangaDetailPage") ?: return emptyList() + val routeValue = fetchRscRoute(url, "pages/MangaDetailPage") + + val dataMap = routeValue.asMap("data") + ?: throw ParseException("Missing 'data' in detail route", url) val related = mutableListOf<Manga>() - (data["suggestions"] as? List<*>)?.filterIsInstance<Map<String, Any?>>()?.mapTo(related) { parseMangaFromList(it) } - val relationsData = data.asMap("relationsData") + (dataMap["suggestions"] as? List<*>) + ?.filterIsInstance<Map<String, Any?>>() + ?.mapTo(related) { parseMangaFromList(JSONObject(it)) } + + val relationsData = dataMap.asMap("relationsData") val relations = relationsData?.asMap("relations") as? Map<String, List<Map<String, Any?>>> - relations?.values?.forEach { list -> list.mapTo(related) { parseMangaFromList(it) } } + relations?.values?.forEach { list -> + list.mapTo(related) { parseMangaFromList(JSONObject(it)) } + } return related } diff --git a/src/main/kotlin/tsuki/site/all/Pixiv.kt b/src/main/kotlin/tsuki/site/all/Pixiv.kt new file mode 100644 index 00000000..d06a7e53 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/Pixiv.kt @@ -0,0 +1,459 @@ +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONObject +import java.util.EnumSet + +private val KNOWN_LOCALES = listOf("en", "ja", "zh", "zh-tw", "ko") + +interface PixivTargetCompanion<out T : PixivTarget> { + val searchPrefix: String + fun fromSearchQuery(query: String): T? { + if (!query.startsWith(searchPrefix)) return null + val id = query.removePrefix(searchPrefix) + if (!id.matches(Regex("\\d+"))) return null + return fromSearchQueryId(id) + } + fun fromSearchQueryId(id: String): T? +} + +sealed class PixivTarget { + companion object { + val BASE_URI = "https://www.pixiv.net".toHttpUrl() + + fun fromSearchQuery(query: String) = sequenceOf(User, Series, Illustration) + .firstNotNullOfOrNull { it.fromSearchQuery(query) } + + fun fromUri(uri: String) = uri.toHttpUrlOrNull()?.let { fromUri(it) } + fun fromUri(uri: HttpUrl): PixivTarget? { + if (!(uri.scheme in listOf(null, "http", "https") && uri.host.let { "pixiv.net" == it.removePrefix("www.") }) + ) return null + var pathSegments = uri.pathSegments.ifEmpty { null } ?: return null + if (KNOWN_LOCALES.contains(pathSegments[0])) { + pathSegments = pathSegments.subList(1, pathSegments.size) + } + if (pathSegments.size < 2) return null + with(pathSegments[0]) { + return when { + equals("artworks") -> Illustration(pathSegments[1]) + equals("users") -> User(pathSegments[1]) + equals("user") && (pathSegments.size >= 4 && pathSegments[2] == "series") -> + Series(pathSegments[3], pathSegments[1]) + else -> null + } + } + } + } + + abstract fun toHttpUrl(): HttpUrl + abstract fun toSearchQuery(): String + + data class User(val userId: String) : PixivTarget() { + companion object : PixivTargetCompanion<User> { + override val searchPrefix = "user:" + override fun fromSearchQueryId(id: String) = User(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("users").addPathSegment(userId).build() + override fun toSearchQuery(): String = searchPrefix + userId + } + + data class Illustration(val illustId: String) : PixivTarget() { + companion object : PixivTargetCompanion<Illustration> { + override val searchPrefix = "aid:" + override fun fromSearchQueryId(id: String) = Illustration(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("artworks").addPathSegment(illustId).build() + override fun toSearchQuery(): String = searchPrefix + illustId + } + + data class Series(val seriesId: String, val authorUserId: String? = null) : PixivTarget() { + companion object : PixivTargetCompanion<Series> { + override val searchPrefix = "sid:" + override fun fromSearchQueryId(id: String) = Series(id) + } + override fun toHttpUrl() = BASE_URI.newBuilder() + .addPathSegment("user").addPathSegment(authorUserId ?: "") + .addPathSegment("series").addPathSegment(seriesId).build() + override fun toSearchQuery(): String = searchPrefix + seriesId + } +} + +@MangaSourceParser("PIXIV_EN", "Pixiv (English)", "en") +internal class PixivEn(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_EN) + +@MangaSourceParser("PIXIV_JA", "Pixiv (Japanese)", "ja") +internal class PixivJa(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_JA) + +@MangaSourceParser("PIXIV_ZH", "Pixiv (Chinese)", "zh") +internal class PixivZh(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_ZH) + +@MangaSourceParser("PIXIV_KO", "Pixiv (Korean)", "ko") +internal class PixivKo(context: MangaLoaderContext) : PixivParser(context, MangaParserSource.PIXIV_KO) + +abstract class PixivParser( + context: MangaLoaderContext, + source: MangaParserSource, +) : PagedMangaParser(context, source, 18) { + + override val configKeyDomain = ConfigKey.Domain("pixiv.net") + + private val baseUrl = "https://$domain" + + private val imageQualityKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + "thumb_mini" to "Thumb Mini", + "small" to "Small", + "regular" to "Regular", + "original" to "Original", + ), + defaultValue = "original", + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(imageQualityKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = + MangaListFilterOptions() + + private suspend fun apiGet(path: String): JSONObject { + val url = (baseUrl + path).toHttpUrlOrNull() + ?: throw IllegalArgumentException("Invalid URL: $path") + val response = webClient.httpGet(url.toString()) + val body = response.body?.string() ?: throw Exception("Empty response") + return JSONObject(body) + } + + private suspend fun fetchRanking(page: Int): List<Manga> { + val rankingJson = apiGet("/touch/ajax/ranking/illust?mode=daily&type=manga&page=$page") + val rankingArray = rankingJson.optJSONObject("body")?.optJSONArray("ranking") + ?: return emptyList() + val illustIds = mutableListOf<String>() + for (i in 0 until rankingArray.length()) { + val entry = rankingArray.getJSONObject(i) + entry.optString("illustId").takeIf { it.isNotEmpty() }?.let { illustIds.add(it) } + } + if (illustIds.isEmpty()) return emptyList() + val detailsUrl = "/touch/ajax/illust/details/many?" + + illustIds.joinToString("&") { "illust_ids[]=$it" } + val detailsJson = apiGet(detailsUrl) + val illustsArray = detailsJson.optJSONObject("body")?.optJSONArray("illust_details") + ?: return emptyList() + return (0 until illustsArray.length()).mapNotNull { i -> + illustsArray.getJSONObject(i).toMangaIllust() + } + } + + private suspend fun fetchLatest(page: Int): List<Manga> { + val json = apiGet("/touch/ajax/latest?type=manga&p=$page") + val illustsArray = json.optJSONObject("body")?.optJSONArray("illusts") + ?: return emptyList() + return (0 until illustsArray.length()).mapNotNull { i -> + illustsArray.getJSONObject(i).toMangaIllust() + } + } + + private suspend fun searchManga(query: String, page: Int): List<Manga> { + val target = PixivTarget.fromUri(query) ?: PixivTarget.fromSearchQuery(query) + when (target) { + is PixivTarget.Illustration -> { + val illust = getIllustCached(target.illustId) + return listOfNotNull(illust?.toMangaIllust()) + } + is PixivTarget.Series -> { + val series = fetchSeries(target.seriesId) + return listOfNotNull(series?.toMangaSeries()) + } + is PixivTarget.User -> { + val user = getUserCached(target.userId) + return listOf(user.toMangaUser(target.userId)) + } + else -> {} + } + + val word = query.trim() + val url = "/touch/ajax/search/illusts?word=${word.urlEncoded()}&s_mode=s_tc&p=$page" + val json = apiGet(url) + val illustsArray = json.optJSONObject("body")?.optJSONArray("illusts") + ?: return emptyList() + val results = mutableListOf<JSONObject>() + for (i in 0 until illustsArray.length()) { + val obj = illustsArray.getJSONObject(i) + if (obj.optInt("is_ad_container") == 1) continue + if (obj.optString("type") == "2") continue + results.add(obj) + } + return results.mapNotNull { it.toMangaIllust() } + } + + private fun String.urlEncoded(): String = java.net.URLEncoder.encode(this, "UTF-8") + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim() ?: "" + if (query.isNotBlank()) { + return searchManga(query, page) + } + return when (order) { + SortOrder.POPULARITY -> fetchRanking(page) + SortOrder.NEWEST -> fetchLatest(page) + else -> fetchLatest(page) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val target = PixivTarget.fromUri(baseUrl + manga.url) ?: return@coroutineScope manga + + val detailsDeferred = async { fetchMangaDetails(target, manga) } + val chaptersDeferred = async { fetchMangaChapters(target) } + + val details = detailsDeferred.await() + val chapters = chaptersDeferred.await() + + details.copy(chapters = chapters) + } + + private suspend fun fetchMangaDetails(target: PixivTarget, manga: Manga): Manga { + return when (target) { + is PixivTarget.User -> { + val user = getUserCached(target.userId) + manga.copy( + title = user.optString("name", manga.title), + authors = setOf(user.optString("name")), + description = user.optString("comment", manga.description ?: ""), + coverUrl = user.optString("imageBig", manga.coverUrl), + ) + } + is PixivTarget.Series -> { + val series = fetchSeries(target.seriesId) ?: return manga + val illusts = getSeriesIllustsCached(target.seriesId) ?: emptyList() + val first = illusts.firstOrNull() + val cover = series.optString("coverImage", null) + ?: first?.optString("url") + ?: manga.coverUrl + val tags = illusts.flatMap { obj -> + val arr = obj.optJSONArray("tags") ?: return@flatMap emptyList() + (0 until arr.length()).map { arr.getString(it) } + }.distinct().map { MangaTag(it, it, source) }.toSet() + manga.copy( + title = series.optString("title", manga.title), + authors = setOfNotNull(first?.optJSONObject("author_details")?.optString("user_name")), + description = series.optString("caption", manga.description ?: ""), + coverUrl = cover, + tags = tags, + ) + } + is PixivTarget.Illustration -> { + val illust = getIllustCached(target.illustId) ?: return manga + val tags = (0 until (illust.optJSONArray("tags")?.length() ?: 0)) + .map { illust.getJSONArray("tags").getString(it) } + .map { MangaTag(it, it, source) }.toSet() + manga.copy( + title = illust.optString("title", manga.title), + authors = setOfNotNull(illust.optJSONObject("author_details")?.optString("user_name")), + description = illust.optString("comment", manga.description ?: ""), + coverUrl = illust.optString("url", manga.coverUrl), + tags = tags, + ) + } + } + } + + private suspend fun fetchMangaChapters(target: PixivTarget): List<MangaChapter> { + val illusts = when (target) { + is PixivTarget.User -> { + val json = apiGet("/touch/ajax/user/illusts?id=${target.userId}&p=1") + val arr = json.optJSONObject("body")?.optJSONArray("illusts") ?: return emptyList() + (0 until arr.length()).map { arr.getJSONObject(it) } + } + is PixivTarget.Series -> getSeriesIllustsCached(target.seriesId) ?: emptyList() + is PixivTarget.Illustration -> listOfNotNull(getIllustCached(target.illustId)) + } + return illusts.mapIndexed { i, obj -> + val id = obj.optString("id", i.toString()) + MangaChapter( + id = generateUid(id), + title = obj.optString("title", "(null)"), + number = (illusts.size - i).toFloat(), + volume = 0, + url = "/artworks/$id", + scanlator = null, + uploadDate = obj.optLongOrNull("upload_timestamp")?.let { it * 1000 } ?: 0, + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val illustId = chapter.url.substringAfterLast('/') + val json = apiGet("/ajax/illust/$illustId/pages") + val pagesArray = json.optJSONArray("body") ?: return emptyList() + val quality = config[imageQualityKey] ?: "original" + return (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + val urls = pageObj.optJSONObject("urls") ?: JSONObject() + val imageUrl = getImageUrl(urls, quality) + MangaPage( + id = generateUid("${chapter.id}-$i"), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private fun getImageUrl(urls: JSONObject, quality: String): String { + val sizeOrder = listOf("thumb_mini", "small", "regular", "original") + val startIndex = sizeOrder.indexOf(quality).takeIf { it >= 0 } ?: sizeOrder.lastIndex + return sizeOrder.drop(startIndex).firstNotNullOf { size -> + urls.optString(size).takeIf { it.isNotEmpty() } + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getIllustCached(illustId: String): JSONObject? { + val json = apiGet("/touch/ajax/illust/details?illust_id=$illustId") + return json.optJSONObject("body")?.optJSONObject("illust_details") + } + + private suspend fun getUserCached(userId: String): JSONObject { + val json = apiGet("/ajax/user/$userId?full=1") + return json.optJSONObject("body") ?: json + } + + private suspend fun getSeriesIllustsCached(seriesId: String): List<JSONObject>? { + val result = mutableListOf<JSONObject>() + var lastOrder = 0 + while (true) { + val json = apiGet("/touch/ajax/illust/series_content/$seriesId?last_order=$lastOrder") + val arr = json.optJSONObject("body")?.optJSONArray("series_contents") ?: break + if (arr.length() == 0) break + for (i in 0 until arr.length()) { + result.add(arr.getJSONObject(i)) + } + lastOrder += arr.length() + } + return result.takeIf { it.isNotEmpty() } + } + + private suspend fun fetchSeries(seriesId: String): JSONObject? { + val json = apiGet("/touch/ajax/illust/series/$seriesId") + return json.optJSONObject("body")?.optJSONObject("series") + } + + private fun JSONObject.optLongOrNull(key: String): Long? = + try { optLong(key) } catch (_: Exception) { null } + + private fun JSONObject.toMangaIllust(): Manga? { + val id = optString("id").takeIf { it.isNotEmpty() } ?: return null + val series = optJSONObject("series") + val url = if (series != null) { + val userId = series.optString("userId", optJSONObject("author_details")?.optString("user_id") ?: "") + "/user/$userId/series/${series.optString("id")}" + } else { + "/artworks/$id" + } + val tags = optJSONArray("tags")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) }.map { MangaTag(it, it, source) }.toSet() + } ?: emptySet() + val authors = setOfNotNull(optJSONObject("author_details")?.optString("user_name")) + return Manga( + id = generateUid(url), + title = optString("title", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = if (optString("x_restrict") == "1") ContentRating.ADULT else ContentRating.SAFE, + coverUrl = optString("url", ""), + tags = tags, + state = null, + authors = authors, + largeCoverUrl = null, + description = optString("comment"), + source = source, + ) + } + + private fun JSONObject.toMangaSeries(): Manga? { + val id = optString("id").takeIf { it.isNotEmpty() } ?: return null + val userId = optString("userId").takeIf { it.isNotEmpty() } ?: return null + val url = "/user/$userId/series/$id" + return Manga( + id = generateUid(url), + title = optString("title", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + coverUrl = optString("coverImage", ""), + tags = emptySet(), + state = null, + authors = emptySet(), + largeCoverUrl = null, + description = optString("caption"), + source = source, + ) + } + + private fun JSONObject.toMangaUser(userId: String): Manga { + val url = "/users/$userId" + return Manga( + id = generateUid(url), + title = optString("name", "(null)"), + altTitles = emptySet(), + url = url, + publicUrl = baseUrl + url, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + coverUrl = optString("imageBig", ""), + tags = emptySet(), + state = null, + authors = setOf(optString("name")), + largeCoverUrl = null, + description = optString("comment"), + source = source, + ) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt rename to src/main/kotlin/tsuki/site/all/ProjectSuki.kt index 08fd7947..c4d4f8a5 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/ProjectSuki.kt +++ b/src/main/kotlin/tsuki/site/all/ProjectSuki.kt @@ -1,46 +1,45 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl -import android.util.LruCache import okhttp3.Headers import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONObject import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import java.util.EnumSet -import java.util.LinkedHashMap -import java.util.LinkedHashSet -import java.util.Locale import java.text.SimpleDateFormat import java.util.Calendar +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("PROJECTSUKI", "ProjectSuki") internal class ProjectSuki(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.PROJECTSUKI, pageSize = 30) { override val configKeyDomain = ConfigKey.Domain("projectsuki.com") - private val pagesCache = LruCache<String, List<MangaPage>>(50) override val defaultSortOrder: SortOrder = SortOrder.UPDATED override val availableSortOrders: Set<SortOrder> = EnumSet.of( @@ -397,9 +396,6 @@ internal class ProjectSuki(context: MangaLoaderContext) : } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val cached = pagesCache.get(chapter.url) - if (cached != null) return cached - val parts = chapter.url.toChapterParts() ?: return emptyList() val payload = JSONObject() @@ -440,11 +436,7 @@ internal class ProjectSuki(context: MangaLoaderContext) : ) ) } - - val result = pages.toList() - pagesCache.put(chapter.url, result) - - return result + return pages } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt b/src/main/kotlin/tsuki/site/all/Webtoons.kt similarity index 62% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt rename to src/main/kotlin/tsuki/site/all/Webtoons.kt index 500ae46c..9b906000 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/Webtoons.kt +++ b/src/main/kotlin/tsuki/site/all/Webtoons.kt @@ -1,19 +1,41 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all +package tsuki.site.all + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded -import androidx.collection.arraySetOf import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull +import org.jsoup.parser.Parser +import java.net.SocketException +import java.text.DecimalFormat +import java.util.Calendar import java.util.EnumSet +import kotlin.math.max private const val ID_SEARCH_PREFIX = "id:" @@ -23,7 +45,7 @@ private const val canvas = "canvas" internal abstract class WebtoonsParser( context: MangaLoaderContext, source: MangaParserSource, -) : AbstractMangaParser(context, source) { +) : PagedMangaParser(context, source, 20) { override val configKeyDomain = ConfigKey.Domain("webtoons.com") @@ -38,33 +60,26 @@ internal abstract class WebtoonsParser( defaultValue = webtoons, ) + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(contentKey) + } + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( SortOrder.POPULARITY, - SortOrder.RATING, + SortOrder.RELEVANCE, SortOrder.UPDATED, + SortOrder.RATING, ) override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36") + get() = MangaListFilterCapabilities(isSearchSupported = true) - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), + override val userAgentKey = ConfigKey.UserAgent( + "Mozilla/5.0 (Linux; Android 12; SM-G991B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.120 Mobile Safari/537.36" ) - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(contentKey) - } - - override suspend fun getPageUrl(page: MangaPage): String { - return page.url.toAbsoluteUrl(staticDomain) - } - private val languageCode: String get() = when (val tag = sourceLocale.toLanguageTag()) { "in" -> "id" @@ -72,6 +87,42 @@ internal abstract class WebtoonsParser( else -> tag } + private val localeForCookie: String + get() = if (languageCode == "zh-hant") "zh_TW" else languageCode + + private val customClient by lazy { + context.httpClient.newBuilder() + .addInterceptor { chain -> + val original = chain.request() + val url = original.url + if (url.host.endsWith("webtoons.com")) { + val newRequest = original.newBuilder() + .header("Cookie", "ageGatePass=true; locale=$localeForCookie; needGDPR=false") + .build() + try { + chain.proceed(newRequest) + } catch (_: SocketException) { + chain.proceed(newRequest) + } + } else { + chain.proceed(original) + } + } + .build() + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + + override val webClient by lazy { + tsuki.network.OkHttpWebClient(customClient, source) + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + private suspend fun fetchEpisodes(titleNo: Long, type: String): List<MangaChapter> { val baseUrl = "https://$mobileApiDomain/api/v1/$type/$titleNo/episodes" val url = if (type == "canvas") { @@ -80,27 +131,72 @@ internal abstract class WebtoonsParser( "$baseUrl?pageSize=99999" } val json = webClient.httpGet(url).parseJson() - val episodeList = json.optJSONObject("result")?.optJSONArray("episodeList") ?: throw ParseException("No episodes found for title $titleNo", url) - return episodeList.mapChapters { _, jo -> - val episodeTitle = jo.getStringOrNull("episodeTitle") ?: "" - val episodeNo = jo.getInt("episodeNo") + val numberFormatter = DecimalFormat("#.##") + data class RawEpisode( + val jo: JSONObject, + var chapterNumber: Float = -1f, + var seasonNumber: Int = 1, + ) + + val rawList = (0 until episodeList.length()).map { i -> + RawEpisode(episodeList.getJSONObject(i)) + } + + for (raw in rawList) { + val title = raw.jo.optString("episodeTitle") ?: "" + val match = episodeNoRegex.find(title)?.groupValues + if (match != null && match.getOrNull(6).isNullOrBlank()) { + raw.chapterNumber = match.getOrNull(11)?.toFloatOrNull() ?: -1f + raw.seasonNumber = match.getOrNull(4)?.takeIf(String::isNotBlank)?.toInt() ?: 1 + } + } + + var maxChapterNumber = 0f + var currentSeason = 1 + var seasonOffset = 0f + + for ((idx, raw) in rawList.withIndex()) { + if (raw.chapterNumber != -1f) { + val original = raw.chapterNumber + if (raw.seasonNumber > currentSeason) { + currentSeason = raw.seasonNumber + if (original <= maxChapterNumber) { + seasonOffset = maxChapterNumber + } + } + raw.chapterNumber = seasonOffset + original + maxChapterNumber = max(maxChapterNumber, raw.chapterNumber) + } else { + val prev = rawList.getOrNull(idx - 1) + if (prev != null) { + raw.chapterNumber = prev.chapterNumber + 0.01f + } else { + raw.chapterNumber = 0f + } + } + } + + return rawList.mapIndexed { idx, raw -> + val jo = raw.jo + val episodeTitle = jo.optString("episodeTitle", "") val viewerLink = jo.getString("viewerLink") val hasBgm = jo.optBoolean("hasBgm", false) + val chapterNumber = raw.chapterNumber MangaChapter( - id = generateUid("$titleNo-$episodeNo"), + id = generateUid("$titleNo-$idx"), title = buildString { - append(org.jsoup.parser.Parser.unescapeEntities(episodeTitle, false)) - append(" (ch. $episodeNo)") + append(Parser.unescapeEntities(episodeTitle, false)) + append(" (ch. ", numberFormatter.format(chapterNumber), ")") if (hasBgm) append(" ♫") }, - number = episodeNo.toFloat(), + number = chapterNumber, volume = 0, url = viewerLink, - uploadDate = jo.getLong("exposureDateMillis"), + uploadDate = jo.optLong("exposureDateMillis"), branch = null, scanlator = null, source = source, @@ -108,6 +204,11 @@ internal abstract class WebtoonsParser( }.sortedBy { it.number } } + private val episodeNoRegex = Regex( + """(?:(s(eason)?|saison|part|vol(ume)?)\s*\.?\s*(\d+).*?)?(.*?(mini|bonus|special).*?)?(e(p(isode)?)?|ch(apter)?)\s*\.?\s*(\d+(\.\d+)?)""", + RegexOption.IGNORE_CASE, + ) + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val titleNo = manga.url.toLong() val detailsUrl = manga.publicUrl.ifBlank { @@ -176,113 +277,19 @@ internal abstract class WebtoonsParser( ) } - private fun getSortOrderParam(order: SortOrder): String { - return when (order) { - SortOrder.POPULARITY -> "MANA" - SortOrder.RATING -> "LIKEIT" - SortOrder.UPDATED -> "UPDATE" - else -> "MANA" - } - } - - private fun availableTags() = arraySetOf( - MangaTag("Action", "action", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Drama", "drama", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Horror", "horror", source), - MangaTag("Romance", "romance", source), - MangaTag("Sci-Fi", "sf", source), - MangaTag("Slice of Life", "slice_of_life", source), - MangaTag("Sports", "sports", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Historical", "historical", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Superhero", "super_hero", source), - MangaTag("Heartwarming", "heartwarming", source), - MangaTag("Graphic Novel", "graphic_novel", source), - MangaTag("Informative", "tiptoon", source), - ) - - private val genreUrlMap: Map<String, String> = availableTags().associate { - it.title.lowercase() to it.key - } - - override suspend fun getList(offset: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val query = filter.query.orEmpty() val currentContent = config[contentKey] ?: webtoons - var directTitleNo: Long? = null - var directType: String? = null - - when { - query.startsWith("https://") -> { - val url = query.toHttpUrl() - if (!url.host.endsWith("webtoons.com")) { - throw Exception("Unsupported domain: ${url.host}. Only webtoons.com and m.webtoons.com are allowed.") - } - directTitleNo = url.queryParameter("title_no")?.toLongOrNull() - ?: throw Exception("Missing 'title_no' parameter in URL") - val path = url.pathSegments.filter(String::isNotEmpty) - if (path.size < 2) throw Exception("Invalid URL path") - if (path[0] != languageCode) { - throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") - } - directType = if (path.getOrNull(1) == "canvas") canvas else webtoons - } - - query.startsWith(ID_SEARCH_PREFIX) -> { - val idPart = query.removePrefix(ID_SEARCH_PREFIX) - val parts = idPart.split(":") - val (type, lang, titleNo) = when (parts.size) { - 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[0]}")) - 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[1]}")) - 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() - ?: throw Exception("Invalid title number: ${parts[2]}")) - else -> throw Exception("Invalid id search format. Use id:type:lang:titleNo, id:type:titleNo, or id:titleNo") - } - if (lang != languageCode) return emptyList() - if (type !in listOf(webtoons, canvas)) { - throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") - } - directTitleNo = titleNo - directType = type - } - - query.toLongOrNull() != null -> { - directTitleNo = query.toLong() - directType = currentContent - } + if (query.startsWith("https://") || query.startsWith(ID_SEARCH_PREFIX)) { + return handleDirectSearch(query, currentContent) } - if (directTitleNo != null && directType != null) { - val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" - val manga = getDetails(Manga( - id = generateUid(directTitleNo), - title = "", - altTitles = emptySet(), - url = directTitleNo.toString(), - publicUrl = detailsUrl, - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = null, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - )) - return listOf(manga) - } - - val page = offset / 20 + 1 - val inPageOffset = offset % 20 val selectedGenre = filter.tags.firstOrNull() val url = buildString { append("https://$domain/$languageCode") + when { query.isNotBlank() -> { append("/search") @@ -307,35 +314,63 @@ internal abstract class WebtoonsParser( } if (searchType != null) append("/$searchType") append("?keyword=${cleanQuery.urlEncoded()}") - if (page > 1) append("&page=$page") + if (page > 1 && searchType != null) append("&page=$page") } + selectedGenre != null -> { val genreUrlPath = genreUrlMap[selectedGenre.key] ?: selectedGenre.key append("/genres/$genreUrlPath") append("?sortOrder=${getSortOrderParam(order)}") if (page > 1) append("&page=$page") } + else -> { - val rankingType = when { - currentContent == canvas -> "canvas" - else -> when (order) { - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "trending" - SortOrder.UPDATED -> "originals" - else -> "popular" + if (currentContent == canvas) { + append("/ranking/canvas") + if (page > 1) append("?page=$page") + } else { + when (order) { + SortOrder.POPULARITY -> { + append("/ranking/popular") + if (page > 1) append("?page=$page") + } + SortOrder.RATING -> { + append("/ranking/trending") + if (page > 1) append("?page=$page") + } + SortOrder.RELEVANCE -> { + append("/ranking/originals") + if (page > 1) append("?page=$page") + } + SortOrder.UPDATED -> { + val day = Calendar.getInstance().get(Calendar.DAY_OF_WEEK) + val dayName = when (day) { + Calendar.MONDAY -> "monday" + Calendar.TUESDAY -> "tuesday" + Calendar.WEDNESDAY -> "wednesday" + Calendar.THURSDAY -> "thursday" + Calendar.FRIDAY -> "friday" + Calendar.SATURDAY -> "saturday" + Calendar.SUNDAY -> "sunday" + else -> "sunday" + } + append("/originals/$dayName?sortOrder=UPDATE") + } + else -> throw Exception("Unknown sort order") } } - append("/ranking/$rankingType") - if (page > 1) append("?page=$page") } } } val doc = webClient.httpGet(url).parseHtml() - val items = doc.select(".webtoon_list li a, .card_wrap .card_item a") + return doc.select(".webtoon_list li a, .card_wrap .card_item a") .map { element -> createMangaFromElement(element, source as MangaParserSource, selectedGenre) } + } - return items.drop(inPageOffset).take(20) + private fun extractTitleNoFromUrl(url: String): Long { + return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() + ?: throw ParseException("Could not extract title_no from URL: $url", url) } private fun createMangaFromElement( @@ -366,14 +401,80 @@ internal abstract class WebtoonsParser( ) } - private fun extractTitleNoFromUrl(url: String): Long { - return Regex("title_no=(\\d+)").find(url)?.groupValues?.get(1)?.toLong() - ?: throw ParseException("Could not extract title_no from URL: $url", url) + private fun getSortOrderParam(order: SortOrder): String = when (order) { + SortOrder.POPULARITY -> "MANA" + SortOrder.RELEVANCE -> "LIKEIT" + SortOrder.RATING -> "LIKEIT" + SortOrder.UPDATED -> "UPDATE" + else -> "MANA" + } + + private suspend fun handleDirectSearch(query: String, currentContent: String): List<Manga> { + var directTitleNo: Long? = null + var directType: String? = null + + when { + query.startsWith("https://") -> { + val url = query.toHttpUrl() + if (!url.host.endsWith("webtoons.com")) throw Exception("Unsupported domain: ${url.host}") + directTitleNo = url.queryParameter("title_no")?.toLongOrNull() + ?: throw Exception("Missing 'title_no' parameter in URL") + val path = url.pathSegments.filter(String::isNotEmpty) + if (path.size < 2) throw Exception("Invalid URL path") + if (path[0] != languageCode) { + throw Exception("URL language (${path[0]}) does not match parser language ($languageCode)") + } + directType = if (path.getOrNull(1) == "canvas") "canvas" else "webtoon" + } + query.startsWith(ID_SEARCH_PREFIX) -> { + val idPart = query.removePrefix(ID_SEARCH_PREFIX) + val parts = idPart.split(":") + val (type, lang, titleNo) = when (parts.size) { + 1 -> Triple(currentContent, languageCode, parts[0].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[0]}")) + 2 -> Triple(parts[0], languageCode, parts[1].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[1]}")) + 3 -> Triple(parts[0], parts[1], parts[2].toLongOrNull() + ?: throw Exception("Invalid title number: ${parts[2]}")) + else -> throw Exception("Invalid id search format") + } + if (lang != languageCode) return emptyList() + if (type !in listOf("webtoons", "canvas")) throw Exception("Unknown type '$type'. Use 'webtoons' or 'canvas'.") + directTitleNo = titleNo + directType = type + } + query.toLongOrNull() != null -> { + directTitleNo = query.toLong() + directType = currentContent + } + } + + if (directTitleNo != null && directType != null) { + val detailsUrl = "https://$domain/$languageCode/$directType/placeholder/list?title_no=$directTitleNo" + val manga = getDetails( + Manga( + id = generateUid(directTitleNo), + title = "", + altTitles = emptySet(), + url = directTitleNo.toString(), + publicUrl = detailsUrl, + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + return listOf(manga) + } + return emptyList() } override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val absUrl = chapter.url.toAbsoluteUrl(domain) val doc = try { - val absUrl = chapter.url.toAbsoluteUrl(domain) webClient.httpGet(absUrl).parseHtml() } catch (e: Exception) { throw ParseException("Failed to get pages for chapter: ${chapter.title}", chapter.url, e) @@ -394,14 +495,10 @@ internal abstract class WebtoonsParser( if (images.isEmpty()) { val motionPages = fetchMotionToonPages(doc) - if (motionPages.isNotEmpty()) { - images.addAll(motionPages) - } + if (motionPages.isNotEmpty()) images.addAll(motionPages) } - if (images.isEmpty()) { - throw ParseException("No images found in chapter.", chapter.url) - } + if (images.isEmpty()) throw ParseException("No images found in chapter.", chapter.url) return images } @@ -432,24 +529,48 @@ internal abstract class WebtoonsParser( } } - @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", type = ContentType.MANGA) + private fun availableTags() = setOf( + MangaTag("Action", "action", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Drama", "drama", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Horror", "horror", source), + MangaTag("Romance", "romance", source), + MangaTag("Sci-Fi", "sf", source), + MangaTag("Slice of Life", "slice_of_life", source), + MangaTag("Sports", "sports", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Historical", "historical", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Superhero", "super_hero", source), + MangaTag("Heartwarming", "heartwarming", source), + MangaTag("Graphic Novel", "graphic_novel", source), + MangaTag("Informative", "tiptoon", source), + ) + + private val genreUrlMap: Map<String, String> = availableTags().associate { + it.title.lowercase() to it.key + } + + @MangaSourceParser("WEBTOONS_EN", "Webtoons (English)", "en", ContentType.MANGA) class English(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_EN) - @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ID", "Webtoons (Indonesia)", "id", ContentType.MANGA) class Indonesian(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ID) - @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ES", "Webtoons (Spanish)", "es",ContentType.MANGA) class Spanish(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ES) - @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_FR", "Webtoons (French)", "fr", ContentType.MANGA) class French(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_FR) - @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_TH", "Webtoons (Thai)", "th", ContentType.MANGA) class Thai(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_TH) - @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_ZH", "Webtoons (Chinese)", "zh", ContentType.MANGA) class Chinese(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_ZH) - @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", type = ContentType.MANGA) + @MangaSourceParser("WEBTOONS_DE", "Webtoons (German)", "de", ContentType.MANGA) class German(context: MangaLoaderContext) : WebtoonsParser(context, MangaParserSource.WEBTOONS_DE) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt similarity index 67% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt rename to src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt index 5f2ed071..2be0d833 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/AsmHentai.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/AsmHentai.kt @@ -1,15 +1,17 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.removeSuffix -import java.util.* +import java.util.Locale @MangaSourceParser("ASMHENTAI", "AsmHentai", type = ContentType.HENTAI) internal class AsmHentai(context: MangaLoaderContext) : diff --git a/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt b/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt new file mode 100644 index 00000000..3d7d8e6a --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/BuonDua.kt @@ -0,0 +1,19 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.GalleryParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("BUONDUA", "Buon Dua", type = ContentType.OTHER) +internal class BuonDua(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.BUONDUA, "buondua.com") { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain( + "buondua.com", + "buondua.us", + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt new file mode 100644 index 00000000..2de9a82c --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/Danbooru.kt @@ -0,0 +1,224 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("DANBOORU", "Danbooru", type = ContentType.HENTAI) +internal class Danbooru(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DANBOORU, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("danbooru.donmai.us") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient: WebClient by lazy { + OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(this) + .build(), + source + ) + } + + override fun getRequestHeaders(): Headers { + return super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, // "updated_at" + SortOrder.ALPHABETICAL, // "name" + SortOrder.NEWEST, // "created_at" + SortOrder.POPULARITY, // "post_count" + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.startsWith("http://") || query.startsWith("https://")) { + val url = query.toHttpUrlOrNull() + if (url != null && url.host == domain.toHttpUrl().host && url.pathSegments.size >= 2 && url.pathSegments[0] == "pools") { + val poolId = url.pathSegments[1] + return listOf( + Manga( + id = generateUid(poolId), + url = "/pools/$poolId", + publicUrl = "https://$domain/pools/$poolId", + title = poolId, + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + } + } + + val sortParam = when (order) { + SortOrder.UPDATED -> "updated_at" + SortOrder.ALPHABETICAL -> "name" + SortOrder.NEWEST -> "created_at" + SortOrder.POPULARITY -> "post_count" + else -> "updated_at" + } + + val url = "https://$domain/pools/gallery".toHttpUrl().newBuilder() + .addQueryParameter("search[category]", "series") + .addQueryParameter("search[order]", sortParam) + .addQueryParameter("page", page.toString()) + .apply { + if (query.isNotBlank()) { + addQueryParameter("search[name_contains]", query) + } + } + .build() + + val doc = webClient.httpGet(url).parseHtml() + val entries = doc.select("article.post-preview") + return entries.mapNotNull { element -> + val link = element.selectFirst(".post-preview-link") ?: return@mapNotNull null + val href = link.attr("href") + val title = element.selectFirst("div.text-center")?.text() ?: return@mapNotNull null + val cover = element.selectFirst("source")?.attr("srcset") + ?.substringAfterLast(',')?.trim() + ?.substringBeforeLast(' ')?.trimStart() + Manga( + id = generateUid(href), + url = href, + publicUrl = "https://$domain$href", + title = title, + altTitles = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.publicUrl + val doc = webClient.httpGet(fullUrl).parseHtml() + val title = doc.selectFirst(".pool-category-series, .pool-category-collection")?.text() + ?: doc.selectFirst("h1")?.text() ?: manga.title + val description = doc.getElementById("description")?.wholeText() + val author = doc.selectFirst("#description a[href*=artists]")?.ownText() + val artists = setOfNotNull(author) + + val poolJson = webClient.httpGet("$fullUrl.json").parseJson() + val postIds = poolJson.getJSONArray("post_ids") + val updatedAt = poolJson.optString("updated_at", null) + + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until postIds.length()) { + val postId = postIds.getInt(i) + chapters.add( + MangaChapter( + id = generateUid("/posts/$postId"), + title = "Post ${i + 1}", + number = (i + 1).toFloat(), + volume = 0, + url = "/posts/$postId", + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + ) + } + if (chapters.isNotEmpty() && updatedAt != null) { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT) + val date = dateFormat.parseSafe(updatedAt) + chapters[0] = chapters[0].copy(uploadDate = date) + } + + return manga.copy( + title = title, + description = description, + authors = artists, + chapters = chapters.sortedBy { it.number }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val postId = chapter.url.substringAfterLast("/") + val json = webClient.httpGet("https://$domain/posts/$postId.json").parseJson() + val imageUrl = json.optString("file_url", null) + ?: json.optString("large_file_url", null) + ?: json.optString("preview_file_url", null) + ?: throw Exception("Image URL not found for post $postId") + val fullImageUrl = if (imageUrl.startsWith("http")) imageUrl else "https://$domain$imageUrl" + return listOf( + MangaPage( + id = generateUid(fullImageUrl), + url = fullImageUrl, + preview = null, + source = source, + ) + ) + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + if (request.url.host == "cdn.donmai.us") { + val newRequest = request.newBuilder() + .removeHeader("Cookie") + .header("Accept", "image/avif,image/webp,image/png,image/svg+xml,image/*;q=0.8,*/*;q=0.5") + .header("Sec-Fetch-Dest", "image") + .header("Sec-Fetch-Mode", "no-cors") + .header("Sec-Fetch-Site", "same-site") + .build() + return chain.proceed(newRequest) + } + return chain.proceed(request) + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } +} diff --git a/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt new file mode 100644 index 00000000..aa26836c --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/EXHentai.kt @@ -0,0 +1,537 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.bitmap.Rect +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.AuthRequiredException +import tsuki.exception.TooManyRequestExceptions + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.ChaptersListBuilder +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.copyCookies +import tsuki.util.cssUrl +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.headersContentLength +import tsuki.util.insertCookies +import tsuki.util.isNumeric +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.ownTextOrNull +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.selectFirstOrThrow +import tsuki.util.styleValueOrNull +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder + +import androidx.collection.ArraySet +import androidx.collection.MutableIntLongMap +import androidx.collection.MutableIntObjectMap +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Collections.emptyList +import java.util.EnumSet +import java.util.Locale +import java.util.concurrent.TimeUnit +import kotlin.collections.orEmpty + +private const val DOMAIN_UNAUTHORIZED = "e-hentai.org" +private const val DOMAIN_AUTHORIZED = "exhentai.org" +private val TAG_PREFIXES = arrayOf("male:", "female:", "other:") +private const val BANNED_RESPONSE_LENGTH = 256L + +@MangaSourceParser("EXHENTAI", "EXHentai / Ehentai", type = ContentType.HENTAI) +internal class EXHentai(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.EXHENTAI, pageSize = 25), MangaParserAuthProvider, Interceptor { + + override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) + + override val configKeyDomain: ConfigKey.Domain + get() { + val isAuthorized = checkAuth() + return ConfigKey.Domain( + if (isAuthorized) DOMAIN_AUTHORIZED else DOMAIN_UNAUTHORIZED, + if (isAuthorized) DOMAIN_UNAUTHORIZED else DOMAIN_AUTHORIZED, + ) + } + + override val authUrl: String + get() = "https://${domain}/bounce_login.php" + + private val ratingPattern = Regex("-?[0-9]+px") + private val titleCleanupPattern = Regex("(\\[.*?]|\\([C0-9]*\\))") + private val spacesCleanupPattern = Regex("(^\\s+|\\s+$|\\s+(?=\\s))") + private val authCookies = arrayOf("ipb_member_id", "ipb_pass_hash") + private val suspiciousContentKey = ConfigKey.ShowSuspiciousContent(false) + private val nextPages = MutableIntObjectMap<MutableIntLongMap>() + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun isAuthorized(): Boolean = checkAuth() + + init { + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "nw=1", "sl=dm_2") + context.cookieJar.insertCookies(DOMAIN_UNAUTHORIZED, "nw=1", "sl=dm_2") + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = mapTags(), + availableContentTypes = EnumSet.of( + ContentType.DOUJINSHI, + ContentType.MANGA, + ContentType.ARTIST_CG, + ContentType.GAME_CG, + ContentType.COMICS, + ContentType.IMAGE_SET, + ContentType.OTHER, + ), + availableLocales = setOf( + Locale.JAPANESE, + Locale.ENGLISH, + Locale.CHINESE, + Locale("nl"), + Locale.FRENCH, + Locale.GERMAN, + Locale("hu"), + Locale.ITALIAN, + Locale("kr"), + Locale("pl"), + Locale("pt"), + Locale("ru"), + Locale("es"), + Locale("th"), + Locale("vi"), + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + return getListPage(page, order, filter, updateDm = false) + } + + private suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + updateDm: Boolean, + ): List<Manga> { + val next = synchronized(nextPages) { + nextPages[filter.hashCode()]?.getOrDefault(page, 0L) ?: 0L + } + + if (page > 0 && next == 0L) { + assert(false) { "Page timestamp not found" } + return emptyList() + } + + val url = urlBuilder() + url.addEncodedQueryParameter("next", next.toString()) + url.addQueryParameter("f_search", filter.toSearchQuery()) + + val fCats = filter.types.toFCats() + if (fCats != 0) { + url.addEncodedQueryParameter("f_cats", (1023 - fCats).toString()) + } + if (updateDm) { + // by unknown reason cookie "sl=dm_2" is ignored, so, we should request it again + url.addQueryParameter("inline_set", "dm_e") + } + url.addQueryParameter("advsearch", "1") + if (config[suspiciousContentKey]) { + url.addQueryParameter("f_sh", "on") + } + val body = webClient.httpGet(url.build()).parseHtml().body() + val root = body.selectFirst("table.itg")?.selectFirst("tbody") ?: if (updateDm) { + if (body.getElementsContainingText("No hits found").isNotEmpty()) { + return emptyList() + } else { + body.parseFailed("Cannot find root") + } + } else { + return getListPage(page, order, filter, updateDm = true) + } + val nextTimestamp = getNextTimestamp(body) + synchronized(nextPages) { + nextPages.getOrPut(filter.hashCode()) { + MutableIntLongMap() + }.put(page + 1, nextTimestamp) + } + + return root.children().mapNotNull { tr -> + if (tr.childrenSize() != 2) return@mapNotNull null + val (td1, td2) = tr.children() + val gLink = td2.selectFirstOrThrow("div.glink") + val a = gLink.parents().select("a").first() ?: gLink.parseFailed("link not found") + val href = a.attrAsRelativeUrl("href") + val tagsDiv = gLink.nextElementSibling() ?: gLink.parseFailed("tags div not found") + val rawTitle = gLink.text() + val author = tagsDiv.getElementsContainingOwnText("artist:").first() + ?.nextElementSibling()?.textOrNull() + Manga( + id = generateUid(href), + title = rawTitle.cleanupTitle(), + altTitles = emptySet(), + url = href, + publicUrl = a.absUrl("href"), + rating = td2.selectFirst("div.ir")?.parseRating() ?: RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = td1.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + tags = tagsDiv.parseTags(), + state = when { + rawTitle.contains("(ongoing)", ignoreCase = true) -> MangaState.ONGOING + else -> null + }, + authors = setOfNotNull(author), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().selectFirstOrThrow("div.gm") + val cover = root.getElementById("gd1")?.children()?.first() + val title = root.getElementById("gd2") + val tagList = root.getElementById("taglist") + val tabs = doc.body().selectFirst("table.ptt")?.selectFirst("tr") + val gd3 = root.getElementById("gd3") + val lang = gd3 + ?.selectFirst("tr:contains(Language)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + val uploadDate = gd3 + ?.selectFirst("tr:contains(Posted)") + ?.selectFirst(".gdt2")?.ownTextOrNull() + .let { SimpleDateFormat("yyyy-MM-dd HH:mm", sourceLocale).parseSafe(it) } + val uploader = gd3 + ?.getElementsByAttributeValueContaining("href", "/uploader/") + ?.firstOrNull() + ?.ownTextOrNull() + val tags = tagList?.parseTags().orEmpty() + + return manga.copy( + title = title?.getElementById("gn")?.text()?.cleanupTitle() ?: manga.title, + altTitles = setOfNotNull(title?.getElementById("gj")?.text()?.cleanupTitle()?.nullIfEmpty()), + publicUrl = doc.baseUri().ifEmpty { manga.publicUrl }, + rating = root.getElementById("rating_label")?.text() + ?.substringAfterLast(' ') + ?.toFloatOrNull() + ?.div(5f) ?: manga.rating, + largeCoverUrl = cover?.styleValueOrNull("background")?.cssUrl(), + tags = manga.tags + tags, + description = tagList?.select("tr")?.joinToString("<br>") { tr -> + val (tc, td) = tr.children() + val subTags = td.select("a").joinToString { it.html() } + "<b>${tc.html()}</b> $subTags" + }, + chapters = tabs?.select("a")?.findLast { a -> + a.text().toIntOrNull() != null + }?.let { a -> + val count = a.text().toInt() + val chapters = ChaptersListBuilder(count) + for (i in 1..count) { + val url = "${manga.url}?p=${i - 1}" + chapters += MangaChapter( + id = generateUid(url), + title = null, + number = i.toFloat(), + volume = 0, + url = url, + uploadDate = uploadDate, + source = source, + scanlator = uploader, + branch = lang, + ) + } + chapters.toList() + }, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("gdt") + return root.select("a").map { a -> + val url = a.attrAsRelativeUrl("href") + MangaPage( + id = generateUid(url), + url = url, + preview = a.children().firstOrNull()?.extractPreview(), + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.body().requireElementById("img").attrAsAbsoluteUrl("src") + } + + @Suppress("SpellCheckingInspection") + private val tags: String + get() = "ahegao,anal,angel,apron,bandages,bbw,bdsm,beauty mark,big areolae,big ass,big breasts,big clit,big lips," + + "big nipples,bikini,blackmail,bloomers,blowjob,bodysuit,bondage,breast expansion,bukkake,bunny girl,business suit," + + "catgirl,centaur,cheating,chinese dress,christmas,collar,corset,cosplaying,cowgirl,crossdressing,cunnilingus," + + "dark skin,daughter,deepthroat,defloration,demon girl,double penetration,dougi,dragon,drunk,elf,exhibitionism,farting," + + "females only,femdom,filming,fingering,fishnets,footjob,fox girl,furry,futanari,garter belt,ghost,giantess," + + "glasses,gloves,goblin,gothic lolita,growth,guro,gyaru,hair buns,hairy,hairy armpits,handjob,harem,hidden sex," + + "horns,huge breasts,humiliation,impregnation,incest,inverted nipples,kemonomimi,kimono,kissing,lactation," + + "latex,leg lock,leotard,lingerie,lizard girl,maid,masked face,masturbation,midget,miko,milf,mind break," + + "mind control,monster girl,mother,muscle,nakadashi,netorare,nose hook,nun,nurse,oil,paizuri,panda girl," + + "pantyhose,piercing,pixie cut,policewoman,ponytail,pregnant,rape,rimjob,robot,scat,lolicon,schoolgirl uniform," + + "sex toys,shemale,sister,small breasts,smell,sole dickgirl,sole female,squirting,stockings,sundress,sweating," + + "swimsuit,swinging,tail,tall girl,teacher,tentacles,thigh high boots,tomboy,transformation,twins,twintails," + + "unusual pupils,urination,vore,vtuber,widow,wings,witch,wolf girl,x-ray,yuri,zombie,sole male,males only,yaoi," + + "tomgirl,tall man,oni,shotacon,prostate massage,policeman,males only,huge penis,fox boy,feminization,dog boy,dickgirl on male,big penis" + + private fun mapTags(): Set<MangaTag> { + val tagElements = tags.split(",") + val result = ArraySet<MangaTag>(tagElements.size) + for (tag in tagElements) { + val el = tag.trim() + if (el.isEmpty()) continue + result += MangaTag( + title = el.toTitleCase(Locale.ENGLISH), + key = el, + source = source, + ) + } + return result + } + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.headersContentLength() <= BANNED_RESPONSE_LENGTH) { + val text = response.peekBody(BANNED_RESPONSE_LENGTH).use { it.string() } + if (text.contains("IP address has been temporarily banned", ignoreCase = true)) { + val hours = Regex("([0-9]+) hours?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val minutes = Regex("([0-9]+) minutes?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + val seconds = Regex("([0-9]+) seconds?").find(text)?.groupValues?.getOrNull(1)?.toLongOrNull() ?: 0 + response.closeQuietly() + throw TooManyRequestExceptions( + url = response.request.url.toString(), + retryAfter = TimeUnit.HOURS.toMillis(hours) + + TimeUnit.MINUTES.toMillis(minutes) + + TimeUnit.SECONDS.toMillis(seconds), + ) + } + } + val imageRect = response.request.url.fragment?.split(',') + if (imageRect != null && imageRect.size == 4) { + // rect: top,left,right,bottom + return context.redrawImageResponse(response) { bitmap -> + val srcRect = Rect( + left = imageRect[0].toInt(), + top = imageRect[1].toInt(), + right = imageRect[2].toInt(), + bottom = imageRect[3].toInt(), + ) + val dstRect = Rect(0, 0, srcRect.width, srcRect.height) + val result = context.createBitmap(dstRect.width, dstRect.height) + result.drawBitmap(bitmap, srcRect, dstRect) + result + } + } + return response + } + + private fun Locale.toLanguagePath() = when (language) { + else -> getDisplayLanguage(Locale.ENGLISH).lowercase() + } + + override suspend fun getUsername(): String { + val doc = webClient.httpGet("https://forums.$DOMAIN_UNAUTHORIZED/").parseHtml().body() + val username = doc.getElementById("userlinks") + ?.getElementsByAttributeValueContaining("href", "showuser=") + ?.firstOrNull() + ?.ownText() + ?: if (doc.getElementById("userlinksguest") != null) { + throw AuthRequiredException(source) + } else { + doc.parseFailed() + } + return username + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(suspiciousContentKey) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val query = seed.title + return getListPage( + page = 0, + order = defaultSortOrder, + filter = MangaListFilter(query = query), + ) + } + + private fun isAuthorized(domain: String): Boolean { + val cookies = context.cookieJar.getCookies(domain).mapToSet { x -> x.name } + return authCookies.all { it in cookies } + } + + private fun Element.parseRating(): Float { + return runCatching { + val style = requireNotNull(attr("style")) + val (v1, v2) = ratingPattern.findAll(style).toList() + var p1 = v1.groupValues.first().dropLast(2).toInt() + val p2 = v2.groupValues.first().dropLast(2).toInt() + if (p2 != -1) { + p1 += 8 + } + (80 - p1) / 80f + }.getOrDefault(RATING_UNKNOWN) + } + + private fun String.cleanupTitle(): String { + return replace(titleCleanupPattern, "") + .replace(spacesCleanupPattern, "") + } + + private fun Element.parseTags(): Set<MangaTag> { + + fun Element.parseTag() = textOrNull()?.let { + MangaTag(title = it.toTitleCase(Locale.ENGLISH), key = it, source = source) + } + + val result = ArraySet<MangaTag>() + for (prefix in TAG_PREFIXES) { + getElementsByAttributeValueStarting("id", "ta_$prefix").mapNotNullTo(result, Element::parseTag) + getElementsByAttributeValueStarting("title", prefix).mapNotNullTo(result, Element::parseTag) + } + return result + } + + private fun Element.extractPreview(): String? { + val style = attr("style") + val attrs = style.split(';').mapNotNull { part -> + val trimmed = part.trim() + if (trimmed.isEmpty()) { + null + } else { + val idx = trimmed.indexOf(':') + if (idx < 0) null else { + trimmed.substring(0, idx) to trimmed.substring(idx + 1).trim() + } + } + }.toMap() + + val width = attrs["width"]?.removeSuffix("px")?.toIntOrNull() ?: return null + val height = attrs["height"]?.removeSuffix("px")?.toIntOrNull() ?: return null + + val bgPart = attrs["background"] ?: return null + val bgTokens = bgPart.substringAfter("url").split(Regex("\\s+")).filter { it.isNotBlank() } + val url = bgTokens.firstOrNull()?.removeSurrounding("(", ")") ?: return null + val x = bgTokens.getOrNull(1)?.removeSuffix("px")?.toIntOrNull() ?: 0 + val y = bgTokens.getOrNull(2)?.removeSuffix("px")?.toIntOrNull() ?: 0 + + val left = -x + val right = left + width + val bottom = y + height + + return "$url#$left,$y,$right,$bottom" + } + + private fun getNextTimestamp(root: Element): Long { + return root.getElementById("unext") + ?.attrAsAbsoluteUrlOrNull("href") + ?.toHttpUrlOrNull() + ?.queryParameter("next") + ?.toLongOrNull() ?: 1 + } + + private fun MangaListFilter.toSearchQuery(): String? { + if (isEmpty()) return null + + val parts = mutableListOf<String>() + + query?.let { parts.add(it) } + + for (tag in tags) { + if (tag.key.isNumeric()) continue + parts.add("tag:\"${tag.key}\"") + } + + for (tag in tagsExclude) { + if (tag.key.isNumeric()) continue + parts.add("-tag:\"${tag.key}\"") + } + + locale?.let { lc -> + parts.add("language:\"${lc.toLanguagePath()}\"") + } + + author?.let { + parts.add("artist:\"${it}\"") + } + + return parts.joinToString(" ").nullIfEmpty() + } + + private fun Collection<ContentType>.toFCats(): Int = fold(0) { acc, ct -> + val cat: Int = when (ct) { + ContentType.DOUJINSHI -> 2 + ContentType.MANGA -> 4 + ContentType.ARTIST_CG -> 8 + ContentType.GAME_CG -> 16 + ContentType.COMICS -> 512 + ContentType.IMAGE_SET -> 32 + else -> 449 // 1 or 64 or 128 or 256 + } + acc or cat + } + + private fun checkAuth(): Boolean { + val authorized = isAuthorized(DOMAIN_UNAUTHORIZED) + if (authorized) { + if (!isAuthorized(DOMAIN_AUTHORIZED)) { + context.cookieJar.copyCookies( + DOMAIN_UNAUTHORIZED, + DOMAIN_AUTHORIZED, + authCookies, + ) + context.cookieJar.insertCookies(DOMAIN_AUTHORIZED, "yay=louder") + } + return true + } + return false + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt rename to src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt index 53bdbbab..b33087df 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiEnvy.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEnvy.kt @@ -1,14 +1,23 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.SortOrder + +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import java.util.EnumSet +import java.util.Locale -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.urlEncoded -import java.util.* @MangaSourceParser("HENTAIENVY", "HentaiEnvy", type = ContentType.HENTAI) internal class HentaiEnvy(context: MangaLoaderContext) : @@ -43,9 +52,8 @@ internal class HentaiEnvy(context: MangaLoaderContext) : append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?s_key=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) append("&") } @@ -78,4 +86,4 @@ internal class HentaiEnvy(context: MangaLoaderContext) : } return parseMangaList(webClient.httpGet(url).parseHtml()) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt new file mode 100644 index 00000000..4a83e09b --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiEra.kt @@ -0,0 +1,153 @@ +package tsuki.site.all.nsfw + +import org.jsoup.nodes.Element +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("HENTAIERA", "HentaiEra", type = ContentType.HENTAI) +internal class HentaiEra(context: MangaLoaderContext) : + GalleryAdultsParser(context, MangaParserSource.HENTAIERA, "hentaiera.com", 25) { + override val selectTags = ".tags_section" + override val selectTag = ".galleries_info li:contains(Tags) div.info_tags" + override val selectAuthor = ".galleries_info li:contains(Artists) span.item_name" + override val selectLanguageChapter = ".galleries_info li:contains(Languages) div.info_tags .item_name" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = super.getFilterOptions().copy( + availableLocales = setOf( + Locale.ENGLISH, + Locale.FRENCH, + Locale.JAPANESE, + Locale("es"), + Locale("ru"), + Locale("ko"), + Locale.GERMAN, + ), + ) + + override fun Element.parseTags() = select("a.tag, .gallery_title a").mapToSet { + val key = it.attr("href").removeSuffix('/').substringAfterLast('/') + val name = it.selectFirst(".item_name")?.text() ?: it.text() + MangaTag( + key = key, + title = name, + source = source, + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + append("/search/?key=") + append(filter.query.urlEncoded()) + append("&") + } + else -> { + if (filter.tags.size > 1 || (filter.tags.isNotEmpty() && filter.locale != null)) { + append("/search/?key=") + append(buildQuery(filter.tags, filter.locale)) + + append("&") + } else if (filter.tags.isNotEmpty()) { + filter.tags.oneOrThrowIfMany()?.let { + append("/tag/") + append(it.key) + } + append("/") + + append("?") + } else if (filter.locale != null) { + append("/language/") + append(filter.locale?.toLanguagePath()) + append("/") + + append("?") + } else { + append("/?") + } + } + } + append("page=") + append(page.toString()) + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun buildQuery(tags: Collection<MangaTag>, locale: Locale?): String { + val queryDefault = + "&search=&mg=1&dj=1&ws=1&is=1&ac=1&gc=1&en=0&jp=0&es=0&fr=0&kr=0&de=0&ru=0<=1&dl=0&pp=0&tr=0" + val tag = tags.joinToString(" ", postfix = " ") { it.key } + val queryMod = when (val lp = locale?.toLanguagePath()) { + "english" -> queryDefault.replace("en=0", "en=1") + "japanese" -> queryDefault.replace("jp=0", "jp=1") + "spanish" -> queryDefault.replace("es=0", "es=1") + "french" -> queryDefault.replace("fr=0", "fr=1") + "korean" -> queryDefault.replace("kr=0", "kr=1") + "russian" -> queryDefault.replace("ru=0", "ru=1") + "german" -> queryDefault.replace("de=0", "de=1") + null -> "&search=&mg=1&dj=1&ws=1&is=1&ac=1&gc=1&en=1&jp=1&es=1&fr=1&kr=1&de=1&ru=1<=1&dl=0&pp=0&tr=0" + else -> throw IllegalArgumentException("Unsupported locale: $lp") + } + return tag + queryMod + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val urlChapters = doc.selectFirstOrThrow("#cover a, .cover a, .left_cover a").attr("href") + val tag = doc.selectFirst(selectTag)?.parseTags() + val branch = doc.select(selectLanguageChapter).joinToString(separator = " / ") { + it.text() + } + val author = doc.selectFirst(selectAuthor)?.text() + return manga.copy( + tags = tag.orEmpty(), + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 1f, + volume = 0, + url = urlChapters, + scanlator = null, + uploadDate = 0, + branch = branch, + source = source, + ), + ), + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt similarity index 74% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt rename to src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt index d0fe0b91..b93bd857 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HentaiFox.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HentaiFox.kt @@ -1,22 +1,26 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType +import tsuki.model.MangaTag +import tsuki.util.mapToSet +import tsuki.util.removeSuffix +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons -import org.jsoup.internal.StringUtil import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.util.EnumSet import java.util.Locale @@ -34,6 +38,15 @@ internal class HentaiFox(context: MangaLoaderContext) : isMultipleTagsSupported = true, ) + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/images/favicon.ico", 16, null), + ), + domain, + ) + } + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { @@ -42,9 +55,8 @@ internal class HentaiFox(context: MangaLoaderContext) : append(domain) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("/search/?q=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) if (page > 1) { append("&page=") append(page.toString()) @@ -121,13 +133,9 @@ internal class HentaiFox(context: MangaLoaderContext) : } private fun buildQuery(tags: Collection<MangaTag>, language: Locale?): String { - val joiner = StringUtil.StringJoiner(" ") - tags.forEach { tag -> - joiner.add(tag.key) - } - language?.let { lc -> - joiner.add(lc.toLanguagePath()) - } - return joiner.complete() + val parts = mutableListOf<String>() + tags.forEach { tag -> parts += tag.key } + language?.let { parts += it.toLanguagePath() } + return parts.joinToString(" ") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt similarity index 62% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt rename to src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt index 2943533d..6bbe24e6 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/HitomiLaParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/HitomiLaParser.kt @@ -1,4 +1,39 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.suspendlazy.suspendLazy +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseBytes +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.parseSafe +import tsuki.util.runCatchingCancellable +import tsuki.util.selectFirstOrThrow +import tsuki.util.splitByWhitespace +import tsuki.util.toAbsoluteUrl +import tsuki.util.toCamelCase +import tsuki.util.toTitleCase +import tsuki.util.urlDecode import androidx.collection.ArraySet import kotlinx.coroutines.async @@ -10,38 +45,6 @@ import okhttp3.Headers import org.json.JSONArray import org.json.JSONObject import org.jsoup.Jsoup -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.parseBytes -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseRaw -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.runCatchingCancellable -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.splitByWhitespace -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toCamelCase -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlDecode import java.nio.ByteBuffer import java.nio.ByteOrder import java.security.MessageDigest @@ -50,7 +53,6 @@ import java.util.EnumSet import java.util.LinkedList import java.util.Locale import kotlin.math.min -import kotlin.text.get @OptIn(ExperimentalUnsignedTypes::class) @MangaSourceParser("HITOMILA", "Hitomi.La", type = ContentType.HENTAI) @@ -103,14 +105,14 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser ) override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - ) + isMultipleTagsSupported = true, + isSearchSupported = true, + ) override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = fetchAvailableTags(), - availableLocales = localeMap.keys, - ) + availableTags = fetchAvailableTags(), + availableLocales = localeMap.keys, + ) private fun Locale?.getSiteLang(): String = when (this) { null -> "all" @@ -118,34 +120,34 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { - ('a'..'z').map { alphabet -> - async { - val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() - - doc.select(".posts > li").mapNotNull { element -> - val num = - element.ownText().let { - Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 - } - - if (num > 100) { - val url = element.selectFirst("a") - val href = - url?.attrAsRelativeUrl("href") - ?: return@mapNotNull null - - MangaTag( - title = url.ownText().toTagTitle(), - key = href.tagUrlToTag(), - source = source, - ) - } else { - null - } - } - } - }.awaitAll().flatten().toSet() - } + ('a'..'z').map { alphabet -> + async { + val doc = webClient.httpGet("https://$domain/alltags-$alphabet.html").parseHtml() + + doc.select(".posts > li").mapNotNull { element -> + val num = + element.ownText().let { + Regex("""\((\d+)\)""").find(it)?.groupValues?.get(1)?.toIntOrNull() ?: 0 + } + + if (num > 100) { + val url = element.selectFirst("a") + val href = + url?.attrAsRelativeUrl("href") + ?: return@mapNotNull null + + MangaTag( + title = url.ownText().toTagTitle(), + key = href.tagUrlToTag(), + source = source, + ) + } else { + null + } + } + } + }.awaitAll().flatten().toSet() + } private var cachedSearchIds: List<Int> = emptyList() @@ -220,80 +222,80 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } private suspend fun hitomiSearch( - query: String, - sortByPopularity: SortOrder = SortOrder.UPDATED, - language: String = "all", + query: String, + sortByPopularity: SortOrder = SortOrder.UPDATED, + language: String = "all", ): Set<Int> = - coroutineScope { - val terms = query - .trim() - .replace(Regex("""^\?"""), "") - .lowercase() - .splitByWhitespace() - .map { - it.replace('_', ' ') - } - - val positiveTerms = LinkedList<String>() - val negativeTerms = LinkedList<String>() - - for (term in terms) { - if (term.startsWith("-")) { - negativeTerms.push(term.removePrefix("-")) - } else if (term.isNotBlank()) { - positiveTerms.push(term) - } - } - - val positiveResults = positiveTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val negativeResults = negativeTerms.map { - async { - runCatchingCancellable { - getGalleryIDsForQuery(it, language) - }.getOrDefault(emptySet()) - } - } - - val results = when { - sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) - sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) - sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) - sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) - sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) - positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) - else -> emptySet() - }.toMutableSet() - - fun filterPositive(newResults: Set<Int>) { - when { - results.isEmpty() -> results.addAll(newResults) - else -> results.retainAll(newResults) - } - } - - fun filterNegative(newResults: Set<Int>) { - results.removeAll(newResults) - } - - // positive results - positiveResults.forEach { - filterPositive(it.await()) - } - - // negative results - negativeResults.forEach { - filterNegative(it.await()) - } - - results - } + coroutineScope { + val terms = query + .trim() + .replace(Regex("""^\?"""), "") + .lowercase() + .splitByWhitespace() + .map { + it.replace('_', ' ') + } + + val positiveTerms = LinkedList<String>() + val negativeTerms = LinkedList<String>() + + for (term in terms) { + if (term.startsWith("-")) { + negativeTerms.push(term.removePrefix("-")) + } else if (term.isNotBlank()) { + positiveTerms.push(term) + } + } + + val positiveResults = positiveTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val negativeResults = negativeTerms.map { + async { + runCatchingCancellable { + getGalleryIDsForQuery(it, language) + }.getOrDefault(emptySet()) + } + } + + val results = when { + sortByPopularity == SortOrder.UPDATED -> getGalleryIDsFromNozomi(null, "index", language) + sortByPopularity == SortOrder.POPULARITY_TODAY -> getGalleryIDsFromNozomi("popular", "today", language) + sortByPopularity == SortOrder.POPULARITY_WEEK -> getGalleryIDsFromNozomi("popular", "week", language) + sortByPopularity == SortOrder.POPULARITY_MONTH -> getGalleryIDsFromNozomi("popular", "month", language) + sortByPopularity == SortOrder.POPULARITY_YEAR -> getGalleryIDsFromNozomi("popular", "year", language) + positiveTerms.isEmpty() -> getGalleryIDsFromNozomi(null, "index", language) + else -> emptySet() + }.toMutableSet() + + fun filterPositive(newResults: Set<Int>) { + when { + results.isEmpty() -> results.addAll(newResults) + else -> results.retainAll(newResults) + } + } + + fun filterNegative(newResults: Set<Int>) { + results.removeAll(newResults) + } + + // positive results + positiveResults.forEach { + filterPositive(it.await()) + } + + // negative results + negativeResults.forEach { + filterNegative(it.await()) + } + + results + } // search.js private suspend fun getGalleryIDsForQuery( @@ -450,8 +452,8 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } private val galleriesIndexVersion = suspendLazy { - webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() - } + webClient.httpGet("$ltnBaseUrl/galleriesindex/version?_=${System.currentTimeMillis()}").parseRaw() + } private data class Node( val keys: List<UByteArray>, @@ -528,39 +530,39 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser } private suspend fun Collection<Int>.toMangaList(): List<Manga> = coroutineScope { - map { id -> - async { - runCatchingCancellable { - val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { - val baseUri = it.request.url.toString() - val html = it.parseRaw() - Jsoup.parse(rewriteTnPaths(html), baseUri) - } - - Manga( - id = generateUid(id.toString()), - title = doc.selectFirstOrThrow("h1").text(), - url = id.toString(), - coverUrl = - "https:" + - doc.selectFirstOrThrow("picture > img") - .attr("data-src"), - publicUrl = - doc.selectFirstOrThrow("h1 > a") - .attrAsRelativeUrl("href") - .toAbsoluteUrl(domain), - authors = emptySet(), - tags = emptySet(), - contentRating = ContentRating.ADULT, - rating = RATING_UNKNOWN, - altTitles = emptySet(), - state = null, - source = source, - ) - }.getOrNull() - } - }.awaitAll().filterNotNull() - } + map { id -> + async { + runCatchingCancellable { + val doc = webClient.httpGet("$ltnBaseUrl/galleryblock/$id.html").let { + val baseUri = it.request.url.toString() + val html = it.parseRaw() + Jsoup.parse(rewriteTnPaths(html), baseUri) + } + + Manga( + id = generateUid(id.toString()), + title = doc.selectFirstOrThrow("h1").text(), + url = id.toString(), + coverUrl = + "https:" + + doc.selectFirstOrThrow("picture > img") + .attr("data-src"), + publicUrl = + doc.selectFirstOrThrow("h1 > a") + .attrAsRelativeUrl("href") + .toAbsoluteUrl(domain), + authors = emptySet(), + tags = emptySet(), + contentRating = ContentRating.ADULT, + rating = RATING_UNKNOWN, + altTitles = emptySet(), + state = null, + source = source, + ) + }.getOrNull() + } + }.awaitAll().filterNotNull() + } override suspend fun getDetails(manga: Manga): Manga { val json = webClient.httpGet("$ltnBaseUrl/galleries/${manga.url}.js") @@ -604,17 +606,17 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser ?.let(::addAll) }, chapters = listOf( - MangaChapter( - id = generateUid(manga.url), - url = manga.url, - title = json.getStringOrNull("title"), - scanlator = json.getString("type").toTitleCase(), - number = 1f, - volume = 0, - branch = json.getString("language_localname"), - source = source, - uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), - ), + MangaChapter( + id = generateUid(manga.url), + url = manga.url, + title = json.getStringOrNull("title"), + scanlator = json.getString("type").toTitleCase(), + number = 1f, + volume = 0, + branch = json.getString("language_localname"), + source = source, + uploadDate = dateFormat.parseSafe(json.getString("date").substringBeforeLast("-")), + ), ), ) } @@ -625,19 +627,19 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser val tags = ArraySet<MangaTag>(length()) mapJSON { MangaTag( - title = - it.getString(key).toCamelCase().let { title -> - if (it.getStringOrNull("female")?.toIntOrNull() == 1) { - "$title ♀" - } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { - "$title ♂" - } else { - title - } - }, - key = it.getString("url").tagUrlToTag(), - source = source, - ).let(tags::add) + title = + it.getString(key).toCamelCase().let { title -> + if (it.getStringOrNull("female")?.toIntOrNull() == 1) { + "$title ♀" + } else if (it.getStringOrNull("male")?.toIntOrNull() == 1) { + "$title ♂" + } else { + title + } + }, + key = it.getString("url").tagUrlToTag(), + source = source, + ).let(tags::add) } return tags } @@ -682,12 +684,12 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser val imageId = imageIdFromHash(hash) val subDomain = subdomainOffset(imageId) + 1 val thumbSubdomain = 'a' + subdomainOffset(imageId) - MangaPage( - id = generateUid(hash), - url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", - preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", - source = source, - ) + MangaPage( + id = generateUid(hash), + url = "https://a${subDomain}.$cdnDomain/$commonId$imageId/$hash.avif", + preview = "https://${thumbSubdomain}tn.$cdnDomain/webpsmallsmalltn/${thumbPathFromHash(hash)}/$hash.webp", + source = source, + ) } } @@ -793,4 +795,4 @@ internal class HitomiLaParser(context: MangaLoaderContext) : AbstractMangaParser .replace("♂", "(male)") .replace("♀", "(female)") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt similarity index 78% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt rename to src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt index 4c2f22f4..b12d9cf4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/IMHentai.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/IMHentai.kt @@ -1,41 +1,44 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.ownTextOrNull +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectFirstParentOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import androidx.collection.ArraySet import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.flattenTo -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.ownTextOrNull -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireElementById -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.selectFirstParentOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.util.ArrayList import java.util.EnumSet import java.util.Locale @@ -81,11 +84,7 @@ internal class ImHentai(context: MangaLoaderContext) : keys.add(userAgentKey) } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val url = buildString { append("https://") append(domain) @@ -93,9 +92,8 @@ internal class ImHentai(context: MangaLoaderContext) : append(page.toString()) when { !filter.query.isNullOrEmpty() -> { - val query = filter.query append("&key=") - append(query?.urlEncoded()) + append(filter.query?.urlEncoded()) } else -> { @@ -161,8 +159,9 @@ internal class ImHentai(context: MangaLoaderContext) : } } - //Tags are deliberately reduced because there are too many and this slows down the application. - //only the most popular ones are taken. + /** + * Reduced tags way too many + */ private suspend fun fetchAvailableTags(): Set<MangaTag> { return coroutineScope { (1..3).map { page -> @@ -268,4 +267,4 @@ internal class ImHentai(context: MangaLoaderContext) : val img = doc.body().requireElementById("gimg") return img.requireSrc() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt b/src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt similarity index 63% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt rename to src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt index bbe55a7a..1b07b72a 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Kiutaku.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Kiutaku.kt @@ -1,13 +1,14 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.network.CommonHeaders +import tsuki.parsers.GalleryParser import okhttp3.Interceptor import okhttp3.Response -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.parsers.GalleryParser @MangaSourceParser("KIUTAKU", "Kiutaku", type = ContentType.OTHER) internal class Kiutaku(context: MangaLoaderContext) : diff --git a/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt new file mode 100644 index 00000000..aab9bbb0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/MangaCrazy.kt @@ -0,0 +1,16 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.util.Locale + +@MangaSourceParser("MANGACRAZY", "MangaCrazy", "", ContentType.HENTAI) +internal class MangaCrazy(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGACRAZY, "mangacrazy.net") { + override val sourceLocale: Locale = Locale.ENGLISH +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt similarity index 76% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt rename to src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt index 17a4be9a..ed9c8f59 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Misskon.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Misskon.kt @@ -1,28 +1,31 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + import java.util.EnumSet @MangaSourceParser("MISSKON", "MissKon", type = ContentType.HENTAI) @@ -127,7 +130,7 @@ internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, // Single page gallery return doc.select("div.post-inner > div.entry > p > img") .mapNotNull { img -> img.absUrl("data-src") } - .mapIndexed { i, url -> + .mapIndexed { _, url -> MangaPage( id = generateUid(url), url = url, @@ -162,4 +165,4 @@ internal class Misskon(context: MangaLoaderContext) : PagedMangaParser(context, } return pages } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt similarity index 74% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt rename to src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt index 0faf0501..5f5015d9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/Multporn.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/Multporn.kt @@ -1,35 +1,37 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.CommonHeaders + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.splitByWhitespace +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import okhttp3.Headers import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.splitByWhitespace -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.util.EnumSet import java.util.Locale @@ -75,14 +77,13 @@ internal class Multporn(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?search_api_views_fulltext=") - val encodedQuery = query.splitByWhitespace().joinToString(separator = "+") { part -> + val encodedQuery = filter.query.splitByWhitespace().joinToString(separator = "+") { part -> part.urlEncoded() } append(encodedQuery) @@ -157,12 +158,35 @@ internal class Multporn(context: MangaLoaderContext) : } val doc = webClient.httpGet(url).parseHtml() - return doc.select(".masonry-item").map { div -> - val href = div.selectFirstOrThrow(".views-field-title a").attrAsRelativeUrl("href") - val coverUrl = div.selectFirstOrThrow(".views-field img").requireSrc() + val items = doc.select(".masonry-item") + .ifEmpty { doc.select(".views-row") } + .ifEmpty { doc.select(".node") } + + return items.mapNotNull { div -> + val linkElement = div.selectFirst(".views-field-title a") + ?: div.selectFirst(".views-field-name a") + ?: div.selectFirst(".field-content a") + ?: div.selectFirst("h2 a") + ?: div.selectFirst("h3 a") + ?: return@mapNotNull null + + val href = linkElement.attrAsRelativeUrl("href") + if (href.isBlank()) return@mapNotNull null + val coverElement = div.selectFirst(".views-field-field-files img") + ?: div.selectFirst(".views-field-field-game-preview img") + ?: div.selectFirst(".views-field-field-preview-1 img") + ?: div.selectFirst(".views-field-field-picture-preview img") + ?: div.selectFirst(".views-field img") + ?: div.selectFirst("img") + ?: return@mapNotNull null + + val coverUrl = coverElement.requireSrc() + val title = linkElement.text() + .ifEmpty { div.selectFirst(".views-field-title")?.text() ?: "Unknown" } + Manga( id = generateUid(href), - title = div.select(".views-field-title").text(), + title = title, altTitles = emptySet(), url = href, publicUrl = href.toAbsoluteUrl(domain), @@ -251,4 +275,4 @@ internal class Multporn(context: MangaLoaderContext) : document.select(".$className a").map { it.text().trim() } } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt rename to src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt index f23e3df5..1684790d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiParser.kt @@ -1,15 +1,31 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.urlEncoded + +import org.json.JSONObject +import java.util.EnumSet +import java.util.Locale // removed first server, it has always problems private const val TWO_IMG_SERVER = "2" @@ -839,7 +855,6 @@ internal class NHentaiParser(context: MangaLoaderContext) : ): List<Manga> { ensureNhConfig() - // Direct ID search val directId = extractGalleryId(filter.query) if (directId != null) { return try { @@ -857,31 +872,38 @@ internal class NHentaiParser(context: MangaLoaderContext) : return (0 until items.length()).map { i -> items.getJSONObject(i).toManga() } } - val lang = filter.locale?.let { loc -> - LANG_MAP[loc.language] + val lang = filter.locale?.let { LANG_MAP[it.language] } + val searchQuery = buildString { + if (!filter.query.isNullOrBlank()) { + append("title:\"${filter.query}\"") + } + if (lang != null) { + if (isNotEmpty()) append(" ") + append("language:$lang") + } } - val url = when { - // Search with language - lang != null -> { - val sort = when (order) { - SortOrder.POPULARITY -> "popular" - else -> "date" + val url: String + when (order) { + SortOrder.UPDATED -> { + url = if (searchQuery.isEmpty()) { + "https://$domain/api/v2/galleries?page=$page" + } else { + "https://$domain/api/v2/search?query=${searchQuery.urlEncoded()}&sort=date&page=$page" } - "https://$domain/api/v2/search?query=language:$lang&sort=$sort&page=$page" } - // Latest - order == SortOrder.UPDATED -> "https://$domain/api/v2/galleries?page=$page" - // Popular (all / today / week / month) else -> { val sort = when (order) { + SortOrder.POPULARITY -> "popular" SortOrder.POPULARITY_TODAY -> "popular-today" - SortOrder.POPULARITY_WEEK -> "popular-week" + SortOrder.POPULARITY_WEEK -> "popular-week" SortOrder.POPULARITY_MONTH -> "popular-month" - SortOrder.POPULARITY -> "popular" else -> "popular" } - "https://$domain/api/v2/search?query=%22%22&sort=$sort&page=$page" + val finalQuery = searchQuery.ifBlank { + lang?.let { "language:$it" } ?: "language:english" + } + url = "https://$domain/api/v2/search?query=${finalQuery.urlEncoded()}&sort=$sort&page=$page" } } @@ -1023,7 +1045,7 @@ internal class NHentaiParser(context: MangaLoaderContext) : publicUrl = "https://$domain/g/$id/", rating = RATING_UNKNOWN, contentRating = ContentRating.ADULT, - coverUrl = "$thumbServer/$thumbPath", + coverUrl = "$thumbServer/$coverPath", largeCoverUrl = "$imageServer/$coverPath", tags = tags, state = null, diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt rename to src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt index 5591672b..064e54fa 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/all/hentais/NHentaiXXXParser.kt +++ b/src/main/kotlin/tsuki/site/all/nsfw/NHentaiXXXParser.kt @@ -1,15 +1,39 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.all.hentais +package tsuki.site.all.nsfw + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded -import org.jsoup.internal.StringUtil import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.ErrorMessages -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.* -import java.util.* +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("NHENTAIXXX", "NHentai.xxx", type = ContentType.HENTAI) internal class NHentaiXXXParser(context: MangaLoaderContext) : @@ -44,7 +68,6 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) @@ -65,15 +88,9 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : else -> { append("/search/?key=") - val joiner = StringUtil.StringJoiner("+") - tags.forEach { tag -> - joiner.add(tag.title) - } - - if (!query.isNullOrEmpty()) { - joiner.add(query.urlEncoded()) - } - append(joiner.complete()) + val parts = tags.map { it.title }.toMutableList() + if (!filter.query.isNullOrEmpty()) parts.add(filter.query.urlEncoded()) + append(parts.joinToString("+")) } } @@ -141,7 +158,7 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : return try { webClient.httpHead(url) false - } catch (e: Exception) { + } catch (_: Exception) { true } } @@ -155,8 +172,8 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : try { webClient.httpHead(newUrl) newUrl - } catch (e: Exception) { - null // Skip errors and continue checking other formats + } catch (_: Exception) { + null } } } @@ -181,4 +198,4 @@ internal class NHentaiXXXParser(context: MangaLoaderContext) : url.substringBeforeLast("/") + "/$newPageNumber" } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt b/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt new file mode 100644 index 00000000..be80883e --- /dev/null +++ b/src/main/kotlin/tsuki/site/all/nsfw/SexKomik2COM.kt @@ -0,0 +1,302 @@ +package tsuki.site.all.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient + +import tsuki.model.ContentType +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("SEXKOMIX2COM", "SexKomik2COM", type = ContentType.HENTAI) +class SexKomik2COM(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SEXKOMIX2COM, 24) { + + override val configKeyDomain = tsuki.config.ConfigKey.Domain("sexkomix2.com") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, // date + SortOrder.POPULARITY, // prosmotr (views) + SortOrder.RATING, // like + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + private val cookieInterceptor = Interceptor { chain -> + val original = chain.request() + val request = original.newBuilder() + .header("Cookie", "confirm=true") + .header("Referer", "https://$domain/") + .build() + chain.proceed(request) + } + + override val webClient by lazy { + val client = context.httpClient.newBuilder() + .addInterceptor(cookieInterceptor) + .build() + OkHttpWebClient(client, source) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = setOf( + MangaTag("3D", "3D", source), + MangaTag("Adventure", "Adventure", source), + MangaTag("AI Generated", "AI Generated", source), + MangaTag("BDSM", "BDSM", source), + MangaTag("Big boobs", "Big boobs", source), + MangaTag("Big dick", "Big dick", source), + MangaTag("Black girls", "Black girls", source), + MangaTag("Blondes", "Blondes", source), + MangaTag("Brunettes", "Brunettes", source), + MangaTag("Cheating", "Cheating", source), + MangaTag("Double penetration", "Double penetration", source), + MangaTag("Fantasy", "Fantasy", source), + MangaTag("Fat girls", "Fat girls", source), + MangaTag("Fitness", "Fitness", source), + MangaTag("Furry", "Furry", source), + MangaTag("Gays Yaoi", "Gays Yaoi", source), + MangaTag("Giants", "Giants", source), + MangaTag("Hentai Manga", "Hentai Manga", source), + MangaTag("Inter Komix", "Inter Komix", source), + MangaTag("Interracial sex", "Interracial sex", source), + MangaTag("Lesbians", "Lesbians", source), + MangaTag("Milf", "Milf", source), + MangaTag("Monsters", "Monsters", source), + MangaTag("Neighbor", "Neighbor", source), + MangaTag("NFT", "NFT", source), + MangaTag("Normal boobs", "Normal boobs", source), + MangaTag("Nurses", "Nurses", source), + MangaTag("Parodies", "Parodies", source), + MangaTag("Redheads", "Redheads", source), + MangaTag("Shemale", "Shemale", source), + MangaTag("Simpsons", "Simpsons", source), + MangaTag("Super heroes", "Super heroes", source), + MangaTag("Teacher", "Teacher", source), + MangaTag("Teen", "Teen", source), + MangaTag("Threesome", "Threesome", source), + MangaTag("Unique VIP", "Unique VIP", source), + MangaTag("With sex toys", "With sex toys", source), + ), + availableStates = emptySet(), + availableContentTypes = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.NEWEST -> "date" + SortOrder.POPULARITY -> "prosmotr" + SortOrder.RATING -> "like" + else -> "date" + } + + val tag = filter.tags.firstOrNull() + val basePath = if (tag != null) "categories" else "search" + + val url = "https://$domain/$basePath/".toHttpUrl().newBuilder().apply { + addQueryParameter("lang", "en") + + if (tag != null) { + addQueryParameter("cat", tag.key) + } + + filter.query?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("q", it) + } + + addQueryParameter("page", page.toString()) + addQueryParameter("sort", sortParam) + }.build().toString() + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + val listContainer = doc.selectFirst("#comix_directory") + ?: return emptyList() + + return listContainer.select("li.comix").mapNotNull { element -> + parseMangaItem(element) + } + } + + private fun parseMangaItem(element: Element): Manga? { + val link = element.selectFirst("a[href]") ?: return null + val href = link.attr("abs:href") + if (href.isBlank()) return null + + val titleElement = element.selectFirst(".comix_title h2 p") + val title = titleElement?.text()?.trim() ?: link.attr("title").ifBlank { link.text() }.trim() + if (title.isBlank()) return null + + val img = element.selectFirst(".comix_img_box img") + val cover = img?.imgAttr() ?: "" + + val tagElements = element.select(".tags_ul li a") + val tags = tagElements.mapNotNull { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + }.toSet() + + return Manga( + id = generateUid(href), + title = title, + altTitles = emptySet(), + url = href, + publicUrl = href, + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = tags, + state = null, + authors = emptySet(), + source = source, + ) + } + + private fun Element.imgAttr(): String = when { + hasAttr("data-src") -> attr("abs:data-src") + hasAttr("data-url") -> attr("abs:data-url") + hasAttr("data-zoom-src") -> attr("abs:data-zoom-src") + hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") + hasAttr("data-cfsrc") -> attr("abs:data-cfsrc") + else -> attr("abs:src") + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url).parseHtml() + + val title = doc.selectFirst(".info_box h1 a")?.text()?.trim() + ?: doc.selectFirst("h1")?.text()?.trim() + if (title.isNullOrBlank()) { + throw ParseException( + "Failed to extract title from manga page: ${manga.url}\n" + + "HTML snippet: ${doc.html().take(300)}", + manga.url + ) + } + + val description = doc.selectFirst(".info_box p")?.text()?.trim() ?: "" + + val coverElement = doc.selectFirst("#comix_cover_img") + val cover = coverElement?.imgAttr()?.takeIf { it.isNotBlank() } + if (cover == null) { + throw ParseException( + "Failed to extract cover image from manga page: ${manga.url}\n" + + "HTML snippet: ${doc.html().take(300)}", + manga.url + ) + } + + val categoryElements = doc.select(".info_box:has(.tags_ul) .tags_ul li a") + val tags = categoryElements.mapNotNull { a -> + val text = a.text().trim() + if (text.isNotBlank()) MangaTag(key = text.lowercase(), title = text, source = source) else null + }.toSet() + + val studioElement = doc.selectFirst(".studio_translator_box .link_button a") + val authors = studioElement?.text()?.trim()?.let { setOf(it) } ?: emptySet() + + val chapters = listOf( + MangaChapter( + id = generateUid(manga.url), + url = manga.url, + title = title, + number = 1f, + volume = 0, + uploadDate = 0L, + scanlator = null, + branch = null, + source = source, + ) + ) + + return manga.copy( + title = title, + description = description, + coverUrl = cover, + tags = tags, + authors = authors, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url).parseHtml() + + val fancyboxLinks = doc.select("#comix_pages_ul li a.fancybox[href]") + if (fancyboxLinks.isNotEmpty()) { + return fancyboxLinks.mapIndexed { _, element -> + val url = element.attr("abs:href") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + val images = doc.select("#comix_pages_ul li img") + if (images.isNotEmpty()) { + return images.mapIndexed { _, img -> + val url = img.imgAttr() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + val scriptMatch = Regex("\"images\"\\s*:\\s*(\\[.*?])", RegexOption.DOT_MATCHES_ALL) + .find(doc.html()) + if (scriptMatch != null) { + val jsonArray = JSONArray(scriptMatch.groupValues[1]) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return emptyList() + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + return emptyList() + } +} diff --git a/src/main/kotlin/tsuki/site/ar/Azora.kt b/src/main/kotlin/tsuki/site/ar/Azora.kt new file mode 100644 index 00000000..f47fc5f0 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Azora.kt @@ -0,0 +1,438 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.MangaParserSource +import tsuki.model.MangaChapter +import tsuki.model.Manga +import tsuki.model.MangaTag +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.json.toJSONObjectOrNull +import tsuki.util.mapNotNullToSet +import tsuki.util.parseSafe +import tsuki.util.src +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Document +import org.jsoup.parser.Parser +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("AZORAMANGA", "Azora Manga", "ar", ContentType.MANGA) +internal class Azora(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.AZORAMANGA, 24) { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL) + + override val configKeyDomain = ConfigKey.Domain("azorafly.com") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://api.") + append(domain) + append("/api/query") + + val params = mutableListOf<String>() + + params.add("page=$page") + params.add("perPage=24") + + if (!filter.query.isNullOrEmpty()) { + params.add("searchTerm=${filter.query!!.urlEncoded()}") + } + + if (filter.tags.isNotEmpty()) { + val genreIds = filter.tags.joinToString(",") { it.key } + params.add("genreIds=$genreIds") + } + + val (orderBy, orderDirection) = when (order) { + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.NEWEST -> "createdAt" to "desc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + else -> "lastChapterAddedAt" to "desc" + } + params.add("orderBy=$orderBy") + params.add("orderDirection=$orderDirection") + + if (params.isNotEmpty()) { + append("?") + append(params.joinToString("&")) + } + } + + val response = webClient.httpGet(url) + val body = response.body.string() + + val jsonArray = try { + JSONArray(body) + } catch (_: Exception) { + try { + val jsonObject = JSONObject(body) + when { + jsonObject.has("posts") -> jsonObject.getJSONArray("posts") + jsonObject.has("data") -> jsonObject.getJSONArray("data") + jsonObject.has("results") -> jsonObject.getJSONArray("results") + else -> JSONArray() + } + } catch (_: Exception) { + JSONArray() + } + } + return parseMangaList(jsonArray) + } + + private fun parseMangaList(json: JSONArray): List<Manga> { + val result = mutableListOf<Manga>() + + for (i in 0 until json.length()) { + val obj = json.getJSONObject(i) + val slug = obj.getString("slug") + val url = "/series/$slug" + val title = obj.getString("postTitle") + val coverUrl = obj.optString("featuredImage") + + val seriesStatus = obj.optString("seriesStatus", "") + val state = when (seriesStatus.uppercase()) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + else -> null + } + + val genresArray = obj.optJSONArray("genres") + var isNovel = false + val tags = if (genresArray != null) { + buildSet { + for (idx in 0 until genresArray.length()) { + val genre = genresArray.getJSONObject(idx) + val genreName = genre.getString("name") + if (genreName == "رواية") isNovel = true + add(MangaTag( + key = genre.getInt("id").toString(), + title = genreName, + source = source, + )) + } + } + } else { + emptySet() + } + + if (obj.optString("postType") == "رواية" || obj.optString("type") == "رواية") isNovel = true + + if (obj.optString("seriesType").equals("NOVEL", ignoreCase = true)) isNovel = true + + if (slug.contains("novel", ignoreCase = true)) isNovel = true + + if (isNovel) continue + + result.add( + Manga( + id = generateUid(url), + title = title, + altTitles = emptySet(), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = tags, + state = state, + authors = emptySet(), + source = source, + ) + ) + } + + return result + } + + private fun fetchAvailableTags(): Set<MangaTag> { + return setOf( + MangaTag("أكشن", "1", source), + MangaTag("حريم", "2", source), + MangaTag("زمكاني", "3", source), + MangaTag("سحر", "4", source), + MangaTag("شونين", "5", source), + MangaTag("مغامرات", "6", source), + MangaTag("خيال", "7", source), + MangaTag("رومانسي", "8", source), + MangaTag("كوميدي", "9", source), + MangaTag("مانهوا", "10", source), + MangaTag("إثارة", "11", source), + MangaTag("دراما", "12", source), + MangaTag("تاريخي", "13", source), + MangaTag("راشد", "14", source), + MangaTag("سينين", "15", source), + MangaTag("خارق للطبيعة", "16", source), + MangaTag("شياطين", "17", source), + MangaTag("حياة مدرسية", "18", source), + MangaTag("جوسي", "19", source), + MangaTag("مانها", "20", source), + MangaTag("ويبتون", "21", source), + MangaTag("شينين", "22", source), + MangaTag("قوة خارقة", "23", source), + MangaTag("خيال علمي", "24", source), + MangaTag("غموض", "25", source), + MangaTag("مأساة", "26", source), + MangaTag("شريحة من الحياة", "27", source), + MangaTag("Ùنون قتالية", "28", source), + MangaTag("شوجو", "29", source), + MangaTag("ايسكاي", "30", source), + MangaTag("مصاصي الدماء", "31", source), + MangaTag("اسبوعي", "32", source), + MangaTag("لعبة", "33", source), + MangaTag("Ù†Ùسي", "34", source), + MangaTag("وحوش", "35", source), + MangaTag("الحياة اليومية", "36", source), + MangaTag("الحياة المدرسية", "37", source), + MangaTag("رعب", "38", source), + MangaTag("عسكري", "39", source), + MangaTag("رياضي", "40", source), + MangaTag("اتشي", "41", source), + MangaTag("ايشي", "42", source), + MangaTag("دموي", "43", source), + MangaTag("زومبي", "44", source), + MangaTag("مميز", "45", source), + MangaTag("ايسيكاي", "46", source), + MangaTag("Ùنتازيا", "47", source), + MangaTag("اشباح", "48", source), + MangaTag("إعادة إحياء", "49", source), + MangaTag("بطل غير اعتيادي", "50", source), + MangaTag("ثأر", "51", source), + MangaTag("اثارة", "52", source), + MangaTag("تراجيدي", "53", source), + MangaTag("طبخ", "54", source), + MangaTag("تناسخ", "55", source), + MangaTag("عودة بالزمن", "56", source), + MangaTag("انتقام", "57", source), + MangaTag("تجسيد", "58", source), + MangaTag("Ùانتازيا", "59", source), + MangaTag("عائلي", "60", source), + MangaTag("تجسد", "61", source), + MangaTag("العاب", "62", source), + MangaTag("عالم اخر", "63", source), + MangaTag("السÙر عبر الزمن", "64", source), + MangaTag("خيالي", "65", source), + MangaTag("زمنكاني", "66", source), + MangaTag("مغامرة", "67", source), + MangaTag("طبي", "68", source), + MangaTag("عصور وسطى", "69", source), + MangaTag("ساموراي", "70", source), + MangaTag("ماÙيا", "71", source), + MangaTag("نظام", "72", source), + MangaTag("هوس", "73", source), + MangaTag("عصري", "74", source), + MangaTag("بطل مجنون", "75", source), + MangaTag("رعاية اطÙال", "76", source), + MangaTag("زواج مدبر", "77", source), + MangaTag("تشويق", "78", source), + MangaTag("مكتبي", "79", source), + MangaTag("قوى خارقه", "80", source), + MangaTag("تحقيق", "81", source), + MangaTag("أيتام", "82", source), + MangaTag("جوسين", "83", source), + MangaTag("موسيقي", "84", source), + MangaTag("قصة حقيقة", "85", source), + MangaTag("موريم", "86", source), + MangaTag("موظÙين", "87", source), + MangaTag("Ùيكتوري", "88", source), + MangaTag("مأساوي", "89", source), + MangaTag("عصر حديث", "90", source), + MangaTag("ندم", "91", source), + MangaTag("حياة جامعية", "92", source), + MangaTag("حاصد", "93", source), + MangaTag("الأرواح", "94", source), + MangaTag("جريمة", "95", source), + MangaTag("عاطÙÙŠ", "96", source), + MangaTag("أكاديمي", "97", source), + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + if (doc.selectFirst("span.inline-block.border.text-foreground")?.text() == "رواية") { + return manga.copy( + state = MangaState.FINISHED, + chapters = emptyList(), + ) + } + + val chapters = loadChapters(manga, doc) + + val coverUrl = doc.selectFirst("section img")?.src() ?: manga.coverUrl + + val ratingText = doc.selectFirst("div:contains(التقييم) + div, span:contains(Rating)")?.text() + val rating = ratingText?.substringBefore("/")?.trim()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN + + val statusText = doc.selectFirst("div:contains(الحالة) + div, span:contains(Status)")?.text() + val state = when { + statusText?.contains("مستمر", ignoreCase = true) == true || + statusText?.contains("ongoing", ignoreCase = true) == true -> MangaState.ONGOING + statusText?.contains("مكتمل", ignoreCase = true) == true || + statusText?.contains("completed", ignoreCase = true) == true -> MangaState.FINISHED + else -> null + } + + val description = doc.selectFirst("meta[name=description]")?.attr("content") + + val tags = doc.select("a[href*='/series/?genres='], span.genre").mapNotNullToSet { element -> + val genreName = element.text().trim() + val genreId = element.attr("href").substringAfter("genres=").substringBefore("&") + .ifEmpty { genreName } + + if (genreName.isNotEmpty()) { + MangaTag( + key = genreId, + title = genreName, + source = source, + ) + } else { + null + } + } + + return manga.copy( + coverUrl = coverUrl, + rating = rating, + state = state, + tags = tags, + description = description, + chapters = chapters, + ) + } + + private fun loadChapters(manga: Manga, doc: Document): List<MangaChapter> { + val seriesSlug = manga.url.substringAfter("/series/", "").substringBefore('/') + if (seriesSlug.isEmpty()) return emptyList() + + val chaptersJson = extractChaptersFromAstroProps(doc) ?: return emptyList() + + val chapters = ArrayList<MangaChapter>(chaptersJson.length()) + for (i in 0 until chaptersJson.length()) { + val chapter = unboxSerializedValue(chaptersJson.opt(i)) as? JSONObject ?: continue + val slug = (unboxSerializedValue(chapter.opt("slug")) as? String) + ?.takeUnless { it.isBlank() || it == "null" } + ?: continue + val url = "/series/$seriesSlug/$slug" + val number = when (val rawNumber = unboxSerializedValue(chapter.opt("number"))) { + is Number -> rawNumber.toFloat() + is String -> rawNumber.toFloatOrNull() ?: 0f + else -> 0f + } + val title = (unboxSerializedValue(chapter.opt("title")) as? String) + ?.takeUnless { it.isBlank() || it == "null" } + val createdAt = unboxSerializedValue(chapter.opt("createdAt")) as? String + + chapters += MangaChapter( + id = generateUid(url), + title = when { + !title.isNullOrBlank() && number > 0f -> "الÙصل ${formatChapterNumber(number)} - $title" + !title.isNullOrBlank() -> title + number > 0f -> "الÙصل ${formatChapterNumber(number)}" + else -> slug + }, + number = number, + volume = 0, + url = url, + scanlator = null, + uploadDate = parseChapterDate(createdAt), + branch = null, + source = source, + ) + } + return chapters.sortedBy { it.number } + } + + private fun extractChaptersFromAstroProps(doc: Document): JSONArray? { + val propsEscaped = doc + .selectFirst("section[data-series-tab-panel=chapters] astro-island[props]") + ?.attr("props") + ?.takeUnless { it.isBlank() } + ?: return null + + val props = Parser.unescapeEntities(propsEscaped, false) + val root = props.toJSONObjectOrNull() ?: return null + (unboxSerializedValue(root.opt("initialChap")) as? JSONArray)?.let { return it } + val post = unboxSerializedValue(root.opt("post")) as? JSONObject ?: return null + return unboxSerializedValue(post.opt("chapters")) as? JSONArray + } + + private fun unboxSerializedValue(value: Any?): Any? { + if (value !is JSONArray || value.length() != 2) { + return value + } + return when (value.opt(0)) { + 0, 1, "0", "1" -> unboxSerializedValue(value.opt(1)) + else -> value + } + } + + private fun formatChapterNumber(number: Float): String { + return if (number % 1f == 0f) { + number.toInt().toString() + } else { + number.toString() + } + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + return synchronized(chapterDateFormat) { chapterDateFormat.parseSafe(date) } + } + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select("section[itemprop=articleBody] figure img").mapNotNull { img -> + val imageUrl = img.src()?.takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/ar/DespairManga.kt b/src/main/kotlin/tsuki/site/ar/DespairManga.kt new file mode 100644 index 00000000..af695454 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/DespairManga.kt @@ -0,0 +1,11 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("DESPAIRMANGA", "Despair Manga", "ar") +internal class DespairManga(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.DESPAIRMANGA, "despair-manga.net") diff --git a/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt new file mode 100644 index 00000000..5a7c52b1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/DetectiveConan.kt @@ -0,0 +1,23 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +@MangaSourceParser("DETECTIVECONAN", "DetectiveConan", "ar", ContentType.MANGA) +internal class DetectiveConan(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.DETECTIVECONAN, "manga.detectiveconanar.com") { + override val datePattern = "MMMM dd, yyyy" + + override suspend fun fetchAvailableTags(): Set<MangaTag> = emptySet() + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = false, + isYearRangeSupported = false, + ) +} diff --git a/src/main/kotlin/tsuki/site/ar/Hijala.kt b/src/main/kotlin/tsuki/site/ar/Hijala.kt new file mode 100644 index 00000000..aa15f851 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Hijala.kt @@ -0,0 +1,145 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia +import tsuki.network.OkHttpWebClient + +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.Protocol +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.text.SimpleDateFormat +import java.util.Locale +import javax.imageio.ImageIO + +@MangaSourceParser("HIJALA", "Hijala", "ar") +internal class Hijala(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.HIJALA, "hijala.com") { // adjust domain if needed + + override val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("ar")) + + override val webClient = OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(::unscrambleInterceptor) + .build(), + source + ) + + private fun unscrambleInterceptor(chain: Interceptor.Chain): Response { + val request = chain.request() + val url = request.url + + if (url.host != SCRAMBLED_HOST) { + return chain.proceed(request) + } + + val leftUrl = url.queryParameter("leftImage") ?: error("Missing leftImage") + val rightUrl = url.queryParameter("rightImage") ?: error("Missing rightImage") + + // Fetch both pieces + val leftResponse = chain.proceed(request.newBuilder().url(leftUrl).build()) + val rightResponse = chain.proceed(request.newBuilder().url(rightUrl).build()) + + return try { + val leftBytes = leftResponse.body?.bytes() ?: error("Left image empty") + val rightBytes = rightResponse.body?.bytes() ?: error("Right image empty") + + val leftImage = ImageIO.read(ByteArrayInputStream(leftBytes)) + val rightImage = ImageIO.read(ByteArrayInputStream(rightBytes)) + + val width = leftImage.width + rightImage.width + val height = maxOf(leftImage.height, rightImage.height) + + val combined = BufferedImage(width, height, BufferedImage.TYPE_INT_RGB) + val g = combined.createGraphics() + g.drawImage(leftImage, 0, 0, null) + g.drawImage(rightImage, leftImage.width, 0, null) + g.dispose() + + val outputStream = ByteArrayOutputStream() + ImageIO.write(combined, "jpeg", outputStream) + val combinedBytes = outputStream.toByteArray() + + Response.Builder() + .request(request) + .protocol(Protocol.HTTP_1_1) + .code(200) + .message("OK") + .body(combinedBytes.toResponseBody("image/jpeg".toMediaType())) + .build() + } finally { + leftResponse.close() + rightResponse.close() + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + + val pages = mutableListOf<MangaPage>() + val jsonRegex = Regex(""""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + val jsonMatch = jsonRegex.find(doc.html()) + if (jsonMatch != null) { + val jsonArray = org.json.JSONArray(jsonMatch.groupValues[1]) + for (i in 0 until jsonArray.length()) { + val url = jsonArray.getString(i) + pages.add(MangaPage(id = generateUid(url), url = url, preview = null, source = source)) + } + } else { + doc.select(pageSelector).forEach { img -> + val url = img.imgAttr() + if (url.isNotBlank()) { + pages.add(MangaPage(id = generateUid(url), url = url, preview = null, source = source)) + } + } + } + + // if chapter is not scrambled return + if (doc.selectFirst("#chapter-pages-js-before") == null) return pages + + // group into left/right pairs and create synthetic URLs + val pairs = mutableListOf<List<String>>() + var i = 0 + while (i < pages.size) { + val pair = pages.subList(i, minOf(i + 2, pages.size)) + if (pair.size == 2) { + pairs.add(pair.map { it.url }) + } + i += 2 + } + + return pairs.mapIndexed { _, pair -> + val imageUrl = HttpUrl.Builder() + .scheme("http") + .host(SCRAMBLED_HOST) + .addQueryParameter("leftImage", pair[0]) + .addQueryParameter("rightImage", pair[1]) + .build() + .toString() + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source + ) + } + } + + companion object { + private const val SCRAMBLED_HOST = "127.0.0.1" + } +} diff --git a/src/main/kotlin/tsuki/site/ar/KawiiManga.kt b/src/main/kotlin/tsuki/site/ar/KawiiManga.kt new file mode 100644 index 00000000..32babbe9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/KawiiManga.kt @@ -0,0 +1,185 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded + +import java.time.Instant +import java.util.EnumSet +import kotlinx.coroutines.coroutineScope +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject + +@MangaSourceParser("KAWIIMANGA", "Kawaii Manga", "ar") +internal class KawiiManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.KAWIIMANGA, 20) { + + override val configKeyDomain = ConfigKey.Domain("kawaiimanga.org") + + private val apiUrl = "https://manga-api.kawaii-anime.com/api/manga/own" + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("x-app-key", "km_2026_live") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + init { + paginator.firstPage = 1 + searchPaginator.firstPage = 1 + } + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query.orEmpty() + val url = if (query.isNotBlank()) { + "$apiUrl?action=search&q=${query.urlEncoded()}" + } else { + val sort = when (order) { + SortOrder.POPULARITY -> "views" + else -> "latest" + } + "$apiUrl?action=browse&page=$page&sort=$sort" + } + val json = webClient.httpGet(url).parseJsonObject() + return parseMangaList(json) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val slug = manga.url + val json = webClient.httpGet("$apiUrl?action=series&slug=$slug").parseJsonObject() + val detailedManga = parseManga(json) + val chapters = parseChapters(json.optJSONArray("chapters"), slug) + detailedManga.copy(chapters = chapters) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfterLast('#') + val json = webClient.httpGet("$apiUrl?action=pages&chapterId=$chapterId").parseJsonObject() + val pagesArray = json.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun Response.parseJsonObject(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + private fun parseMangaList(json: JSONObject): List<Manga> { + val results = json.optJSONArray("results") ?: return emptyList() + return (0 until results.length()).map { i -> + parseManga(results.getJSONObject(i)) + } + } + + private fun parseManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("title") + val cover = obj.optString("coverUrl").takeIf { it.isNotBlank() } + val author = obj.optString("author").takeIf { it.isNotBlank() && it != "unknown" } + val artist = obj.optString("artist").takeIf { it.isNotBlank() && it != "unknown" } + val description = obj.optString("description").takeIf { it.isNotBlank() } + val type = obj.optString("type") + val status = obj.optString("status") + val genresArray = obj.optJSONArray("genres") + + val tags = buildSet { + when (type) { + "manga" -> add(MangaTag("manga", "Manga", source)) + "manhua" -> add(MangaTag("manhua", "Manhua", source)) + "manhwa" -> add(MangaTag("manhwa", "Manhwa", source)) + } + if (genresArray != null) { + for (i in 0 until genresArray.length()) { + val genre = genresArray.getString(i) + add(MangaTag(genre.lowercase(), genre, source)) + } + } + } + + val state = when (status) { + "ongoing", "coming_soon" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "cancelled", "dropped" -> MangaState.ABANDONED + else -> null + } + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = emptySet(), + authors = setOfNotNull(author, artist), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = tags, + state = state, + description = description, + contentRating = null, + source = source, + ) + } + + private fun parseChapters(jsonArray: JSONArray?, slug: String): List<MangaChapter> { + if (jsonArray == null) return emptyList() + return (0 until jsonArray.length()).map { i -> + val obj = jsonArray.getJSONObject(i) + val id = obj.getString("id") + val title = obj.optString("title") + val number = obj.getInt("number") + val createdAt = obj.optString("createdAt") + val chapterName = if (title.isNotBlank() && title != "$number") { + "الÙصل $number - $title" + } else { + "الÙصل $number" + } + val date = runCatching { Instant.parse(createdAt).toEpochMilli() }.getOrDefault(0L) + MangaChapter( + id = generateUid("$slug/$number#$id"), + url = "$slug/$number#$id", + title = chapterName, + number = number.toFloat(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } +} diff --git a/src/main/kotlin/tsuki/site/ar/LavaScans.kt b/src/main/kotlin/tsuki/site/ar/LavaScans.kt new file mode 100644 index 00000000..9c84d9de --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/LavaScans.kt @@ -0,0 +1,239 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Locale +import kotlin.time.Duration.Companion.hours + +@MangaSourceParser("LAVASCANS", "Lava Scans", "ar") +internal class LavaScans(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.LAVASCANS, "lavascans.com") { + + override val dateFormat = SimpleDateFormat("yyyy/MM/dd", Locale("ar")) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + private val slugRegex = Regex("""^\d+-""") + private val listUrl = "https://$domain/$mangaDirectory/list-mode/" + private val listSelector = "div#content div.soralist ul li a.series" + + private var urlMapCache: Map<String, String> = emptyMap() + private var mapFetchedAt: Long = 0L + private val mapMutex = Mutex() + + private suspend fun getUrlMap(): Map<String, String> { + if (System.currentTimeMillis() - mapFetchedAt < 1.hours.inWholeMilliseconds) return urlMapCache + return mapMutex.withLock { + if (System.currentTimeMillis() - mapFetchedAt < 1.hours.inWholeMilliseconds) return urlMapCache + urlMapCache = fetchUrlMap() + mapFetchedAt = System.currentTimeMillis() + urlMapCache + } + } + + private suspend fun fetchUrlMap(): Map<String, String> { + return try { + val doc = webClient.httpGet(listUrl).parseHtml() + doc.select(listSelector).associate { a -> + val fullUrl = a.attr("abs:href") + val slug = fullUrl.removeSuffix("/").substringAfterLast("/") + val permaSlug = slug.replaceFirst(slugRegex, "") + permaSlug to slug + } + } catch (_: Exception) { + emptyMap() + } + } + + private val browseSelector = ".listupd .manga-card-v" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val page = page + 1 + + val url = if (query.isNotEmpty()) { + val pagePath = if (page > 1) "page/$page/" else "" + "https://$domain/$pagePath?s=${query.urlEncoded()}" + } else { + buildString { + append("https://$domain/$mangaDirectory/") + append("?page=$page") + append("&order=") + append(when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ADDED -> "latest" + SortOrder.ALPHABETICAL -> "title" + SortOrder.ALPHABETICAL_DESC -> "titlereverse" + else -> "update" + }) + filter.states.firstOrNull()?.let { + append("&status=") + append(when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "dropped" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + append("&type=") + append(when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.NOVEL -> "novel" + else -> "" + }) + } + filter.tags.forEach { append("&genre[]=${it.key.urlEncoded()}") } + } + } + + val doc = webClient.httpGet(url).parseHtml() + val selector = if (query.isNotEmpty()) ".legend-card" else browseSelector + val elements = doc.select(selector) + if (elements.isEmpty()) return emptyList() + + val list = elements.mapNotNull { element -> + parseMangaElement(element, isSearch = query.isNotEmpty()) + } + + return list.map { manga -> + if (manga.url.startsWith("/$mangaDirectory/")) { + val slug = manga.url.removeSuffix("/").substringAfterLast("/") + val permaSlug = slug.replaceFirst(slugRegex, "") + manga.copy(url = "/$mangaDirectory/$permaSlug/") + } else { + manga + } + } + } + + private fun parseMangaElement(element: Element, isSearch: Boolean): Manga? { + return if (isSearch) { + val link = element.selectFirst(".legend-title a") + ?: element.selectFirst(".legend-poster a") ?: return null + val title = link.text().trim().ifBlank { link.attr("title") } + val cover = element.selectFirst(".legend-poster img")?.imgAttr() + val href = link.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } else { + val titleEl = element.selectFirst("h3.card-v-title a") ?: return null + val title = titleEl.text().trim() + val cover = element.selectFirst(".card-v-cover img")?.imgAttr() + val href = titleEl.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val permaSlug = manga.url.removeSuffix("/").substringAfterLast("/") + val randomSlug = getUrlMap()[permaSlug] ?: permaSlug + val fullUrl = "https://$domain/$mangaDirectory/$randomSlug/" + val doc = webClient.httpGet(fullUrl).parseHtml() + + val detailsContainer = doc.selectFirst("div.lh-container") ?: return manga + val title = detailsContainer.select(".lh-title").text() + val cover = detailsContainer.select(".lh-poster img").first()?.imgAttr() ?: manga.coverUrl + val genres = detailsContainer.select(".lh-genres a").map { it.text() }.toSet() + val desc = detailsContainer.select("#manga-story").textOrNull()?.trim() + val statusText = detailsContainer.select(".status-badge-lux").text() + val state = parseStatus(statusText) + val chapters = loadChapters(doc, fullUrl) + + return manga.copy( + title = title, + coverUrl = cover, + description = desc, + tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = state, + chapters = chapters, + altTitles = emptySet(), + authors = emptySet() + ) + } + + override val chapterListSelector = "#chapters-list-container .ch-item:not(.locked)" + + override suspend fun loadChapters(doc: Document, mangaUrl: String): List<MangaChapter> { + val elements = doc.select(chapterListSelector) + return elements.map { el -> + val a = el.selectFirst("a") ?: throw Exception("Missing chapter link") + val href = a.attrAsRelativeUrl("href") + val name = el.selectFirst(".ch-num")?.text().orEmpty().ifBlank { + a.selectFirst(".chapternum")?.text() ?: a.ownText() + } + val dateStr = el.selectFirst(".ch-date")?.text() + val uploadDate = dateStr?.let { dateFormat.parse(it)?.time ?: 0L } ?: 0L + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } +} diff --git a/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt b/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt new file mode 100644 index 00000000..16abf0d9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Manga3Asq.kt @@ -0,0 +1,14 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA3ASQ", "3Asq", "ar") +internal class Manga3Asq(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGA3ASQ, "3asq.online") { + override val datePattern = "d MMM\u060c yyy" + override val withoutAjax = true +} diff --git a/src/main/kotlin/tsuki/site/ar/Mangalek.kt b/src/main/kotlin/tsuki/site/ar/Mangalek.kt new file mode 100644 index 00000000..db8adf8f --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/Mangalek.kt @@ -0,0 +1,113 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource +import tsuki.model.MangaChapter +import tsuki.model.Manga +import tsuki.model.MangaTag + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.removeSuffix + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("MANGALEK", "Mangalek", "ar") +internal class Mangalek(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGALEK, "mangalik.net") { + + override val datePattern = "MMMM dd, yyyy" + override val stylePage = "" + + private val altDateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.US) + + private fun parseDateTwoFormats(primary: java.text.DateFormat, date: String?): Long { + date ?: return 0L + return try { + primary.parse(date)?.time ?: 0L + } catch (_: Exception) { + try { + altDateFormat.parse(date)?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.body().select(selectChapter).map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage // stylePage is empty, so just href + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = link, + uploadDate = parseDateTwoFormats(dateFormat, dateText), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> { + val doc = if (postReq) { + val mangaId = document.select("div#manga-chapters-holder").attr("data-id") + val url = "https://$domain/wp-admin/admin-ajax.php" + val postData = postDataReq + mangaId + webClient.httpPost(url, postData).parseHtml() + } else { + val url = mangaUrl.toAbsoluteUrl(domain).removeSuffix('/') + "/ajax/chapters/" + webClient.httpPost(url, emptyMap()).parseHtml() + } + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + return doc.select(selectChapter).map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attrAsRelativeUrl("href") + val link = href + stylePage + val dateText = li.selectFirst("a.c-new-tag")?.attr("title") + ?: li.selectFirst(selectDate)?.text() + val name = a.selectFirst("p")?.text() ?: a.ownText() + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = name.extractChapterNumber(), + volume = 0, + branch = null, + uploadDate = parseDateTwoFormats(dateFormat, dateText), + scanlator = null, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/$listUrl").parseHtml() + return doc.selectFirst("div.genres") + ?.select("a") + .orEmpty() + .mapNotNull { a -> + val genre = a.ownText() + if (genre.isBlank()) null + else MangaTag(key = genre.lowercase(), title = genre, source = source) + } + .toSet() + } +} diff --git a/src/main/kotlin/tsuki/site/ar/RocksManga.kt b/src/main/kotlin/tsuki/site/ar/RocksManga.kt new file mode 100644 index 00000000..803d8c92 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/RocksManga.kt @@ -0,0 +1,285 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded + +import kotlinx.coroutines.coroutineScope +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("ROCKSMANGA", "Rocks Manga", "ar") +internal class RocksManga(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.ROCKSMANGA, "rocksmanga.com") { + + override val datePattern = "MMMM d, yyyy" + override val withoutAjax = true + override val stylePage = "" + override val selectBodyPage = "div.reading-content" + override val selectPage = "img" + override val selectDesc = ".description" + override val selectGenre = "div.genres-content a" + override val selectChapter = "ul.scroll-sm li.item" + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val pages = page + 1 + val url = buildString { + append("https://") + append(domain) + + if (!filter.query.isNullOrEmpty()) { + // Text search + if (pages > 1) { + append("/page/") + append(pages) + } + append("/?s=") + append(filter.query!!.urlEncoded()) + append("&post_type=wp-manga") + } else { + // Browse with optional genre / type filter + val genreTag = filter.tags.firstOrNull() + val type = filter.types.firstOrNull() + + if (genreTag != null) { + append("/manga-genre/") + append(genreTag.key) + append("/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + } else if (type != null) { + val typeKey = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + ContentType.ONE_SHOT -> "one-shot" + else -> null + } + if (typeKey != null) { + append("/manga-type/") + append(typeKey) + append("/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + } + } else { + // Default browse – only one sort order is available, so hardcode it + append("/manga") + if (pages > 1) { + append("/page/") + append(pages) + } + append("/?m_orderby=latest") + } + } + } + + val doc = webClient.httpGet(url).parseHtml() + if (doc.location().removeSuffix("/").equals("https://$domain", ignoreCase = true)) { + return emptyList() + } + return parseMangaList(doc) + } + + override fun parseMangaList(doc: Document): List<Manga> { + val items = doc.select("div.original.card-lg div.unit") + return items.map { unit -> + val posterLink = unit.selectFirstOrThrow("a.poster") + val href = posterLink.attr("href").toRelativeUrl(domain) + val img = posterLink.selectFirst("img") + val info = unit.selectFirst("div.info") + val titleLink = info?.selectFirst("a") + val title = titleLink?.text()?.trim().orEmpty() + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = img?.src(), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("div.info h1")?.text() ?: manga.title + val coverUrl = doc.selectFirst("div.poster img")?.src() ?: manga.coverUrl + val description = doc.selectFirst("div.description")?.html()?.takeIf { it.isNotBlank() } + + val metaElements = doc.select("div.meta > div") + var author: String? = null + var status: MangaState? = null + val altTitles = mutableSetOf<String>() + + metaElements.forEach { element -> + val label = element.selectFirst("span")?.text()?.trim() ?: return@forEach + val value = element.selectFirst("a")?.text()?.trim() ?: element.ownText().trim() + + when { + label.contains("المؤلÙ") || label.contains("الكاتب") -> author = value + label.contains("الحالة") || label.contains("الوضع") -> { + status = when (value.lowercase()) { + "مستمر", "مستمرة", "ongoing" -> MangaState.ONGOING + "مكتمل", "مكتملة", "completed", "complete" -> MangaState.FINISHED + "متوقÙ", "متوقÙØ©", "hiatus" -> MangaState.PAUSED + "ملغي", "ملغية", "cancelled", "dropped" -> MangaState.ABANDONED + else -> null + } + } + label.contains("الأسماء البديلة") || label.contains("أسماء أخرى") -> { + value.split(",", "ØŒ", ";").forEach { name -> + val trimmed = name.trim() + if (trimmed.isNotEmpty()) altTitles.add(trimmed) + } + } + } + } + + val tags = doc.select("div.genres a, div.tags a").mapNotNullToSet { a -> + val href = a.attr("href").removeSuffix("/").substringAfterLast("/") + val name = a.text().trim() + if (href.isNotEmpty() && name.isNotEmpty()) { + MangaTag(key = href, title = name, source = source) + } else null + } + + val ratingText = doc.selectFirst("div.rating span.score")?.text() + val rating = ratingText?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN + + val chapters = getChapters(manga, doc) + + val isAdult = doc.selectFirst("div.adult-content") != null || + tags.any { it.key in setOf("adult", "mature", "18+", "ecchi", "smut") } + + manga.copy( + title = title, + altTitles = manga.altTitles + altTitles, + coverUrl = coverUrl, + largeCoverUrl = coverUrl, + description = description, + tags = tags, + state = status, + authors = setOfNotNull(author), + rating = rating, + chapters = chapters, + contentRating = if (isAdult) ContentRating.ADULT else ContentRating.SAFE, + publicUrl = fullUrl, + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + val dateFormat = SimpleDateFormat("MMMM d, yyyy", Locale("ar")) + fun SimpleDateFormat.parseSafeOrZero(date: String): Long = + try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } + + return doc.body().select("ul.scroll-sm li.item").map { li -> + val a = li.selectFirstOrThrow("a") + val href = a.attr("href").toRelativeUrl(domain) + val link = href + stylePage + + val chapterNum = a.attr("title").toFloatOrNull() ?: 0f + val name = "Chapter ${chapterNum.toInt().toString().removeSuffix(".0")}" + + val dateText = li.selectFirst("span.time")?.text().orEmpty() + val scanlator = li.selectFirst("span.user span")?.text() + + MangaChapter( + id = generateUid(href), + url = link, + title = name, + number = chapterNum, + volume = 0, + branch = null, + uploadDate = dateFormat.parseSafeOrZero(dateText), + scanlator = scanlator, + source = source, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val container = doc.selectFirst("#ch-images") + ?: doc.parseFailed("Container #ch-images not found") + val images = container.select("img.preload-image") + + return images.mapNotNull { img -> + val url = img.attr("data-src").ifEmpty { img.attr("src") } + .takeIf { it.isNotEmpty() } ?: return@mapNotNull null + val relativeUrl = url.toRelativeUrl(domain) + MangaPage( + id = generateUid(relativeUrl), + url = relativeUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val menu = doc.selectFirst("#nav-menu li:contains(التصنيÙات) ul") + ?: return emptySet() + return menu.select("li a").mapNotNull { a -> + val href = a.attr("href") + val slug = href.substringAfter("/manga-genre/").substringBefore("/") + if (slug.isEmpty()) null + else MangaTag(key = slug, title = a.text().trim(), source = source) + }.toSet() + } +} diff --git a/src/main/kotlin/tsuki/site/ar/StellarSaber.kt b/src/main/kotlin/tsuki/site/ar/StellarSaber.kt new file mode 100644 index 00000000..07ded6fc --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/StellarSaber.kt @@ -0,0 +1,364 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.await +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.toAbsoluteUrl + +import okhttp3.FormBody +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.MultipartBody +import okhttp3.Request +import okhttp3.Response +import okhttp3.ResponseBody.Companion.asResponseBody +import okio.buffer +import okio.cipherSource +import org.jsoup.nodes.Document +import java.util.EnumSet +import java.util.Base64 as JavaBase64 +import java.util.Locale +import javax.crypto.Cipher +import javax.crypto.spec.GCMParameterSpec +import javax.crypto.spec.SecretKeySpec +import kotlin.time.Duration +import kotlin.time.Duration.Companion.days +import kotlin.time.Duration.Companion.hours +import kotlin.time.Duration.Companion.minutes + +@MangaSourceParser("STELLARSABER", "StellarSaber", "ar") +internal class StellarSaber(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.STELLARSABER, pageSize = 24), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("stellarsaber.pro") + private val baseUrl = "https://$domain" + + private var nonce: String? = null + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = false, + isMultipleTagsSupported = false, + ) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.isNotEmpty()) { + return searchManga(page, query) + } + val url = when (order) { + SortOrder.POPULARITY -> "$baseUrl/manga/${page.pageNumber()}?sort=rating" + else -> "$baseUrl/manga/${page.pageNumber()}?sort=latest" + } + val doc = webClient.httpGet(url).parseHtml() + fetchNonce(doc) + return parseMangaList(doc) + } + + private suspend fun searchManga(page: Int, query: String): List<Manga> { + val nonce = fetchNonce() + val body = MultipartBody.Builder().apply { + setType(MultipartBody.FORM) + addFormDataPart("action", "flavor_ajax_filter_content") + addFormDataPart("nonce", nonce) + addFormDataPart("page", page.toString()) + addFormDataPart("keyword", query) + }.build() + + val request = Request.Builder() + .url("$baseUrl/wp-admin/admin-ajax.php") + .post(body) + .headers(getRequestHeaders()) + .build() + + val response = context.httpClient.newCall(request).await() + val json = response.parseJson() + val results = json.optJSONObject("data")?.optJSONArray("results") ?: return emptyList() + + val list = mutableListOf<Manga>() + for (i in 0 until results.length()) { + val item = results.getJSONObject(i) + val type = item.optString("type") + if (type in listOf("novel", "anime")) continue + list.add( + Manga( + id = generateUid(item.getString("url")), + url = item.getString("url"), + publicUrl = item.getString("url").toAbsoluteUrl(domain), + title = item.getString("title"), + coverUrl = item.optString("cover").takeIf { it.isNotBlank() }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + ) + } + return list + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select(".card-grid .card").mapNotNull { element -> + val a = element.select("a").firstOrNull() ?: return@mapNotNull null + val title = element.selectFirst(".card__title")?.text() ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + coverUrl = element.selectFirst("img")?.attr("abs:src"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun fetchNonce(): String { + nonce?.let { return it } + val doc = webClient.httpGet("$baseUrl/manga/").parseHtml() + return fetchNonce(doc) + } + + private fun fetchNonce(doc: Document): String { + nonce?.let { return it } + val script = doc.selectFirst("#flavor-ajax-js-extra")?.data() + ?: throw ParseException("Nonce script not found", "$baseUrl/manga/") + val nonceValue = script.extract("nonce\":\"", '"') + nonce = nonceValue + return nonceValue + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val updated = parseMangaDetails(doc, manga) + val chapters = parseChapterList(doc) + return updated.copy(chapters = chapters) + } + + private fun parseMangaDetails(doc: Document, original: Manga): Manga { + val element = doc.selectFirst(".detail-content") + ?: return original + + val title = element.selectFirst(".detail-info__title")?.text() ?: original.title + val cover = element.selectFirst(".detail-poster img")?.attr("abs:src") + val genres = element.select(".detail-genres a").map { it.text() }.toSet() + val description = element.select("#detail-desc p").joinToString { it.wholeText().trim() } + val otherName = element.select(".detail-info__alt-title").text() + val authors = mutableSetOf<String>() + val artists = mutableSetOf<String>() + var state: MangaState? = null + + element.select(".detail-meta__label").forEach { labelEl -> + val label = labelEl.text() + val value = labelEl.nextElementSibling()?.text() ?: return@forEach + when { + label.contains("المؤلÙ") -> authors.add(value) + label.contains("الرسّام") -> artists.add(value) + label.contains("الحالة") -> state = when { + "مستمر" in value -> MangaState.ONGOING + "مكتمل" in value -> MangaState.FINISHED + "ملغا" in value -> MangaState.ABANDONED + "متوقÙ" in value -> MangaState.PAUSED + else -> null + } + } + } + + return original.copy( + title = title, + coverUrl = cover ?: original.coverUrl, + description = buildString { + if (description.isNotBlank()) append(description) + if (otherName.isNotBlank()) { + if (isNotEmpty()) append("\n\n") + append(otherName) + } + }.trim().ifBlank { null }, + authors = authors + artists, + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT).replace(" ", "-"), source) }.toSet(), + state = state, + ) + } + + private fun parseChapterList(doc: Document): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + doc.select(".volume-group").forEach { volume -> + val volumeName = volume.selectFirst(".volume-group__label")?.text() + volume.select(".chapter-item").forEach { element -> + val a = element.selectFirst("a") ?: return@forEach + val href = a.attrAsRelativeUrl("href") + val title = element.selectFirst(".chapter-item__title")?.text() ?: return@forEach + val number = element.selectFirst(".chapter-item__number")?.text().orEmpty() + val name = buildString { + if (!volumeName.isNullOrEmpty()) append("$volumeName - ") + if (number.isNotEmpty() && !title.contains(number)) append("$number: ") + append(title) + } + chapters.add( + MangaChapter( + id = generateUid(href), + title = name, + number = number.toFloatOrNull() ?: -1f, + volume = 0, + url = href, + uploadDate = parseRelativeDate(element.selectFirst(".chapter-item__date")?.text()), + scanlator = element.selectFirst(".chapter-item__team")?.text(), + branch = null, + source = source, + ) + ) + } + } + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val key = fetchCdnKey(doc) + return doc.select("img.reader__page[data-cdn-url]") + .mapIndexed { index, img -> + val url = img.attr("abs:data-cdn-url") + MangaPage( + id = generateUid(url), + url = "$url#$key", + preview = null, + source = source, + ) + } + } + + private suspend fun fetchCdnKey(doc: Document): String { + val script = doc.selectFirst("script:containsData(flavorReaderData)")?.data() + ?: throw ParseException("Script not found", doc.location()) + val nonce = script.extract("cdnNonce: '", '\'') + val chapterId = script.extract("chapterId: ", ',') + + val formBody = FormBody.Builder() + .add("action", "flavor_cdn_get_key") + .add("nonce", nonce) + .add("chapter_id", chapterId) + .build() + + val request = Request.Builder() + .url("$baseUrl/wp-admin/admin-ajax.php") + .post(formBody) + .headers(getRequestHeaders()) + .build() + + val response = context.httpClient.newCall(request).await() + val json = response.parseJson() + return json.getJSONObject("data").getString("key") + } + + private fun parseRelativeDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val trimmed = date.trim() + val now = System.currentTimeMillis() + val number = Regex("""(\d+)""").find(trimmed)?.value?.toIntOrNull() + + val duration: Duration? = if (number == null) { + when { + trimmed.contains("دقيقتين") -> 2.minutes + trimmed.contains("دقيقة") -> 1.minutes + trimmed.contains("ساعتين") -> 2.hours + trimmed.contains("ساعة") -> 1.hours + trimmed.contains("يومين") -> 2.days + trimmed.contains("يوم") -> 1.days + trimmed.contains("أسبوعين") -> 14.days + trimmed.contains("أسبوع") -> 7.days + trimmed.contains("شهرين") -> 60.days + trimmed.contains("شهر") -> 30.days + trimmed.contains("سنتين") -> 730.days + trimmed.contains("سنة") -> 365.days + else -> null + } + } else { + when { + trimmed.contains("دقيقة") || trimmed.contains("دقائق") -> number.minutes + trimmed.contains("ساعة") || trimmed.contains("ساعات") -> number.hours + trimmed.contains("يوم") || trimmed.contains("أيام") -> number.days + trimmed.contains("أسبوع") || trimmed.contains("أسابيع") -> (number * 7).days + trimmed.contains("شهر") || trimmed.contains("أشهر") -> (number * 30).days + trimmed.contains("سنة") || trimmed.contains("سنوات") -> (number * 365).days + else -> null + } + } + return duration?.let { now - it.inWholeMilliseconds } ?: 0L + } + + private fun String.extract(startKey: String, endChar: Char): String { + val startIndex = indexOf(startKey) + require(startIndex != -1) { "$startKey not found" } + val valueStart = startIndex + startKey.length + val valueEnd = indexOf(endChar, valueStart) + require(valueEnd != -1) { "End char not found for $startKey" } + return substring(valueStart, valueEnd) + } + + private fun Int.pageNumber() = if (this > 1) "page/$this/" else "" + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val fragment = request.url.fragment ?: return response + + val keyBytes = JavaBase64.getDecoder().decode(fragment) + val source = response.body?.source() ?: return response + val iv = source.readByteArray(12) + + val cipher = Cipher.getInstance("AES/GCM/NoPadding").apply { + init( + Cipher.DECRYPT_MODE, + SecretKeySpec(keyBytes, "AES"), + GCMParameterSpec(128, iv), + ) + } + + val cipherSource = source.cipherSource(cipher) + return response.newBuilder() + .body(cipherSource.buffer().asResponseBody("image/jpeg".toMediaType(), -1)) + .build() + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/tsuki/site/ar/SwatManga.kt b/src/main/kotlin/tsuki/site/ar/SwatManga.kt new file mode 100644 index 00000000..8de12950 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ar/SwatManga.kt @@ -0,0 +1,248 @@ +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.urlEncoded + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SWATMANGA", "Swat Manga", "ar", ContentType.MANGA) +internal class SwatManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SWATMANGA, 20) { + + override val configKeyDomain = ConfigKey.Domain("meshmanga.com") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override val availableSortOrders: Set<SortOrder> = LinkedHashSet( + listOf( + SortOrder.RELEVANCE, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val response = webClient.httpGet("https://appswat.com/v2/api/v2/genres/").parseJsonArray() + val tags = mutableSetOf<MangaTag>() + for (i in 0 until response.length()) { + val genreObj = response.getJSONObject(i) + tags.add(MangaTag( + key = genreObj.getInt("id").toString(), + title = genreObj.getString("name"), + source = source, + )) + } + return tags + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://appswat.com/v2/api/v2/series/?page_size=20") + if (page > 1) { + val offset = (page - 1) * 20 + append("&offset=$offset") + } + if (!filter.query.isNullOrEmpty()) { + append("&search=${filter.query.urlEncoded()}") + } + when (order) { + SortOrder.POPULARITY -> append("&order_by=followers_count") + SortOrder.RATING -> append("&order_by=-rating") + else -> {} + } + if (filter.tags.isNotEmpty()) { + filter.tags.forEach { tag -> + append("&genres=${tag.key}") + } + } + } + + val response = webClient.httpGet(url).parseJson() + val results = response.getJSONArray("results") + + return (0 until results.length()).map { i -> + val item = results.getJSONObject(i) + parseMangaFromJson(item) + } + } + + private fun parseMangaFromJson(json: JSONObject): Manga { + val id = json.getInt("id") + val title = json.getString("title") + val slug = json.getString("slug") + + val status = json.getJSONObject("status") + val state = when (status.getString("name")) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val poster = json.getJSONObject("poster") + val coverUrl = poster.optString("medium").nullIfEmpty() + ?: poster.optString("thumbnail").nullIfEmpty() + + val rating = json.optString("rating", "0.0").toFloatOrNull() ?: 0f + val normalizedRating = if (rating > 0) rating / 10f else RATING_UNKNOWN + + val genres = json.getJSONArray("genres") + val tags = mutableSetOf<MangaTag>() + for (i in 0 until genres.length()) { + val genre = genres.getJSONObject(i) + tags.add(MangaTag( + key = genre.getInt("id").toString(), + title = genre.getString("name"), + source = source, + )) + } + + val authors = mutableSetOf<String>() + json.optJSONObject("translator")?.let { translator -> + authors.add(translator.getString("name")) + } + json.optJSONObject("editor")?.let { editor -> + authors.add(editor.getString("name")) + } + + return Manga( + id = generateUid(id.toString()), + url = "/series/$id", + publicUrl = "https://meshmanga.com/series/$slug/", + coverUrl = coverUrl, + title = title, + altTitles = emptySet(), + rating = normalizedRating, + tags = tags, + authors = authors, + state = state, + source = source, + contentRating = ContentRating.SAFE, + ) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val seriesId = manga.url.substringAfter("/series/") + val chaptersDeferred = async { getChapters(seriesId) } + + val detailUrl = "https://appswat.com/v2/api/v2/series/?page_size=100" + val response = webClient.httpGet(detailUrl).parseJson() + val results = response.getJSONArray("results") + + var updatedManga = manga + for (i in 0 until results.length()) { + val item = results.getJSONObject(i) + if (item.getInt("id").toString() == seriesId) { + updatedManga = parseMangaFromJson(item) + break + } + } + + return@coroutineScope updatedManga.copy( + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfter("/chapters/") + val chapterUrl = "https://appswat.com/v2/api/v2/chapters/$chapterId/" + + val response = webClient.httpGet(chapterUrl).parseJson() + val images = response.getJSONArray("images") + + return (0 until images.length()).map { i -> + val imageObj = images.getJSONObject(i) + val imageUrl = imageObj.getString("image") + val order = imageObj.getInt("order") + + MangaPage( + id = generateUid("$chapterId-$order"), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun getChapters(seriesId: String): List<MangaChapter> { + val allChapters = mutableListOf<JSONObject>() + var page = 1 + + while (true) { + val chaptersUrl = "https://appswat.com/v2/api/v2/chapters/?page=$page&serie=$seriesId&order_by=-order&page_size=20" + val response = webClient.httpGet(chaptersUrl).parseJson() + val results = response.getJSONArray("results") + + if (results.length() == 0) break + + for (i in 0 until results.length()) { + allChapters.add(results.getJSONObject(i)) + } + + val hasNext = !response.isNull("next") + if (!hasNext) break + page++ + } + + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + + return allChapters.mapIndexed { index, item -> + val chapterId = item.getInt("id") + val chapterNumber = item.optString("chapter", "").toFloatOrNull() ?: (index + 1f) + val title = item.getString("title") + val createdAt = item.getString("created_at") + + val uploadDate = try { + dateFormat.parse(createdAt)?.time ?: 0L + } catch (_: Exception) { + 0L + } + + MangaChapter( + id = generateUid(chapterId.toString()), + title = title, + number = chapterNumber, + volume = 0, + url = "/chapters/$chapterId", + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt similarity index 87% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt rename to src/main/kotlin/tsuki/site/ar/TeamXNovel.kt index 55bccbd8..699e97c9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/ar/TeamXNovel.kt +++ b/src/main/kotlin/tsuki/site/ar/TeamXNovel.kt @@ -1,17 +1,46 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.ar +package tsuki.site.ar + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireElementById +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("TEAMXNOVEL", "TeamXNovel", "ar") internal class TeamXNovel(context: MangaLoaderContext) : @@ -141,8 +170,8 @@ internal class TeamXNovel(context: MangaLoaderContext) : val mangaUrl = manga.url.toAbsoluteUrl(domain) val maxPageChapterSelect = doc.select(".pagination .page-item a") var maxPageChapter = 1 - if (!maxPageChapterSelect.isNullOrEmpty()) { - maxPageChapterSelect.map { + if (maxPageChapterSelect.isNotEmpty()) { + maxPageChapterSelect.forEach { val i = it.attr("href").substringAfterLast("=").toInt() if (i > maxPageChapter) { maxPageChapter = i @@ -269,4 +298,4 @@ internal class TeamXNovel(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/de/MangaTube.kt b/src/main/kotlin/tsuki/site/de/MangaTube.kt new file mode 100644 index 00000000..23dd282c --- /dev/null +++ b/src/main/kotlin/tsuki/site/de/MangaTube.kt @@ -0,0 +1,334 @@ +package tsuki.site.de + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.urlEncoded + +import okhttp3.FormBody +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.concurrent.TimeUnit + +@MangaSourceParser("MANGATUBE", "Manga Tube", "de") +internal class MangaTube(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGATUBE, 20) { + + override val configKeyDomain = ConfigKey.Domain("manga-tube.me") + internal val baseUrl = "https://$domain" + + internal val baseClient: OkHttpClient = context.httpClient.newBuilder() + .connectTimeout(1, TimeUnit.MINUTES) + .readTimeout(1, TimeUnit.MINUTES) + .writeTimeout(1, TimeUnit.MINUTES) + .build() + + override val webClient: OkHttpWebClient by lazy { + val client = baseClient.newBuilder() + .addInterceptor(ChallengeInterceptor(this)) + .addInterceptor { chain -> + val req = chain.request().newBuilder() + .header("User-Agent", getUserAgent()) + .build() + chain.proceed(req) + } + .build() + OkHttpWebClient(client, source) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + internal fun getUserAgent(): String = config[userAgentKey] + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY + ) + override val filterCapabilities = MangaListFilterCapabilities(isSearchSupported = true) + override suspend fun getFilterOptions() = MangaListFilterOptions() + + private val apiHeaders: Headers + get() = Headers.Builder() + .add("Accept", "application/json") + .add("Referer", baseUrl) + .build() + + private fun mangaApiHeaders(slug: String) = apiHeaders.newBuilder() + .add("Referer", "$baseUrl/series/$slug") + .add("Use-Parameter", "manga_slug") + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf(String::isNotEmpty) + if (query != null) return searchManga(query) + + return when (order) { + SortOrder.POPULARITY -> if (page == 1) popularManga() else emptyList() + SortOrder.UPDATED -> latestUpdates(page) + else -> emptyList() + } + } + + private suspend fun popularManga(): List<Manga> { + val json = webClient.httpGet("$baseUrl/api/home/top-manga", apiHeaders).parseJson() + return json?.getJSONObject("data")?.getJSONArray("manga")?.toMangaList().orEmpty() + } + + private suspend fun latestUpdates(page: Int): List<Manga> { + val offset = (page - 1) * LATEST_PAGE_SIZE + val json = webClient.httpGet("$baseUrl/api/home/updates?offset=$offset", apiHeaders).parseJson() + val published = json?.getJSONObject("data")?.getJSONArray("published") ?: return emptyList() + val mangas = (0 until published.length()).map { i -> + published.getJSONObject(i).getJSONObject("manga").toManga() + }.distinctBy { it.url } + return if (mangas.size < LATEST_PAGE_SIZE) mangas else mangas + } + + private suspend fun searchManga(query: String): List<Manga> { + val json = webClient.httpGet("$baseUrl/api/manga/quick-search?query=${query.urlEncoded()}", apiHeaders).parseJson() + return json?.getJSONArray("data")?.toMangaList().orEmpty() + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.substringAfterLast("/") + val json = webClient.httpGet("$baseUrl/api/manga/$slug", mangaApiHeaders(slug)).parseJson() + val data = json?.getJSONObject("data")?.getJSONObject("manga") ?: return manga + + val title = data.getString("title") + val cover = data.optString("cover", manga.coverUrl ?: "") + val description = data.optString("description", "").ifBlank { null } + val state = when (data.optInt("status", 0)) { 1 -> MangaState.ONGOING; 2 -> MangaState.FINISHED; else -> null } + val authors = data.optJSONArray("author")?.toNameList().orEmpty() + val artists = data.optJSONArray("artist")?.toNameList().orEmpty() + + val chaptersJson = webClient.httpGet("$baseUrl/api/manga/$slug/chapters", mangaApiHeaders(slug)).parseJson() + val chapters = chaptersJson?.getJSONObject("data")?.getJSONArray("chapters")?.toChapterList(slug).orEmpty() + .sortedWith(compareBy({ it.volume }, { it.number })) + + return manga.copy( + title = title, + coverUrl = cover, + largeCoverUrl = cover, + description = description, + state = state, + authors = authors + artists, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val slug = chapter.url.substringAfter("/api/manga/").substringBefore("/chapter/") + val json = webClient.httpGet("$baseUrl${chapter.url}", mangaApiHeaders(slug)).parseJson() + val pagesArray = json?.getJSONObject("data")?.getJSONObject("chapter")?.getJSONArray("pages") ?: return emptyList() + val pageList = (0 until pagesArray.length()).map { i -> + val obj = pagesArray.getJSONObject(i) + val url = obj.optString("url", "").ifEmpty { obj.optString("alt_source", "") } + obj.optInt("page", 0) to url + }.sortedBy { it.first } + return pageList.mapIndexed { _, (_, url) -> + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun JSONObject.toManga(): Manga = Manga( + id = generateUid(getString("url")), + url = getString("url"), + publicUrl = baseUrl + getString("url"), + title = getString("title"), + coverUrl = optString("cover", ""), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + + private fun JSONArray.toMangaList() = (0 until length()).map { getJSONObject(it).toManga() } + private fun JSONArray.toNameList() = (0 until length()).map { getJSONObject(it).getString("name") }.toSet() + + private fun JSONArray.toChapterList(slug: String) = (0 until length()).map { i -> + getJSONObject(i).toChapter(slug) + } + + private fun JSONObject.toChapter(slug: String): MangaChapter { + val id = getLong("id") + val number = optDouble("number", 0.0).toFloat() + val subNumber = optDouble("subNumber", 0.0).toFloat() + val volume = optDouble("volume", 0.0).toFloat() + val name = optString("name", "") + val publishedAt = optString("publishedAt", "") + + val title = buildString { + if (volume > 0) append("Vol. ${volume.toString().removeSuffix(".0")} ") + append("Ch. ${number.toString().removeSuffix(".0")}") + if (subNumber > 0) append(".${subNumber.toString().removeSuffix(".0")}") + if (name.isNotEmpty()) append(" - $name") + } + + return MangaChapter( + id = generateUid("/api/manga/$slug/chapter/$id"), + title = title, + number = number + subNumber / 10f, + volume = volume.toInt(), + url = "/api/manga/$slug/chapter/$id", + uploadDate = dateFormat.parseSafe(publishedAt), + source = source, + scanlator = null, + branch = null, + ) + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) + + companion object { + private const val LATEST_PAGE_SIZE = 40 + } +} + +private class ChallengeInterceptor( + private val parser: MangaTube, +) : Interceptor { + + override fun intercept(chain: Interceptor.Chain): Response { + val originalRequest = chain.request() + if (originalRequest.header(CHALLENGE_BYPASSED_HEADER) == "1") { + return chain.proceed(originalRequest) + } + + if (originalRequest.url.encodedPath.startsWith("/api/") && needsBootstrap(originalRequest.url)) { + bootstrapSession() + } + + val response = chain.proceed(originalRequest) + val bodyString = response.peekBody(Long.MAX_VALUE).string() + if (!isChallengePage(bodyString)) { + return response + } + + response.close() + solveChallenge(bodyString) + + return chain.proceed( + originalRequest.newBuilder() + .header(CHALLENGE_BYPASSED_HEADER, "1") + .build() + ) + } + + private fun isChallengePage(body: String): Boolean = + body.contains("window.__challange") || body.contains("_challange =") + + private fun needsBootstrap(url: HttpUrl): Boolean { + val cookies = parser.context.cookieJar.loadForRequest(url) + return REQUIRED_API_COOKIES.any { required -> cookies.none { it.name == required } } + } + + private fun bootstrapSession() { + val request = Request.Builder() + .url(parser.baseUrl) + .headers(sourceHeaders()) + .build() + val response = parser.baseClient.newCall(request).execute() + val bodyString = response.peekBody(Long.MAX_VALUE).string() + if (isChallengePage(bodyString)) { + response.close() + solveChallenge(bodyString) + parser.baseClient.newCall(request).execute().close() + return + } + response.close() + } + + private fun solveChallenge(body: String) { + val regex = Regex("""_challange = (.+?);""") + val match = regex.find(body) + ?: throw IOException("Challenge payload not found") + + val json = JSONObject(match.groupValues[1]) + val tk = json.getString("tk") + val arg1 = json.getString("arg1") + val arg2 = json.getString("arg2") + val arg3 = json.getString("arg3") + val solution = solve(arg1, arg2, arg3) + + val challengeHeaders = sourceHeaders().newBuilder() + .add("Content-Type", "application/x-www-form-urlencoded") + .add("x-challange-token", tk) + .add("x-challange-arg1", arg1) + .add("x-challange-arg2", arg2) + .add("x-challange-arg3", arg3) + .add("x-challange-arg4", solution) + .build() + + Thread.sleep(1000) + + val postBody = FormBody.Builder().build() + val postRequest = Request.Builder() + .url("${parser.baseUrl}/") + .post(postBody) + .headers(challengeHeaders) + .build() + + parser.baseClient.newCall(postRequest).execute().use { resp -> + if (!resp.isSuccessful) throw IOException("Challenge validation failed: ${resp.code}") + } + } + + private fun solve(arg1: String, arg2: String, op: String): String { + val left = arg1.toLong(16).toDouble() + val right = arg2.toLong(16).toDouble() + val result = when (op) { + "a" -> left / right + "b" -> left * right + "c" -> left - right + "d" -> left + right + else -> throw IOException("Unknown challenge op: $op") + } + return if (result == result.toLong().toDouble()) result.toLong().toString() else result.toString() + } + + private fun sourceHeaders(): Headers = Headers.Builder() + .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + .add("User-Agent", parser.getUserAgent()) + .build() + + companion object { + private val REQUIRED_API_COOKIES = setOf("XSRF-TOKEN", "manga_tube_beta_session", "__mtbpass") + private const val CHALLENGE_BYPASSED_HEADER = "X-MangaTube-Challenge-Bypassed" + } +} diff --git a/src/main/kotlin/tsuki/site/en/AnisaScans.kt b/src/main/kotlin/tsuki/site/en/AnisaScans.kt new file mode 100644 index 00000000..0f8d79f7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/AnisaScans.kt @@ -0,0 +1,21 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.MadaraParser +import tsuki.network.UserAgents + +import tsuki.model.MangaParserSource + +@MangaSourceParser("ANISASCANS", "AnisaScans", "en") +internal class AnisaScans(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.ANISASCANS, "anisascans.in") { + override val datePattern = "d MMMM, yyyy" + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt b/src/main/kotlin/tsuki/site/en/AquaReader.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt rename to src/main/kotlin/tsuki/site/en/AquaReader.kt index 079c25df..0ce6e8ec 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AquaManga.kt +++ b/src/main/kotlin/tsuki/site/en/AquaReader.kt @@ -1,16 +1,34 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone -@MangaSourceParser("AQUAMANGA", "AquaManga", "en") -internal class AquaManga(context: MangaLoaderContext) : +@MangaSourceParser("AQUAMANGA", "AquaReader", "en") +internal class AquaReader(context: MangaLoaderContext) : MadaraParser(context, MangaParserSource.AQUAMANGA, "aquareader.org", 20) { override val withoutAjax = true diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt b/src/main/kotlin/tsuki/site/en/AsuraScan.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt rename to src/main/kotlin/tsuki/site/en/AsuraScan.kt index c9bf0358..e2223bd9 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/AsuraScan.kt +++ b/src/main/kotlin/tsuki/site/en/AsuraScan.kt @@ -1,19 +1,49 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectOrThrow +import tsuki.util.src +import tsuki.util.substringBetween +import tsuki.util.toAbsoluteUrl +import tsuki.util.json.toJSONArrayOrNull +import tsuki.util.json.toJSONObjectOrNull import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.toJSONArrayOrNull -import org.koitharu.kotatsu.parsers.util.json.toJSONObjectOrNull import java.text.DateFormat import java.text.SimpleDateFormat -import java.util.* +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import java.util.TreeMap -@MangaSourceParser("ASURASCANS", "AsuraComic", "en") +@MangaSourceParser("ASURASCANS", "Asura Scans", "en") internal class AsuraScansParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.ASURASCANS, pageSize = 20) { @@ -199,7 +229,7 @@ internal class AsuraScansParser(context: MangaLoaderContext) : ?.groupValues ?.getOrNull(1) ?.toFloatOrNull() - ?: i + 1f + ?: (i + 1f) val dateText = a.selectFirst("span.text-sm.text-white\\/40") ?.text() ?.trim() @@ -346,4 +376,4 @@ internal class AsuraScansParser(context: MangaLoaderContext) : ) private val asuraGenreKeyRegex = Regex("[^a-z0-9]+") } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt b/src/main/kotlin/tsuki/site/en/Atsumaru.kt similarity index 57% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt rename to src/main/kotlin/tsuki/site/en/Atsumaru.kt index a5aa24db..5d0f74be 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Atsumaru.kt +++ b/src/main/kotlin/tsuki/site/en/Atsumaru.kt @@ -1,18 +1,35 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import okhttp3.Headers import okhttp3.HttpUrl.Companion.toHttpUrl import org.json.JSONArray import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* -import kotlinx.coroutines.async -import kotlinx.coroutines.coroutineScope +import java.util.EnumSet @MangaSourceParser("ATSUMARU", "Atsumaru", "en") internal class Atsumaru(context: MangaLoaderContext) : @@ -20,6 +37,7 @@ internal class Atsumaru(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("atsu.moe") private val baseUrl = "https://atsu.moe" + private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) private val apiHeaders: Headers by lazy { Headers.Builder() @@ -29,11 +47,19 @@ internal class Atsumaru(context: MangaLoaderContext) : .build() } + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showAdultKey) + } + override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.POPULARITY, // trending (views) SortOrder.UPDATED, // recently added (dateAdded) SortOrder.ADDED, // recently new added mangas - SortOrder.RELEVANCE // chapters count + SortOrder.RELEVANCE, // rising + SortOrder.ALPHABETICAL, + SortOrder.RATING, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -134,22 +160,75 @@ internal class Atsumaru(context: MangaLoaderContext) : override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = (allGenres.map { (name, id) -> MangaTag(name, "genre:$id", source) } + allTags.map { (name, id) -> MangaTag(name, "tag:$id", source) }).toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), ) - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> = getSearchPage(page, order, filter) + private suspend fun fetchRising(page: Int) = fetchList("rising", page) + private suspend fun fetchTopRated(page: Int) = fetchList("topRated", page) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + return when (order) { + SortOrder.RELEVANCE -> fetchRising(page) + SortOrder.RATING -> fetchTopRated(page) + else -> getSearchPage(page, order, filter) + } + } + + private suspend fun fetchList(path: String, page: Int): List<Manga> { + val offset = (page - 1) * pageSize + val url = "$baseUrl/api/home2/$path".toHttpUrl().newBuilder() + .addQueryParameter("offset", offset.toString()) + .addQueryParameter("limit", pageSize.toString()) + .build() + val json = webClient.httpGet(url, apiHeaders).parseJson() + val items = json.optJSONArray("items") ?: return emptyList() + val showAdult = config[showAdultKey] + return (0 until items.length()).mapNotNull { i -> + val obj = items.getJSONObject(i) + if (obj.optString("medium") == "Novel") return@mapNotNull null + val isAdult = obj.optBoolean("isAdult", false) + if (!showAdult && isAdult) return@mapNotNull null + val id = obj.getString("id") + val title = obj.getString("title") + val image = obj.optString("mediumImage").takeIf { it.isNotEmpty() } ?: obj.optString("image") + val coverUrl = if (image.isNotEmpty()) { + if (image.startsWith("http") || image.startsWith("//")) image + else "https://$domain/static/$image" + } else null + val type = obj.optString("type") + val rating = obj.optDouble("mbRating", -1.0).takeIf { it >= 0 }?.let { (it / 10.0).toFloat() } ?: RATING_UNKNOWN + Manga( + id = generateUid(id), + title = title, + altTitles = emptySet(), + url = "/manga/$id", + publicUrl = "$baseUrl/manga/$id", + rating = rating, + coverUrl = coverUrl, + tags = setOfNotNull(type.takeIf { it.isNotEmpty() }?.let { MangaTag(it, "type:$it", source) }), + source = source, + contentRating = if (isAdult) ContentRating.ADULT else null, + state = null, + authors = emptySet(), + ) + } + } private suspend fun getSearchPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query ?: "*" val url = "$baseUrl/collections/manga/documents/search".toHttpUrl().newBuilder().apply { - addQueryParameter("q", query) - if (query != "*") { + addQueryParameter("q", filter.query) + if (filter.query != "*") { addQueryParameter("query_by", "title,englishTitle,otherNames,authors") addQueryParameter("query_by_weights", "4,3,2,1") addQueryParameter("num_typos", "4,3,2,1") @@ -157,20 +236,22 @@ internal class Atsumaru(context: MangaLoaderContext) : addQueryParameter("page", page.toString()) addQueryParameter("per_page", pageSize.toString()) - val filterParts = mutableListOf("hidden:!=true", "views:>0") + val filterParts = mutableListOf("hidden:!=true", "views:>0", "medium:!=[`Novel`]") - val ratings = filter.contentRating - when { - ratings.contains(ContentRating.ADULT) && !ratings.contains(ContentRating.SAFE) -> { - filterParts.add("isAdult:=true") - } - ratings.contains(ContentRating.SAFE) && !ratings.contains(ContentRating.ADULT) -> { + val showAdult = config[showAdultKey] + + val explicitAdultFilter = filter.contentRating.contains(ContentRating.ADULT) + val explicitSafeFilter = filter.contentRating.contains(ContentRating.SAFE) + + if (explicitAdultFilter && !explicitSafeFilter) { + filterParts.add("isAdult:=true") + } else if (explicitSafeFilter && !explicitAdultFilter) { + filterParts.add("isAdult:=false") + } else { + if (!showAdult) { filterParts.add("isAdult:=false") } - // both or none: no filter shows all - else -> { /* no isAdult filter */ } } - filterParts.add("(mbContentRating:=[`Safe`,`Suggestive`,`Erotica`] || mbContentRating:!=*)") val types = filter.types.mapNotNull { ct -> when (ct) { @@ -203,21 +284,19 @@ internal class Atsumaru(context: MangaLoaderContext) : val includedTags = mutableListOf<String>() val excludedTags = mutableListOf<String>() - for (tag in filter.tags) { - val key = tag.key - when { - key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) - } - } - for (tag in filter.tagsExclude) { - val key = tag.key - when { - key.startsWith("genre:") -> excludedGenres.add(key.removePrefix("genre:")) - key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + fun processTags(tags: Collection<MangaTag>, genreAcc: MutableList<String>, tagAcc: MutableList<String>) { + for (tag in tags) { + val key = tag.key + when { + key.startsWith("genre:") -> genreAcc.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> tagAcc.add(key.removePrefix("tag:")) + } } } + processTags(filter.tags, includedGenres, includedTags) + processTags(filter.tagsExclude, excludedGenres, excludedTags) + if (includedGenres.isNotEmpty()) { filterParts.add(includedGenres.joinToString(" && ") { "genreIds:=`$it`" }) } @@ -237,11 +316,12 @@ internal class Atsumaru(context: MangaLoaderContext) : SortOrder.POPULARITY -> "trending:desc" SortOrder.UPDATED -> "released:desc" SortOrder.ADDED -> "dateAdded:desc" - SortOrder.RELEVANCE -> "chapterCount:desc" + SortOrder.RELEVANCE -> "rising:desc" + SortOrder.ALPHABETICAL -> "title:desc" + SortOrder.RATING -> "topRated:desc" else -> "released:desc" } addQueryParameter("sort_by", sort) - addQueryParameter("sort_by", sort) }.build() val json = webClient.httpGet(url, apiHeaders).parseJson() @@ -251,130 +331,133 @@ internal class Atsumaru(context: MangaLoaderContext) : } } - // limit tags to 20 - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val result = coroutineScope { - val mangaId = manga.url.substringAfterLast("/") + private val detailsCacheLock = Any() - val pageDeferred = async { - webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() - } - val chaptersDeferred = async { - webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() - } - - val pageJson = pageDeferred.await() - val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga - - val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } - val description = mangaPage.optString("synopsis") ?: manga.description - - val rawRating = mangaPage.optDouble("avgRating", -1.0) - val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN - - val posterObj = mangaPage.optJSONObject("poster") - val posterImage = posterObj?.optString("mediumImage") - val coverUrl = if (!posterImage.isNullOrEmpty()) { - "https://$domain/static/$posterImage" - } else manga.coverUrl - - val authors = mangaPage.optJSONArray("authors")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() - } ?: emptySet() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } - val statusText = mangaPage.optString("status").orEmpty() - val state = when (statusText.lowercase()) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "hiatus" -> MangaState.PAUSED - "canceled" -> MangaState.ABANDONED - else -> manga.state - } + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + val result = try { + coroutineScope { + val mangaId = manga.url.substringAfterLast("/") + + val pageDeferred = async { webClient.httpGet("$baseUrl/api/manga/page?id=$mangaId", apiHeaders).parseJson() } + val pageJson = pageDeferred.await() + val mangaPage = pageJson.optJSONObject("mangaPage") ?: return@coroutineScope manga + + val title = mangaPage.optString("title").ifEmpty { mangaPage.optString("englishTitle", manga.title) } + val description = mangaPage.optString("synopsis") ?: manga.description + + val rawRating = mangaPage.optDouble("avgRating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val posterObj = mangaPage.optJSONObject("poster") + val posterImage = posterObj?.optString("mediumImage") + val coverUrl = if (!posterImage.isNullOrEmpty()) { + "https://$domain/static/$posterImage" + } else manga.coverUrl + + val authors = mangaPage.optJSONArray("authors")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optJSONObject(it)?.optString("name") }.toSet() + } ?: emptySet() + + val statusText = mangaPage.optString("status").orEmpty() + val state = when (statusText.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> manga.state + } - val tagSet = mutableSetOf<MangaTag>() - mangaPage.optJSONArray("genres")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) + val tagSet = mutableSetOf<MangaTag>() + mangaPage.optJSONArray("genres")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "genre:${obj.optString("id")}", source)) + } } - } - mangaPage.optJSONArray("tags")?.let { arr -> - for (i in 0 until minOf(arr.length(), 10)) { - val obj = arr.optJSONObject(i) ?: continue - val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue - tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) + mangaPage.optJSONArray("tags")?.let { arr -> + for (i in 0 until minOf(arr.length(), 10)) { + val obj = arr.optJSONObject(i) ?: continue + val name = obj.optString("name").takeIf { it.isNotEmpty() } ?: continue + tagSet.add(MangaTag(name, "tag:${obj.optString("id")}", source)) + } } - } - - val chaptersJson = chaptersDeferred.await() - val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() - val scanlators = mutableMapOf<String, String>() - mangaPage.optJSONArray("scanlators")?.let { scanArr -> - for (i in 0 until scanArr.length()) { - val sc = scanArr.optJSONObject(i) ?: continue - val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue - val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue - scanlators[id] = name + val scanlators = mutableMapOf<String, String>() + mangaPage.optJSONArray("scanlators")?.let { scanArr -> + for (i in 0 until scanArr.length()) { + val sc = scanArr.optJSONObject(i) ?: continue + val id = sc.optString("id").takeIf { it.isNotEmpty() } ?: continue + val name = sc.optString("name").takeIf { it.isNotEmpty() } ?: continue + scanlators[id] = name + } } - } - val chapters = (0 until chaptersArray.length()).map { i -> - val ch = chaptersArray.getJSONObject(i) - val chId = ch.getString("id") - val number = ch.optDouble("number", 0.0).toFloat() - val chTitle = ch.optString("title") - val scanId = ch.optString("scanlationMangaId").nullIfEmpty() - val scanName = scanId?.let { scanlators[it] } - val createdAt = ch.optLong("createdAt", 0L) - - MangaChapter( - id = generateUid("$mangaId/$chId"), - title = chTitle.ifBlank { "Chapter $number" }, - number = number, - volume = 0, - url = "$mangaId/$chId", - uploadDate = createdAt, - source = source, - scanlator = scanName, - branch = scanName, + val chapters = loadChapters(mangaId, scanlators) + + val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> + (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() + } ?: emptySet() + + manga.copy( + title = title, + description = description, + coverUrl = coverUrl, + authors = authors, + state = state, + tags = tagSet, + rating = rating, + chapters = chapters, + altTitles = altTitles, ) - }.reversed() - - val groupedChapters = if (chapters.map { it.branch }.distinct().size > 1) { - chapters.map { it.copy(branch = it.branch ?: "Unknown") } - } else { - chapters.map { it.copy(branch = null, scanlator = null) } } - - val altTitles = mangaPage.optJSONArray("otherNames")?.let { arr -> - (0 until arr.length()).mapNotNull { arr.optString(it) }.toSet() - } ?: emptySet() - - manga.copy( - title = title, - description = description, - coverUrl = coverUrl, - authors = authors, - state = state, - tags = tagSet, - rating = rating, - chapters = groupedChapters, - altTitles = altTitles, - ) + } catch (_: Exception) { + manga + } + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result } - - detailsCache.put(manga.url, result) return result } - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean { - return size > 10 + private suspend fun loadChapters(mangaId: String, scanlators: Map<String, String>): List<MangaChapter> { + val chaptersJson = webClient.httpGet("$baseUrl/api/manga/allChapters?mangaId=$mangaId", apiHeaders).parseJson() + val chaptersArray = chaptersJson.optJSONArray("chapters") ?: JSONArray() + + val chapters = (0 until chaptersArray.length()).map { i -> + val ch = chaptersArray.getJSONObject(i) + val chId = ch.getString("id") + val number = ch.optDouble("number", 0.0).toFloat() + val chTitle = ch.optString("title") + val scanId = ch.optString("scanlationMangaId").nullIfEmpty() + val scanName = scanId?.let { scanlators[it] } + val createdAt = ch.optLong("createdAt", 0L) + + MangaChapter( + id = generateUid("$mangaId/$chId"), + title = chTitle.ifBlank { "Chapter $number" }, + number = number, + volume = 0, + url = "$mangaId/$chId", + uploadDate = createdAt, + source = source, + scanlator = scanName, + branch = scanName, + ) + }.sortedBy { it.number } + + val distinctBranches = chapters.map { it.branch }.distinct() + return if (distinctBranches.size > 1) { + chapters.map { it.copy(branch = it.branch ?: "Unknown") } + } else { + chapters.map { it.copy(branch = null, scanlator = null) } } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt b/src/main/kotlin/tsuki/site/en/Batcave.kt similarity index 61% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt rename to src/main/kotlin/tsuki/site/en/Batcave.kt index 7699d15d..d6972123 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Batcave.kt +++ b/src/main/kotlin/tsuki/site/en/Batcave.kt @@ -1,251 +1,340 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.FormBody -import okhttp3.MediaType.Companion.toMediaTypeOrNull -import okhttp3.OkHttpClient -import okhttp3.Request -import okhttp3.RequestBody.Companion.toRequestBody -import okhttp3.Response -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) -internal class Batcave(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { - - override val configKeyDomain = ConfigKey.Domain("batcave.biz") - - private var publishers: List<Pair<String, Int>> = emptyList() - private var genres: List<Pair<String, Int>> = emptyList() - private var filterParseFailed = false - - private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.US) - - private val apiClient: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor(::refererInterceptor) - .addInterceptor(::dleGuardInterceptor) - .build() - OkHttpWebClient(httpClient, source) - } - - private val rawHttpClient: OkHttpClient by lazy { context.httpClient.newBuilder().build() } - - private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { - val request = chain.request() - val url = request.url.toString() - val referer = when { - url.contains("readcomicsonline.ru") -> "https://readcomicsonline.ru/" - else -> "https://$domain/" - } - return chain.proceed(request.newBuilder().header("Referer", referer).build()) - } - - private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { - val originalRequest = chain.request() - val response = chain.proceed(originalRequest) - if (response.request.url.pathSegments.firstOrNull() != "_c") { - return response - } - response.close() - val url = if (originalRequest.method == "GET") { - originalRequest.url.toString() - } else { - "https://$domain/" - } - context.requestBrowserAction(this, url) - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = emptySet(), - ) - } - - private fun parseFilterValues(filterItems: JSONObject, key: String): List<Pair<String, Int>> { - val obj = filterItems.optJSONObject(key) ?: return emptyList() - val values = obj.optJSONArray("values") ?: return emptyList() - val result = mutableListOf<Pair<String, Int>>() - for (i in 0 until values.length()) { - val item = values.getJSONObject(i) - result.add(item.getString("value") to item.getInt("id")) - } - return result - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - if (query != null) return searchManga(query, page) - - val urlBuilder = StringBuilder().apply { - append("https://$domain/ComicList/") - if (filter.yearFrom != YEAR_UNKNOWN) append("y[from]=${filter.yearFrom}/") - if (filter.yearTo != YEAR_UNKNOWN) append("y[to]=${filter.yearTo}/") - val pIds = filter.tags.filter { it.key.startsWith("p_") }.map { it.key.removePrefix("p_") } - val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } - if (pIds.isNotEmpty()) append("p=${pIds.joinToString(",")}/") - if (gIds.isNotEmpty()) append("g=${gIds.joinToString(",")}/") - append("sort") - if (page > 1) append("/page/$page/") - } - val url = urlBuilder.toString() - - val sortPair = when (order) { - SortOrder.POPULARITY -> "rating" to "desc" - SortOrder.UPDATED -> "date" to "desc" - else -> "" to "" - } - - return if (sortPair.first.isEmpty()) { - parseMangaList(apiClient.httpGet(url).parseHtml()) - } else { - val formBody = FormBody.Builder() - .add("dlenewssortby", sortPair.first) - .add("dledirection", sortPair.second) - .add("set_new_sort", "dle_sort_xfilter") - .add("set_direction_sort", "dle_direction_xfilter") - .build() - val request = Request.Builder().url(url).post(formBody).build() - val response = rawHttpClient.newCall(request).execute() - parseMangaList(response.parseHtml()) - } - } - - private suspend fun searchManga(query: String, page: Int): List<Manga> { - val encoded = query.urlEncoded() - val url = if (page == 1) "https://$domain/search/$encoded/" - else "https://$domain/search/$encoded/page/$page/" - return parseMangaList(apiClient.httpGet(url).parseHtml()) - } - - private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { - return doc.select("#dle-content > .readed").map { element -> - val a = element.selectFirst(".readed__title > a") ?: return@map null - val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") - Manga( - id = generateUid(a.attrAsRelativeUrl("href")), - url = a.attrAsRelativeUrl("href"), - publicUrl = a.absUrl("href"), - title = a.ownText(), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - }.filterNotNull() - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - - val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title - val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") - val description = doc.selectFirst("div.page__text")?.text() - val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() - val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { - "Ongoing" -> MangaState.ONGOING - else -> MangaState.FINISHED - } - - val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() - ?: throw ParseException("Chapter data script not found", manga.url) - val json = JSONObject( - script.substringAfter("window.__DATA__ = ").substringBeforeLast(";").trim() - ) - val newsId = json.getLong("news_id") - val chaptersArray = json.getJSONArray("chapters") - val chapters = (0 until chaptersArray.length()).map { i -> - val ch = chaptersArray.getJSONObject(i) - MangaChapter( - id = generateUid("/reader/$newsId/${ch.getInt("id")}"), - url = "/reader/$newsId/${ch.getInt("id")}", - title = ch.optString("title"), - number = ch.optDouble("posi", 0.0).toFloat(), - volume = 0, - scanlator = null, - uploadDate = dateFormat.parseSafe(ch.optString("date")), - branch = null, - source = source, - ) - }.reversed() - - val result = manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - description = description, - authors = setOfNotNull(author), - state = state, - tags = emptySet(), - chapters = chapters, - ) - detailsCache[manga.url] = result - return result - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) - val id = Regex("^\\d+").find(rawId)?.value ?: rawId - val jsonBody = JSONObject().apply { - put("news_id", newsId) - put("chapter_id", id) - }.toString() - - val request = Request.Builder() - .url("https://$domain/engine/ajax/controller.php?mod=api&action=reader/getChapterData") - .header("Referer", "https://$domain/") - .header("Content-Type", "application/json") - .post( - jsonBody - .toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull()) - ) - .build() - - val response = rawHttpClient.newCall(request).execute() - val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) - val data = json.getJSONObject("data") - val images = data.getJSONArray("images") - - return (0 until images.length()).map { i -> - var img = images.getString(i).trim() - if (!img.startsWith("http")) img = "https://$domain$img" - MangaPage(id = generateUid(img), url = img, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - - private fun SimpleDateFormat.parseSafe(date: String?): Long = - if (date.isNullOrBlank()) 0L else runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + +import okhttp3.FormBody +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import org.json.JSONObject +import org.jsoup.Jsoup +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("BATCAVE", "Batcave", "en", ContentType.COMICS) +internal class Batcave(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BATCAVE, 20) { + + override val configKeyDomain = ConfigKey.Domain("batcave.biz") + + private val dateFormat = SimpleDateFormat("dd.MM.yyyy", Locale.ENGLISH) + + @Volatile + private var genreList: List<Pair<String, Int>>? = null + private var filterFetchFailed = false + + private val rawHttpClient: OkHttpClient by lazy { + context.httpClient.newBuilder() + .addInterceptor(::refererInterceptor) + .addInterceptor(::dleGuardInterceptor) + .build() + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Sec-Fetch-Dest", "document") + .set("Sec-Fetch-Mode", "navigate") + .set("Sec-Fetch-Site", "none") + .set("Sec-Fetch-User", "?1") + .build() + + private val apiClient: OkHttpWebClient by lazy { + OkHttpWebClient(rawHttpClient, source) + } + + private fun refererInterceptor(chain: okhttp3.Interceptor.Chain): Response { + val request = chain.request() + val url = request.url.toString() + val referer = when { + url.contains("readcomicsonline.ru") -> "https://readcomicsonline.ru/" + else -> "https://$domain/" + } + return chain.proceed(request.newBuilder().header("Referer", referer).build()) + } + + private fun dleGuardInterceptor(chain: okhttp3.Interceptor.Chain): Response { + val originalRequest = chain.request() + val response = chain.proceed(originalRequest) + if (response.request.url.pathSegments.firstOrNull() != "_c") { + return response + } + response.close() + val url = if (originalRequest.method == "GET") { + originalRequest.url.toString() + } else { + "https://$domain/" + } + context.requestBrowserAction(this, url) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + isYearSupported = false, + ) + + private suspend fun fetchFilters() { + if (genreList != null) return + if (filterFetchFailed) return + + try { + val doc = apiClient.httpGet("https://$domain/comix").parseHtml() + val script = doc.selectFirst("script:containsData(window.__XFILTER__)")?.data() + ?: throw ParseException("Filter data not found", "https://$domain/comix") + + val rawJson = script + .substringAfter("window.__XFILTER__ = ") + .substringBeforeLast(";") + .trim() + val root = JSONObject(rawJson) + val filterItems = root.getJSONObject("filter_items") + + genreList = parseFilterValues(filterItems) + } catch (_: Exception) { + filterFetchFailed = true + } + } + + private fun parseFilterValues(filterItems: JSONObject): List<Pair<String, Int>> { + val obj = filterItems.optJSONObject("g") ?: return emptyList() + val values = obj.optJSONArray("values") ?: return emptyList() + val result = mutableListOf<Pair<String, Int>>() + for (i in 0 until values.length()) { + val item = values.getJSONObject(i) + result.add(item.getString("value") to item.getInt("id")) + } + return result + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + fetchFilters() + val tags = mutableSetOf<MangaTag>() + genreList?.forEach { (name, id) -> + tags += MangaTag(name, "g_$id", source) + } + return MangaListFilterOptions( + availableTags = tags, + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrBlank()) { + return searchManga(page, filter.query!!) + } + + val urlBuilder = StringBuilder().apply { + append("https://$domain/ComicList/") + val gIds = filter.tags.filter { it.key.startsWith("g_") }.map { it.key.removePrefix("g_") } + if (gIds.isNotEmpty()) append("g=${gIds.joinToString(",")}/") + append("sort") + if (page > 1) append("/page/$page/") + } + val url = urlBuilder.toString() + + val sortPair = when (order) { + SortOrder.POPULARITY -> "rating" to "desc" + SortOrder.UPDATED -> "date" to "desc" + else -> "" to "" + } + + return if (sortPair.first.isEmpty()) { + parseMangaList(apiClient.httpGet(url).parseHtml()) + } else { + val formBody = FormBody.Builder() + .add("dlenewssortby", sortPair.first) + .add("dledirection", sortPair.second) + .add("set_new_sort", "dle_sort_xfilter") + .add("set_direction_sort", "dle_direction_xfilter") + .build() + val request = Request.Builder().url(url).post(formBody).build() + val response = rawHttpClient.newCall(request).execute() + parseMangaList(response.parseHtml()) + } + } + + private suspend fun searchManga(page: Int, query: String): List<Manga> { + val url = "https://$domain".toHttpUrl().newBuilder() + .addPathSegment("search") + .addPathSegment(query) + .apply { + if (page > 1) { + addPathSegment("page") + addPathSegment(page.toString()) + addPathSegment("") + } + } + .build() + val html = apiClient.httpGet(url.toString()).body?.string() ?: return emptyList() + val doc = Jsoup.parse(html, "https://$domain") + return doc.select("#dle-content > .readed").map { element -> + val a = element.selectFirst(".readed__title > a") ?: return@map null + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.absUrl("href"), + title = a.ownText(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = element.selectFirst(".readed__img img")?.attrAsAbsoluteUrl("data-src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("#dle-content > .readed").map { element -> + val a = element.selectFirst(".readed__title > a") ?: return@map null + val cover = element.selectFirst("img")?.attrAsAbsoluteUrl("data-src") + Manga( + id = generateUid(a.attrAsRelativeUrl("href")), + url = a.attrAsRelativeUrl("href"), + publicUrl = a.absUrl("href"), + title = a.ownText(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("header.page__header h1")?.text() ?: manga.title + val cover = doc.selectFirst("div.page__poster img")?.absUrl("src") + val description = doc.selectFirst("div.page__text")?.text() + + val author = doc.selectFirst(".page__list > li:has(> div:contains(Writer)) a")?.text() + ?: doc.selectFirst(".page__list > li:has(> div:contains(Writer))")?.ownText() + val state = when (doc.selectFirst(".page__list > li:has(> div:contains(Release type))")?.ownText()?.trim()) { + "Ongoing" -> MangaState.ONGOING + else -> MangaState.FINISHED + } + val tags = doc.select("div.page__tags a").map { a -> + MangaTag(a.text(), a.text().lowercase().replace(" ", "-"), source) + }.toSet() + MangaTag("Comic", "comic", source) + + val script = doc.selectFirst("script:containsData(window.__DATA__)")?.data() + ?: throw ParseException("Chapter data script not found", manga.url) + val json = JSONObject( + script.substringAfter("window.__DATA__ = ").substringBeforeLast(";").trim() + ) + val newsId = json.getLong("news_id") + val chaptersArray = json.getJSONArray("chapters") + val chapters = (0 until chaptersArray.length()).map { i -> + val ch = chaptersArray.getJSONObject(i) + MangaChapter( + id = generateUid("/reader/$newsId/${ch.getInt("id")}"), + url = "/reader/$newsId/${ch.getInt("id")}", + title = ch.optString("title"), + number = ch.optDouble("posi", 0.0).toFloat(), + volume = 0, + scanlator = null, + uploadDate = dateFormat.parseSafe(ch.optString("date")), + branch = null, + source = source, + ) + }.reversed() + + val result = manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + description = description, + authors = setOfNotNull(author), + state = state, + tags = tags, + chapters = chapters, + ) + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val (newsId, rawId) = chapter.url.substringAfter("reader/").split("/", limit = 2) + val id = Regex("^\\d+").find(rawId)?.value ?: rawId + val jsonBody = JSONObject().apply { + put("news_id", newsId) + put("chapter_id", id) + }.toString() + + val request = Request.Builder() + .url("https://$domain/engine/ajax/controller.php?mod=api&action=reader/getChapterData") + .header("Referer", "https://$domain/") + .header("Content-Type", "application/json") + .post(jsonBody.toRequestBody("application/json; charset=utf-8".toMediaTypeOrNull())) + .build() + + val response = rawHttpClient.newCall(request).execute() + val json = JSONObject(response.body?.string() ?: throw ParseException("Empty response", chapter.url)) + val data = json.getJSONObject("data") + val images = data.optJSONArray("images") ?: throw ParseException("No images found", chapter.url) + + return (0 until images.length()).map { i -> + var img = images.getString(i).trim() + if (!img.startsWith("http")) img = "https://$domain$img" + MangaPage(id = generateUid(img), url = img, preview = null, source = source) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private fun SimpleDateFormat.parseSafe(date: String?): Long = + if (date.isNullOrBlank()) 0L else runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) +} diff --git a/src/main/kotlin/tsuki/site/en/Chikari.kt b/src/main/kotlin/tsuki/site/en/Chikari.kt new file mode 100644 index 00000000..a8f0e9f8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Chikari.kt @@ -0,0 +1,420 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.YEAR_UNKNOWN + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.oneOrThrowIfMany + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.time.Instant +import java.util.EnumSet + +@MangaSourceParser("CHIKARI", "Chikari", "en") +class Chikari(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CHIKARI, pageSize = 36) { + + override val configKeyDomain = ConfigKey.Domain("chikari.moe") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = true, + ) + + private val tags by lazy { + GENRES.entries.map { (slug, title) -> + MangaTag(key = slug, title = title, source = source) + }.toSet() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER, // OEL + ), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val offset = (page - 1) * pageSize + val url = buildApiUrl(offset, order, filter) + val response = webClient.httpGet(url).parseJson() + return parseMangaList(response) + } + + private fun buildApiUrl(offset: Int, order: SortOrder, filter: MangaListFilter): String { + val sortParam = when (order) { + SortOrder.NEWEST -> "added" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "title" + SortOrder.UPDATED -> "updated" + SortOrder.RATING -> "top_rated" + else -> "popular" + } + + return "https://$domain/api/series".toHttpUrl().newBuilder().apply { + addQueryParameter("sort", sortParam) + addQueryParameter("limit", pageSize.toString()) + addQueryParameter("offset", offset.toString()) + addQueryParameter("adult", (filter.contentRating.oneOrThrowIfMany() == ContentRating.ADULT).toString()) + + filter.query?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("q", it) + } + + filter.tags.forEach { tag -> + addQueryParameter("genre", tag.key) + } + + filter.tagsExclude.forEach { tag -> + addQueryParameter("genre_exclude", tag.key) + } + + filter.types.forEach { type -> + addQueryParameter("type", type.toQueryParam()) + } + + filter.states.firstOrNull()?.let { state -> + addQueryParameter("status", state.toQueryParam()) + } + + if (filter.year != YEAR_UNKNOWN) { + addQueryParameter("year", filter.year.toString()) + } + }.build().toString() + } + + private fun parseMangaList(json: JSONObject): List<Manga> { + val items = json.getJSONArray("items") + return (0 until items.length()).mapNotNull { index -> + parseMangaItem(items.getJSONObject(index)) + } + } + + private fun parseStatus(status: String?): MangaState? = when (status?.lowercase()) { + "releasing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + } + + private fun normalizeRating(rating: Double): Float = + if (rating > 0) rating.toFloat() / 10f else RATING_UNKNOWN + + private fun parseContentRating(isNsfw: Boolean): ContentRating = + if (isNsfw) ContentRating.ADULT else ContentRating.SAFE + + private fun parseMangaItem(item: JSONObject): Manga? { + val slug = item.optString("slug").takeIf { it.isNotEmpty() } ?: return null + val title = item.optString("title").takeIf { it.isNotEmpty() } ?: return null + + val url = "/series/$slug".toAbsoluteUrl(domain) + val state = parseStatus(item.optString("status")) + val rating = normalizeRating(item.optDouble("rating", 0.0)) + val contentRating = parseContentRating(item.optBoolean("is_nsfw", false)) + val coverUrl = item.optString("cover_url").takeIf { it.isNotEmpty() } + + return Manga( + id = generateUid(slug), + title = title, + altTitles = emptySet(), + url = url, + publicUrl = url, + rating = rating, + contentRating = contentRating, + coverUrl = coverUrl, + tags = emptySet(), + state = state, + authors = emptySet(), + source = source, + ) + } + + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val result = try { + coroutineScope { + val slug = manga.url.substringAfterLast("/") + val detailsJson = webClient.httpGet("https://$domain/api/series/$slug").parseJson() + val data = detailsJson.optJSONObject("data") ?: detailsJson + + val chaptersDeferred = async { fetchChapters(slug, data) } + + val title = data.optString("title", manga.title) + val cover = data.optString("cover_url").takeIf { it.isNotEmpty() } ?: manga.coverUrl + val description = data.optString("description").takeIf { it.isNotEmpty() } ?: "" + + val state = parseStatus(data.optString("status")) + + val allTags = mutableSetOf<MangaTag>() + + data.optJSONArray("genres")?.let { genresArray -> + for (i in 0 until genresArray.length()) { + val genreObj = genresArray.optJSONObject(i) ?: continue + val genreSlug = genreObj.optString("slug") + val genreName = genreObj.optString("name") + if (genreSlug.isNotBlank() && genreName.isNotBlank()) { + allTags.add(MangaTag(key = genreSlug, title = genreName, source = source)) + } + } + } + + val authors = mutableSetOf<String>() + data.optJSONArray("authors")?.let { authorsArray -> + for (i in 0 until authorsArray.length()) { + val authorObj = authorsArray.optJSONObject(i) ?: continue + val authorName = authorObj.optString("name") + if (authorName.isNotBlank()) { + authors.add(authorName) + } + } + } + + val altTitles = mutableSetOf<String>() + data.optJSONArray("alt_titles")?.let { altArray -> + for (i in 0 until altArray.length()) { + altArray.optString(i)?.takeIf { it.isNotBlank() }?.let { altTitles.add(it) } + } + } + + val rating = normalizeRating(data.optDouble("rating", 0.0)) + val contentRating = parseContentRating(data.optBoolean("is_nsfw", false)) + + val chapters = chaptersDeferred.await() + + manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = authors, + tags = allTags, + altTitles = altTitles, + state = state, + rating = rating, + contentRating = contentRating, + chapters = chapters, + ) + } + } catch (_: Exception) { + manga + } + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private suspend fun fetchChapters(slug: String, detailsJson: JSONObject? = null): List<MangaChapter> { + try { + val url = "https://$domain/api/series/$slug/chapters".toHttpUrl().newBuilder() + .addQueryParameter("order", "desc") + .addQueryParameter("limit", "9999") + .addQueryParameter("offset", "0") + .build() + .toString() + + val response = webClient.httpGet(url).parseJson() + response.optJSONArray("items")?.let { items -> + return parseChapterArray(items, slug) + } + } catch (_: Exception) { + } + + if (detailsJson != null) { + val allChapters = mutableListOf<MangaChapter>() + detailsJson.optJSONArray("chapters_head")?.let { allChapters.addAll(parseChapterArray(it, slug)) } + detailsJson.optJSONArray("chapters_tail")?.let { allChapters.addAll(parseChapterArray(it, slug)) } + return allChapters.distinctBy { it.number }.sortedBy { it.number } + } + return emptyList() + } + + private fun parseChapterArray(array: JSONArray, slug: String): List<MangaChapter> { + return (0 until array.length()).mapNotNull { index -> + val ch = array.optJSONObject(index) ?: return@mapNotNull null + val number = ch.optDouble("number", 0.0).toFloat() + val title = ch.optString("title").takeIf { it.isNotEmpty() } ?: "Chapter $number" + val createdAt = parseDate(ch.optString("created_at")) + val chapterUrl = "/reader/$slug/$number".toAbsoluteUrl(domain) + + MangaChapter( + id = generateUid("$slug-$number"), + url = chapterUrl, + title = title, + number = number, + volume = 0, + uploadDate = createdAt, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + private fun parseDate(dateStr: String): Long = + try { + Instant.parse(dateStr).toEpochMilli() + } catch (_: Exception) { + 0L + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + try { + val segments = chapter.url.trimEnd('/').split('/') + if (segments.size < 2) return emptyList() + val slug = segments[segments.size - 2] + val number = segments[segments.size - 1] + val response = webClient.httpGet("https://$domain/api/series/$slug/chapters/$number").parseJson() + val pagesArray = response.optJSONArray("pages") ?: return emptyList() + + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } catch (_: Exception) { + return emptyList() + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + try { + val slug = seed.url.substringAfterLast("/") + val response = webClient.httpGet("https://$domain/api/series/$slug/similar") + val array = response.toJSONArray() + return (0 until array.length()).mapNotNull { index -> + parseMangaItem(array.getJSONObject(index)) + } + } catch (_: Exception) { + return emptyList() + } + } + + private fun okhttp3.Response.toJSONArray(): JSONArray { + val body = body?.string() ?: throw tsuki.exception.ParseException("Empty response body", request.url.toString()) + return JSONArray(body) + } + + private fun ContentType.toQueryParam(): String = when (this) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.OTHER -> "oel" + else -> "manga" + } + + private fun MangaState.toQueryParam(): String = when (this) { + MangaState.ONGOING -> "releasing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "cancelled" + else -> "upcoming" + } + + private fun String.toAbsoluteUrl(domain: String): String = + if (startsWith("http")) this else "https://$domain$this" + + companion object { + private val GENRES = mapOf( + "action" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "award-winning" to "Award Winning", + "boys-love" to "Boys Love", + "comedy" to "Comedy", + "doujinshi" to "Doujinshi", + "drama" to "Drama", + "ecchi" to "Ecchi", + "erotica" to "Erotica", + "fantasy" to "Fantasy", + "gender-bender" to "Gender Bender", + "girls-love" to "Girls Love", + "gourmet" to "Gourmet", + "harem" to "Harem", + "historical" to "Historical", + "horror" to "Horror", + "josei" to "Josei", + "mahou-shoujo" to "Mahou Shoujo", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "music" to "Music", + "mystery" to "Mystery", + "psychological" to "Psychological", + "romance" to "Romance", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "shounen-ai" to "Shounen Ai", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "sports" to "Sports", + "supernatural" to "Supernatural", + "suspense" to "Suspense", + "thriller" to "Thriller", + "tragedy" to "Tragedy", + "yaoi" to "Yaoi", + "yuri" to "Yuri", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt b/src/main/kotlin/tsuki/site/en/DemonicScan.kt similarity index 88% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt rename to src/main/kotlin/tsuki/site/en/DemonicScan.kt index 2b252806..1028bc3e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/DemonicScan.kt +++ b/src/main/kotlin/tsuki/site/en/DemonicScan.kt @@ -1,15 +1,38 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.textOrNull +import tsuki.util.toTitleCase import androidx.collection.arraySetOf -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import org.jsoup.nodes.Element import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale @MangaSourceParser("DEMONICSCANS", "DemonicScans", "en") internal class DemonicScans(context: MangaLoaderContext) : @@ -160,16 +183,17 @@ internal class DemonicScans(context: MangaLoaderContext) : else -> null } - val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { i, el -> + val chapters = doc.select("div#chapters-list a.chplinks").mapChapters(reversed = true) { _, el -> val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ENGLISH) val href = el.attr("href").let { if (it.startsWith("http")) it else "https://$domain${if (it.startsWith("/")) it else "/$it"}" } val date = el.selectFirst("span")?.text() + MangaChapter( id = generateUid(href), title = el.ownText(), - number = i + 1f, + number = el.ownText().extractChapterNumber(), volume = 0, url = href, scanlator = null, @@ -241,4 +265,4 @@ internal class DemonicScans(context: MangaLoaderContext) : MangaTag("Tragedy", "18", source), MangaTag("Webtoons", "24", source), ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/en/DragonTea.kt b/src/main/kotlin/tsuki/site/en/DragonTea.kt new file mode 100644 index 00000000..4f727a71 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/DragonTea.kt @@ -0,0 +1,25 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.MadaraParser +import tsuki.network.UserAgents + +import tsuki.model.MangaParserSource + +@MangaSourceParser("DRAGONTEA", "DragonTea", "en") +internal class DragonTea(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.DRAGONTEA, "dragontea.ink") { + override val datePattern = "MM/dd/yyyy" + override val postReq = false + override val listUrl = "novel/" + override val tagPrefix = "novel-genre/" + + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.remove(userAgentKey) + } +} diff --git a/src/main/kotlin/tsuki/site/en/DynastyScans.kt b/src/main/kotlin/tsuki/site/en/DynastyScans.kt new file mode 100644 index 00000000..328a415a --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/DynastyScans.kt @@ -0,0 +1,455 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.flattenTo +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +private const val SERIES_TYPE = "Series" +private const val ANTHOLOGY_TYPE = "Anthology" +private const val DOUJIN_TYPE = "Doujin" +private const val ISSUE_TYPE = "Issue" + +private const val SERIES_DIR = "series" +private const val CHAPTERS_DIR = "chapters" +private const val ANTHOLOGIES_DIR = "anthologies" +private const val DOUJINS_DIR = "doujins" +private const val ISSUES_DIR = "issues" + +private val dateFormat = SimpleDateFormat("yyyy-MM-dd", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") +} + +@MangaSourceParser("DYNASTYSCANS", "Dynasty Scans", "en") +internal class DynastyScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DYNASTYSCANS, 20) { + + override val configKeyDomain = ConfigKey.Domain("dynasty-scans.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + + private suspend fun fetchAvailableTags(): Set<MangaTag> = coroutineScope { + if (tagsCache != null) return@coroutineScope tagsCache!! + val tags = (1..3).map { page -> async { getTags(page) } }.awaitAll().flattenTo(LinkedHashSet()) + tagsCache = tags + tags + } + + private suspend fun getTags(page: Int): Set<MangaTag> { + val url = "https://$domain/tags?page=$page" + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".tag-list a").mapToSet { + MangaTag( + key = it.attr("href").removeSuffix('/').substringAfterLast('/'), + title = it.text().toTitleCase(sourceLocale), + source = source, + ) + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrEmpty()) { + return searchList(page, filter) + } + + if (filter.tags.isNotEmpty()) { + val tag = filter.tags.oneOrThrowIfMany() + val url = "https://$domain/tags/${tag?.key}?view=groupings&page=$page" + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + val url = "https://$domain/chapters/added.json?page=$page" + val json = webClient.httpGet(url).parseJson() + val chaptersArray = json.getJSONArray("chapters") + val totalPages = json.getInt("total_pages") + + val entries = LinkedHashSet<MangaEntry>() + for (i in 0 until chaptersArray.length()) { + val chapter = chaptersArray.getJSONObject(i) + val tags = chapter.getJSONArray("tags") + var isSeries = false + + for (j in 0 until tags.length()) { + val tag = tags.getJSONObject(j) + val type = tag.getString("type") + val name = tag.getString("name") + val permalink = tag.getString("permalink") + + if (type in setOf(SERIES_TYPE, ANTHOLOGY_TYPE, DOUJIN_TYPE, ISSUE_TYPE)) { + entries.add(MangaEntry(url = "/${type.directory()}/$permalink", title = name)) + isSeries = isSeries || type == SERIES_TYPE + } + } + + if (!isSeries) { + entries.add( + MangaEntry( + url = "/$CHAPTERS_DIR/${chapter.getString("permalink")}", + title = chapter.getString("title") + ) + ) + } + } + + return entries.map { entry -> + Manga( + id = generateUid(entry.url), + url = entry.url, + publicUrl = entry.url.toAbsoluteUrl(domain), + title = entry.title, + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.take(if (page < totalPages) pageSize else entries.size) + } + + private fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("li.span2").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = div.selectFirstOrThrow("div.caption").text(), + altTitles = emptySet(), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = div.selectFirstOrThrow("img").attrAsAbsoluteUrl("src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private suspend fun searchList(page: Int, filter: MangaListFilter): List<Manga> { + val query = filter.query.orEmpty() + val url = buildString { + append("https://$domain/search?q=") + append(query.urlEncoded()) + append("&classes[]=Series&page=$page") + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select("dl.chapter-list dd").mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = a.text(), + altTitles = emptySet(), + coverUrl = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val (directory, permalink) = manga.url.trim('/').split('/').let { it[0] to it[1] } + val jsonUrl = "https://$domain/$directory/$permalink.json" + val json = webClient.httpGet(jsonUrl).parseJson() + + if (directory == CHAPTERS_DIR) { + return chapterDetails(json, manga) + } + + val data = parseMangaResponse(json) + val authors = LinkedHashSet<String>() + val tagNames = LinkedHashSet<String>() + val statuses = LinkedHashSet<String>() + + data.tags.forEach { tag -> + when (tag.type) { + "Author" -> authors.add(tag.name) + "General" -> tagNames.add(tag.name) + "Status" -> statuses.add(tag.name) + } + } + + data.taggings.filterIsInstance<ChapterItem.Chapter>().forEach { chapter -> + chapter.tags.forEach { tag -> + when (tag.type) { + "Author" -> authors.add(tag.name) + "General" -> tagNames.add(tag.name) + } + } + } + + val chapters = fetchAllChapters(data) + + return manga.copy( + title = data.name, + coverUrl = data.cover?.let { "https://$domain$it" }, + description = buildString { + data.description?.let { desc -> + append(org.jsoup.Jsoup.parseBodyFragment(desc).wholeText().trim()) + append("\n\n") + } + append("Type: ${data.type}\n") + if (authors.isNotEmpty()) { + append("Authors: ${authors.joinToString(", ")}\n") + } + }.trim(), + authors = authors.toSet(), + tags = tagNames.map { MangaTag(it, it.lowercase().replace(" ", "-"), source) }.toSet(), + state = when { + statuses.any { it.equals("Ongoing", ignoreCase = true) } -> MangaState.ONGOING + statuses.any { it.equals("Completed", ignoreCase = true) } -> MangaState.FINISHED + statuses.any { it.equals("On Hiatus", ignoreCase = true) } -> MangaState.PAUSED + statuses.any { it == "Dropped" || it == "Cancelled" || it == "Abandoned" } -> MangaState.ABANDONED + else -> null + }, + chapters = chapters, + ) + } + + private fun chapterDetails(json: JSONObject, manga: Manga): Manga { + val data = parseChapterResponse(json) + return manga.copy( + title = data.title, + coverUrl = data.pages.firstOrNull()?.let { "https://$domain${it.url}" }, + description = "Type: Chapter\nReleased: ${data.releasedOn}", + state = MangaState.FINISHED, + chapters = listOf( + MangaChapter( + id = generateUid("/$CHAPTERS_DIR/${data.permalink}"), + title = "Chapter", + number = 1f, + url = "/$CHAPTERS_DIR/${data.permalink}", + uploadDate = dateFormat.parseSafe(data.releasedOn), + source = source, + scanlator = data.tags.filter { it.type == "Scanlator" }.joinToString { it.name }.nullIfEmpty(), + volume = 0, + branch = null, + ) + ), + ) + } + + private suspend fun fetchAllChapters(data: MangaResponse): List<MangaChapter> = coroutineScope { + val firstPage = data.taggings + val totalPages = data.totalPages + + val deferred = (2..totalPages).map { page -> + async { + val url = "https://$domain/${data.directory}/${data.permalink}.json?page=$page" + parseMangaResponse(webClient.httpGet(url).parseJson()).taggings + } + } + val allTaggings = firstPage + deferred.awaitAll().flatten() + + var header: String? = null + val result = mutableListOf<MangaChapter>() + + allTaggings.forEach { item -> + when (item) { + is ChapterItem.Header -> header = item.header + is ChapterItem.Chapter -> { + var name = header?.let { "$it ${item.title}" } ?: item.title + if (data.type != SERIES_TYPE) { + name += item.tags.filter { it.type == "Author" } + .joinToString(prefix = " by ", separator = " and ") { it.name } + } + result.add( + MangaChapter( + id = generateUid("/$CHAPTERS_DIR/${item.permalink}"), + title = name, + number = (result.size + 1).toFloat(), + url = "/$CHAPTERS_DIR/${item.permalink}", + uploadDate = dateFormat.parseSafe(item.releasedOn), + source = source, + scanlator = item.tags.filter { it.type == "Scanlator" }.joinToString { it.name }.nullIfEmpty(), + volume = 0, + branch = null, + ) + ) + } + } + } + + if (data.type != DOUJIN_TYPE) result.sortedBy { it.number } else result + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val permalink = chapter.url.substringAfterLast('/') + val jsonUrl = "https://$domain/$CHAPTERS_DIR/$permalink.json" + val json = webClient.httpGet(jsonUrl).parseJson() + val data = parseChapterResponse(json) + return data.pages.mapIndexed { _, page -> + MangaPage( + id = generateUid(page.url), + url = "https://$domain${page.url}", + preview = null, + source = source, + ) + } + } + + private data class MangaResponse( + val name: String, + val type: String, + val permalink: String, + val directory: String, + val cover: String?, + val description: String?, + val tags: List<BrowseTag>, + val taggings: List<ChapterItem>, + val totalPages: Int, + ) + + private sealed class ChapterItem { + data class Header(val header: String) : ChapterItem() + data class Chapter( + val title: String, + val permalink: String, + val releasedOn: String, + val tags: List<BrowseTag>, + ) : ChapterItem() + } + + private data class ChapterResponse( + val title: String, + val permalink: String, + val releasedOn: String, + val tags: List<BrowseTag>, + val pages: List<Page>, + ) + + private data class Page(val url: String) + private data class BrowseTag(val type: String, val name: String, val permalink: String) + private data class MangaEntry(val url: String, val title: String) + + private fun parseMangaResponse(json: JSONObject): MangaResponse { + val name = json.getString("name") + val type = json.getString("type") + val permalink = json.getString("permalink") + val directory = when (type) { + SERIES_TYPE -> SERIES_DIR + ANTHOLOGY_TYPE -> ANTHOLOGIES_DIR + DOUJIN_TYPE -> DOUJINS_DIR + ISSUE_TYPE -> ISSUES_DIR + else -> throw Exception("Unknown type: $type") + } + val cover = json.optString("cover", null) + val description = json.optString("description", null) + val tags = json.getJSONArray("tags").parseTags() + val taggings = json.optJSONArray("taggings")?.parseTaggings() ?: emptyList() + val totalPages = json.optInt("total_pages", 1) + return MangaResponse(name, type, permalink, directory, cover, description, tags, taggings, totalPages) + } + + private fun parseChapterResponse(json: JSONObject): ChapterResponse { + val title = json.getString("title") + val permalink = json.getString("permalink") + val releasedOn = json.getString("released_on") + val tags = json.getJSONArray("tags").parseTags() + val pagesArray = json.getJSONArray("pages") + val pages = (0 until pagesArray.length()).map { i -> + val pageObj = pagesArray.getJSONObject(i) + Page(pageObj.getString("url")) + } + return ChapterResponse(title, permalink, releasedOn, tags, pages) + } + + private fun JSONArray.parseTags(): List<BrowseTag> = (0 until length()).map { i -> + val obj = getJSONObject(i) + BrowseTag(obj.getString("type"), obj.getString("name"), obj.getString("permalink")) + } + + private fun JSONArray.parseTaggings(): List<ChapterItem> = (0 until length()).map { i -> + val obj = getJSONObject(i) + if (obj.has("header")) { + ChapterItem.Header(obj.getString("header")) + } else { + ChapterItem.Chapter( + title = obj.getString("title"), + permalink = obj.getString("permalink"), + releasedOn = obj.getString("released_on"), + tags = obj.getJSONArray("tags").parseTags(), + ) + } + } + + private fun String.directory() = when (this) { + SERIES_TYPE -> SERIES_DIR + ANTHOLOGY_TYPE -> ANTHOLOGIES_DIR + DOUJIN_TYPE -> DOUJINS_DIR + ISSUE_TYPE -> ISSUES_DIR + else -> throw IllegalArgumentException("Unknown type: $this") + } +} diff --git a/src/main/kotlin/tsuki/site/en/EnThunderScans.kt b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt new file mode 100644 index 00000000..c8097aee --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/EnThunderScans.kt @@ -0,0 +1,14 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("ENTHUNDERSCANS", "EnThunderScans", "en") +internal class EnThunderScans(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.ENTHUNDERSCANS, "en-thunderscans.com", pageSize = 30) { + override val mangaDirectory = "comics" + override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt b/src/main/kotlin/tsuki/site/en/FlameComics.kt similarity index 62% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt rename to src/main/kotlin/tsuki/site/en/FlameComics.kt index 4f0520b2..a275c8dc 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/FlameComics.kt +++ b/src/main/kotlin/tsuki/site/en/FlameComics.kt @@ -1,17 +1,45 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.findGroupValue +import tsuki.util.generateUid +import tsuki.util.longOf +import tsuki.util.mapToSet +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.json.asTypedList +import tsuki.util.json.entries +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNull +import tsuki.util.json.toStringSet +import tsuki.util.suspendlazy.suspendLazy -import androidx.collection.ArraySet import okhttp3.HttpUrl import org.json.JSONArray +import org.json.JSONException import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.SinglePageMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import org.koitharu.kotatsu.parsers.util.suspendlazy.suspendLazy import java.util.EnumSet import java.util.concurrent.TimeUnit @@ -23,56 +51,89 @@ internal class FlameComics(context: MangaLoaderContext) : private val removeSpecialCharsRegex = Regex("[^A-Za-z0-9 ]") override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + SortOrder.UPDATED, ) override val filterCapabilities: MangaListFilterCapabilities get() = MangaListFilterCapabilities( isMultipleTagsSupported = true, - isTagsExclusionSupported = true, + isTagsExclusionSupported = false, isSearchSupported = true, - isSearchWithFiltersSupported = true, + isSearchWithFiltersSupported = false, ) override val configKeyDomain = ConfigKey.Domain("flamecomics.xyz") + private fun JSONObject.getJSONArraySafe(key: String): JSONArray? { + return try { + val value = get(key) + value as? JSONArray + } catch (_: JSONException) { + null + } + } + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = urlBuilder() - .addPathSegment("_next") - .addPathSegment("data") - .addPathSegment(commonPrefix.get()) - .addPathSegment("browse.json") - .build() + val hasSearchQuery = !filter.query.isNullOrEmpty() + val hasTagFilter = filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() - val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") + val rawMangas = if (hasSearchQuery || hasTagFilter || order != SortOrder.UPDATED) { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("browse.json") + .build() + val json = webClient.httpGet(url).parseJson().getJSONObject("pageProps").getJSONArray("series") + + json.mapJSONNotNull { jo -> + val manga = parseManga(jo) + val lastEdit = jo.getLongOrDefault("last_edit", 0L) + val views = jo.getLongOrDefault("views", 0L) + Triple(manga, lastEdit, views) + } + } else { + val url = urlBuilder() + .addPathSegment("_next") + .addPathSegment("data") + .addPathSegment(commonPrefix.get()) + .addPathSegment("index.json") + .build() + val json = webClient.httpGet(url).parseJson() + val seriesArray = json + .getJSONObject("pageProps") + .getJSONObject("latestEntries") + .getJSONArray("blocks") + .getJSONObject(0) + .getJSONArray("series") - val allManga = json.mapJSONNotNull { jo -> - parseManga(jo).takeIf { it.tags.matches(filter) } + seriesArray.mapJSONNotNull { jo -> + val manga = parseManga(jo) + Triple(manga, 0L, 0L) + } } - // Filter by search if provided - val query = filter.query - val filteredManga = if (!query.isNullOrEmpty()) { - val normalizedQuery = removeSpecialCharsRegex.replace(query.lowercase(), "") - allManga.filter { manga -> + val filteredByTags = rawMangas.filter { (manga, _, _) -> manga.tags.matches(filter) } + + val filteredManga = if (hasSearchQuery) { + val normalizedQuery = removeSpecialCharsRegex.replace(filter.query!!.lowercase(), "") + filteredByTags.filter { (manga, _, _) -> val titles = mutableListOf(manga.title) titles.addAll(manga.altTitles) - titles.any { title -> normalizedQuery in removeSpecialCharsRegex.replace(title.lowercase(), "") } } } else { - allManga + filteredByTags } - - return filteredManga - .let { list -> - when (order) { - SortOrder.ALPHABETICAL -> list.sortedBy { it.title } - else -> list - } - } + val sorted = when (order) { + SortOrder.UPDATED -> filteredManga.sortedByDescending { it.second } + SortOrder.POPULARITY -> filteredManga.sortedByDescending { it.third } + else -> filteredManga + } + return sorted.map { it.first } } override suspend fun getDetails(manga: Manga): Manga = getDetailsImpl(manga.url.toLong()) @@ -147,21 +208,26 @@ internal class FlameComics(context: MangaLoaderContext) : .addPathSegment(commonPrefix.get()) .addPathSegment("browse.json") .build() - return webClient.httpGet(url).parseJson() + val seriesArray = webClient.httpGet(url).parseJson() .getJSONObject("pageProps") .getJSONArray("series") - .mapJSONNotNull { it.getStringOrNull("categories") } - .flatMapTo(ArraySet()) { - JSONArray(it).asTypedList<String>().mapToSet { tagName -> tagName.toMangaTag() } + + val tags = mutableSetOf<MangaTag>() + for (i in 0 until seriesArray.length()) { + val jo = seriesArray.getJSONObject(i) + val categoriesArray = jo.getJSONArraySafe("categories") + categoriesArray?.asTypedList<String>()?.forEach { tagName -> + tags += tagName.toMangaTag() } + } + return tags } private fun parseManga(jo: JSONObject): Manga { - // Try different possible field names for series ID, including string conversion val seriesId = jo.getLongOrDefault("series_id", -1L).takeIf { it != -1L } - ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } // Handle novels + ?: jo.getLongOrDefault("novel_id", -1L).takeIf { it != -1L } ?: jo.getStringOrNull("series_id")?.toLongOrNull() - ?: jo.getStringOrNull("novel_id")?.toLongOrNull() // Handle novels as string + ?: jo.getStringOrNull("novel_id")?.toLongOrNull() ?: jo.getLongOrDefault("id", -1L).takeIf { it != -1L } ?: jo.getStringOrNull("id")?.toLongOrNull() ?: jo.getLongOrDefault("seriesId", -1L).takeIf { it != -1L } @@ -175,7 +241,13 @@ internal class FlameComics(context: MangaLoaderContext) : "JSON sample: ${jo.toString().take(500)}" ) val cover = jo.getStringOrNull("cover") - val author = jo.getStringOrNull("author") + + val authorArray = jo.getJSONArraySafe("author") + val authors = authorArray?.asTypedList<String>()?.toSet() ?: emptySet() + + val categoriesArray = jo.getJSONArraySafe("categories") + val tags = categoriesArray?.asTypedList<String>()?.mapToSet { it.toMangaTag() } ?: emptySet() + return Manga( id = generateUid(seriesId), title = jo.getString("title"), @@ -188,12 +260,8 @@ internal class FlameComics(context: MangaLoaderContext) : contentRating = null, coverUrl = if (cover != null) { imageUrl(seriesId, cover, 384) - } else { - null - }, - tags = jo.getStringOrNull("categories")?.let { - JSONArray(it).asTypedList<String>().mapToSet { tagName -> tagName.toMangaTag() } - }.orEmpty(), + } else null, + tags = tags, state = when (jo.getStringOrNull("status")) { "Dropped" -> MangaState.ABANDONED "Completed" -> MangaState.FINISHED @@ -201,12 +269,10 @@ internal class FlameComics(context: MangaLoaderContext) : "Ongoing" -> MangaState.ONGOING else -> null }, - authors = setOfNotNull(author), + authors = authors, largeCoverUrl = if (cover != null) { imageUrl(seriesId, cover, 720) - } else { - null - }, + } else null, description = jo.getStringOrNull("description"), source = source, ) @@ -260,4 +326,4 @@ internal class FlameComics(context: MangaLoaderContext) : key = this.lowercase().trim(), source = source, ) -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/en/HiveToons.kt b/src/main/kotlin/tsuki/site/en/HiveToons.kt new file mode 100644 index 00000000..017e0d43 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/HiveToons.kt @@ -0,0 +1,337 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray + +import org.json.JSONArray +import org.json.JSONObject +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import kotlin.collections.isNotEmpty + +@MangaSourceParser("HIVETOONS", "HiveToons", "en") +internal class HiveToons(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.HIVETOONS, 20) { + + override val configKeyDomain = ConfigKey.Domain("hivetoons.org") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://api.$domain" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .build() + + private var cachedGenres: List<Pair<String, String>>? = null + + private suspend fun fetchGenres(): List<Pair<String, String>> { + if (cachedGenres != null) return cachedGenres!! + val arr = webClient.httpGet("$apiBaseUrl/api/genres", getRequestHeaders()).parseJsonArray() + val list = mutableListOf<Pair<String, String>>() + for (i in 0 until arr.length()) { + val obj = arr.optJSONObject(i) ?: continue + list.add(obj.optString("name") to obj.optString("id")) + } + cachedGenres = list + return list + } + + private val statusTags = listOf( + "Ongoing" to "status_ongoing", + "Completed" to "status_completed", + "Canceled" to "status_canceled", + "Dropped" to "status_dropped", + "Coming Soon" to "status_coming_soon", + ) + + private val typeTags = listOf( + "Manga" to "type_manga", + "Manhua" to "type_manhua", + "Manhwa" to "type_manhwa", + "Russian" to "type_russian", + "Spanish" to "type_spanish", + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.ALPHABETICAL, // asc + SortOrder.ALPHABETICAL_DESC // desc + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + val tags = mutableSetOf<MangaTag>() + + statusTags.forEach { (name, key) -> tags.add(MangaTag(name, key, source)) } + typeTags.forEach { (name, key) -> tags.add(MangaTag(name, key, source)) } + genres.forEach { (name, id) -> tags.add(MangaTag(name, "genre_$id", source)) } + + return MangaListFilterOptions( + availableTags = tags, + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val status = filter.tags.firstOrNull { it.key.startsWith("status_") }?.key?.removePrefix("status_") + val type = filter.tags.firstOrNull { it.key.startsWith("type_") }?.key?.removePrefix("type_") + val genreIds = filter.tags.filter { it.key.startsWith("genre_") }.map { it.key.removePrefix("genre_") } + + val (orderBy, orderDir) = when (order) { + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.ADDED -> "createdAt" to "desc" + SortOrder.ADDED_ASC -> "createdAt" to "asc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + SortOrder.ALPHABETICAL_DESC -> "postTitle" to "desc" + else -> "totalViews" to "desc" + } + + val baseHttpUrl = "$apiBaseUrl/api/query".toHttpUrlOrNull() ?: return emptyList() + val urlBuilder = baseHttpUrl.newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + if (query.isNotEmpty()) addQueryParameter("searchTerm", query) + if (!status.isNullOrEmpty()) addQueryParameter("seriesStatus", status.uppercase()) + if (!type.isNullOrEmpty()) addQueryParameter("seriesType", type.uppercase()) + if (genreIds.isNotEmpty()) addQueryParameter("genreIds", genreIds.joinToString(",")) + addQueryParameter("orderBy", orderBy) + addQueryParameter("orderDirection", orderDir) + } + + val (mangas, _) = fetchSearchPageWithFallback(urlBuilder, page) + return mangas + } + + private suspend fun fetchSearchPageWithFallback(initialUrlBuilder: HttpUrl.Builder, startPage: Int): Pair<List<Manga>, Boolean> { + var currentPage = startPage + while (true) { + val url = initialUrlBuilder + .setQueryParameter("page", currentPage.toString()) + .build() + .toString() + + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val posts = json.optJSONArray("posts") ?: JSONArray() + val totalCount = json.optInt("totalCount", 0) + val mangas = posts.toMangaList() + + val hasNext = totalCount > (currentPage * pageSize) + + if (mangas.isNotEmpty() || !hasNext) { + return mangas to hasNext + } + currentPage++ + } + } + + private fun JSONArray.toMangaList(): List<Manga> { + val list = mutableListOf<Manga>() + for (i in 0 until length()) { + val obj = optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + list.add(obj.toManga()) + } + return list + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val slug = getString("slug") + val title = getString("postTitle") + val cover = optString("featuredImage", "") + return Manga( + id = generateUid("$slug#$id"), + url = "$slug#$id", + publicUrl = "$baseUrl/series/$slug", + title = title, + coverUrl = cover.ifEmpty { null }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val (slug, id) = manga.url.split("#") + val detailsUrl = "$apiBaseUrl/api/post?postSlug=$slug" + val json = webClient.httpGet(detailsUrl, getRequestHeaders()).parseJson() + val data = json.optJSONObject("post") ?: return manga + if (data.optBoolean("isNovel", false)) throw Exception("Novels not supported") + + val title = data.optString("postTitle", manga.title) + val cover = data.optString("featuredImage", "") + val desc = data.optString("postContent", "") + val altTitles = data.optString("alternativeTitles", "") + val author = data.optString("author", "") + val artist = data.optString("artist", "") + val seriesType = data.optString("seriesType", "") + val seriesStatus = data.optString("seriesStatus", "") + val genresArr = data.optJSONArray("genres") ?: JSONArray() + val chaptersArr = data.optJSONArray("chapters") ?: JSONArray() + val totalChapters = json.optInt("totalChapterCount", chaptersArr.length()) + + val state = when (seriesStatus.uppercase()) { + "ONGOING", "COMING_SOON", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (seriesType.isNotEmpty()) { + tags.add(MangaTag(seriesType.lowercase(), seriesType, source)) + } + for (i in 0 until genresArr.length()) { + val g = genresArr.optJSONObject(i) ?: continue + val name = g.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name.lowercase(), name, source)) + } + + val plainDesc = if (desc.isNotEmpty()) { + org.jsoup.Jsoup.parse(desc.replace("\n", "<br>")).text() + } else "" + val fullDesc = buildString { + append(plainDesc) + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Alternative Names: $altTitles") + } + }.trim() + + val finalChaptersArr = if (totalChapters > chaptersArr.length()) { + val chaptersJson = webClient.httpGet("$apiBaseUrl/api/chapters?postId=${id.toInt()}", getRequestHeaders()).parseJson() + chaptersJson.optJSONObject("post")?.optJSONArray("chapters") ?: JSONArray() + } else { + chaptersArr + } + + val chapters = parseChaptersFromArray(finalChaptersArr, slug) + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = fullDesc.takeIf { it.isNotEmpty() }, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private fun parseChaptersFromArray(arr: JSONArray, mangaSlug: String): List<MangaChapter> { + val list = mutableListOf<MangaChapter>() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + if (!ch.optBoolean("isAccessible", false)) continue + val id = ch.getInt("id") + val number = ch.optString("number", "") + val title = ch.optString("title", "") + val slug = ch.optString("slug", "") + val dateStr = ch.optString("createdAt", "") + val uploadDate = try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + + val cleanTitle = if (title == "null" || title.isEmpty()) null else title + val fullTitle = buildString { + append("Chapter $number") + if (cleanTitle != null) append(" - $cleanTitle") + } + + list.add( + MangaChapter( + id = generateUid("$mangaSlug#$id"), + title = fullTitle, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug#$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("#") + if (parts.size != 2) return emptyList() + val chapterId = parts[1].toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/chapter?chapterId=$chapterId", getRequestHeaders()).parseJson() + val chData = json.optJSONObject("chapter") ?: return emptyList() + if (chData.optBoolean("isShortLinkLocked")) throw Exception("Chapter locked (short link)") + if (chData.optBoolean("isLockedByCoins")) throw Exception("Chapter locked (coins required)") + if (chData.optBoolean("isPermanentlyLocked")) throw Exception("Chapter permanently locked") + + val imagesArr = chData.optJSONArray("images") ?: return emptyList() + val pages = mutableListOf<JSONObject>() + for (i in 0 until imagesArr.length()) { + pages.add(imagesArr.optJSONObject(i) ?: continue) + } + pages.sortBy { it.optInt("order", Int.MAX_VALUE) } + + return pages.mapIndexed { _, imgObj -> + val imgUrl = imgObj.optString("url", "").replace(" ", "%20") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val (_, id) = seed.url.split("#") + val postId = id.toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/recommendations?postId=$postId&limit=25", getRequestHeaders()).parseJson() + val recs = json.optJSONArray("recommendations") ?: return emptyList() + return recs.toMangaList() + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt b/src/main/kotlin/tsuki/site/en/KaliScan.kt similarity index 80% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt rename to src/main/kotlin/tsuki/site/en/KaliScan.kt index b11da65b..6ddeac26 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/KaliScan.kt +++ b/src/main/kotlin/tsuki/site/en/KaliScan.kt @@ -1,36 +1,41 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadthemeParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.removeSuffix import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadthemeParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet -@MangaSourceParser("KALISCAN", "kaliscan.io", "en") +@MangaSourceParser("KALISCAN", "Kaliscan.io", "en") internal class KaliScan(context: MangaLoaderContext) : MadthemeParser(context, MangaParserSource.KALISCAN, "kaliscan.io", pageSize=24) { @@ -41,6 +46,15 @@ internal class KaliScan(context: MangaLoaderContext) : override val selectDate = ".chapter-update" override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/static/sites/icons/favicon-32x32.png", 32, null), + ), + domain, + ) + } + override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( isAuthorSearchSupported = true, @@ -227,4 +241,4 @@ internal class KaliScan(context: MangaLoaderContext) : ) } ?: emptySet() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/en/KingofShojo.kt b/src/main/kotlin/tsuki/site/en/KingofShojo.kt new file mode 100644 index 00000000..63cc6a90 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/KingofShojo.kt @@ -0,0 +1,11 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("KINGOFSHOJO", "King of Shojo", "en") +internal class KingofShojo(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KINGOFSHOJO, "kingofshojo.com") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt b/src/main/kotlin/tsuki/site/en/LikeManga.kt similarity index 54% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt rename to src/main/kotlin/tsuki/site/en/LikeManga.kt index fc19cc98..8169e037 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/LikeManga.kt +++ b/src/main/kotlin/tsuki/site/en/LikeManga.kt @@ -1,20 +1,22 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.CommonHeaders +import tsuki.parsers.LikeMangaParser + +import tsuki.model.MangaParserSource -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.LikeMangaParser import okhttp3.Interceptor import okhttp3.Response -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.network.CommonHeaders private const val USER_AGENT = "Usagi/1 (Android)" @MangaSourceParser("LIKEMANGA", "LikeManga", "en") internal class LikeManga(context: MangaLoaderContext) : LikeMangaParser(context, MangaParserSource.LIKEMANGA, "likemanga.ink") { - override val userAgentKey = ConfigKey.UserAgent(USER_AGENT,) + override val userAgentKey = ConfigKey.UserAgent(USER_AGENT) override fun intercept(chain: Interceptor.Chain): Response { val request = chain.request() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt b/src/main/kotlin/tsuki/site/en/MangaFox.kt similarity index 88% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt rename to src/main/kotlin/tsuki/site/en/MangaFox.kt index 8af146a5..4c08337f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaFox.kt +++ b/src/main/kotlin/tsuki/site/en/MangaFox.kt @@ -1,236 +1,259 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGAFOX", "MangaFox", "en") -internal class MangaFox(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAFOX, pageSize = 24) { - - private val baseUrl = "fanfox.net" - private val mobileUrl = "https://m.fanfox.net" - private val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH) - - override val configKeyDomain = ConfigKey.Domain(baseUrl) - - private val apiClient: OkHttpWebClient by lazy { - val httpClient = context.httpClient.newBuilder() - .addInterceptor { chain -> - val request = chain.request().newBuilder() - .header("Referer", "https://$baseUrl/") - .build() - chain.proceed(request) - } - .build() - OkHttpWebClient(httpClient, source) - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true - ) - - @Volatile - private var tagsCache: Set<MangaTag>? = null - private val tagsMutex = Mutex() - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = getOrFetchTags() - ) - } - - private suspend fun getOrFetchTags(): Set<MangaTag> { - tagsCache?.let { return it } - return tagsMutex.withLock { - tagsCache ?: fetchTags().also { tagsCache = it } - } - } - - private fun fetchTags(): Set<MangaTag> { - return listOf( - "action", "adventure", "comedy", "drama", "fantasy", - "harem", "romance", "school-life", "shounen", "shoujo", - "seinen", "sci-fi", "slice-of-life", "supernatural", - "ecchi", "mystery", "psychological", "historical", - "martial-arts", "mecha", "sports" - ).map { - MangaTag(key = it, title = it.replace("-", " "), source = source) - }.toSet() - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildUrl(page, filter, order) - val doc = apiClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun buildUrl(page: Int, filter: MangaListFilter, order: SortOrder): String { - val query = filter.query?.trim().orEmpty() - val tag = filter.tags.firstOrNull()?.key - return when { - !tag.isNullOrBlank() && query.isEmpty() -> - "https://m.fanfox.net/search/cate/$tag" - query.isNotEmpty() -> - "https://m.fanfox.net/search?k=${query.urlEncoded()}" - else -> - buildString { - append("https://$baseUrl/directory/") - if (page > 1) append("$page.html") - if (order == SortOrder.UPDATED) append("?latest") - } - } - } - - private fun parseMangaList(doc: Document): List<Manga> { - val searchItems = doc.select(".post-list li") - if (searchItems.isNotEmpty()) { - return searchItems.mapNotNull { el -> - val a = el.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = a.attr("title").ifBlank { a.selectFirst(".cover-info .title")?.text() ?: a.text() }, - url = href, - publicUrl = a.absUrl("href"), - coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - source = source - ) - } - } - - val dirSelectors = listOf( - "ul.manga-list-1-list li", - "ul.manga-list-4-list li", - "ul.manga-list-2-list li" - ) - for (sel in dirSelectors) { - val items = doc.select(sel) - if (items.isNotEmpty()) { - return items.mapNotNull { el -> - val a = el.selectFirst("a") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - Manga( - id = generateUid(href), - title = a.attr("title").ifBlank { a.text() }, - url = href, - publicUrl = a.absUrl("href"), - coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - source = source - ) - } - } - } - return emptyList() - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() - val info = doc.selectFirst(".detail-info-right")!! - - val chapters = doc.select("ul.detail-main-list li").mapIndexedNotNull { i, el -> - val a = el.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("p")?.text() ?: a.ownText() - val rawDate = el.text() - .let { text -> - Regex("""(Today|Yesterday|\w{3}\s\d{1,2},\s\d{4}|\d{4}-\d{2}-\d{2})""") - .find(text)?.value - } - val parsedDate = parseChapterDate(rawDate) - MangaChapter( - id = generateUid(href), - title = name, - number = extractChapterNumber(name), - volume = 0, - url = href, - uploadDate = parsedDate.takeIf { it > 0L } - ?: (System.currentTimeMillis() - (i * 86_400_000L)), - scanlator = null, - branch = null, - source = source - ) - }.sortedBy { it.number } - - return manga.copy( - title = doc.selectFirst("h1")?.text() ?: manga.title, - authors = info.select(".detail-info-right-say a").mapToSet { it.text() }, - tags = info.select(".detail-info-right-tag-list a").mapToSet { - MangaTag(key = it.text().lowercase(), title = it.text(), source = source) - }, - description = info.selectFirst("p.fullcontent")?.text(), - state = parseStatus(info.selectFirst(".detail-info-right-title-tip")?.text().orEmpty()), - coverUrl = doc.selectFirst(".detail-info-cover-img")?.attr("abs:src").orEmpty(), - chapters = chapters - ) - } - - private fun parseStatus(status: String): MangaState? = when { - status.contains("Ongoing", true) -> MangaState.ONGOING - status.contains("Completed", true) -> MangaState.FINISHED - else -> null - } - - private fun extractChapterNumber(name: String): Float { - return Regex("""(?i)(chapter|ch\.)\s*(\d+(\.\d+)?)""") - .find(name) - ?.groupValues - ?.getOrNull(2) - ?.toFloatOrNull() - ?: 0f - } - - private fun parseChapterDate(date: String?): Long { - if (date.isNullOrBlank()) return 0L - val clean = date.trim() - return when { - "Today" in clean -> Calendar.getInstance().timeInMillis - "Yesterday" in clean -> Calendar.getInstance().apply { add(Calendar.DATE, -1) }.timeInMillis - clean.matches(Regex("""\d{4}-\d{2}-\d{2}""")) -> runCatching { - val parts = clean.split("-") - Calendar.getInstance().apply { set(parts[0].toInt(), parts[1].toInt() - 1, parts[2].toInt()) }.timeInMillis - }.getOrDefault(0L) - else -> runCatching { dateFormat.parse(clean)?.time ?: 0L }.getOrDefault(0L) - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val mobilePath = chapter.url.replace("/manga/", "/roll_manga/") - val url = "$mobileUrl$mobilePath" - val doc = apiClient.httpGet(url).parseHtml() - return doc.select("#viewer img").map { - val imgUrl = it.attr("abs:data-original") - MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) - } - } - - override suspend fun getPageUrl(page: MangaPage): String = page.url - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAFOX", "MangaFox", "en") +internal class MangaFox(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAFOX, pageSize = 24) { + + private val baseUrl = "fanfox.net" + private val mobileUrl = "https://m.fanfox.net" + private val dateFormat = SimpleDateFormat("MMM dd, yyyy", Locale.ENGLISH) + + override val configKeyDomain = ConfigKey.Domain(baseUrl) + + private val apiClient: OkHttpWebClient by lazy { + val httpClient = context.httpClient.newBuilder() + .addInterceptor { chain -> + val request = chain.request().newBuilder() + .header("Referer", "https://$baseUrl/") + .build() + chain.proceed(request) + } + .build() + OkHttpWebClient(httpClient, source) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED) + + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + private val tagsMutex = Mutex() + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = getOrFetchTags() + ) + } + + private suspend fun getOrFetchTags(): Set<MangaTag> { + tagsCache?.let { return it } + return tagsMutex.withLock { + tagsCache ?: fetchTags().also { tagsCache = it } + } + } + + private fun fetchTags(): Set<MangaTag> { + return listOf( + "action", "adventure", "comedy", "drama", "fantasy", + "harem", "romance", "school-life", "shounen", "shoujo", + "seinen", "sci-fi", "slice-of-life", "supernatural", + "ecchi", "mystery", "psychological", "historical", + "martial-arts", "mecha", "sports" + ).map { + MangaTag(key = it, title = it.replace("-", " "), source = source) + }.toSet() + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildUrl(page, filter, order) + val doc = apiClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun buildUrl(page: Int, filter: MangaListFilter, order: SortOrder): String { + val query = filter.query?.trim().orEmpty() + val tag = filter.tags.firstOrNull()?.key + return when { + !tag.isNullOrBlank() && query.isEmpty() -> + "https://m.fanfox.net/search/cate/$tag" + query.isNotEmpty() -> + "https://m.fanfox.net/search?k=${query.urlEncoded()}" + else -> + buildString { + append("https://$baseUrl/directory/") + if (page > 1) append("$page.html") + if (order == SortOrder.UPDATED) append("?latest") + } + } + } + + private fun parseMangaList(doc: Document): List<Manga> { + val searchItems = doc.select(".post-list li") + if (searchItems.isNotEmpty()) { + return searchItems.mapNotNull { el -> + val a = el.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = a.attr("title").ifBlank { a.selectFirst(".cover-info .title")?.text() ?: a.text() }, + url = href, + publicUrl = a.absUrl("href"), + coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + source = source + ) + } + } + + val dirSelectors = listOf( + "ul.manga-list-1-list li", + "ul.manga-list-4-list li", + "ul.manga-list-2-list li" + ) + for (sel in dirSelectors) { + val items = doc.select(sel) + if (items.isNotEmpty()) { + return items.mapNotNull { el -> + val a = el.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = a.attr("title").ifBlank { a.text() }, + url = href, + publicUrl = a.absUrl("href"), + coverUrl = el.selectFirst("img")?.attr("abs:src").orEmpty(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + source = source + ) + } + } + } + return emptyList() + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = apiClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() + val info = doc.selectFirst(".detail-info-right")!! + + val chapters = doc.select("ul.detail-main-list li").mapIndexedNotNull { i, el -> + val a = el.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("p")?.text() ?: a.ownText() + val rawDate = el.text() + .let { text -> + Regex("""(Today|Yesterday|\w{3}\s\d{1,2},\s\d{4}|\d{4}-\d{2}-\d{2})""") + .find(text)?.value + } + val parsedDate = parseChapterDate(rawDate) + MangaChapter( + id = generateUid(href), + title = name, + number = extractChapterNumber(name), + volume = 0, + url = href, + uploadDate = parsedDate.takeIf { it > 0L } + ?: (System.currentTimeMillis() - (i * 86_400_000L)), + scanlator = null, + branch = null, + source = source + ) + }.sortedBy { it.number } + + return manga.copy( + title = doc.selectFirst("h1")?.text() ?: manga.title, + authors = info.select(".detail-info-right-say a").mapToSet { it.text() }, + tags = info.select(".detail-info-right-tag-list a").mapToSet { + MangaTag(key = it.text().lowercase(), title = it.text(), source = source) + }, + description = info.selectFirst("p.fullcontent")?.text(), + state = parseStatus(info.selectFirst(".detail-info-right-title-tip")?.text().orEmpty()), + coverUrl = doc.selectFirst(".detail-info-cover-img")?.attr("abs:src").orEmpty(), + chapters = chapters + ) + } + + private fun parseStatus(status: String): MangaState? = when { + status.contains("Ongoing", true) -> MangaState.ONGOING + status.contains("Completed", true) -> MangaState.FINISHED + else -> null + } + + private fun extractChapterNumber(name: String): Float { + return Regex("""(?i)(chapter|ch\.)\s*(\d+(\.\d+)?)""") + .find(name) + ?.groupValues + ?.getOrNull(2) + ?.toFloatOrNull() + ?: 0f + } + + private fun parseChapterDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + val clean = date.trim() + return when { + "Today" in clean -> Calendar.getInstance().timeInMillis + "Yesterday" in clean -> Calendar.getInstance().apply { add(Calendar.DATE, -1) }.timeInMillis + clean.matches(Regex("""\d{4}-\d{2}-\d{2}""")) -> runCatching { + val parts = clean.split("-") + Calendar.getInstance().apply { set(parts[0].toInt(), parts[1].toInt() - 1, parts[2].toInt()) }.timeInMillis + }.getOrDefault(0L) + else -> runCatching { dateFormat.parse(clean)?.time ?: 0L }.getOrDefault(0L) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val mobilePath = chapter.url.replace("/manga/", "/roll_manga/") + val url = "$mobileUrl$mobilePath" + val doc = apiClient.httpGet(url).parseHtml() + return doc.select("#viewer img").map { + val imgUrl = it.attr("abs:data-original") + MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt b/src/main/kotlin/tsuki/site/en/MangaGG.kt similarity index 80% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt rename to src/main/kotlin/tsuki/site/en/MangaGG.kt index 92bab010..4cbede7e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGG.kt +++ b/src/main/kotlin/tsuki/site/en/MangaGG.kt @@ -1,19 +1,22 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl import java.text.SimpleDateFormat import kotlinx.coroutines.awaitAll @@ -153,4 +156,4 @@ internal class MangaGG(context: MangaLoaderContext) : branch = null, ) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt b/src/main/kotlin/tsuki/site/en/MangaGeko.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt rename to src/main/kotlin/tsuki/site/en/MangaGeko.kt index c99da478..8f518e2d 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaGeko.kt +++ b/src/main/kotlin/tsuki/site/en/MangaGeko.kt @@ -1,242 +1,274 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.jsoup.Jsoup -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGAKEKO", "MangaGeko", "en") -internal class MangaGeko(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGAKEKO, 30) { - - override val configKeyDomain = ConfigKey.Domain("www.mgeko.cc") - private val baseUrl = "https://$domain" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.NEWEST, // recently_added - SortOrder.UPDATED, // latest - SortOrder.POPULARITY, // popular_all_time - SortOrder.RATING, // rating - SortOrder.ALPHABETICAL, // az - SortOrder.ALPHABETICAL_DESC, // za - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - ) - - private val dateFormats = listOf( - SimpleDateFormat("MMMM dd, yyyy, h:mm a", Locale.ENGLISH), - SimpleDateFormat("MMMM dd, yyyy, h a", Locale.ENGLISH), - ) - - private val allGenres = setOf( - "Action", "Adventure", "Comedy", "Cooking", "Manga", "Drama", "Fantasy", - "Gender bender", "Harem", "Historical", "Horror", "Isekai", "Josei", - "Manhua", "Manhwa", "Martial arts", "Mature", "Mecha", "Medical", - "Mystery", "One shot", "Psychological", "Romance", "School life", - "Sci fi", "Seinen", "Shoujo", "Shounen", "Slice of life", "Sports", - "Supernatural", "Tragedy", "Webtoons", "Ladies", - ) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override suspend fun getFilterOptions(): MangaListFilterOptions { - val tags = allGenres.map { MangaTag(it, it.lowercase().replace(" ", "_"), source) }.toSet() - return MangaListFilterOptions( - availableTags = tags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS), - availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), - ) - } - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - if (!filter.query.isNullOrEmpty()) { - return searchManga(page, filter.query!!) - } - return browseManga(page, order, filter) - } - - private suspend fun searchManga(page: Int, query: String): List<Manga> { - val url = "$baseUrl/search/".toHttpUrl().newBuilder() - .addQueryParameter("search", query) - .addQueryParameter("results", page.toString()) - .build() - val doc = webClient.httpGet(url).parseHtml() - return doc.select(".novel-item").mapNotNull { it.toManga() } - } - - private suspend fun browseManga(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val sort = when (order) { - SortOrder.NEWEST -> "recently_added" - SortOrder.UPDATED -> "latest" - SortOrder.POPULARITY -> "popular_all_time" - SortOrder.RATING -> "rating" - SortOrder.ALPHABETICAL -> "az" - SortOrder.ALPHABETICAL_DESC -> "za" - else -> "latest" - } - - val url = "$baseUrl/browse-comics/data/".toHttpUrl().newBuilder().apply { - addQueryParameter("page", page.toString()) - addQueryParameter("sort", sort) - - val safeMode = if (ContentRating.ADULT !in filter.contentRating) "1" else "0" - addQueryParameter("safe_mode", safeMode) - - if (filter.tags.isNotEmpty()) { - addQueryParameter("include_genres", filter.tags.joinToString(",") { it.key.replace("_", " ") }) - } - if (filter.tagsExclude.isNotEmpty()) { - addQueryParameter("exclude_genres", filter.tagsExclude.joinToString(",") { it.key.replace("_", " ") }) - } - - if (filter.states.isNotEmpty()) { - val status = when (filter.states.first()) { - MangaState.ONGOING -> "Ongoing" - MangaState.FINISHED -> "Completed" - MangaState.PAUSED -> "Hiatus" - else -> null - } - status?.let { addQueryParameter("status", it) } - } - - if (filter.types.isNotEmpty()) { - val type = when (filter.types.first()) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Webtoon" - else -> null - } - type?.let { addQueryParameter("type", it) } - } - }.build() - - val response = webClient.httpGet(url) - val json = JSONObject(response.body.string()) - val htmlFragment = json.getString("results_html") - val doc = Jsoup.parseBodyFragment(htmlFragment, baseUrl) - return doc.select(".comic-card").map { it.toManga() } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val header = doc.selectFirstOrThrow(".novel-header") - - val title = header.selectFirst(".novel-title")?.text()?.trim() ?: manga.title - val cover = header.selectFirst(".cover img")?.let { - it.attr("data-src").ifBlank { it.attr("src") } - } ?: manga.coverUrl - - val author = header.selectFirst(".author a")?.attr("title")?.trim()?.takeIf { - it.lowercase() != "updating" - } - - val state = when { - header.selectFirst("div.header-stats strong.completed") != null -> MangaState.FINISHED - header.selectFirst("div.header-stats strong.ongoing") != null -> MangaState.ONGOING - else -> null - } - - val tags = doc.select(".categories a[href*=genre]").mapToSet { a -> - val name = a.ownText().trim().split(" ").joinToString(" ") { word -> - word.lowercase().replaceFirstChar { it.uppercase() } - } - MangaTag(name, name.lowercase().replace(" ", "_"), source) - } - - val chapters = fetchChapters(manga.url) - - return manga.copy( - title = title, - altTitles = emptySet(), - coverUrl = cover, - largeCoverUrl = cover, - state = state, - authors = setOfNotNull(author), - description = null, - tags = tags, - chapters = chapters, - ) - } - - private suspend fun fetchChapters(mangaUrl: String): List<MangaChapter> { - val doc = webClient.httpGet(mangaUrl.toAbsoluteUrl(domain) + "all-chapters/").parseHtml() - return doc.select("ul.chapter-list > li").mapNotNull { li -> - val a = li.selectFirst("a") ?: return@mapNotNull null - val url = a.attrAsRelativeUrl("href") - val titleEl = li.selectFirst(".chapter-title, .chapter-number") ?: return@mapNotNull null - val chapterName = titleEl.ownText() - val dateStr = li.selectFirst(".chapter-update")?.attr("datetime") - ?.replace(".", "")?.replace("Sept", "Sep") ?: "" - val date = dateFormats.firstNotNullOfOrNull { fmt -> fmt.parseSafe(dateStr) } ?: 0L - MangaChapter( - id = generateUid(url), - title = "Chapter $chapterName", - number = 0f, - volume = 0, - url = url, - uploadDate = date, - source = source, - scanlator = null, - branch = null, - ) - }.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-reader img").mapNotNull { img -> - val src = img.absUrl("src") - if (src.isEmpty()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun Element.toManga(): Manga { - val titleEl = selectFirst(".comic-card__title a") ?: selectFirst(".novel-title") - val title = titleEl?.text()?.trim() ?: "" - val href = selectFirst("a")?.attrAsRelativeUrl("href") ?: "" - val cover = selectFirst(".comic-card__cover img, .novel-cover img")?.let { - it.attr("data-src").ifBlank { it.attr("src") } - } ?: "" - return Manga( - id = generateUid(href), - url = href, - publicUrl = href.toAbsoluteUrl(domain), - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = cover, - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAKEKO", "MangaGeko", "en") +internal class MangaGeko(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAKEKO, 30) { + + override val configKeyDomain = ConfigKey.Domain("www.mgeko.cc") + private val baseUrl = "https://$domain" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, // recently_added + SortOrder.UPDATED, // latest + SortOrder.POPULARITY, // popular_all_time + SortOrder.RATING, // rating + SortOrder.ALPHABETICAL, // az + SortOrder.ALPHABETICAL_DESC, // za + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + private val dateFormats = listOf( + SimpleDateFormat("MMMM dd, yyyy, h:mm a", Locale.ENGLISH), + SimpleDateFormat("MMMM dd, yyyy, h a", Locale.ENGLISH), + ) + + private val allGenres = setOf( + "Action", "Adventure", "Comedy", "Cooking", "Manga", "Drama", "Fantasy", + "Gender bender", "Harem", "Historical", "Horror", "Isekai", "Josei", + "Manhua", "Manhwa", "Martial arts", "Mature", "Mecha", "Medical", + "Mystery", "One shot", "Psychological", "Romance", "School life", + "Sci fi", "Seinen", "Shoujo", "Shounen", "Slice of life", "Sports", + "Supernatural", "Tragedy", "Webtoons", "Ladies", + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val tags = allGenres.map { MangaTag(it, it.lowercase().replace(" ", "_"), source) }.toSet() + return MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.COMICS + ), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.ADULT), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrEmpty()) { + return searchManga(page, filter.query!!) + } + return browseManga(page, order, filter) + } + + private suspend fun searchManga(page: Int, query: String): List<Manga> { + val url = "$baseUrl/search/".toHttpUrl().newBuilder() + .addQueryParameter("search", query) + .addQueryParameter("results", page.toString()) + .build() + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".novel-item").mapNotNull { it.toManga() } + } + + private suspend fun browseManga(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.NEWEST -> "recently_added" + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "popular_all_time" + SortOrder.RATING -> "rating" + SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" + else -> "latest" + } + + val url = "$baseUrl/browse-comics/data/".toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("sort", sort) + + val safeMode = if (ContentRating.ADULT !in filter.contentRating) "1" else "0" + addQueryParameter("safe_mode", safeMode) + + if (filter.tags.isNotEmpty()) { + addQueryParameter("include_genres", filter.tags.joinToString(",") { it.key.replace("_", " ") }) + } + if (filter.tagsExclude.isNotEmpty()) { + addQueryParameter("exclude_genres", filter.tagsExclude.joinToString(",") { it.key.replace("_", " ") }) + } + + if (filter.states.isNotEmpty()) { + val status = when (filter.states.first()) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + else -> null + } + status?.let { addQueryParameter("status", it) } + } + + if (filter.types.isNotEmpty()) { + val type = when (filter.types.first()) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Webtoon" + else -> null + } + type?.let { addQueryParameter("type", it) } + } + }.build() + + val response = webClient.httpGet(url) + val json = JSONObject(response.body.string()) + val htmlFragment = json.getString("results_html") + val doc = Jsoup.parseBodyFragment(htmlFragment, baseUrl) + return doc.select(".comic-card").map { it.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val header = doc.selectFirstOrThrow(".novel-header") + + val title = header.selectFirst(".novel-title")?.text()?.trim() ?: manga.title + val cover = header.selectFirst(".cover img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + } ?: manga.coverUrl + val altTitle = doc.selectFirst(".alternative-title")?.textOrNull() + val description = doc.selectFirst(".description")?.html() + + val author = header.selectFirst(".author a")?.attr("title")?.trim()?.takeIf { + it.lowercase() != "updating" + } + + val state = when { + header.selectFirst("div.header-stats strong.completed") != null -> MangaState.FINISHED + header.selectFirst("div.header-stats strong.ongoing") != null -> MangaState.ONGOING + else -> null + } + + val tags = doc.select(".categories a[href*=genre]").mapToSet { a -> + val name = a.ownText().trim().split(" ").joinToString(" ") { word -> + word.lowercase().replaceFirstChar { it.uppercase() } + } + MangaTag(name, name.lowercase().replace(" ", "_"), source) + } + + val chapters = fetchChapters(manga.url) + + return manga.copy( + title = title, + altTitles = setOfNotNull(altTitle), + coverUrl = cover, + largeCoverUrl = cover, + state = state, + authors = setOfNotNull(author), + description = description, + tags = tags, + chapters = chapters, + ) + } + + private suspend fun fetchChapters(mangaUrl: String): List<MangaChapter> { + val doc = webClient.httpGet(mangaUrl.toAbsoluteUrl(domain) + "all-chapters/").parseHtml() + return doc.select("ul.chapter-list > li").mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val url = a.attrAsRelativeUrl("href") + val titleEl = li.selectFirst(".chapter-title, .chapter-number") ?: return@mapNotNull null + val chapterName = titleEl.ownText() + val dateStr = li.selectFirst(".chapter-update")?.attr("datetime") + ?.replace(".", "")?.replace("Sept", "Sep") ?: "" + val date = dateFormats.firstNotNullOfOrNull { fmt -> fmt.parseSafe(dateStr) } ?: 0L + MangaChapter( + id = generateUid(url), + title = "Chapter $chapterName", + number = 0f, + volume = 0, + url = url, + uploadDate = date, + source = source, + scanlator = null, + branch = null, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("center img") + .mapNotNull { it.attr("src").takeIf { src -> src.isNotBlank() } } + .filterNot { it.startsWith("data:image") || it.contains("credits-mgeko.png") } + .distinct().map { url -> + val finalUrl = url.toRelativeUrl(domain) + MangaPage( + id = generateUid(finalUrl), + url = finalUrl, + preview = null, + source = source, + ) + } + } + + private fun Element.toManga(): Manga { + val titleEl = selectFirst(".comic-card__title a") ?: selectFirst(".novel-title") + val title = titleEl?.text()?.trim() ?: "" + val href = selectFirst("a")?.attrAsRelativeUrl("href") ?: "" + val cover = selectFirst(".comic-card__cover img, .novel-cover img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + } ?: "" + val coverUrl = cover.takeIf { it.isNotEmpty() }?.toAbsoluteUrl(domain) + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt b/src/main/kotlin/tsuki/site/en/MangaJinx.kt similarity index 84% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt rename to src/main/kotlin/tsuki/site/en/MangaJinx.kt index 28783ee2..c935e554 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaJinx.kt +++ b/src/main/kotlin/tsuki/site/en/MangaJinx.kt @@ -1,21 +1,37 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadthemeParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.removeSuffix import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadthemeParser -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat import java.util.EnumSet @@ -30,6 +46,15 @@ internal class MangaJinx(context: MangaLoaderContext) : override val selectDate = ".chapter-update" override val selectChapter = "#chapter-list > li, #chapter-list-inner .chapter-list > li" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/static/sites/icons/favicon-32x32.png", 32, null), + ), + domain, + ) + } + override val filterCapabilities: MangaListFilterCapabilities get() = super.filterCapabilities.copy( isAuthorSearchSupported = true, @@ -216,4 +241,4 @@ internal class MangaJinx(context: MangaLoaderContext) : ) } ?: emptySet() } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt b/src/main/kotlin/tsuki/site/en/MangaK.kt similarity index 76% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt rename to src/main/kotlin/tsuki/site/en/MangaK.kt index 508a8099..74c6b9ce 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaK.kt +++ b/src/main/kotlin/tsuki/site/en/MangaK.kt @@ -1,36 +1,41 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONNotNullToSet +import tsuki.util.json.mapJSONToSet + +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope import org.json.JSONObject -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Demographic -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.json.mapJSONNotNullToSet -import org.koitharu.kotatsu.parsers.util.json.mapJSONToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -43,7 +48,7 @@ internal class MangaK(context: MangaLoaderContext) : override val configKeyDomain = ConfigKey.Domain("mangak.io") - private val apiUrl = "https://api.mangak.io" + private val apiUrl = "https://api.$domain" private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) @@ -67,8 +72,10 @@ internal class MangaK(context: MangaLoaderContext) : override val availableSortOrders: Set<SortOrder> = EnumSet.of( SortOrder.UPDATED, SortOrder.NEWEST, - SortOrder.POPULARITY, SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, ) override val filterCapabilities: MangaListFilterCapabilities @@ -84,8 +91,6 @@ internal class MangaK(context: MangaLoaderContext) : availableStates = EnumSet.of( MangaState.ONGOING, MangaState.FINISHED, -// MangaState.PAUSED, -// MangaState.ABANDONED, ), availableContentTypes = EnumSet.of( ContentType.MANGA, @@ -116,34 +121,35 @@ internal class MangaK(context: MangaLoaderContext) : } } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val allExcludedTags = filter.tagsExclude.map { it.key }.toMutableSet() - val query = filter.query val url = buildString { append(apiUrl) append("/titles/search?page=") append(page) append("&limit=") append(pageSize) - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.NEWEST -> "newest" - SortOrder.POPULARITY -> "popular" - SortOrder.RATING -> "rating" - else -> "latest" - }, - ) - if (!query.isNullOrEmpty()) { + // To have "Best match" results + if (filter.query.isNullOrBlank()) { + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "newest" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "rating" + SortOrder.POPULARITY_WEEK -> "views_7days" + SortOrder.POPULARITY_MONTH -> "views_30days" + else -> "latest" + } + ) + } + + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { q -> append("&q=") - // api does not like commans and character limits to 60-70 - append(query.replace(",", "").urlEncoded()) + // API rejects queries longer than ~50 chars + append(q.take(50).replace(",", "").urlEncoded()) } if (filter.tags.isNotEmpty()) { @@ -198,8 +204,6 @@ internal class MangaK(context: MangaLoaderContext) : when (it) { MangaState.ONGOING -> "ongoing" MangaState.FINISHED -> "completed" -// MangaState.PAUSED -> "hiatus" -// MangaState.ABANDONED -> "cancelled" else -> null } ?: return@let ) @@ -231,7 +235,6 @@ internal class MangaK(context: MangaLoaderContext) : ) } - private fun String?.toContentRating() = when (this) { "safe" -> ContentRating.SAFE "suggestive" -> ContentRating.SUGGESTIVE @@ -239,7 +242,6 @@ internal class MangaK(context: MangaLoaderContext) : else -> null } - private fun JSONObject.parseAltTitles(): Set<String> { val result = mutableSetOf<String>() optJSONArray("alt_names")?.let { arr -> @@ -251,25 +253,37 @@ internal class MangaK(context: MangaLoaderContext) : return result } - private fun JSONObject.parseTags(): Set<MangaTag> = optJSONArray("genres")?.mapJSONNotNullToSet { genre -> val slug = genre.optString("slug").nullIfEmpty() ?: return@mapJSONNotNullToSet null MangaTag(genre.getString("name").toTitleCase(sourceLocale), slug, source) }.orEmpty() - - - override suspend fun getDetails(manga: Manga): Manga { + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { val json = webClient.httpGet("$apiUrl/titles/${manga.url}").parseJson() .getJSONObject("data").getJSONObject("title") + val cv = json.optLong("cv") + val chaptersDeferred = async { + fetchChapters(manga.url, cv) + } val authors = json.optJSONArray("authors")?.mapJSONNotNullToSet { it.optString("name").nullIfEmpty() }.orEmpty() val artists = json.optJSONArray("artists")?.mapJSONNotNullToSet { it.optString("name").nullIfEmpty() }.orEmpty() - return manga.copy( + + val contentRating = json.optString("content_rating").nullIfEmpty()?.let { + when (it) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + } + + manga.copy( title = json.optString("name").nullIfEmpty() ?: manga.title, altTitles = json.parseAltTitles().ifEmpty { manga.altTitles }, description = json.optString("summary").nullIfEmpty() ?: manga.description, @@ -277,11 +291,11 @@ internal class MangaK(context: MangaLoaderContext) : authors = authors + artists, state = json.optString("status").toMangaState() ?: manga.state, rating = json.optDouble("rating", 0.0).toRating(), - chapters = fetchChapters(manga.url, cv), + contentRating = contentRating ?: manga.contentRating, + chapters = chaptersDeferred.await(), ) } - private suspend fun fetchChapters(id: String, cv: Long): List<MangaChapter> { val json = webClient.httpGet("$apiUrl/titles/$id/chapters?cv=$cv").parseJson() val chapters = json.getJSONObject("data").getJSONArray("chapters").mapJSON { it } @@ -302,7 +316,6 @@ internal class MangaK(context: MangaLoaderContext) : }.reversed() } - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { val doc = webClient.httpGet("https://$domain${chapter.url}").parseHtml() val raw = doc.selectFirst("script#__NEXT_DATA__")?.data() @@ -323,7 +336,6 @@ internal class MangaK(context: MangaLoaderContext) : } } - private fun Double.toRating() = if (this > 0.0) (this / 5.0).toFloat() else RATING_UNKNOWN private fun String?.toMangaState() = when (this?.lowercase(Locale.US)) { diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt b/src/main/kotlin/tsuki/site/en/MangaKatana.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt rename to src/main/kotlin/tsuki/site/en/MangaKatana.kt index a4b9072b..473b5a61 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaKatana.kt +++ b/src/main/kotlin/tsuki/site/en/MangaKatana.kt @@ -1,396 +1,362 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.Locale - -private const val two = "?sv=mk" -private const val three = "?sv=3" - -@MangaSourceParser("MANGAKATANA", "MangaKatana", "en") -internal class MangaKatana(context: MangaLoaderContext): - PagedMangaParser(context, MangaParserSource.MANGAKATANA, pageSize = 24) { - - private val baseUrl = "https://mangakatana.com" - - override val configKeyDomain = ConfigKey.Domain("mangakatana.com") - - override val availableSortOrders = EnumSet.of( - SortOrder.UPDATED, - SortOrder.NEWEST, - )!! - - private val preferredServerKey = ConfigKey.PreferredImageServer( - presetValues = mapOf( - two to "First server", - three to "Second server", - ), - defaultValue = two, - ) - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - keys.add(preferredServerKey) - } - - override suspend fun getFilterOptions(): MangaListFilterOptions { - return MangaListFilterOptions( - availableTags = availableTags(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - MangaState.ABANDONED, - ), - ) - } - - private fun MangaState.toMKStatus(): Int { - return when (this) { - MangaState.ONGOING -> 1 - MangaState.FINISHED -> 2 - MangaState.ABANDONED -> 0 - else -> 1 - } - } - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true - ) - - private val dateFormat = SimpleDateFormat("MMM-dd-yyyy", Locale.US) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - - val url = buildUrl(page, order, filter) - val doc = webClient.httpGet(url).parseHtml() - - return doc.select("div#book_list > div.item").map { element -> - - val link = element.selectFirst("div.text > h3 > a")!! - val thumb = element.selectFirst("img")!! - - Manga( - id = generateUid(link.absUrl("href")), - title = link.ownText().ifEmpty { link.text() }, - url = link.absUrl("href"), - publicUrl = link.absUrl("href"), - - coverUrl = thumb.absUrl("src"), - - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - authors = emptySet(), - state = null, - altTitles = emptySet(), - largeCoverUrl = null, - description = null, - - source = source - ) - } - } - - private fun buildUrl( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): String { - val query = filter.query?.trim().orEmpty() - val tags = filter.tags.map { it.key }.filter { it.isNotBlank() } - val status = filter.states.firstOrNull() - - return when { - query.isNotEmpty() -> { - if (page == 1) { - "$baseUrl/?search=${query.urlEncoded()}&search_by=m_name" - } else { - "$baseUrl/page/$page?search=${query.urlEncoded()}&search_by=m_name" - } - } - - status != null -> { - "$baseUrl/genres/?" + - "filter=1" + - "&include_mode=and" + - "&bookmark_opts=off" + - "&chapters=1" + - "&order=latest" + - "&status=${status.toMKStatus()}" + - "&page=$page" - } - - tags.isNotEmpty() -> - "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" - - else -> when (order) { - SortOrder.NEWEST -> "$baseUrl/new-manga?page=$page" - SortOrder.UPDATED -> "$baseUrl/latest/page/$page" - else -> "$baseUrl/latest/page/$page" - } - } - } - - override suspend fun getDetails(manga: Manga): Manga { - - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() - - val chapters = doc.select("tr:has(.chapter)") - .mapNotNull { tr -> - val a = tr.selectFirst("a") ?: return@mapNotNull null - val href = a.absUrl("href") - - if (href.isBlank() || href == "#" || !href.startsWith("http")) return@mapNotNull null - - MangaChapter( - id = generateUid(href), - title = a.text(), - number = extractChapterNumber(href), - volume = 0, - url = href, - uploadDate = parseDate(tr.selectFirst(".update_time")?.text()), - scanlator = null, - branch = null, - source = source - ) - } - .sortedBy { it.number } - - return manga.copy( - title = doc.selectFirst("h1")?.text() - ?: doc.selectFirst("h1.heading")?.text() - ?: manga.title, - - authors = doc.select(".author").eachText().toSet(), - - tags = doc.select(".genres a").map { - MangaTag( - key = it.text().lowercase(), - title = it.text(), - source = source - ) - }.toSet(), - - description = buildString { - append(doc.select(".summary > p").text()) - val alt = doc.select(".alt_name").text() - if (alt.isNotBlank()) { - append("\n\nAlt name(s): $alt") - } - }.trim(), - - state = parseStatus(doc.select(".value.status").text()), - - coverUrl = doc.select("div.media div.cover img").attr("abs:src"), - chapters = chapters - ) - } - - private fun parseStatus(status: String): MangaState? { - return when { - status.contains("Ongoing", true) -> MangaState.ONGOING - status.contains("Completed", true) -> MangaState.FINISHED - else -> null - } - } - - private fun extractChapterNumber(url: String): Float { - return Regex("""/c(\d+(\.\d+)?)""") - .find(url) - ?.groupValues - ?.getOrNull(1) - ?.toFloatOrNull() - ?: 0f - } - - private fun parseDate(text: String?): Long { - if (text.isNullOrBlank()) return 0L - return runCatching { - dateFormat.parse(text)?.time ?: 0L - }.getOrDefault(0L) - } - - - private fun availableTags() = setOf( - MangaTag("4 koma", "4-koma", source), - MangaTag("Action", "action", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Artbook", "artbook", source), - MangaTag("Award winning", "award-winning", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Doujinshi", "doujinshi", source), - MangaTag("Drama", "drama", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Erotica", "erotica", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Loli", "loli", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("One shot", "one-shot", source), - MangaTag("Overpowered MC", "overpowered-mc", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Romance", "romance", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Sexual violence", "sexual-violence", source), - MangaTag("Shota", "shota", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujo-ai", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Shounen Ai", "shounen-ai", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Sports", "sports", source), - MangaTag("Super power", "super-power", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Survival", "survival", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Yaoi", "yaoi", source), - MangaTag("Yuri", "yuri", source), - ) - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - - val url = applyServer(chapter.url) - val doc = webClient.httpGet(url).parseHtml() - - parseThzq(doc)?.let { - if (it.isNotEmpty()) return it - } - parseHtmlImages(doc)?.let { - if (it.isNotEmpty()) return it - } - val fallbackDoc = webClient.httpGet(chapter.url).parseHtml() - parseThzq(fallbackDoc)?.let { - if (it.isNotEmpty()) return it - } - - return emptyList() - } - - private fun applyServer(url: String): String { - val server = config[preferredServerKey] - return url + server - } - - private fun parseThzq(doc: Document): List<MangaPage>? { - - val script = doc.select("script") - .asSequence() - .map { it.data() } - .firstOrNull { it.contains("thzq") } - ?: return null - - val start = script.indexOf("var thzq") - if (start == -1) return null - - val arrStart = script.indexOf('[', start) - if (arrStart == -1) return null - - var depth = 0 - var end = -1 - - for (i in arrStart until script.length) { - when (script[i]) { - '[' -> depth++ - ']' -> { - if (--depth == 0) { - end = i - break - } - } - } - } - - if (end == -1) return null - - val raw = script.substring(arrStart, end) - - val urls = ArrayList<String>(32) - - var i = 0 - while (i < raw.length) { - if (raw[i] == '\'' || raw[i] == '"') { - val quote = raw[i] - val startIdx = ++i - - while (i < raw.length && raw[i] != quote) { - i++ - } - - if (i > startIdx) { - val url = raw.substring(startIdx, i) - if (url.startsWith("http")) { - urls.add(url) - } - } - } - i++ - } - - if (urls.isEmpty()) return null - - return urls.map { url -> - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - } - - private fun parseHtmlImages(doc: Document): List<MangaPage>? { - - val pages = doc.select("div.wrap_img img") - .asSequence() - .mapNotNull { img -> - - val url = img.attr("data-src") - .ifEmpty { img.attr("src") } - - if (url.isEmpty() || !url.startsWith("http")) return@mapNotNull null - - MangaPage( - id = generateUid(url), - url = url, - preview = null, - source = source - ) - } - .toList() - - return pages.ifEmpty { null } - } -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import java.text.SimpleDateFormat + +import java.util.EnumSet +import java.util.Locale +import kotlin.collections.isNotEmpty +import org.jsoup.nodes.Document + +private const val two = "?sv=mk" +private const val three = "?sv=3" + +@MangaSourceParser("MANGAKATANA", "MangaKatana", "en") +internal class MangaKatana(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.MANGAKATANA, pageSize = 24) { + + private val baseUrl = "https://mangakatana.com" + + override val configKeyDomain = ConfigKey.Domain("mangakatana.com") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + ) + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + two to "First server", + three to "Second server", + ), + defaultValue = two, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(preferredServerKey) + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = availableTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + ), + ) + } + + private fun MangaState.toMKStatus(): Int { + return when (this) { + MangaState.ONGOING -> 1 + MangaState.FINISHED -> 2 + MangaState.ABANDONED -> 0 + else -> 1 + } + } + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true + ) + + private val dateFormat = SimpleDateFormat("MMM-dd-yyyy", Locale.US) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter, ): List<Manga> { + val url = buildUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + return doc.select("div#book_list > div.item").map { element -> + val link = element.selectFirst("div.text > h3 > a")!! + val thumb = element.selectFirst("img")!! + Manga( + id = generateUid(link.absUrl("href")), + title = link.ownText().ifEmpty { link.text() }, + url = link.absUrl("href"), + publicUrl = link.absUrl("href"), + coverUrl = thumb.absUrl("src"), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + state = null, + altTitles = emptySet(), + largeCoverUrl = null, + description = null, + source = source + ) + } + } + + private fun buildUrl(page: Int, order: SortOrder, filter: MangaListFilter, ): String { + val query = filter.query?.trim().orEmpty() + val tags = filter.tags.map { it.key }.filter { it.isNotBlank() } + val status = filter.states.firstOrNull() + return when { + query.isNotEmpty() -> { + if (page == 1) { + "$baseUrl/?search=${query.urlEncoded()}&search_by=m_name" + } else { + "$baseUrl/page/$page?search=${query.urlEncoded()}&search_by=m_name" + } + } + status != null -> { + "$baseUrl/genres/?" + + "filter=1" + + "&include_mode=and" + + "&bookmark_opts=off" + + "&chapters=1" + + "&order=latest" + + "&status=${status.toMKStatus()}" + + "&page=$page" + } + tags.isNotEmpty() -> "$baseUrl/manga/page/$page?filter=1&include=${tags.joinToString("_")}" + else -> when (order) { + SortOrder.NEWEST -> "$baseUrl/new-manga?page=$page" + SortOrder.UPDATED -> "$baseUrl/latest/page/$page" + else -> "$baseUrl/latest/page/$page" + } + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(baseUrl)).parseHtml() + val chapters = doc.select("tr:has(.chapter)") + .mapNotNull { tr -> + val a = tr.selectFirst("a") ?: return@mapNotNull null + val href = a.absUrl("href") + if (href.isBlank() || href == "#" || !href.startsWith("http")) return@mapNotNull null + MangaChapter( + id = generateUid(href), + title = a.text(), + number = extractChapterNumber(href), + volume = 0, + url = href, + uploadDate = parseDate(tr.selectFirst(".update_time")?.text()), + scanlator = null, + branch = null, + source = source + ) + } + .sortedBy { it.number } + + return manga.copy( + title = doc.selectFirst("h1")?.text() + ?: doc.selectFirst("h1.heading")?.text() + ?: manga.title, + authors = doc.select(".author").eachText().toSet(), + tags = doc.select(".genres a").map { + MangaTag( + key = it.text().lowercase(), + title = it.text(), + source = source + ) + }.toSet(), + description = buildString { + append(doc.select(".summary > p").text()) + val alt = doc.select(".alt_name").text() + if (alt.isNotBlank()) { + append("\n\nAlt name(s): $alt") + } + }.trim(), + state = parseStatus(doc.select(".value.status").text()), + coverUrl = doc.select("div.media div.cover img").attr("abs:src"), + chapters = chapters + ) + } + + private fun parseStatus(status: String): MangaState? { + return when { + status.contains("Ongoing", true) -> MangaState.ONGOING + status.contains("Completed", true) -> MangaState.FINISHED + else -> null + } + } + + private fun extractChapterNumber(url: String): Float { + return Regex("""/c(\d+(\.\d+)?)""") + .find(url) + ?.groupValues + ?.getOrNull(1) + ?.toFloatOrNull() + ?: 0f + } + + private fun parseDate(text: String?): Long { + if (text.isNullOrBlank()) return 0L + return runCatching { + dateFormat.parse(text)?.time ?: 0L + }.getOrDefault(0L) + } + + private fun availableTags() = setOf( + MangaTag("4 koma", "4-koma", source), + MangaTag("Action", "action", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Artbook", "artbook", source), + MangaTag("Award winning", "award-winning", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Drama", "drama", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Erotica", "erotica", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Loli", "loli", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("One shot", "one-shot", source), + MangaTag("Overpowered MC", "overpowered-mc", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Romance", "romance", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Sexual violence", "sexual-violence", source), + MangaTag("Shota", "shota", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujo-ai", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Shounen Ai", "shounen-ai", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Sports", "sports", source), + MangaTag("Super power", "super-power", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Survival", "survival", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Yuri", "yuri", source), + ) + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = applyServer(chapter.url) + val doc = webClient.httpGet(url).parseHtml() + parseThzq(doc)?.let { + if (it.isNotEmpty()) return it + } + parseHtmlImages(doc)?.let { + if (it.isNotEmpty()) return it + } + val fallbackDoc = webClient.httpGet(chapter.url).parseHtml() + parseThzq(fallbackDoc)?.let { + if (it.isNotEmpty()) return it + } + return emptyList() + } + + private fun applyServer(url: String): String { + val server = config[preferredServerKey] + return url + server + } + + private fun parseThzq(doc: Document): List<MangaPage>? { + val script = doc.select("script") + .asSequence() + .map { it.data() } + .firstOrNull { it.contains("thzq") } + ?: return null + val start = script.indexOf("var thzq") + if (start == -1) return null + val arrStart = script.indexOf('[', start) + if (arrStart == -1) return null + var depth = 0 + var end = -1 + for (i in arrStart until script.length) { + when (script[i]) { + '[' -> depth++ + ']' -> { + if (--depth == 0) { + end = i + break + } + } + } + } + if (end == -1) return null + val raw = script.substring(arrStart, end) + val urls = ArrayList<String>(32) + var i = 0 + while (i < raw.length) { + if (raw[i] == '\'' || raw[i] == '"') { + val quote = raw[i] + val startIdx = ++i + while (i < raw.length && raw[i] != quote) { + i++ + } + if (i > startIdx) { + val url = raw.substring(startIdx, i) + if (url.startsWith("http")) { + urls.add(url) + } + } + } + i++ + } + if (urls.isEmpty()) return null + return urls.map { url -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + } + + private fun parseHtmlImages(doc: Document): List<MangaPage>? { + val pages = doc.select("div.wrap_img img") + .asSequence() + .mapNotNull { img -> + val url = img.attr("data-src") + .ifEmpty { img.attr("src") } + if (url.isEmpty() || !url.startsWith("http")) return@mapNotNull null + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source + ) + } + .toList() + return pages.ifEmpty { null } + } +} diff --git a/src/main/kotlin/tsuki/site/en/MangaPill.kt b/src/main/kotlin/tsuki/site/en/MangaPill.kt new file mode 100644 index 00000000..db1e1da3 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MangaPill.kt @@ -0,0 +1,256 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.util.EnumSet + +@MangaSourceParser("MANGAPILL", "MangaPill", "en") +internal class MangaPill(context: MangaLoaderContext): + PagedMangaParser(context, MangaParserSource.MANGAPILL, 50) { + + override val configKeyDomain = ConfigKey.Domain("mangapill.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.ABANDONED, + MangaState.UPCOMING + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.NOVEL, + ContentType.DOUJINSHI, + ContentType.ONE_SHOT, + ContentType.OTHER, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrEmpty() || filter.tags.isNotEmpty() || filter.types.isNotEmpty() || filter.states.isNotEmpty()) { + return fetchSearch(page, filter) + } + return when (order) { + SortOrder.POPULARITY -> fetchPopular() + SortOrder.UPDATED -> fetchLatest() + else -> fetchSearch(page, filter) + } + } + + private suspend fun fetchPopular(): List<Manga> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + return doc.select("div:has(h4:contains(Trending)) > .grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private suspend fun fetchLatest(): List<Manga> { + val doc = webClient.httpGet("https://$domain/chapters").parseHtml() + return doc.select(".grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private suspend fun fetchSearch(page: Int, filter: MangaListFilter): List<Manga> { + val url = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .apply { + if (!filter.query.isNullOrEmpty()) addQueryParameter("q", filter.query) + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.NOVEL -> "novel" + ContentType.DOUJINSHI -> "doujinshi" + ContentType.ONE_SHOT -> "one-shot" + ContentType.OTHER -> "oel" + else -> "" + }) + } + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "publishing" + MangaState.FINISHED -> "finished" + MangaState.ABANDONED -> "discontinued" + MangaState.PAUSED -> "on hiatus" + MangaState.UPCOMING -> "not yet published" + else -> "" + }) + } + filter.tags.forEach { tag -> + addQueryParameter("genre", tag.key) + } + } + .build() + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".grid > div:not([class])") + .map { latestUpdatesFromElement(it) } + } + + private fun latestUpdatesFromElement(element: org.jsoup.nodes.Element): Manga { + val a = element.selectFirst("a[href^='/manga/']") ?: return Manga( + id = generateUid(""), + url = "", + publicUrl = "", + title = "", + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + val href = a.attr("href").removeSuffix("/") + val title = element.selectFirst("div.line-clamp-2")?.text() ?: "" + val img = element.selectFirst("img") ?: return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + val cover = img.attr("data-src") + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val cover = doc.selectFirst("div.container > div:first-child > div:first-child > img") + ?.attr("data-src") ?: manga.coverUrl + val altTitle = doc.selectFirst("div.text-sm.text-secondary")?.text()?.nullIfEmpty() + val description = doc.select("div.container > div:first-child > div:last-child > div:nth-child(2) > p").text() + val statusText = doc.select("label.text-secondary").firstOrNull { it.text() == "Status" } + ?.nextElementSibling()?.text() + val state = when (statusText) { + "publishing" -> MangaState.ONGOING + "finished" -> MangaState.FINISHED + "discontinued" -> MangaState.ABANDONED + "on hiatus" -> MangaState.PAUSED + "not yet published" -> MangaState.UPCOMING + else -> null + } + val tags = doc.select("div").firstOrNull { + it.selectFirst("label.text-secondary")?.text() == "Genres" + }?.select("a.text-sm.mr-1.text-brand")?.mapNotNullToSet { element -> + val key = element.attr("href").substringAfter("/search?genre=") + if (key.isBlank()) null + else MangaTag(key = key, title = element.text(), source = source) + } ?: emptySet() + + val chapters = doc.select("div#chapters a").mapNotNull { element -> + val href = element.attrAsRelativeUrl("href") + val rawTitle = element.text() + val chapterNumber = rawTitle.substringAfter("Chapter ").toFloatOrNull() ?: 0f + + MangaChapter( + id = generateUid(href), + title = rawTitle, + url = href, + number = chapterNumber, + volume = 0, + scanlator = null, + uploadDate = 0, + branch = null, + source = source, + ) + }.reversed() + + return manga.copy( + coverUrl = cover ?: manga.coverUrl, + altTitles = altTitle?.let { setOf(it) } ?: emptySet(), + description = description.nullIfEmpty(), + state = state, + tags = tags, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("picture img").map { img -> + val url = img.attr("data-src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + return doc.select("div.m-1 label input").mapNotNull { input -> + val title = input.attr("value") + val key = title.replace(" ", "+") + MangaTag(key = key, title = title, source = source) + }.toSet() + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt b/src/main/kotlin/tsuki/site/en/MangaTown.kt similarity index 90% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt rename to src/main/kotlin/tsuki/site/en/MangaTown.kt index 2ecac3ad..212af601 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/MangaTown.kt +++ b/src/main/kotlin/tsuki/site/en/MangaTown.kt @@ -1,375 +1,400 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("MANGATOWN", "MangaTown", "en") -internal class MangaTown(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MANGATOWN, 30) { - - override val configKeyDomain = ConfigKey.Domain("www.mangatown.com") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = Headers.Builder() - .add("Referer", "https://$domain/") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.RATING, - SortOrder.POPULARITY, - SortOrder.UPDATED, - ) - - override val filterCapabilities: MangaListFilterCapabilities - get() = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = true, - isYearSupported = false, - ) - - - @Volatile - private var allTagsCache: List<MangaTag>? = null - - private suspend fun getAllTags(): List<MangaTag> { - if (allTagsCache == null) { - allTagsCache = fetchAvailableTags().toList() - } - return allTagsCache!! - } - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getAllTags().toSet(), - availableStates = EnumSet.of( - MangaState.ONGOING, - MangaState.FINISHED, - ), - availableDemographics = EnumSet.of( - Demographic.SHOUJO, - Demographic.SEINEN, - Demographic.SHOUNEN, - Demographic.JOSEI, - ), - availableContentTypes = EnumSet.of( - ContentType.MANGA, - ContentType.MANHUA, - ContentType.MANHWA, - ), - ) - - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim() - val hasFilters = filter.types.isNotEmpty() || filter.demographics.isNotEmpty() || - filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || - filter.states.isNotEmpty() - - val url = when { - hasFilters -> { - val builder = "https://$domain/search".toHttpUrl().newBuilder() - .addQueryParameter("name_method", "cw") - .addQueryParameter("name", query ?: "") - .addQueryParameter("author_method", "cw") - .addQueryParameter("author", "") - .addQueryParameter("artist_method", "cw") - .addQueryParameter("artist", "") - - builder.addQueryParameter("type", - if (filter.types.size == 1) when (filter.types.first()) { - ContentType.MANGA -> "manga" - ContentType.MANHWA -> "manhwa" - ContentType.MANHUA -> "manhua" - else -> "" - } else "" - ) - - builder.addQueryParameter("demographic", - if (filter.demographics.size == 1) when (filter.demographics.first()) { - Demographic.SEINEN -> "SEINEN" - Demographic.SHOUJO -> "SHOUJO" - Demographic.SHOUNEN -> "SHOUNEN" - Demographic.JOSEI -> "JOSEI" - else -> "" - } else "" - ) - - if (filter.states.size == 1) { - when (filter.states.first()) { - MangaState.FINISHED -> builder.addQueryParameter("is_completed", "1") - MangaState.ONGOING -> builder.addQueryParameter("is_completed", "0") - else -> builder.addQueryParameter("is_completed", "") - } - } else { - builder.addQueryParameter("is_completed", "") - } - - val allTags = getAllTags() - val includedKeys = filter.tags.map { it.key }.toSet() - val excludedKeys = filter.tagsExclude.map { it.key }.toSet() - for (tag in allTags) { - val value = when (tag.key) { - in includedKeys -> "1" - in excludedKeys -> "2" - else -> "0" - } - builder.addQueryParameter("genres[${tag.title}]", value) - } - - builder.addQueryParameter("released_method", "eq") - builder.addQueryParameter("released", "") - builder.addQueryParameter("rating_method", "eq") - builder.addQueryParameter("rating", "") - builder.addQueryParameter("advopts", "1") - builder.addQueryParameter("page", page.toString()) - - builder.build().toString() - } - !query.isNullOrEmpty() -> { - // Simple search - "https://$domain/search?page=$page&name=${query.urlEncoded()}" - } - order == SortOrder.UPDATED -> { - "https://$domain/latest/$page.htm" - } - else -> { - // Directory browse - buildString { - append("https://$domain/directory/") - append("0-") - append( - if (filter.tags.isNotEmpty()) filter.tags.oneOrThrowIfMany()?.key ?: "0" - else "0" - ) - append("-0-") - append( - if (filter.states.isNotEmpty()) { - filter.states.oneOrThrowIfMany()?.let { - when (it) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> "0" - } - } ?: "0" - } else "0" - ) - append("-0-0/") - append(page).append(".htm") - append( - when (order) { - SortOrder.POPULARITY -> "" - SortOrder.ALPHABETICAL -> "?name.az" - SortOrder.RATING -> "?rating.za" - else -> "?last_chapter_time.za" - } - ) - } - } - } - - val doc = webClient.httpGet(url, getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("ul.manga_pic_list") ?: return emptyList() - val manga = root.select("li") - if (manga.isEmpty()) return emptyList() - - return manga.mapNotNull { li -> - val a = li.selectFirst("a.manga_cover") ?: return@mapNotNull null - val href = a.attrAsRelativeUrl("href") - val views = li.select("p.view") - val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } - ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) - val author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } - ?.substringAfter(':') - ?.trim() - Manga( - id = generateUid(href), - title = a.attr("title"), - coverUrl = a.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), - source = source, - altTitles = emptySet(), - rating = li.selectFirst("p.score")?.selectFirst("b") - ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, - authors = setOfNotNull(author), - state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - }, - tags = li.selectFirst("p.keyWord")?.select("a")?.mapToSet tags@{ x -> - MangaTag( - title = x.attr("title").toTitleCase(), - key = x.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), - source = source, - ) - }.orEmpty(), - url = href, - contentRating = null, - publicUrl = href.toAbsoluteUrl(a.host ?: domain), - ) - } - } - - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("section.main") - ?.selectFirst("div.article_content") ?: return manga - val info = root.selectFirst("div.detail_info")?.selectFirst("ul") - val chaptersList = root.selectFirst("div.chapter_content") - ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - - if (root.select("div.chapter_content:contains(has been licensed)").isNotEmpty()) { - return manga.copy(state = MangaState.ABANDONED, chapters = bypassLicensedChapters(manga)) - } - - return manga.copy( - tags = manga.tags + (info?.select("li")?.find { x -> - x.selectFirst("b")?.ownText() == "Genre(s):" - }?.select("a")?.mapNotNull { a -> - MangaTag( - title = a.attr("title").toTitleCase(), - key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), - source = source, - ) - }.orEmpty()), - description = info?.getElementById("show")?.ownText(), - chapters = chaptersList?.mapChapters { i, li -> - val linkEl = li.selectFirst("a") ?: return@mapChapters null - val href = linkEl.attrAsRelativeUrl("href") - val name = buildString { - append(linkEl.text()) - li.select("span").filter { span -> - !span.hasClass("time") && !span.hasClass("new") - }.forEach { span -> - if (isNotEmpty()) append(" ") - append(span.text()) - } - } - MangaChapter( - id = generateUid(href), - url = href, - source = source, - number = i + 1f, - volume = 0, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - title = name.nullIfEmpty(), - scanlator = null, - branch = null, - ) - } ?: emptyList(), - ) - } - - - private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> { - val subdomain = "m." + domain.removePrefix("www.") - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(subdomain), getRequestHeaders()).parseHtml() - val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() - val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) - return list.select("li").asReversed().mapIndexedNotNull { i, li -> - val a = li.selectFirst("a") ?: return@mapIndexedNotNull null - val href = a.attrAsRelativeUrl("href") - val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { a.ownText() } - MangaChapter( - id = generateUid(href), - url = href, - source = source, - number = i + 1f, - volume = 0, - uploadDate = parseChapterDate( - dateFormat, - li.selectFirst("span.time")?.text(), - ), - title = name.nullIfEmpty(), - scanlator = null, - branch = null, - ) - } - } - - - private fun parseChapterDate(dateFormat: SimpleDateFormat, date: String?): Long { - return when { - date.isNullOrEmpty() -> 0L - date.contains("Today") -> Calendar.getInstance().timeInMillis - date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis - else -> dateFormat.parseSafe(date) - } - } - - - private suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - val root = doc.body().selectFirst("aside.right") - ?.getElementsContainingOwnText("Genres") - ?.first() - ?.nextElementSibling() ?: doc.parseFailed("Root not found") - return root.select("li").mapNotNullToSet { li -> - val a = li.selectFirst("a") ?: return@mapNotNullToSet null - val key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-") - MangaTag( - source = source, - key = key, - title = a.text().toTitleCase(), - ) - } - } - - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val fullUrl = chapter.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl, getRequestHeaders()).parseHtml() - val pageSelect = doc.body().selectFirst("div.page_select select") ?: return emptyList() - val options = pageSelect.select("option").filterNot { it.attr("value").endsWith("featured.html") } - if (options.isNotEmpty()) { - return options.mapIndexed { _, option -> - val href = option.attrAsRelativeUrl("value") - MangaPage( - id = generateUid(href), - url = href, - preview = null, - source = source, - ) - } - } - val imgElements = doc.select("div#viewer img") - return imgElements.map { img -> - val src = img.attrAsAbsoluteUrl("src") - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - - override suspend fun getPageUrl(page: MangaPage): String { - if (page.url.startsWith("http")) return page.url - val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() - return doc.selectFirst("div#viewer img")?.attrAsAbsoluteUrl("src") - ?: throw Exception("Could not find image") - } - - - private fun String.nullIfEmpty(): String? = ifEmpty { null } -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Demographic +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet +import java.util.Locale +import kotlin.collections.toList +import kotlin.collections.toSet + +@MangaSourceParser("MANGATOWN", "MangaTown", "en") +internal class MangaTown(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGATOWN, 30) { + + override val configKeyDomain = ConfigKey.Domain("www.mangatown.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.RATING, + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isYearSupported = false, + ) + + + @Volatile + private var allTagsCache: List<MangaTag>? = null + + private suspend fun getAllTags(): List<MangaTag> { + if (allTagsCache == null) { + allTagsCache = fetchAvailableTags().toList() + } + return allTagsCache!! + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getAllTags().toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableDemographics = EnumSet.of( + Demographic.SHOUJO, + Demographic.SEINEN, + Demographic.SHOUNEN, + Demographic.JOSEI, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHUA, + ContentType.MANHWA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim() + val hasFilters = filter.types.isNotEmpty() || filter.demographics.isNotEmpty() || + filter.tags.isNotEmpty() || filter.tagsExclude.isNotEmpty() || + filter.states.isNotEmpty() + + val url = when { + hasFilters -> { + val builder = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("name_method", "cw") + .addQueryParameter("name", query ?: "") + .addQueryParameter("author_method", "cw") + .addQueryParameter("author", "") + .addQueryParameter("artist_method", "cw") + .addQueryParameter("artist", "") + + builder.addQueryParameter("type", + if (filter.types.size == 1) when (filter.types.first()) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } else "" + ) + + builder.addQueryParameter("demographic", + if (filter.demographics.size == 1) when (filter.demographics.first()) { + Demographic.SEINEN -> "SEINEN" + Demographic.SHOUJO -> "SHOUJO" + Demographic.SHOUNEN -> "SHOUNEN" + Demographic.JOSEI -> "JOSEI" + else -> "" + } else "" + ) + + if (filter.states.size == 1) { + when (filter.states.first()) { + MangaState.FINISHED -> builder.addQueryParameter("is_completed", "1") + MangaState.ONGOING -> builder.addQueryParameter("is_completed", "0") + else -> builder.addQueryParameter("is_completed", "") + } + } else { + builder.addQueryParameter("is_completed", "") + } + + val allTags = getAllTags() + val includedKeys = filter.tags.map { it.key }.toSet() + val excludedKeys = filter.tagsExclude.map { it.key }.toSet() + for (tag in allTags) { + val value = when (tag.key) { + in includedKeys -> "1" + in excludedKeys -> "2" + else -> "0" + } + builder.addQueryParameter("genres[${tag.title}]", value) + } + + builder.addQueryParameter("released_method", "eq") + builder.addQueryParameter("released", "") + builder.addQueryParameter("rating_method", "eq") + builder.addQueryParameter("rating", "") + builder.addQueryParameter("advopts", "1") + builder.addQueryParameter("page", page.toString()) + + builder.build().toString() + } + !query.isNullOrEmpty() -> { + // Simple search + "https://$domain/search?page=$page&name=${query.urlEncoded()}" + } + order == SortOrder.UPDATED -> { + "https://$domain/latest/$page.htm" + } + else -> { + // Directory browse + buildString { + append("https://$domain/directory/") + append("0-") + append( + if (filter.tags.isNotEmpty()) filter.tags.oneOrThrowIfMany()?.key ?: "0" + else "0" + ) + append("-0-") + append( + if (filter.states.isNotEmpty()) { + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> "0" + } + } ?: "0" + } else "0" + ) + append("-0-0/") + append(page).append(".htm") + append( + when (order) { + SortOrder.POPULARITY -> "" + SortOrder.ALPHABETICAL -> "?name.az" + SortOrder.RATING -> "?rating.za" + else -> "?last_chapter_time.za" + } + ) + } + } + } + + val doc = webClient.httpGet(url, getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("ul.manga_pic_list") ?: return emptyList() + val manga = root.select("li") + if (manga.isEmpty()) return emptyList() + + return manga.mapNotNull { li -> + val a = li.selectFirst("a.manga_cover") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val views = li.select("p.view") + val status = views.firstNotNullOfOrNull { it.ownText().takeIf { x -> x.startsWith("Status:") } } + ?.substringAfter(':')?.trim()?.lowercase(Locale.ROOT) + val author = views.firstNotNullOfOrNull { it.text().takeIf { x -> x.startsWith("Author:") } } + ?.substringAfter(':') + ?.trim() + Manga( + id = generateUid(href), + title = a.attr("title"), + coverUrl = a.selectFirst("img")?.attrAsAbsoluteUrlOrNull("src"), + source = source, + altTitles = emptySet(), + rating = li.selectFirst("p.score")?.selectFirst("b") + ?.ownText()?.toFloatOrNull()?.div(5f) ?: RATING_UNKNOWN, + authors = setOfNotNull(author), + state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + tags = li.selectFirst("p.keyWord")?.select("a")?.mapToSet tags@{ x -> + MangaTag( + title = x.attr("title").toTitleCase(), + key = x.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), + source = source, + ) + }.orEmpty(), + url = href, + contentRating = null, + publicUrl = href.toAbsoluteUrl(a.host ?: domain), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("section.main") + ?.selectFirst("div.article_content") ?: return manga + val info = root.selectFirst("div.detail_info")?.selectFirst("ul") + val chaptersList = root.selectFirst("div.chapter_content") + ?.selectFirst("ul.chapter_list")?.select("li")?.asReversed() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + + if (root.select("div.chapter_content:contains(has been licensed)").isNotEmpty()) { + return manga.copy(state = MangaState.ABANDONED, chapters = bypassLicensedChapters(manga)) + } + + return manga.copy( + tags = manga.tags + (info?.select("li")?.find { x -> + x.selectFirst("b")?.ownText() == "Genre(s):" + }?.select("a")?.mapNotNull { a -> + MangaTag( + title = a.attr("title").toTitleCase(), + key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-"), + source = source, + ) + }.orEmpty()), + description = info?.getElementById("show")?.ownText(), + chapters = chaptersList?.mapChapters { i, li -> + val linkEl = li.selectFirst("a") ?: return@mapChapters null + val href = linkEl.attrAsRelativeUrl("href") + val name = buildString { + append(linkEl.text()) + li.select("span").filter { span -> + !span.hasClass("time") && !span.hasClass("new") + }.forEach { span -> + if (isNotEmpty()) append(" ") + append(span.text()) + } + } + MangaChapter( + id = generateUid(href), + url = href, + source = source, + number = i + 1f, + volume = 0, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + title = name.nullIfEmpty(), + scanlator = null, + branch = null, + ) + } ?: emptyList(), + ) + } + + private suspend fun bypassLicensedChapters(manga: Manga): List<MangaChapter> { + val subdomain = "m." + domain.removePrefix("www.") + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(subdomain), getRequestHeaders()).parseHtml() + val list = doc.body().selectFirst("ul.detail-ch-list") ?: return emptyList() + val dateFormat = SimpleDateFormat("MMM dd,yyyy", Locale.US) + return list.select("li").asReversed().mapIndexedNotNull { i, li -> + val a = li.selectFirst("a") ?: return@mapIndexedNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.selectFirst("span.vol")?.text().orEmpty().ifEmpty { a.ownText() } + MangaChapter( + id = generateUid(href), + url = href, + source = source, + number = i + 1f, + volume = 0, + uploadDate = parseChapterDate( + dateFormat, + li.selectFirst("span.time")?.text(), + ), + title = name.nullIfEmpty(), + scanlator = null, + branch = null, + ) + } + } + + private fun parseChapterDate(dateFormat: SimpleDateFormat, date: String?): Long { + return when { + date.isNullOrEmpty() -> 0L + date.contains("Today") -> Calendar.getInstance().timeInMillis + date.contains("Yesterday") -> Calendar.getInstance().apply { add(Calendar.DAY_OF_MONTH, -1) }.timeInMillis + else -> dateFormat.parseSafe(date) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("/directory/".toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + val root = doc.body().selectFirst("aside.right") + ?.getElementsContainingOwnText("Genres") + ?.first() + ?.nextElementSibling() ?: doc.parseFailed("Root not found") + return root.select("li").mapNotNullToSet { li -> + val a = li.selectFirst("a") ?: return@mapNotNullToSet null + val key = a.attr("href").substringAfter("/directory/0-").substringBefore("-0-") + MangaTag( + source = source, + key = key, + title = a.text().toTitleCase(), + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl, getRequestHeaders()).parseHtml() + val pageSelect = doc.body().selectFirst("div.page_select select") ?: return emptyList() + val options = pageSelect.select("option").filterNot { it.attr("value").endsWith("featured.html") } + if (options.isNotEmpty()) { + return options.mapIndexed { _, option -> + val href = option.attrAsRelativeUrl("value") + MangaPage( + id = generateUid(href), + url = href, + preview = null, + source = source, + ) + } + } + val imgElements = doc.select("div#viewer img") + return imgElements.map { img -> + val src = img.attrAsAbsoluteUrl("src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + if (page.url.startsWith("http")) return page.url + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain), getRequestHeaders()).parseHtml() + return doc.selectFirst("div#viewer img")?.attrAsAbsoluteUrl("src") + ?: throw Exception("Could not find image") + } + + private fun String.nullIfEmpty(): String? = ifEmpty { null } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt b/src/main/kotlin/tsuki/site/en/Mangago.kt similarity index 91% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt rename to src/main/kotlin/tsuki/site/en/Mangago.kt index 0d9fbed8..9f6b90dd 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangago.kt +++ b/src/main/kotlin/tsuki/site/en/Mangago.kt @@ -1,18 +1,43 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.bitmap.Bitmap +import tsuki.bitmap.Rect + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.requireElementById +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.extractChapterNumber import okhttp3.Interceptor import okhttp3.Response import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.bitmap.Bitmap -import org.koitharu.kotatsu.parsers.bitmap.Rect -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale import javax.crypto.Cipher import javax.crypto.spec.IvParameterSpec import javax.crypto.spec.SecretKeySpec @@ -90,13 +115,12 @@ internal class MangagoParser(context: MangaLoaderContext) : private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query.orEmpty() - if (!query.isNullOrEmpty()) { + if (filter.query != null) { val url = buildString { append("https://") append(domain) append("/r/l_search?name=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page) } @@ -229,7 +253,7 @@ internal class MangagoParser(context: MangaLoaderContext) : authors = author?.let { setOf(it) } ?: emptySet(), tags = genres, state = state, - chapters = chapters, + chapters = chapters.sortedBy { it.number }, ) } @@ -255,13 +279,18 @@ internal class MangagoParser(context: MangaLoaderContext) : val scanlator = element.selectFirst("td.no a, td.uk-table-shrink a")?.text()?.trim() ?.ifEmpty { null } - ChapterParseData(name, url, dateUpload, scanlator) + ChapterParseData( + name, + url, + dateUpload, + scanlator + ) } - .reversed() return buildChapterList(rawChapters) } + @Suppress("ControlFlowWithEmptyBody") private fun buildChapterList(chapters: List<ChapterParseData>): List<MangaChapter> { val scanlatorCounts = mutableMapOf<String, Int>() for (chapter in chapters) { @@ -339,21 +368,16 @@ internal class MangagoParser(context: MangaLoaderContext) : return result } + private fun extractChapterNumber(title: String): Float? { + val num = title.extractChapterNumber() + return if (num == 0f) null else num + } + private fun extractTitleSuffix(title: String): String? { val regex = Regex("""(?:ch\.?|chapter)\s*\d+(?:\.\d+)?\s*[:\-]\s*(.+)""", RegexOption.IGNORE_CASE) return regex.find(title)?.groupValues?.get(1)?.trim()?.takeIf { it.isNotEmpty() } } - private fun extractChapterNumber(title: String): Float? { - return try { - val regex = Regex("""(?:ch\.?|chapter|vol\.?\s*\d+\s+ch\.?)\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) - val match = regex.find(title) - match?.groupValues?.get(1)?.toFloat() - } catch (e: Exception) { - null - } - } - private val jsCache = mutableMapOf<String, Pair<String, Long>>() private val maxCacheTime = 1000 * 60 * 5 // 5 minutes @@ -385,7 +409,7 @@ internal class MangagoParser(context: MangaLoaderContext) : baseUrl = cleanUrl buildBatchUrl = { batchNum -> "$baseUrl/$batchNum/" } } - + val batchSize = 5 val allImages = mutableListOf<String>() var batchStart = 1 @@ -396,7 +420,7 @@ internal class MangagoParser(context: MangaLoaderContext) : while (allImages.size < pagesCount) { val batchUrl = buildBatchUrl(batchStart) val batchDoc = if (batchStart == 1) doc else webClient.httpGet(batchUrl).parseHtml() - val batchImages = decryptImageList(batchDoc, batchUrl) + val batchImages = decryptImageList(batchDoc) if (batchImages.isEmpty()) { break @@ -430,7 +454,7 @@ internal class MangagoParser(context: MangaLoaderContext) : val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() if (imgsrcsScript != null) { - val images = decryptImageList(doc, fullUrl) + val images = decryptImageList(doc) val cols = getColsFromDoc(doc) ?: "" val js = getDeobfuscatedJS(doc) ?: throw Exception("Could not get JS") @@ -467,7 +491,7 @@ internal class MangagoParser(context: MangaLoaderContext) : override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - private suspend fun decryptImageList(doc: Document, sourceUrl: String): List<String> { + private suspend fun decryptImageList(doc: Document): List<String> { val imgsrcsScript = doc.selectFirst("script:containsData(imgsrcs)")?.html() ?: throw Exception("Could not find imgsrcs") @@ -615,7 +639,7 @@ internal class MangagoParser(context: MangaLoaderContext) : } imgList[loc].toString().toInt() } - } catch (e: NumberFormatException) { + } catch (_: NumberFormatException) { return imgList } diff --git a/src/main/kotlin/tsuki/site/en/Mangaread.kt b/src/main/kotlin/tsuki/site/en/Mangaread.kt new file mode 100644 index 00000000..8bf4f0a9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/Mangaread.kt @@ -0,0 +1,14 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGAREAD", "MangaRead", "en") +internal class MangaRead(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGAREAD, "www.mangaread.org") { + override val tagPrefix = "genres/" + override val datePattern = "dd.MM.yyyy" +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt b/src/main/kotlin/tsuki/site/en/Mangataro.kt similarity index 91% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt rename to src/main/kotlin/tsuki/site/en/Mangataro.kt index 205ae6ea..e7ff1e32 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mangataro.kt +++ b/src/main/kotlin/tsuki/site/en/Mangataro.kt @@ -1,35 +1,38 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.Headers import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseRaw -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl import okhttp3.HttpUrl import java.security.MessageDigest import java.time.ZoneOffset @@ -41,6 +44,7 @@ import java.util.EnumSet import java.util.HashSet import java.util.LinkedHashSet import java.util.Locale +import kotlin.collections.isNotEmpty @MangaSourceParser("MANGATAROORG", "Mangataro", "en", ContentType.MANGA) internal class Mangataro(context: MangaLoaderContext) : @@ -74,29 +78,29 @@ internal class Mangataro(context: MangaLoaderContext) : override suspend fun getFilterOptions() = MangaListFilterOptions( availableTags = genreTags, - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.OTHER), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.OTHER + ), ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val payload = JSONObject().apply { put("page", page) put("search", filter.query.orEmpty()) - // Year val yearArray = JSONArray() if (filter.year > 0) yearArray.put(filter.year.toString()) put("years", yearArray.toString()) - // Genres (tags) put("genres", filter.tags.toGenrePayload()) - // Types val typeArray = JSONArray() filter.types.forEach { ct -> val apiType = when (ct) { @@ -110,7 +114,6 @@ internal class Mangataro(context: MangaLoaderContext) : } put("types", typeArray.toString()) - // Statuses val statusArray = JSONArray() filter.states.forEach { state -> val apiStatus = when (state) { @@ -198,13 +201,8 @@ internal class Mangataro(context: MangaLoaderContext) : return result } - private fun buildChapterUrl( - mangaId: String, - offset: Int, - limit: Int, - token: String, - timestamp: String, - ): String { + @Suppress("SameParameterValue") + private fun buildChapterUrl(mangaId: String, offset: Int, limit: Int, token: String, timestamp: String): String { return HttpUrl.Builder() .scheme("https") .host(domain) diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt b/src/main/kotlin/tsuki/site/en/Manhuaus.kt similarity index 86% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt rename to src/main/kotlin/tsuki/site/en/Manhuaus.kt index ffb883a6..f169d4b1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Manhuaus.kt +++ b/src/main/kotlin/tsuki/site/en/Manhuaus.kt @@ -1,78 +1,80 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import androidx.collection.arraySetOf -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.MadaraParser - -@MangaSourceParser("MANHUAUS", "Manhuaus", "en") -internal class Manhuaus(context: MangaLoaderContext): - MadaraParser(context, MangaParserSource.MANHUAUS, "manhuaus.com") { - - override val postReq = false - - override val filterCapabilities = MangaListFilterCapabilities( - isMultipleTagsSupported = true, - isSearchSupported = true, - isAuthorSearchSupported = true, - isSearchWithFiltersSupported = true, - isYearSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - private fun availableTags() = arraySetOf( - MangaTag("Action", "action", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Anime", "anime", source), - MangaTag("Cartoon", "cartoon", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Comic", "comic", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Detective", "detective", source), - MangaTag("Doujinshi", "doujinshi", source), - MangaTag("Drama", "drama", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Josei", "josei", source), - MangaTag("Live Action", "live-action", source), - MangaTag("Manga", "manga", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mature", "mature", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Mystery", "mystery", source), - MangaTag("One Shot", "one-shot", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Romance", "romance", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujo-ai", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Shounen Ai", "shounen-ai", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Smut", "smut", source), - MangaTag("Soft Yaoi", "soft-yaoi", source), - MangaTag("Soft Yuri", "soft-yuri", source), - MangaTag("Sports", "sports", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Yaoi", "yaoi", source), - MangaTag("Yuri", "yuri", source), - ) -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.model.MangaListFilterCapabilities +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import androidx.collection.arraySetOf + +@MangaSourceParser("MANHUAUS", "Manhuaus", "en") +internal class Manhuaus(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.MANHUAUS, "manhuaus.com") { + + override val postReq = false + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isAuthorSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + private fun availableTags() = arraySetOf( + MangaTag("Action", "action", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Anime", "anime", source), + MangaTag("Cartoon", "cartoon", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Comic", "comic", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Detective", "detective", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Drama", "drama", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Josei", "josei", source), + MangaTag("Live Action", "live-action", source), + MangaTag("Manga", "manga", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mature", "mature", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Mystery", "mystery", source), + MangaTag("One Shot", "one-shot", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Romance", "romance", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujo-ai", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Shounen Ai", "shounen-ai", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Smut", "smut", source), + MangaTag("Soft Yaoi", "soft-yaoi", source), + MangaTag("Soft Yuri", "soft-yuri", source), + MangaTag("Sports", "sports", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Yuri", "yuri", source), + ) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ManhwaNex.kt b/src/main/kotlin/tsuki/site/en/ManhwaNex.kt new file mode 100644 index 00000000..7e34539f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ManhwaNex.kt @@ -0,0 +1,15 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWANEX", "ManhwaNex", "en") +internal class ManhwaNex(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.MANHWANEX, "manhwanex.com") { + + override val postReq = false + +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/ManhwaTop.kt b/src/main/kotlin/tsuki/site/en/ManhwaTop.kt new file mode 100644 index 00000000..9d9506d7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ManhwaTop.kt @@ -0,0 +1,11 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWATOP", "ManhwaTop", "en") +internal class ManhwaTop(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANHWATOP, "manhwatop.com") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt similarity index 80% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt rename to src/main/kotlin/tsuki/site/en/ManhwaZ.kt index f00bc585..af8d3061 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ManhwaZ.kt +++ b/src/main/kotlin/tsuki/site/en/ManhwaZ.kt @@ -1,10 +1,30 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded @MangaSourceParser("MANHWAZ", "ManhwaZ", "en", ContentType.MANGA) internal class Manhwaz(context: MangaLoaderContext) : @@ -26,14 +46,13 @@ internal class Manhwaz(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?s=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page.toString()) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt b/src/main/kotlin/tsuki/site/en/Mgread.kt similarity index 68% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt rename to src/main/kotlin/tsuki/site/en/Mgread.kt index 3a7f8f4c..491a1e5b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/Mgread.kt +++ b/src/main/kotlin/tsuki/site/en/Mgread.kt @@ -1,295 +1,427 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en - -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.network.UserAgents -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* -import okhttp3.Interceptor -import okhttp3.Response - -// TODO -// add filter options - -@MangaSourceParser("MGREADIO", "Mgread.io", "en") -internal class MgreadIo(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.MGREADIO, 24) { - - override val configKeyDomain = ConfigKey.Domain("mgread.io") - override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request().newBuilder() - .header("Referer", "https://$domain/") - .header("Origin", "https://$domain") - .build() - return chain.proceed(request) - } - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - ) - - private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US) - private val restChapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US).apply { - timeZone = TimeZone.getTimeZone("GMT+7") - } - - override suspend fun getFilterOptions() = MangaListFilterOptions() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } - - if (query != null) { - val url = "https://$domain/wp-json/initlise/v1/search".toHttpUrl().newBuilder() - .addQueryParameter("term", query) - .addQueryParameter("page", page.toString()) - .build() - val jsonArray = JSONArray(webClient.httpGet(url).parseRaw()) - return (0 until jsonArray.length()).mapNotNull { i -> - val obj = jsonArray.getJSONObject(i) - val title = obj.getString("title").let { org.jsoup.Jsoup.parse(it).text() } - val mangaUrl = obj.getString("url").trim().takeIf(String::isNotEmpty) ?: return@mapNotNull null - val thumb = obj.optString("thumb", null) - Manga( - id = generateUid(mangaUrl), - title = title, - url = mangaUrl, - publicUrl = mangaUrl.toAbsoluteUrl(domain), - coverUrl = thumb, - source = source, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - ) - }.filterNot { it.isAnime() } - } - - val slug = if (order == SortOrder.POPULARITY) "manga-ranking" else "recently-updated" - val url = if (page == 1) "https://$domain/$slug/" else "https://$domain/$slug/page/$page/" - val doc = webClient.httpGet(url).parseHtml() - return parseMangaGrid(doc).filterNot { it.isAnime() } - } - - private fun parseMangaGrid(doc: Document): List<Manga> { - return doc.select(".manga-item-grid").map { element -> - val titleElement = element.selectFirst("h2 a[href*='/manga/']") - ?: element.selectFirst("a[href*='/manga/']:not([href*='/chapter-'])") - ?: return@map null - val href = titleElement.absUrl("href") - Manga( - id = generateUid(href), - title = titleElement.text(), - url = href.toHttpUrl().encodedPath, - publicUrl = href, - coverUrl = element.selectFirst("img")?.imageUrl(), - source = source, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - tags = emptySet(), - state = null, - authors = emptySet(), - ) - }.filterNotNull() - } - - private fun Manga.isAnime(): Boolean { - val t = title.lowercase() - return t.startsWith("anime -") || t.startsWith("anime –") || - url.substringAfter("/manga/", "").startsWith("anime-") - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst("#manga-title")?.ownText() - ?: doc.selectFirst("meta[property=og:title]")?.attr("content")?.substringBefore(" [Ch.")?.trim() - ?: manga.title - - val cover = doc.selectFirst(".story-cover img")?.imageUrl() - ?: doc.selectFirst("meta[property=og:image]")?.attr("content") - ?: manga.coverUrl - - val descElement = doc.selectFirst("#manga-description") - val descriptionText = descElement?.wholeText()?.trim() - ?: doc.selectFirst("meta[name=description]")?.attr("content")?.trim() - - val genre = doc.select("#genre-tags a[href*='/genre/']") - .joinToString { it.ownText().ifEmpty { it.text() } } - - val statusText = doc.selectFirst("#manga-status")?.text()?.lowercase(Locale.US)?.trim() - val state = when (statusText) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - "season end", "source hiatus", "caught up" -> MangaState.PAUSED - "dropped" -> MangaState.ABANDONED - else -> null - } - - val metaRow = doc.selectFirst("#manga-title + div") - val metadata = buildList { - metaRow?.ownText()?.substringBefore("Chapters")?.trim()?.takeIf(String::isNotEmpty)?.let { - add("Chapters: $it") - } - doc.selectFirst("#comic-othername")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Alternative title: $it") - } - doc.selectFirst(".init-review-info")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Rating: $it") - } - metaRow?.selectFirst(".init-plugin-suite-view-count-number")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Views: $it") - } - doc.selectFirst("#last-updated")?.text()?.takeIf(String::isNotEmpty)?.let { - add("Last updated: $it") - } - } - - val description = buildString { - if (descriptionText != null) append(descriptionText) - if (metadata.isNotEmpty()) { - if (isNotEmpty()) append("\n\n") - metadata.joinTo(this, separator = "\n") - } - }.trim().takeIf(String::isNotEmpty) - - val tags = genre.split(",").mapNotNull { it.trim().takeIf(String::isNotEmpty) }.map { name -> - MangaTag(name, name.lowercase(), source) - }.toSet() - - val mangaId = doc.selectFirst("#manga-title[data-id], #chapter-search-input[data-manga-id]") - ?.attr("data-id")?.ifEmpty { doc.selectFirst("#chapter-search-input[data-manga-id]")?.attr("data-manga-id") } - ?.toIntOrNull() - - val chapters = if (mangaId != null) { - fetchChaptersApi(mangaId, manga.url) - } else { - doc.select(".chapter-list .chapter-item").mapNotNull { chapterFromElement(it) } - } - - return manga.copy( - title = title, - coverUrl = cover, - description = description, - tags = tags, - state = state, - chapters = chapters, - ) - } - - private suspend fun fetchChaptersApi(mangaId: Int, mangaPath: String): List<MangaChapter> { - val result = mutableListOf<MangaChapter>() - var page = 1 - do { - val url = "https://$domain/wp-json/initmanga/v1/chapters".toHttpUrl().newBuilder() - .addQueryParameter("manga_id", mangaId.toString()) - .addQueryParameter("paged", page.toString()) - .addQueryParameter("per_page", "50") - .build() - val json = webClient.httpGet(url).parseJson() - val items = json.optJSONArray("items") ?: break - val totalPages = json.optInt("total_pages", 1) - for (i in 0 until items.length()) { - val item = items.getJSONObject(i) - val number = item.optDouble("number", -1.0).toFloat() - val title = item.optString("title", "") - val slug = item.optString("slug", "") - val createdAt = item.optString("created_at", "") - val chapterName = if (title.isNotEmpty()) "Chapter ${number.toInt()} - $title" else "Chapter ${number.toInt()}" - val cleanPath = mangaPath.substringBefore("/chapter/").trimEnd('/') - result.add( - MangaChapter( - id = generateUid("$cleanPath/$slug/"), - title = chapterName, - number = number, - url = "$cleanPath/$slug/", - uploadDate = restChapterDateFormat.parseSafe(createdAt), - source = source, - volume = 0, - scanlator = null, - branch = null, - ) - ) - } - page++ - } while (page <= totalPages) - return result.sortedBy { it.number } - } - - private fun chapterFromElement(element: Element): MangaChapter? { - val urlElement = element.selectFirst("a[href*='/chapter-']") ?: return null - val chapterUrl = urlElement.absUrl("href") - val name = element.selectFirst("h3")?.text() - ?: chapterUrl.substringBeforeLast('/').substringAfterLast('/').replace('-', ' ').replaceFirstChar { it.uppercase() } - val number = Regex("""/chapter-(\d+(?:\.\d+)?)/""").find(chapterUrl)?.groupValues?.get(1)?.toFloatOrNull() ?: -1f - val dateStr = element.selectFirst("time[datetime]")?.attr("datetime") - val date = dateStr?.parseChapterDate() ?: 0L - return MangaChapter( - id = generateUid(chapterUrl.toHttpUrl().encodedPath), - title = name, - number = number, - url = chapterUrl.toHttpUrl().encodedPath, - uploadDate = date, - source = source, - volume = 0, - scanlator = null, - branch = null, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select("#chapter-content img[src]").mapIndexed { _, img -> - val src = img.absUrl("src") - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun String?.parseChapterDate(): Long { - if (isNullOrBlank()) return 0L - val normalized = this - .replace(Regex("""([+-]\d{2}):(\d{2})$"""), "$1$2") - .replace(Regex("""Z$"""), "+0000") - return chapterDateFormat.parseSafe(normalized) - } - - private fun Element.imageUrl(): String? { - return when (normalName()) { - "meta" -> attr("content") - else -> attr("abs:data-src").ifEmpty { - attr("abs:data-lazy-src").ifEmpty { attr("abs:src") } - } - }.takeIf(String::isNotEmpty) - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - private fun SimpleDateFormat.parseSafe(date: String): Long = - runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) -} +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.UserAgents + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import okhttp3.Interceptor +import okhttp3.Response +import java.util.Locale +import java.util.TimeZone +import kotlin.collections.filterNot +import kotlin.text.isNotEmpty + +@MangaSourceParser("MGREADIO", "Mgread.io", "en") +internal class MgreadIo(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MGREADIO, 24) { + + override val configKeyDomain = ConfigKey.Domain("mgread.io") + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request().newBuilder() + .header("Referer", "https://$domain/") + .header("Origin", "https://$domain") + .build() + return chain.proceed(request) + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.RATING, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isTagsExclusionSupported = false, + isMultipleTagsSupported = true, + isOriginalLocaleSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentRating = EnumSet.of( + ContentRating.SAFE, + ContentRating.SUGGESTIVE, + ContentRating.ADULT, + ), + ) + + private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssZ", Locale.US) + private val restChapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US).apply { + timeZone = TimeZone.getTimeZone("GMT+7") + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + + if (query != null) { + val url = "https://$domain/wp-json/initlise/v1/search".toHttpUrl().newBuilder() + .addQueryParameter("term", query) + .addQueryParameter("page", page.toString()) + .build() + val jsonArray = JSONArray(webClient.httpGet(url).parseRaw()) + return (0 until jsonArray.length()).mapNotNull { i -> + val obj = jsonArray.getJSONObject(i) + val title = obj.getString("title").let { org.jsoup.Jsoup.parse(it).text() } + val mangaUrl = obj.getString("url").trim().takeIf(String::isNotEmpty) ?: return@mapNotNull null + val thumb = obj.optString("thumb", null) + Manga( + id = generateUid(mangaUrl), + title = title, + url = mangaUrl, + publicUrl = mangaUrl.toAbsoluteUrl(domain), + coverUrl = thumb, + source = source, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + ) + }.filterNot { it.isAnime() } + } + + val url = buildAdvancedFilterUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + return parseMangaGrid(doc).filterNot { it.isAnime() } + } + + private fun buildAdvancedFilterUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + val builder = "https://$domain/advanced-filter".toHttpUrl().newBuilder() + if (page > 1) { + builder.addPathSegment("page") + builder.addPathSegment(page.toString()) + } + + val sortParam = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "views" + SortOrder.POPULARITY_TODAY -> "views_day" + SortOrder.POPULARITY_WEEK -> "views_week" + SortOrder.POPULARITY_MONTH -> "view_month" + SortOrder.RATING -> "rating" + SortOrder.NEWEST -> "new" + SortOrder.NEWEST_ASC -> "old" + else -> "updated" + } + builder.addQueryParameter("sort", sortParam) + + val statusParam = filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "season_end" + MangaState.ABANDONED -> "dropped" + else -> null + } + } + statusParam?.let { builder.addQueryParameter("status", it) } + + val ageRatingParam = filter.contentRating.firstOrNull()?.let { cr -> + when (cr) { + ContentRating.SAFE -> "all" + ContentRating.SUGGESTIVE -> "16+" + ContentRating.ADULT -> "18+" + } + } + ageRatingParam?.let { builder.addQueryParameter("age_rating", it) } + + filter.tags.forEach { tag -> + builder.addQueryParameter("genre[]", tag.key) + } + + return builder.build().toString() + } + + private fun parseMangaGrid(doc: Document): List<Manga> { + return doc.select(".manga-item-grid, .manga-item-details").map { element -> + val titleElement = element.selectFirst("h2 a[href*='/manga/']") + ?: element.selectFirst("a[href*='/manga/']:not([href*='/chapter-'])") + ?: return@map null + val href = titleElement.absUrl("href") + Manga( + id = generateUid(href), + title = titleElement.text(), + url = href.toHttpUrl().encodedPath, + publicUrl = href, + coverUrl = element.selectFirst("img")?.imageUrl(), + source = source, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + ) + }.filterNotNull() + } + + private fun Manga.isAnime(): Boolean { + val t = title.lowercase() + return t.startsWith("anime -") || t.startsWith("anime –") || + url.substringAfter("/manga/", "").startsWith("anime-") + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("#manga-title")?.ownText() + ?: doc.selectFirst("meta[property=og:title]")?.attr("content")?.substringBefore(" [Ch.")?.trim() + ?: manga.title + + val cover = doc.selectFirst(".story-cover img")?.imageUrl() + ?: doc.selectFirst("meta[property=og:image]")?.attr("content") + ?: manga.coverUrl + + val descElement = doc.selectFirst("#manga-description") + val descriptionText = descElement?.wholeText()?.trim() + ?: doc.selectFirst("meta[name=description]")?.attr("content")?.trim() + + val genre = doc.select("#genre-tags a[href*='/genre/']") + .joinToString { it.ownText().ifEmpty { it.text() } } + + val statusText = doc.selectFirst("#manga-status")?.text()?.lowercase(Locale.US)?.trim() + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "season end", "source hiatus", "caught up" -> MangaState.PAUSED + "dropped" -> MangaState.ABANDONED + else -> null + } + + val metaRow = doc.selectFirst("#manga-title + div") + val metadata = buildList { + metaRow?.ownText()?.substringBefore("Chapters")?.trim()?.takeIf(String::isNotEmpty)?.let { + add("Chapters: $it") + } + doc.selectFirst("#comic-othername")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Alternative title: $it") + } + doc.selectFirst(".init-review-info")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Rating: $it") + } + metaRow?.selectFirst(".init-plugin-suite-view-count-number")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Views: $it") + } + doc.selectFirst("#last-updated")?.text()?.takeIf(String::isNotEmpty)?.let { + add("Last updated: $it") + } + } + + val description = buildString { + if (descriptionText != null) append(descriptionText) + if (metadata.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + metadata.joinTo(this, separator = "\n") + } + }.trim().takeIf(String::isNotEmpty) + + val tags = genre.split(",").mapNotNull { it.trim().takeIf(String::isNotEmpty) }.map { name -> + MangaTag(name, name.lowercase().replace(" ", "-"), source) + }.toSet() + + val mangaId = doc.selectFirst("#manga-title[data-id], #chapter-search-input[data-manga-id]") + ?.attr("data-id")?.ifEmpty { doc.selectFirst("#chapter-search-input[data-manga-id]")?.attr("data-manga-id") } + ?.toIntOrNull() + + val chapters = if (mangaId != null) { + fetchChaptersApi(mangaId, manga.url) + } else { + doc.select(".chapter-list .chapter-item").mapNotNull { chapterFromElement(it) } + } + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + tags = tags, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChaptersApi(mangaId: Int, mangaPath: String): List<MangaChapter> { + val result = mutableListOf<MangaChapter>() + var page = 1 + do { + val url = "https://$domain/wp-json/initmanga/v1/chapters".toHttpUrl().newBuilder() + .addQueryParameter("manga_id", mangaId.toString()) + .addQueryParameter("paged", page.toString()) + .addQueryParameter("per_page", "50") + .build() + val json = webClient.httpGet(url).parseJson() + val items = json.optJSONArray("items") ?: break + val totalPages = json.optInt("total_pages", 1) + for (i in 0 until items.length()) { + val item = items.getJSONObject(i) + val number = item.optDouble("number", -1.0).toFloat() + val title = item.optString("title", "") + val slug = item.optString("slug", "") + val createdAt = item.optString("created_at", "") + val chapterName = if (title.isNotEmpty()) "Chapter ${number.toInt()} - $title" else "Chapter ${number.toInt()}" + val cleanPath = mangaPath.substringBefore("/chapter/").trimEnd('/') + result.add( + MangaChapter( + id = generateUid("$cleanPath/$slug/"), + title = chapterName, + number = number, + url = "$cleanPath/$slug/", + uploadDate = restChapterDateFormat.parseSafe(createdAt), + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + ) + } + page++ + } while (page <= totalPages) + return result.sortedBy { it.number } + } + + private fun chapterFromElement(element: Element): MangaChapter? { + val urlElement = element.selectFirst("a[href*='/chapter-']") ?: return null + val chapterUrl = urlElement.absUrl("href") + val name = element.selectFirst("h3")?.text() + ?: chapterUrl.substringBeforeLast('/').substringAfterLast('/').replace('-', ' ').replaceFirstChar { it.uppercase() } + val number = Regex("""/chapter-(\d+(?:\.\d+)?)/""").find(chapterUrl)?.groupValues?.get(1)?.toFloatOrNull() ?: -1f + val dateStr = element.selectFirst("time[datetime]")?.attr("datetime") + val date = dateStr?.parseChapterDate() ?: 0L + return MangaChapter( + id = generateUid(chapterUrl.toHttpUrl().encodedPath), + title = name, + number = number, + url = chapterUrl.toHttpUrl().encodedPath, + uploadDate = date, + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#chapter-content img[src]").mapIndexed { _, img -> + val src = img.absUrl("src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun String?.parseChapterDate(): Long { + if (isNullOrBlank()) return 0L + val normalized = this + .replace(Regex("""([+-]\d{2}):(\d{2})$"""), "$1$2") + .replace(Regex("""Z$"""), "+0000") + return chapterDateFormat.parseSafe(normalized) + } + + private fun Element.imageUrl(): String? { + return when (normalName()) { + "meta" -> attr("content") + else -> attr("abs:data-src").ifEmpty { + attr("abs:data-lazy-src").ifEmpty { attr("abs:src") } + } + }.takeIf(String::isNotEmpty) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun SimpleDateFormat.parseSafe(date: String): Long = + runCatching { parse(date)?.time ?: 0L }.getOrDefault(0L) + + private fun fetchTags(): Set<MangaTag> { + return GENRES.map { (name, slug) -> + MangaTag( + title = name, + key = slug, + source = source, + ) + }.toSet() + } + + companion object { + private val GENRES = listOf( + "Action" to "action", + "Adaptation" to "adaptation", + "Adventure" to "adventure", + "Anime" to "anime", + "Comedy" to "comedy", + "Cooking" to "cooking", + "Crime" to "crime", + "Drama" to "drama", + "Ecchi" to "ecchi", + "Fantasy" to "fantasy", + "Harem" to "harem", + "Historical" to "historical", + "Horror" to "horror", + "Isekai" to "isekai", + "Josei" to "josei", + "Martial Arts" to "martial-arts", + "Mature" to "mature", + "Mecha" to "mecha", + "Medical" to "medical", + "Music" to "music", + "Mystery" to "mystery", + "Romance" to "romance", + "School Life" to "school-life", + "Shoujo" to "shoujo", + "Shounen" to "shounen", + "Slice of life" to "slice-of-life", + "Smut" to "smut", + "Sports" to "sports", + "Supernatural" to "supernatural", + "Webtoons" to "webtoons", + ) + } +} diff --git a/src/main/kotlin/tsuki/site/en/MistScans.kt b/src/main/kotlin/tsuki/site/en/MistScans.kt new file mode 100644 index 00000000..4a497058 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/MistScans.kt @@ -0,0 +1,339 @@ +package tsuki.site.en + +import tsuki.network.OkHttpWebClient +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.extractChapterNumber + +import java.text.SimpleDateFormat +import java.util.Calendar +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MISTSCANS", "Mist Scans", "en") +internal class MistScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MISTSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("mistscans.com") + private val baseUrl = "https://$domain" + + private val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + @Volatile + private var cachedGenres: List<Genre>? = null + + private data class Genre( + val name: String, + val id: String + ) + + private suspend fun fetchGenres(): List<Genre> { + if (cachedGenres != null) return cachedGenres!! + val html = webClient.httpGet("$baseUrl/series/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + val genres = doc.select("#series_tags_page > button").map { btn -> + Genre(btn.text(), btn.attr("tag")) + } + cachedGenres = genres + return genres + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ADDED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + return MangaListFilterOptions( + availableTags = genres.map { MangaTag(it.name, it.id, source) }.toSet(), + availableStates = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val genreIds = filter.tags.map { it.key } + + if (query.isNotEmpty() || genreIds.isNotEmpty()) { + return searchManga(query, genreIds) + } + + return when (order) { + SortOrder.UPDATED -> fetchLatest() + SortOrder.ADDED -> fetchAdded() + SortOrder.POPULARITY -> fetchPopular() + else -> fetchPopular() + } + } + + private suspend fun fetchPopular(): List<Manga> { + val html = webClient.httpGet(baseUrl, getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select(".series-splide .splide__slide").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun fetchLatest(): List<Manga> { + val html = webClient.httpGet("$baseUrl/latest/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select("div.grid > div.group").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun fetchAdded(): List<Manga> { + val html = webClient.httpGet("$baseUrl/series/", getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + return doc.select("#searched_series_page > button").map { element -> + popularMangaFromElement(element) + } + } + + private suspend fun searchManga(query: String, genreIds: List<String>): List<Manga> { + val urlBuilder = "$baseUrl/series/".toHttpUrlOrNull()?.newBuilder()?.apply { + if (query.isNotEmpty()) addQueryParameter("q", query) + genreIds.forEach { addQueryParameter("genre", it) } + }?.build()?.toString() ?: return emptyList() + + val html = webClient.httpGet(urlBuilder, getRequestHeaders()).body.string() + val doc = Jsoup.parse(html, baseUrl) + + return doc.select("#searched_series_page > button") + .filter { element -> + val title = element.attr("title") + val matchesQuery = query.isEmpty() || title.contains(query, true) + + val entryGenres = try { + element.attr("tags").parseAsList() + } catch (_: Exception) { emptyList() } + val matchesGenres = genreIds.isEmpty() || + genreIds.all { genre -> entryGenres.any { it.equals(genre, true) } } + + matchesQuery && matchesGenres + } + .map { element -> popularMangaFromElement(element) } + } + + private fun popularMangaFromElement(element: Element): Manga { + val a = element.selectFirst("a[href]") ?: throw Exception("Link not found") + val title = a.attr("title") + val url = a.attr("abs:href") + val cover = element.getImageUrl("*[style*=background-image]") ?: "" + return Manga( + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = fetchDocument(manga.url) + + val title = doc.selectFirst("div.grid > h1")?.text() ?: manga.title + val cover = doc.getImageUrl("div[class*=photoURL]") ?: manga.coverUrl ?: "" + val status = doc.selectFirst("div:has(span:containsOwn(Status)) ~ div").parseStatus() + val author = doc.selectFirst("div:has(span:containsOwn(Author)) ~ div")?.text() + val artist = doc.selectFirst("div:has(span:containsOwn(Artist)) ~ div")?.text() + val type = doc.selectFirst("div:has(span:containsOwn(Type)) ~ div")?.text() + val genres = doc.select("div.grid:has(>h1) > div > a:not([title='Status'])").map { it.text() } + val altNames = doc.select("div.font-medium:containsOwn(Alternative titles) ~ div span").mapToSet { it.text() } + val description = buildString { + val synopsis = doc.selectFirst("#expand_content p")?.text().orEmpty() + append(synopsis) + }.takeIf { it.isNotEmpty() } + + val chapters = fetchChapters(doc) + + return manga.copy( + title = title, + altTitles = altNames, + coverUrl = cover, + description = description, + authors = listOfNotNull(author, artist).toSet(), + tags = (genres + listOfNotNull(type)).map { MangaTag(it.lowercase(), it, source) }.toSet(), + state = status, + chapters = chapters.sortedBy { it.number }, + ) + } + + private suspend fun fetchDocument(url: String): Document { + val fullUrl = if (url.startsWith("http")) url else "$baseUrl$url" + val html = webClient.httpGet(fullUrl, getRequestHeaders()).body.string() + return Jsoup.parse(html, fullUrl) + } + + private fun Element?.parseStatus(): MangaState? = when (this?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "dropped" -> MangaState.ABANDONED + "paused" -> MangaState.PAUSED + "completed" -> MangaState.FINISHED + else -> null + } + + private fun fetchChapters(document: Document): List<MangaChapter> { + val selector = "#chapters > a:not(:has(.text-sm span:matches(Upcoming))):not(:has(img[alt~=Coin]))" + return document.select(selector).map { element -> + val link = element.selectFirst("a[href]") ?: return@map null + val url = link.attr("abs:href") + val name = element.selectFirst(".text-sm")?.text() ?: "" + val dateStr = element.selectFirst(".text-xs")?.text()?.trim() ?: "" + val uploadDate = dateStr.parseDate() + + MangaChapter( + id = generateUid(url), + title = name, + number = name.extractChapterNumber(), + volume = 0, + url = url, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.filterNotNull() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = fetchDocument(chapter.url) + + val cdnUrl = getCdnUrl(doc) + val uidPages = doc.select("#pages > img") + .map { it.attr("uid") } + .filter { it.isNotEmpty() } + + if (uidPages.isNotEmpty() && cdnUrl != null) { + return uidPages.mapIndexed { _, uid -> + MangaPage( + id = generateUid(uid), + url = "$cdnUrl/$uid", + preview = null, + source = source, + ) + } + } + + return doc.select("#pages > img").map { it.imgAttr() } + .filter { it.contains(Regex("""^(https?:)?//cdn\d*\.keyoapp\.com""")) } + .mapIndexed { _, imgUrl -> + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + private fun getCdnUrl(document: Document): String? { + val script = document.select("script").firstOrNull { + CDN_HOST_REGEX.containsMatchIn(it.html()) + } ?: return null + val cdnHost = CDN_HOST_REGEX.find(script.html())?.groups?.get(1)?.value + ?.replace(CDN_CLEAN_REGEX, "") ?: return null + return "https://$cdnHost/uploads" + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun Element.getImageUrl(selector: String): String? { + val element = selectFirst(selector) ?: return null + val style = element.attr("style") + val match = IMG_REGEX.find(style) ?: return null + val rawUrl = match.groups[1]?.value ?: return null + val url = rawUrl.toHttpUrlOrNull() ?: return null + return url.newBuilder().setQueryParameter("w", "480").build().toString() + } + + private fun Element.imgAttr(): String = when { + hasAttr("data-lazy-src") -> attr("abs:data-lazy-src") + hasAttr("data-src") -> attr("abs:data-src") + else -> attr("abs:src") + } + + private fun String.parseDate(): Long { + val trimmed = trim() + if (trimmed.contains("ago")) return trimmed.parseRelativeDate() + return try { dateFormat.parse(trimmed)?.time ?: 0L } catch (_: Exception) { 0L } + } + + private fun String.parseRelativeDate(): Long { + val now = Calendar.getInstance().apply { + set(Calendar.SECOND, 0) + set(Calendar.MILLISECOND, 0) + } + val number = split(" ").firstOrNull() + ?.replace("one", "1") + ?.replace("a", "1") + ?.toIntOrNull() ?: return 0L + val text = this.lowercase() + when { + "second" in text -> now.add(Calendar.SECOND, -number) + "minute" in text -> now.add(Calendar.MINUTE, -number) + "hour" in text -> now.add(Calendar.HOUR, -number) + "day" in text -> now.add(Calendar.DAY_OF_YEAR, -number) + "week" in text -> now.add(Calendar.WEEK_OF_YEAR, -number) + "month" in text -> now.add(Calendar.MONTH, -number) + "year" in text -> now.add(Calendar.YEAR, -number) + } + return now.timeInMillis + } + + private fun String.parseAsList(): List<String> { + return removeSurrounding("[", "]") + .split(",") + .map { it.trim().removeSurrounding("\"") } + .filter { it.isNotEmpty() } + } + + companion object { + val CDN_HOST_REGEX = """realUrl\s*=\s*`[^`]+//([^/]+)""".toRegex() + val CDN_CLEAN_REGEX = """\$\{[^}]*\}""".toRegex() + val IMG_REGEX = """url\(['"]?([^(['")])]+)""".toRegex() + } +} diff --git a/src/main/kotlin/tsuki/site/en/PhiliaScans.kt b/src/main/kotlin/tsuki/site/en/PhiliaScans.kt new file mode 100644 index 00000000..05a7c28f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/PhiliaScans.kt @@ -0,0 +1,639 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException +import tsuki.network.OkHttpWebClient +import tsuki.network.WebClient + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSONNotNull +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe +import tsuki.util.runCatchingCancellable +import tsuki.util.toAbsoluteUrl + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.RequestBody.Companion.toRequestBody +import okhttp3.Response +import okhttp3.ResponseBody.Companion.toResponseBody +import org.json.JSONArray +import org.json.JSONObject +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import java.awt.RenderingHints +import java.awt.image.BufferedImage +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.text.SimpleDateFormat +import javax.crypto.Cipher +import javax.crypto.Mac +import javax.crypto.spec.IvParameterSpec +import javax.crypto.spec.SecretKeySpec +import javax.imageio.ImageIO + +@MangaSourceParser("PHILIASCANS", "Philia Scans", "en") +internal class PhiliaScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.PHILIASCANS, pageSize = 20), Interceptor { + + override val configKeyDomain = ConfigKey.Domain("philiascans.org") + private val apiUrl get() = "https://$domain/api" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient: WebClient by lazy { + OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor(this) + .build(), + source + ) + } + + override fun getRequestHeaders(): Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + val ua = config[userAgentKey].takeIf { it.isNotBlank() } + ?: "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:153.0) Gecko/20100101 Firefox/153.0" + builder["User-Agent"] = ua + return builder.build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.POPULARITY_WEEK, + SortOrder.RATING, SortOrder.NEWEST, SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS, + ), + ) + + @Volatile + private var tagsCache: Set<MangaTag>? = null + + private suspend fun fetchTags(): Set<MangaTag> { + tagsCache?.let { return it } + val array = webClient.httpGet("$apiUrl/genres").parseJsonArray() + val tags = array.mapJSONNotNull { item -> + val key = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val title = item.getStringOrNull("name") ?: return@mapJSONNotNull null + MangaTag(key = key, title = title, source = source) + }.toSet() + tagsCache = tags + return tags + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = "$apiUrl/manga".toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + filter.query?.nullIfEmpty()?.let { addQueryParameter("q", it) } + + val (orderBy, direction) = when (order) { + SortOrder.UPDATED -> null to "desc" + SortOrder.POPULARITY -> "views" to "desc" + SortOrder.POPULARITY_WEEK -> "trending" to "desc" + SortOrder.RATING -> "rating" to "desc" + SortOrder.NEWEST -> "added" to "desc" + SortOrder.ALPHABETICAL -> "title" to "asc" + SortOrder.ALPHABETICAL_DESC -> "title" to "desc" + else -> null to "desc" + } + orderBy?.let { addQueryParameter("orderby", it) } + addQueryParameter("order", direction) + + filter.tags.forEach { addQueryParameter("genres", it.key) } + filter.states.forEach { state -> + stateToApi(state)?.let { addQueryParameter("statuses", it) } + } + filter.types.forEach { type -> + typeToApi(type)?.let { addQueryParameter("types", it) } + } + }.build() + + val items = webClient.httpGet(url).parseJson().optJSONArray("items") ?: return emptyList() + return items.mapJSONNotNull { item -> + val slug = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + Manga( + id = generateUid(slug), + url = "/series/$slug", + publicUrl = "https://$domain/series/$slug", + title = item.getStringOrNull("title") ?: return@mapJSONNotNull null, + altTitles = emptySet(), + coverUrl = item.getStringOrNull("coverImageUrl")?.toAbsoluteUrl(domain), + largeCoverUrl = null, + authors = emptySet(), + description = null, + tags = item.optJSONArray("genres")?.toTags().orEmpty(), + state = parseState(item.getStringOrNull("status")), + rating = item.getStringOrNull("ratingAvg")?.toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN, + contentRating = parseContentRating(item.getStringOrNull("contentRating")), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.slug + val details = webClient.httpGet("$apiUrl/manga/$slug").parseJson() + val chapters = webClient.httpGet("$apiUrl/manga/$slug/chapters").parseJson() + .optJSONArray("items") + .parseChapters(slug) + return manga.copy( + title = details.getStringOrNull("title") ?: manga.title, + altTitles = details.optJSONArray("alternativeTitles")?.let { array -> + (0 until array.length()).mapNotNullTo(LinkedHashSet()) { array.optString(it).nullIfEmpty() } + }.orEmpty(), + coverUrl = details.getStringOrNull("coverImageUrl")?.toAbsoluteUrl(domain) + ?: manga.coverUrl, + description = details.getStringOrNull("synopsis"), + tags = details.optJSONArray("genres")?.toTags().orEmpty(), + authors = buildSet { + addAll(details.optJSONArray("authors").toNames()) + addAll(details.optJSONArray("artists").toNames()) + }, + state = parseState(details.getStringOrNull("status")), + contentRating = parseContentRating(details.getStringOrNull("contentRating")), + chapters = chapters, + ) + } + + private fun JSONArray?.parseChapters(mangaSlug: String): List<MangaChapter> { + if (this == null) return emptyList() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + return mapJSONNotNull { item -> + val slug = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val number = item.optString("number") + val isLocked = !item.optBoolean("purchased") && item.optInt("coinPrice") != 0 + val rawTitle = item.getStringOrNull("title") + ?.takeIf { it != "null" && it != number } + val name = if (rawTitle != null) "Chapter $number - $rawTitle" else "Chapter $number" + MangaChapter( + id = generateUid("$mangaSlug/$slug"), + title = if (isLocked) "🔒 $name" else name, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug", + scanlator = item.optJSONObject("team")?.optString("name")?.nullIfEmpty(), + uploadDate = dateFormat.parseSafe(item.optString("publishedAt")), + branch = null, + source = source, + ) + }.reversed() + } + + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val segments = chapter.url.trim('/').split('/') + if (segments.size < 3) throw ParseException("Unexpected chapter url: ${chapter.url}", chapter.url) + val mangaSlug = segments[1] + val chapterSlug = segments[2] + + val viewerResponse = webClient.httpGet("$apiUrl/manga/$mangaSlug/chapters/$chapterSlug") + val viewer = viewerResponse.parseJson() + if (!viewer.optBoolean("hasAccess", true)) + throw ParseException("This chapter has to be unlocked with coins. Sign in through the browser and buy it first.", chapter.url) + val chapterJson = viewer.optJSONObject("chapter") + ?: throw ParseException("No chapter data in response", chapter.url) + val chapterId = chapterJson.optLong("id") + val isScrambled = chapterJson.optBoolean("scrambled") + System.err.println("PhiliaScans: chapterId=$chapterId, scrambled=$isScrambled") + + val token = try { + val content = ByteArray(0) + val request = okhttp3.Request.Builder() + .url("$apiUrl/reader/access-token") + .post(content.toRequestBody(null, 0, content.size)) + .headers(readerHeaders()) + .build() + val response = context.httpClient.newCall(request).execute() + val bodyString = response.body?.string() ?: throw ParseException("Empty response", chapter.url) + val json = JSONObject(bodyString) + json.optString("token").nullIfEmpty() + ?: throw ParseException("Token missing", chapter.url) + } catch (e: Exception) { + System.err.println("PhiliaScans: failed to get reader access token") + e.printStackTrace() + throw ParseException("Could not obtain reader token: ${e.message}", chapter.url) + } + + val headers = readerHeaders().newBuilder() + .add(HEADER_READER_TOKEN, token) + .build() + + val keysResponse = webClient.httpGet("$apiUrl/chapters/$chapterId/page-keys", headers) + val keys = keysResponse.parseJson() + val chapterKeyB64 = keys.optString("chapterKeyB64") + val gridSize = keys.optInt("gridSize", 1) + System.err.println("PhiliaScans: chapterKeyB64=$chapterKeyB64, gridSize=$gridSize") + + var payloadA: String? = null + var payloadB: String? = null + if (keys.optBoolean("sessionDefault")) { + try { + val open = webClient.httpPost("$apiUrl/chapters/$chapterId/open".toHttpUrl(), JSONObject(), headers).parseJson() + payloadA = open.getStringOrNull("payloadA") + val sessionId = open.getStringOrNull("sessionId") + if (sessionId != null) { + payloadB = runCatchingCancellable { + webClient.httpGet("$apiUrl/chapters/$chapterId/get-drm?session=$sessionId", headers) + .parseJson().optString("payloadB").nullIfEmpty() + }.getOrNull() + } + } catch (e: Exception) { + System.err.println("PhiliaScans: DRM key retrieval failed") + e.printStackTrace() + } + } + + val pages = chapterJson.optJSONArray("pages") ?: return emptyList() + return (0 until pages.length()) + .mapNotNull { pages.optJSONObject(it) } + .sortedBy { it.optInt("position") } + .mapIndexed { index, page -> + val url = page.optString("url").toAbsoluteUrl(domain) + val fragment = listOf( + if (isScrambled) "1" else "0", + page.getStringOrNull("mime") ?: "image/webp", + chapterKeyB64, + gridSize.toString(), + payloadA.orEmpty(), + payloadB.orEmpty(), + index.toString(), + ).joinToString(";") + MangaPage( + id = generateUid(url), + url = "$url#$fragment", + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + return try { + interceptInternal(chain) + } catch (e: Exception) { + System.err.println("PhiliaScans: unhandled exception in interceptor") + e.printStackTrace() + chain.proceed(chain.request()) + } + } + + private fun interceptInternal(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val fragment = request.url.fragment + if (fragment.isNullOrEmpty() || !response.isSuccessful || + !PROTECTED_IMAGE_REGEX.matches(request.url.pathSegments.last())) { + return response + } + + System.err.println("PhiliaScans: decrypting ${request.url}") + val parts = fragment.split(';') + if (parts.size < 7) { + System.err.println("PhiliaScans: fragment too short: $fragment") + return response + } + val isScrambled = parts[0] == "1" + val mimeType = parts[1] + val gridSize = parts[3].toIntOrNull() ?: return response + val pageIndex = parts[6].toIntOrNull() ?: return response + val chapterKey = try { + resolveChapterKey(parts[2], parts[4], parts[5]) + } catch (e: Exception) { + System.err.println("PhiliaScans: failed to resolve chapter key") + e.printStackTrace() + return response + } ?: return response + + val body = response.body + val raw = body.bytes() + if (raw.size < 6) { + System.err.println("PhiliaScans: response too small (${raw.size} bytes)") + return response + } + + val scheme = when { + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_AES -> "aesctr:" + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_CHACHA -> "chacha" + raw[0] == MAGIC_HIGH && raw[1] == MAGIC_AES4 -> "aesctr4:" + else -> null + } + System.err.println("PhiliaScans: detected scheme=$scheme") + val offset = if (scheme != null) 2 else 0 + val header = ByteBuffer.wrap(raw, offset, 4).order(ByteOrder.BIG_ENDIAN) + val originalWidth = header.short.toInt() and 0xFFFF + val originalHeight = header.short.toInt() and 0xFFFF + val payload = raw.copyOfRange(offset + 4, raw.size) + + val plain = try { + when (scheme) { + "aesctr:", "aesctr4:" -> aesCtrDecrypt(payload, chapterKey, pageIndex, scheme) + "chacha" -> chaCha20(payload, hmacSha256(chapterKey, "cc:$pageIndex")) + else -> xorKeystream(payload, chapterKey, pageIndex) + } + } catch (e: Exception) { + System.err.println("PhiliaScans: decryption failed for page $pageIndex") + e.printStackTrace() + return response + } + + val contentType = mimeType.toMediaTypeOrNull() ?: body.contentType() + + if (isScrambled && scheme == null && gridSize >= 2) { + return try { + val unscrambledBytes = unscrambleWithAwt( + plain, chapterKey, pageIndex, gridSize, + originalWidth, originalHeight, formatNameFromMime(mimeType) + ) + response.newBuilder() + .body(unscrambledBytes.toResponseBody(contentType)) + .build() + } catch (e: Exception) { + System.err.println("PhiliaScans: unscramble failed for page $pageIndex") + e.printStackTrace() + response.newBuilder() + .body(plain.toResponseBody(contentType)) + .build() + } + } + + return response.newBuilder() + .body(plain.toResponseBody(contentType)) + .build() + } + + private fun unscrambleWithAwt( + imageBytes: ByteArray, + chapterKey: ByteArray, + pageIndex: Int, + gridSize: Int, + originalWidth: Int, + originalHeight: Int, + formatName: String + ): ByteArray { + return try { + val source = ImageIO.read(ByteArrayInputStream(imageBytes)) + ?: throw RuntimeException("Cannot decode image – format may be unsupported") + val tileCount = gridSize * gridSize + val order = IntArray(tileCount) { it } + + val tilesSignature = hmacSha256(chapterKey, "tiles:$pageIndex") + var counter = 0 + var wordIndex = 8 + var randomBlock = ByteArray(0) + fun nextRandom(): Long { + if (wordIndex >= 8) { + randomBlock = hmacSha256(tilesSignature, "perm:${counter++}") + wordIndex = 0 + } + val value = ByteBuffer.wrap(randomBlock, wordIndex * 4, 4) + .order(ByteOrder.LITTLE_ENDIAN).int.toLong() and 0xFFFFFFFFL + wordIndex++ + return value + } + for (i in tileCount - 1 downTo 1) { + val j = (nextRandom() % (i + 1)).toInt() + val tmp = order[i]; order[i] = order[j]; order[j] = tmp + } + + val inverse = IntArray(tileCount) + for (i in 0 until tileCount) inverse[order[i]] = i + + val tileWidth = source.width / gridSize + val tileHeight = source.height / gridSize + val outW = if (originalWidth > 0) originalWidth else source.width + val outH = if (originalHeight > 0) originalHeight else source.height + + val output = BufferedImage(outW, outH, BufferedImage.TYPE_INT_ARGB) + val g = output.createGraphics() + g.setRenderingHint(RenderingHints.KEY_INTERPOLATION, RenderingHints.VALUE_INTERPOLATION_BILINEAR) + + for (target in 0 until tileCount) { + val from = inverse[target] + val srcX = (from % gridSize) * tileWidth + val srcY = (from / gridSize) * tileHeight + val dstX = (target % gridSize) * tileWidth + val dstY = (target / gridSize) * tileHeight + g.drawImage( + source, + dstX, dstY, dstX + tileWidth, dstY + tileHeight, + srcX, srcY, srcX + tileWidth, srcY + tileHeight, + null + ) + } + g.dispose() + + val bos = ByteArrayOutputStream() + ImageIO.write(output, formatName, bos) + bos.toByteArray() + } catch (e: Exception) { + System.err.println("PhiliaScans: unscrambleWithAwt internal error") + e.printStackTrace() + throw e + } + } + + private fun formatNameFromMime(mime: String): String = when { + mime.contains("webp", ignoreCase = true) -> "webp" + mime.contains("jpeg") || mime.contains("jpg") -> "jpeg" + mime.contains("png") -> "png" + else -> "webp" + } + + private fun resolveChapterKey(keyB64: String, payloadA: String, payloadB: String): ByteArray? { + if (payloadA.isNotBlank() && payloadB.isNotBlank()) { + val a = runCatching { context.decodeBase64(payloadA) }.getOrNull() + val b = runCatching { context.decodeBase64(payloadB) }.getOrNull() + if (a != null && b != null && a.size >= 32 && b.size >= 32) + return ByteArray(32) { i -> (a[i].toInt() xor b[i].toInt()).toByte() } + } + return runCatching { context.decodeBase64(keyB64) }.getOrNull()?.takeIf { it.isNotEmpty() } + } + + private fun hmacSha256(key: ByteArray, message: String): ByteArray { + val mac = Mac.getInstance("HmacSHA256") + mac.init(SecretKeySpec(key, "HmacSHA256")) + return mac.doFinal(message.toByteArray(Charsets.UTF_8)) + } + + private fun aesCtrDecrypt(data: ByteArray, chapterKey: ByteArray, pageIndex: Int, prefix: String): ByteArray { + val derived = hmacSha256(chapterKey, "$prefix$pageIndex") + val cipher = Cipher.getInstance("AES/CTR/NoPadding") + cipher.init(Cipher.DECRYPT_MODE, SecretKeySpec(derived, "AES"), IvParameterSpec(ByteArray(16))) + return cipher.doFinal(data) + } + + private fun xorKeystream(data: ByteArray, chapterKey: ByteArray, pageIndex: Int): ByteArray { + val result = data.copyOf() + var blockIndex = 0 + var blockOffset = 0 + var block = hmacSha256(chapterKey, "page:$pageIndex:${0}") + for (i in result.indices) { + if (blockOffset == block.size) { + blockIndex++ + block = hmacSha256(chapterKey, "page:$pageIndex:$blockIndex") + blockOffset = 0 + } + result[i] = (result[i].toInt() xor block[blockOffset].toInt()).toByte() + blockOffset++ + } + return result + } + + private fun chaCha20(data: ByteArray, key: ByteArray): ByteArray { + val result = data.copyOf() + val nonce = ByteArray(12) + var counter = 0 + var block = chaCha20Block(key, nonce, counter) + var offset = 0 + for (i in result.indices) { + if (offset == block.size) { + counter++ + block = chaCha20Block(key, nonce, counter) + offset = 0 + } + result[i] = (result[i].toInt() xor block[offset].toInt()).toByte() + offset++ + } + return result + } + + private fun chaCha20Block(key: ByteArray, nonce: ByteArray, counter: Int): ByteArray { + val state = IntArray(16) + state[0] = 0x61707865; state[1] = 0x3320646e; state[2] = 0x79622d32; state[3] = 0x6b206574 + for (i in 0 until 8) state[4 + i] = key.readIntLe(i * 4) + state[12] = counter; state[13] = nonce.readIntLe(0); state[14] = nonce.readIntLe(4); state[15] = nonce.readIntLe(8) + val working = state.copyOf() + repeat(10) { + quarterRound(working, 0, 4, 8, 12); quarterRound(working, 1, 5, 9, 13) + quarterRound(working, 2, 6, 10, 14); quarterRound(working, 3, 7, 11, 15) + quarterRound(working, 0, 5, 10, 15); quarterRound(working, 1, 6, 11, 12) + quarterRound(working, 2, 7, 8, 13); quarterRound(working, 3, 4, 9, 14) + } + val block = ByteArray(64) + for (i in 0 until 16) block.writeIntLe(i * 4, working[i] + state[i]) + return block + } + + private fun quarterRound(state: IntArray, a: Int, b: Int, c: Int, d: Int) { + state[a] += state[b]; state[d] = Integer.rotateLeft(state[d] xor state[a], 16) + state[c] += state[d]; state[b] = Integer.rotateLeft(state[b] xor state[c], 12) + state[a] += state[b]; state[d] = Integer.rotateLeft(state[d] xor state[a], 8) + state[c] += state[d]; state[b] = Integer.rotateLeft(state[b] xor state[c], 7) + } + + private fun ByteArray.readIntLe(offset: Int) = (this[offset].toInt() and 0xFF) or + ((this[offset + 1].toInt() and 0xFF) shl 8) or + ((this[offset + 2].toInt() and 0xFF) shl 16) or + ((this[offset + 3].toInt() and 0xFF) shl 24) + + private fun ByteArray.writeIntLe(offset: Int, value: Int) { + this[offset] = value.toByte(); this[offset + 1] = (value ushr 8).toByte() + this[offset + 2] = (value ushr 16).toByte(); this[offset + 3] = (value ushr 24).toByte() + } + + private fun readerHeaders() = getRequestHeaders().newBuilder() + .set("Accept", "application/json") + .set("Referer", "https://$domain/") + .set("X-Requested-With", "XMLHttpRequest") + .build() + + private val Manga.slug get() = url.trim('/').substringAfterLast('/') + + private fun JSONArray.toTags(): Set<MangaTag> = mapJSONNotNull { item -> + val key = item.getStringOrNull("slug") ?: return@mapJSONNotNull null + val title = item.getStringOrNull("name") ?: return@mapJSONNotNull null + MangaTag(key = key, title = title, source = source) + }.toSet() + + private fun JSONArray?.toNames(): Set<String> { + if (this == null) return emptySet() + return mapJSONNotNull { it.getStringOrNull("name") }.toSet() + } + + private fun parseState(value: String?): MangaState? = when (value?.uppercase(Locale.ROOT)) { + "ON_GOING", "ONGOING", "RELEASING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "ON_HOLD", "HIATUS" -> MangaState.PAUSED + "CANCELED", "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + private fun stateToApi(state: MangaState): String? = when (state) { + MangaState.ONGOING -> "on_going" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on_hold" + MangaState.ABANDONED -> "canceled" + else -> null + } + + private fun typeToApi(type: ContentType): String? = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comic" + else -> null + } + + private fun parseContentRating(value: String?): ContentRating? = when (value?.lowercase(Locale.ROOT)) { + "safe" -> ContentRating.SAFE + "suggestive" -> ContentRating.SUGGESTIVE + "adult", "erotica", "pornographic" -> ContentRating.ADULT + else -> null + } + + private companion object { + private const val HEADER_READER_TOKEN = "X-Reader-Access-Token" + private val PROTECTED_IMAGE_REGEX = Regex(""".*_s\.[^.]+$""") + private const val MAGIC_HIGH = 0xFF.toByte() + private const val MAGIC_AES = 0x02.toByte() + private const val MAGIC_CHACHA = 0x03.toByte() + private const val MAGIC_AES4 = 0x04.toByte() + } +} diff --git a/src/main/kotlin/tsuki/site/en/QiScans.kt b/src/main/kotlin/tsuki/site/en/QiScans.kt new file mode 100644 index 00000000..e8d27571 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/QiScans.kt @@ -0,0 +1,165 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.EZManhwaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.parseJson +import tsuki.util.json.mapJSON + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import java.util.EnumSet + +@MangaSourceParser("QISCANS", "QiScans", "en") +internal class QiScans(context: MangaLoaderContext) : + EZManhwaParser(context, MangaParserSource.QISCANS, "qimanga.com", "https://api.qimanga.com/api/v1") { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override fun getRequestHeaders(): Headers { + return super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Sec-Fetch-Dest", "empty") + .set("Sec-Fetch-Mode", "cors") + .set("Sec-Fetch-Site", "same-site") + .build() + } + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val base = super.getFilterOptions() + return base.copy( + availableTags = GENRES + .filter { it.first.isNotEmpty() } + .map { (key, name) -> MangaTag(key = key, title = name, source = source) } + .toSet() + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val endpoint = if (filter.query.isNullOrBlank()) "$apiUrl/series" else "$apiUrl/series/search" + val url = endpoint.toHttpUrl().newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + + if (filter.query.isNullOrBlank()) { + val sort = when (order) { + SortOrder.UPDATED, SortOrder.UPDATED_ASC -> "latest" + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popular" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "newest" + SortOrder.ALPHABETICAL, SortOrder.ALPHABETICAL_DESC -> "alphabetical" + else -> "latest" + } + addQueryParameter("sort", sort) + + filter.states.firstOrNull()?.let { + addQueryParameter("status", when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + MangaState.ABANDONED -> "DROPPED" + else -> "" + }) + } + filter.types.firstOrNull()?.let { + addQueryParameter("type", when (it) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + }) + } + filter.tags.firstOrNull()?.key?.let { genreSlug -> + if (genreSlug.isNotBlank()) { + addQueryParameter("genre", genreSlug) + } + } + } else { + addQueryParameter("q", filter.query) + } + }.build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: org.json.JSONObject): List<Manga> { + val data = json.getJSONArray("data") + return data.mapJSON { obj -> + obj.toManga(domain, source, isNsfwSource) + } + } + + companion object { + private val GENRES = listOf( + "" to "All", + "acting" to "Acting", + "action" to "Action", + "action-582" to "Action (Alt)", + "adventure" to "Adventure", + "adventure-589" to "Adventure (Alt)", + "apocalypce" to "Apocalypse", + "comedy" to "Comedy", + "cooking" to "Cooking", + "crazy-mc" to "Crazy MC", + "cultivation" to "Cultivation", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "fantasy-747" to "Fantasy (Alt)", + "fight" to "Fight", + "gender-bender" to "Gender Bender", + "harem" to "Harem", + "hidden" to "Hidden", + "historical" to "Historical", + "horror" to "Horror", + "josei" to "Josei", + "live" to "Live", + "magic" to "Magic", + "manhua" to "Manhua", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medieval-area" to "Medieval Area", + "munchkin" to "Munchkin", + "murim" to "Murim", + "mystery" to "Mystery", + "myth" to "Myth", + "politics" to "Politics", + "psychological" to "Psychological", + "reincarnation" to "Reincarnation", + "revenge" to "Revenge", + "romance" to "Romance", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "shounen" to "Shounen", + "slice-of-life" to "Slice of Life", + "sports" to "Sports", + "supernatural" to "Supernatural", + "superpower" to "Superpower", + "system" to "System", + "taming" to "Taming", + "tower" to "Tower", + "tragedy" to "Tragedy", + "urban" to "Urban", + "vampiers" to "Vampires", + "virtual-reality" to "Virtual Reality", + "wuxia" to "Wuxia", + ) + } +} diff --git a/src/main/kotlin/tsuki/site/en/RavenScans.kt b/src/main/kotlin/tsuki/site/en/RavenScans.kt new file mode 100644 index 00000000..0f07115f --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/RavenScans.kt @@ -0,0 +1,13 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("RAVENSCANS", "RavenScans", "en") +internal class RavensScan(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.RAVENSCANS, "ravenscans.org") { + override val mangaDirectory = "manga" +} diff --git a/src/main/kotlin/tsuki/site/en/RoliaScan.kt b/src/main/kotlin/tsuki/site/en/RoliaScan.kt new file mode 100644 index 00000000..f4c399cf --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/RoliaScan.kt @@ -0,0 +1,562 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons + +import tsuki.util.generateUid +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseRaw +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import org.jsoup.parser.Parser +import java.security.MessageDigest +import java.time.ZoneOffset +import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter +import java.util.EnumSet +import java.util.Locale +import java.util.Calendar +import kotlin.collections.forEach +import kotlin.collections.mapNotNull + +@MangaSourceParser("ROLIASCAN", "Rolia Scan", "en") +internal class RoliaScan(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ROLIASCAN, API_PAGES_PER_PAGE * API_PAGE_SIZE) { + + override val configKeyDomain = ConfigKey.Domain("roliascan.com") + + private val numberRegex = Regex("""(\d+(?:\.\d+)?)""") + private val relativeDateRegex = Regex("""(\d+)\s+(second|minute|hour|day|week|month|year)s?\s+ago""") + private val tokenFormatter = DateTimeFormatter.ofPattern("yyyyMMddHH", Locale.US) + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/content/themes/mangapeak/assets/img/favicon-32x32.png", 32, null), + ), + domain, + ) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .build() + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isYearSupported = true, + ) + + private val genreTags by lazy { + ROLIA_TAGS.mapTo(LinkedHashSet(ROLIA_TAGS.size)) { (title, id) -> + MangaTag(title = title, key = id.toString(), source = source) + } + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = genreTags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.NOVEL + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val start = (page - 1) * API_PAGES_PER_PAGE + 1 + val end = start + API_PAGES_PER_PAGE - 1 + val result = ArrayList<Manga>(pageSize) + val seen = HashSet<String>(pageSize * 2) + + for (apiPage in start..end) { + val yearsParam = if (filter.year > 0) "[${filter.year}]" else "[]" + val payload = JSONObject().apply { + put("page", apiPage) + put("search", filter.query.orEmpty()) + put("years", yearsParam) + put("genres", filter.tags.toGenrePayload()) + put("types", filter.types.toTypePayload()) + put("statuses", filter.states.toStatusPayload()) + put("sort", order.toApiSort()) + put("genreMatchMode", "any") + } + val json = postJsonArray("https://$domain/wp-json/manga/v1/load", payload) + if (json.length() == 0) break + repeat(json.length()) { i -> + val item = json.optJSONObject(i) ?: return@repeat + if (item.optString("type").equals("Novel", true)) return@repeat + val rawUrl = item.optString("url").trim() + if (rawUrl.isEmpty()) return@repeat + val uid = item.optString("id").nullIfEmpty() ?: rawUrl + if (seen.add(uid)) result.add(item.toManga()) + } + if (json.length() < API_PAGE_SIZE) break + } + return result + } + + private fun JSONObject.toManga(): Manga { + val url = optString("url") + val id = optString("id") + val fullUrl = withMangaId(url, id) + return Manga( + id = generateUid(fullUrl), + title = optString("title").unescapeHtml(), + altTitles = emptySet(), + url = fullUrl, + publicUrl = url.nullIfEmpty()?.toAbsoluteUrl(domain) + ?: url.toRelativeUrl(domain).toAbsoluteUrl(domain), + rating = optString("score").toFloatOrNull() ?: RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover").nullIfEmpty(), + tags = emptySet(), + state = optString("status").toMangaState(), + authors = emptySet(), + description = optString("description").nullIfEmpty()?.unescapeHtml(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = fetchDocument(manga.url.urlWithoutFragment().toAbsoluteUrl(domain)) + val ld = doc.findSeriesJsonLd() + val mangaId = extractMangaId(manga.url) + ?: doc.selectFirst("[data-manga-id]")?.attr("data-manga-id") + ?: ld?.optString("identifier")?.nullIfEmpty() + ?: throw ParseException("Cannot find manga id", manga.url) + + val cleanTitle = doc.selectFirst("h1")?.ownText()?.trim()?.nullIfEmpty() + ?: ld?.optString("name")?.nullIfEmpty()?.unescapeHtml() + ?: manga.title + + return manga.copy( + title = cleanTitle, + description = doc.extractDescription(ld)?.unescapeHtml(), + altTitles = doc.extractAltTitles(ld).mapTo(mutableSetOf()) { it.unescapeHtml() }, + tags = doc.extractTags(ld).ifEmpty { manga.tags }, + authors = doc.extractAuthors(ld).ifEmpty { manga.authors }.mapTo(mutableSetOf()) { it.unescapeHtml() }, + state = doc.extractStatus(), + rating = doc.extractRating(ld), + chapters = loadChapters(mangaId, manga.url.toAbsoluteUrl(domain)), + ) + } + + private suspend fun loadChapters(mangaId: String, referer: String): List<MangaChapter> { + val headers = Headers.headersOf("Referer", referer) + val result = ArrayList<MangaChapter>() + val seen = HashSet<String>() + var offset = 0 + val limit = CHAPTER_LIMIT + var total = Int.MAX_VALUE + while (offset < total) { + val (token, ts) = chapterToken() + val raw = webClient.httpGet( + buildChapterUrl(mangaId, offset, limit, token, ts), headers + ).parseRaw().trim() + if (raw.isEmpty()) break + val json = runCatching { JSONObject(raw) }.getOrNull() ?: break + if (!json.optBoolean("success", true)) break + total = json.optInt("total", total) + val arr = json.optJSONArray("chapters") ?: break + if (arr.length() == 0) break + for (i in 0 until arr.length()) { + val ch = arr.getJSONObject(i) + val url = ch.optString("url").nullIfEmpty() ?: continue + if (ch.optString("language").nullIfEmpty()?.equals("en", true) == false) continue + val key = ch.optString("id").ifEmpty { url } + if (!seen.add(key)) continue + result.add(ch.toChapter(url, offset + i + 1f)) + } + offset += arr.length() + if (!json.optBoolean("has_more", true)) break + } + return result.reversed() + } + + private fun JSONObject.toChapter(url: String, fallback: Float): MangaChapter { + val chapterNum = optString("chapter").nullIfEmpty() ?: fallback.toLabel() + val rawTitle = optString("title").trim() + val number = numberRegex.find(chapterNum)?.value?.toFloatOrNull() ?: fallback + + val title = buildString { + append("Ch. ") + append(chapterNum) + val cleanTitle = rawTitle.takeUnless { + it.equals("N/A", true) || it == "—" || it.isEmpty() + }?.unescapeHtml() + if (!cleanTitle.isNullOrBlank()) { + append(" - ") + append(cleanTitle) + } + } + return MangaChapter( + id = generateUid(url), + title = title, + number = number, + volume = 0, + url = url.toRelativeUrl(domain), + scanlator = null, + uploadDate = parseRelativeDate(optString("date")), + branch = null, + source = source, + ) + } + + private fun chapterToken() = Pair( + MessageDigest.getInstance("MD5") + .digest("${System.currentTimeMillis() / 1000L}mng_ch_${ZonedDateTime.now(ZoneOffset.UTC).format(tokenFormatter)}".toByteArray()) + .toHex().substring(0, 16), + (System.currentTimeMillis() / 1000L).toString() + ) + + @Suppress("SameParameterValue") + private fun buildChapterUrl(mangaId: String, offset: Int, limit: Int, token: String, ts: String) = + HttpUrl.Builder().apply { + scheme("https").host(domain) + addPathSegment("auth").addPathSegment("manga-chapters") + addQueryParameter("manga_id", mangaId) + addQueryParameter("offset", offset.toString()) + addQueryParameter("limit", limit.toString()) + addQueryParameter("order", "DESC") + addQueryParameter("_t", token) + addQueryParameter("_ts", ts) + }.build().toString() + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val id = chapter.url.toAbsoluteUrl(domain) + .removeSuffix("/").substringBefore('?').substringAfterLast('/').substringAfterLast('-') + .nullIfEmpty() ?: return emptyList() + val raw = webClient.httpGet( + "https://$domain/auth/chapter-content?chapter_id=$id", getRequestHeaders() + ).parseRaw().trim() + if (raw.isEmpty()) return emptyList() + val images = runCatching { JSONObject(raw).optJSONArray("images") ?: JSONArray(raw) }.getOrNull() ?: return emptyList() + val seen = HashSet<String>(images.length()) + return (0 until images.length()).mapNotNull { i -> + val img = images.optString(i).nullIfEmpty() ?: return@mapNotNull null + val abs = if (img.startsWith("http", true)) img else img.toAbsoluteUrl(domain) + if (seen.add(abs)) MangaPage(id = generateUid(abs), url = abs, preview = null, source = source) else null + } + } + + private fun String?.toMangaState() = when { + this == null -> null + contains("ongoing", true) -> MangaState.ONGOING + contains("complete", true) -> MangaState.FINISHED + contains("hiatus", true) -> MangaState.PAUSED + contains("cancel", true) -> MangaState.ABANDONED + contains("upcoming", true) || contains("tba", true) -> MangaState.UPCOMING + else -> null + } + + private fun parseRelativeDate(raw: String?): Long { + val v = raw?.trim()?.lowercase(Locale.US)?.nullIfEmpty() ?: return 0L + val m = relativeDateRegex.matchEntire(v) ?: return 0L + val (num, unit) = m.destructured + val cal = Calendar.getInstance() + val n = num.toIntOrNull() ?: return 0L + when (unit) { + "second" -> cal.add(Calendar.SECOND, -n) + "minute" -> cal.add(Calendar.MINUTE, -n) + "hour" -> cal.add(Calendar.HOUR_OF_DAY, -n) + "day" -> cal.add(Calendar.DAY_OF_YEAR, -n) + "week" -> cal.add(Calendar.WEEK_OF_YEAR, -n) + "month" -> cal.add(Calendar.MONTH, -n) + "year" -> cal.add(Calendar.YEAR, -n) + } + return cal.timeInMillis + } + + private fun String.unescapeHtml(): String { + var d = this + var p: String + do { + p = d + d = Parser.unescapeEntities(d, false) + } while (d != p) + return d + } + + private fun Float.toLabel() = if (this % 1f == 0f) toInt().toString() else toString().trimEnd('0').removeSuffix(".") + + private fun Collection<MangaState>.toStatusPayload() = if (isEmpty()) "[]" else + mapNotNull { s -> when(s) { MangaState.FINISHED -> "Completed"; MangaState.ONGOING -> "Ongoing"; else -> null } }.toJsonArray() + + private fun Collection<ContentType>.toTypePayload() = if (isEmpty()) "[]" else + mapNotNull { t -> when(t) { ContentType.MANGA -> "Manga"; ContentType.MANHWA -> "Manhwa"; ContentType.MANHUA -> "Manhua"; ContentType.COMICS -> "Comics"; else -> null } }.toJsonArray() + + private fun Collection<String>.toJsonArray() = if (isEmpty()) "[]" else + joinToString(prefix = "[", postfix = "]", separator = ",") { "\"${it.escapeJson()}\"" } + + private fun SortOrder.toApiSort() = when(this) { + SortOrder.POPULARITY -> "popular_desc" + SortOrder.UPDATED -> "post_desc" + SortOrder.NEWEST -> "release_desc" + SortOrder.ALPHABETICAL -> "title_asc" + SortOrder.ALPHABETICAL_DESC -> "title_desc" + else -> "post_desc" + } + + private fun String.escapeJson() = replace("\\", "\\\\").replace("\"", "\\\"") + + private fun Collection<MangaTag>.toGenrePayload(): String { + if (isEmpty()) return "[]" + val ids = mapNotNull { tag -> + tag.key.toIntOrNull()?.toString() + ?: genreTags.find { it.title.equals(tag.title, true) }?.key?.toIntOrNull()?.toString() + } + return ids.toJsonArray() + } + + private fun Document.extractDescription(ld: JSONObject?) = + ld?.optString("description")?.nullIfEmpty() + ?: selectFirst("div[data-description], div#description, div#synopsis")?.nextElementSibling()?.text()?.nullIfEmpty() + ?: selectFirst("meta[name='description']")?.attr("content")?.nullIfEmpty() + ?: selectFirst("meta[property='og:description']")?.attr("content")?.nullIfEmpty() + + private fun Document.extractAltTitles(ld: JSONObject?) = buildSet { + ld?.opt("alternateName")?.let { v -> + when(v) { is JSONArray -> repeat(v.length()) { it -> v.optString(it).nullIfEmpty()?.let { add(it) } }; is String -> v.nullIfEmpty()?.let { add(it) } } + } + if (isEmpty()) selectFirst("h1 + p")?.text()?.split(" / ")?.mapNotNull { it.nullIfEmpty() }?.let { addAll(it) } + } + + private fun Document.extractTags(ld: JSONObject?) = buildSet { + ld?.opt("genre")?.let { v -> + when(v) { is JSONArray -> repeat(v.length()) { it -> v.optString(it).nullIfEmpty()?.let { add(MangaTag(key = it.lowercase().replace(' ', '-'), title = it, source = source)) } }; is String -> v.nullIfEmpty()?.let { add(MangaTag(key = it.lowercase().replace(' ', '-'), title = it, source = source)) } } + } + if (isEmpty()) { + select("a[href*='/genre/']").mapNotNull { a -> + a.text().nullIfEmpty()?.let { title -> + val slug = a.attr("href").trimEnd('/').substringAfterLast('/').substringBefore('?').nullIfEmpty() ?: return@mapNotNull null + add(MangaTag(key = slug, title = title, source = source)) + } + } + } + } + + private fun Document.extractAuthors(ld: JSONObject?) = buildSet { + fun addName(v: Any?) { + when(v) { is JSONArray -> repeat(v.length()) { addName(v.opt(it)) }; is JSONObject -> addName(v.opt("name")); is String -> v.nullIfEmpty()?.let { add(it) } } + } + addName(ld?.opt("author")); addName(ld?.opt("creator")) + if (isEmpty()) select("a[href*='/author/'], a[href*='/artist/']").forEach { it -> it.text().nullIfEmpty()?.let { add(it) } } + } + + private fun Document.extractStatus(): MangaState? { + val label = select("*").firstOrNull { it.ownText().trim().equals("Status", true) } + val text = label?.let { it.nextElementSibling()?.text()?.nullIfEmpty() } + ?: selectFirst("[data-status]")?.attr("data-status")?.nullIfEmpty() + ?: selectFirst("span.status, div.status")?.text()?.nullIfEmpty() + return text?.toMangaState() + } + + private fun Document.extractRating(ld: JSONObject?): Float { + val rawRating = ld?.optJSONObject("aggregateRating")?.optDouble("ratingValue") + ?.takeIf { !it.isNaN() } + ?: selectFirst("svg ~ span.font-semibold")?.text()?.toDoubleOrNull() + return if (rawRating != null && rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + } + + private fun Document.findSeriesJsonLd(): JSONObject? { + for (s in select("script[type=application/ld+json]")) { + val raw = s.data().trim() + val obj = runCatching { JSONObject(raw) }.getOrNull() + if (obj != null && obj.matchesSeries()) return obj + runCatching { JSONArray(raw) }.getOrNull()?.let { arr -> + repeat(arr.length()) { it -> arr.optJSONObject(it)?.takeIf { it.matchesSeries() }?.let { return it } } + } + } + return null + } + + private fun JSONObject.matchesSeries() = optString("@type") in setOf("ComicSeries", "ComicStory", "Book", "CreativeWorkSeries") + + private fun extractMangaId(url: String): String? { + url.substringAfter('#', "").split('&').firstOrNull { it.startsWith("mid=") }?.substringAfter("mid=")?.nullIfEmpty()?.let { return it } + url.substringAfter('?', "").split('&').firstOrNull { it.startsWith("mid=") }?.substringAfter("mid=")?.nullIfEmpty()?.let { return it } + return null + } + + private fun String.urlWithoutFragment() = substringBefore('#') + + private fun withMangaId(rawUrl: String?, id: String?): String { + val url = rawUrl.nullIfEmpty() ?: return "/" + val cleanId = id?.nullIfEmpty() ?: return url + return if (url.contains('#')) { + if (url.contains("mid=")) url else "$url&mid=$cleanId" + } else { + "$url#mid=$cleanId" + } + } + + private suspend fun postJsonArray(url: String, payload: JSONObject): JSONArray { + val raw = webClient.httpPost(url, payload).parseRaw().trim() + return runCatching { JSONArray(raw) }.getOrDefault(JSONArray()) + } + + private suspend fun fetchDocument(url: String) = webClient.httpGet(url).parseHtml() + + private fun ByteArray.toHex(): String { + if (isEmpty()) return "" + val chars = CharArray(size * 2) + for ((i, b) in withIndex()) { + val v = b.toInt() and 0xFF + chars[i * 2] = Character.forDigit(v ushr 4, 16) + chars[i * 2 + 1] = Character.forDigit(v and 0xF, 16) + } + return String(chars) + } + + private companion object { + const val API_PAGES_PER_PAGE = 5 + const val API_PAGE_SIZE = 24 + const val CHAPTER_LIMIT = 9999 + + val ROLIA_TAGS = listOf( + "Action" to 5, + "Adaptation" to 49, + "Adapted to Manhua" to 717, + "Adult Cast" to 119, + "Adventure" to 19, + "Aliens" to 803, + "Animals" to 240, + "Award Winning" to 8, + "Childcare" to 1146, + "Combat Sports" to 358, + "Comedy" to 61, + "Cooking" to 266, + "Crime" to 248, + "Crossdressing" to 724, + "Delinquents" to 228, + "Demons" to 162, + "Detective" to 150, + "Drama" to 26, + "Ecchi" to 117, + "Erotica" to 202, + "Fantasy" to 17, + "Full Color" to 40, + "Gag Humor" to 1068, + "Game" to 1130, + "Gender Bender" to 1190, + "Ghosts" to 215, + "Gore" to 187, + "Gourmet" to 89, + "Harem" to 47, + "Historical" to 66, + "Horror" to 67, + "Isekai" to 55, + "Josei" to 1062, + "Light Novel" to 98, + "Long Strip" to 41, + "Love Status Quo" to 541, + "Mafia" to 356, + "Magic" to 45, + "Magical Sex Shift" to 551, + "Manga" to 97, + "Manhua" to 35, + "Manhwa" to 18, + "Martial Arts" to 56, + "Mature" to 404, + "Mecha" to 396, + "Medical" to 244, + "Military" to 131, + "Monster Girls" to 231, + "Monsters" to 46, + "Music" to 694, + "Mystery" to 34, + "Mythology" to 110, + "Ninja" to 163, + "Office Workers" to 505, + "Official Colored" to 866, + "Organized Crime" to 134, + "Otaku Culture" to 570, + "Parody" to 605, + "Philosophical" to 912, + "Post-Apocalyptic" to 241, + "Psychological" to 149, + "Regression" to 1131, + "Reincarnation" to 29, + "Revenge" to 964, + "Reverse Harem" to 1085, + "Romance" to 2, + "Romantic Subtext" to 486, + "School" to 14, + "School Life" to 27, + "Sci-Fi" to 33, + "Seinen" to 105, + "Self-Published" to 577, + "Sexual Violence" to 536, + "Shoujo" to 1071, + "Shounen" to 11, + "Showbiz" to 429, + "Slice of Life" to 93, + "Smut" to 742, + "Space" to 206, + "Sports" to 9, + "Streaming" to 1132, + "Suggestive" to 1116, + "Super Power" to 6, + "Superhero" to 865, + "Supernatural" to 65, + "Survival" to 236, + "Suspense" to 287, + "Team Sports" to 10, + "Thriller" to 184, + "Time Travel" to 37, + "Tragedy" to 316, + "Transmigiration" to 1133, + "Urban Fantasy" to 120, + "Vampire" to 209, + "Video Game" to 277, + "Video Games" to 616, + "Villainess" to 355, + "Virtual Reality" to 617, + "Web Comic" to 48, + "Webtoon" to 350, + "Workplace" to 138, + "Wuxia" to 68, + "Xianxia" to 718, + "Zombies" to 1115, + ) + } +} diff --git a/src/main/kotlin/tsuki/site/en/ScansGG.kt b/src/main/kotlin/tsuki/site/en/ScansGG.kt new file mode 100644 index 00000000..fa871905 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ScansGG.kt @@ -0,0 +1,297 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("SCANSGG", "ScansGG", "en") +internal class ScansGG(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SCANSGG, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("scans.gg") + + private val apiUrl = "https://api.scans.gg" + private val cdnUrl = "https://cdn.scans.gg/uploads" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + return super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Referer", "https://$domain/") + .build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, SortOrder.POPULARITY + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val tagsMap = mapOf( + 49 to "Regression", 48 to "Male Protagonist", 47 to "Survival", + 46 to "Avant Garde", 45 to "Award Winning", 44 to "Lolicon", + 43 to "Mahou Shoujo", 42 to "Doujinshi", 41 to "Girls Love", + 40 to "Hentai", 39 to "Mecha", 38 to "Shotacon", 37 to "Ecchi", + 36 to "Music", 35 to "Smut", 34 to "Erotica", 33 to "Adult", + 32 to "Gourmet", 31 to "Yuri", 30 to "Shoujo Ai", 29 to "Yaoi", + 28 to "Shounen Ai", 27 to "Boys Love", 26 to "Harem", 25 to "Tragedy", + 24 to "Gender Bender", 23 to "Suspense", 22 to "Psychological", + 21 to "Mature", 20 to "Horror", 19 to "Mystery", 18 to "Martial Arts", + 17 to "Sci-fi", 16 to "Adventure", 15 to "Supernatural", 14 to "Sports", + 13 to "Shounen", 12 to "Historical", 11 to "Seinen", 10 to "Action", + 9 to "Josei", 8 to "Thriller", 7 to "School Life", 6 to "Slice Of Life", + 5 to "Drama", 4 to "Comedy", 3 to "Shoujo", 2 to "Romance", 1 to "Fantasy" + ) + + private val typeToApiId = mapOf( + ContentType.MANGA to 2, ContentType.MANHWA to 3, ContentType.MANHUA to 4, + ContentType.COMICS to 1, ContentType.OTHER to 5 + ) + + private val stateToApiId = mapOf( + MangaState.ONGOING to 1, MangaState.FINISHED to 2, + MangaState.PAUSED to 3, MangaState.ABANDONED to 4 + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tagsMap.map { (id, name) -> MangaTag(key = id.toString(), title = name, source = source) } + .toSet(), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, + ContentType.COMICS, ContentType.OTHER + ), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (order == SortOrder.UPDATED && query.isEmpty()) return fetchLatest(page) + + val limit = if (order == SortOrder.UPDATED) 14 else 21 + val offset = (page - 1) * limit + val url = "$apiUrl/series".toHttpUrl().newBuilder() + .addQueryParameter("limit", limit.toString()) + .addQueryParameter("offset", offset.toString()) + .apply { + if (query.isNotEmpty()) addQueryParameter("q", query) + val typeIds = filter.types.mapNotNull { typeToApiId[it] } + if (typeIds.isNotEmpty()) addQueryParameter("q_type", typeIds.joinToString(",", "[", "]")) + val statusIds = filter.states.mapNotNull { stateToApiId[it] } + if (statusIds.isNotEmpty()) addQueryParameter("q_status", statusIds.joinToString(",", "[", "]")) + val tagIds = filter.tags.mapNotNull { it.key.toIntOrNull() } + if (tagIds.isNotEmpty()) addQueryParameter("q_tags", tagIds.joinToString(",", "[", "]")) + } + .build() + + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private suspend fun fetchLatest(page: Int): List<Manga> { + val url = "$apiUrl/chapters".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("limit", "14") + .addQueryParameter("chapters", "true") + .addQueryParameter("series_details", "true") + .addQueryParameter("group_details", "true") + .addQueryParameter("sort", "date") + .build() + + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + val meta = json.optJSONObject("meta") + val hasMore = meta?.optBoolean("has_more", false) ?: false + return (0 until data.length()).map { i -> data.getJSONObject(i).toManga(cdnUrl) } + .take(if (hasMore) 14 else data.length()) + } + + private fun parseSeriesList(json: JSONObject): List<Manga> { + val data = json.getJSONArray("data") + return (0 until data.length()).map { i -> data.getJSONObject(i).toManga(cdnUrl) } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val seriesUrl = "$apiUrl/series".toHttpUrl().newBuilder() + .addQueryParameter("id", manga.url) + .addQueryParameter("trackers", "true") + .addQueryParameter("sources", "true") + .build() + val seriesJson = webClient.httpGet(seriesUrl).parseJson() + val seriesObj = seriesJson.getJSONObject("data") + val updatedManga = seriesObj.toMangaDetail(cdnUrl, tagsMap) + .copy(id = manga.id, source = source) + + val chapters = fetchChaptersPaginated(manga.url) + updatedManga.copy(chapters = chapters) + } + + private suspend fun fetchChaptersPaginated(seriesId: String): List<MangaChapter> { + val allChapters = mutableListOf<MangaChapter>() + val groupTitles = mutableMapOf<Int, String>() + var page = 1 + var hasMore = true + + while (hasMore) { + val url = "$apiUrl/chapters".toHttpUrl().newBuilder() + .addQueryParameter("series_id", seriesId) + .addQueryParameter("limit", "100") + .addQueryParameter("page", page.toString()) + .addQueryParameter("group_details", "true") + .addQueryParameter("sort_order", "desc") + .build() + val json = webClient.httpGet(url).parseJson() + val data = json.getJSONArray("data") + for (i in 0 until data.length()) { + val ch = data.getJSONObject(i) + val chId = ch.getInt("id") + val number = ch.getDouble("number").toFloat() + val title = ch.optString("title", null) + val createdAt = ch.optString("created_at", null) + val groupId = ch.optInt("group_id", 0) + val groupObj = ch.optJSONObject("group") + val groupTitle = groupObj?.optString("title", null) + if (groupTitle != null) { + groupTitles[groupId] = groupTitle + } + + val name = buildString { + append("Chapter ${number.toString().removeSuffix(".0")}") + if (!title.isNullOrEmpty()) append(" - $title") + } + + allChapters.add( + MangaChapter( + id = generateUid(chId.toString()), + title = name, + number = number, + volume = 0, + url = "$apiUrl/chapter-navigation?series_id=$seriesId&chapter_id=$chId&group_id=$groupId", + uploadDate = dateFormat.parseSafe(createdAt), + scanlator = groupTitle, + branch = null, + source = source, + ) + ) + } + val meta = json.optJSONObject("meta") + hasMore = meta?.optBoolean("has_more", false) ?: false + page++ + } + + return if (groupTitles.size > 1) { + allChapters.map { ch -> + val branch = ch.scanlator ?: "Unknown" + ch.copy(branch = branch) + }.sortedBy { it.number } + } else { + allChapters.sortedBy { it.number } + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val json = webClient.httpGet(chapter.url).parseJson() + val data = json.optJSONObject("data") ?: return emptyList() + val chapterObj = data.optJSONObject("chapter") ?: data + val chapterId = chapterObj.optInt("id", 0) + val pages = chapterObj.optJSONArray("pages") ?: return emptyList() + return (0 until pages.length()).map { i -> + val page = pages.getJSONObject(i) + val path = page.getString("path") + val imageUrl = "$cdnUrl/pages/$chapterId/$path" + MangaPage(id = generateUid(imageUrl), url = imageUrl, preview = null, source = source) + } + } + + private fun JSONObject.toManga(cdnUrl: String): Manga { + val id = getInt("id").toString() + val title = getString("title") + val cover = optString("cover", null) + return Manga( + id = generateUid(id), url = id, + publicUrl = "https://$domain/series/$id", title = title, + altTitles = emptySet(), coverUrl = cover?.let { "$cdnUrl/covers/$it" }, + rating = RATING_UNKNOWN, contentRating = null, + tags = emptySet(), state = null, authors = emptySet(), source = source, + ) + } + + private fun JSONObject.toMangaDetail(cdnUrl: String, tagsMap: Map<Int, String>): Manga { + val id = getInt("id").toString() + val title = getString("title") + val summary = optString("summary", null) + val cover = optString("cover", null) + val authorList = optJSONArray("author")?.let { arr -> (0 until arr.length()).map { arr.getString(it) } } ?: emptyList() + val artistList = optJSONArray("artist")?.let { arr -> (0 until arr.length()).map { arr.getString(it) } } ?: emptyList() + val altTitlesArray = optJSONArray("alternative_titles") + val altTitles = if (altTitlesArray != null) { + (0 until altTitlesArray.length()).map { altTitlesArray.getJSONObject(it).getString("title") }.toSet() + } else emptySet() + val tagsIds = optJSONArray("tags")?.let { arr -> (0 until arr.length()).map { arr.getInt(it) } } ?: emptyList() + val genres = tagsIds.mapNotNull { tagsMap[it] } + val typeId = optInt("type", -1) + val typeName = when (typeId) { + 1 -> "Comic"; 2 -> "Manga"; 3 -> "Manhwa"; 4 -> "Manhua"; 5 -> "Webtoon" + else -> null + } + val tags = mutableSetOf<MangaTag>().apply { + addAll(genres.map { MangaTag(key = it, title = it, source = source) }) + typeName?.let { add(MangaTag(key = "type:$typeId", title = it, source = source)) } + } + val statusId = optInt("status", -1) + val state = when (statusId) { + 1 -> MangaState.ONGOING; 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED; 4, 5 -> MangaState.ABANDONED + else -> null + } + return Manga( + id = generateUid(id), url = id, + publicUrl = "https://$domain/series/$id", title = title, + altTitles = altTitles, coverUrl = cover?.let { "$cdnUrl/covers/$it" }, + description = summary, authors = (authorList + artistList).toSet(), + tags = tags, state = state, rating = RATING_UNKNOWN, + source = source, contentRating = null, + ) + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } +} diff --git a/src/main/kotlin/tsuki/site/en/ScytheScans.kt b/src/main/kotlin/tsuki/site/en/ScytheScans.kt new file mode 100644 index 00000000..42d14f42 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/ScytheScans.kt @@ -0,0 +1,47 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl + +import java.util.Base64 + +@MangaSourceParser("SYCTHESCANS", "Scythe Scans", "en") +internal class ScytheScans(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SYCTHESCANS, "scythescans.com", pageSize = 20) { + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + val script = doc.selectFirst("script[src^=data:text/javascript;base64,dHNfcmVhZGVyLnJ1bih7]") + if (script != null) { + val base64 = script.attr("src").substringAfter("base64,") + val decoded = String(Base64.getDecoder().decode(base64), Charsets.UTF_8) + val imageListJson = Regex("""images"\s*:\s*(\[.*?])""", RegexOption.DOT_MATCHES_ALL) + .find(decoded) + ?.groupValues + ?.getOrNull(1) + .orEmpty() + + val jsonArray = org.json.JSONArray(imageListJson) + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return super.getPages(chapter) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt b/src/main/kotlin/tsuki/site/en/TopManhua.kt similarity index 51% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt rename to src/main/kotlin/tsuki/site/en/TopManhua.kt index ea35e04a..3ca840d1 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/TopManhua.kt +++ b/src/main/kotlin/tsuki/site/en/TopManhua.kt @@ -1,9 +1,10 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource @MangaSourceParser("TOPMANHUA", "ManhuaTop", "en") internal class TopManhua(context: MangaLoaderContext) : diff --git a/src/main/kotlin/tsuki/site/en/VortexScans.kt b/src/main/kotlin/tsuki/site/en/VortexScans.kt new file mode 100644 index 00000000..f8bef529 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/VortexScans.kt @@ -0,0 +1,341 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import org.json.JSONArray +import org.json.JSONObject +import okhttp3.HttpUrl +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +// Todo dedup + +@MangaSourceParser("VORTEXSCANS", "Vortex Scans", "en") +internal class VortexScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VORTEXSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("vortexscans.org") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://api.$domain" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .build() + + private var cachedGenres: List<Pair<String, String>>? = null + + private suspend fun fetchGenres(): List<Pair<String, String>> { + if (cachedGenres != null) return cachedGenres!! + val json = webClient.httpGet("$apiBaseUrl/api/genres", getRequestHeaders()).parseJson() + val arr = json.optJSONArray("data") ?: json.optJSONArray("genres") ?: return emptyList() + val list = mutableListOf<Pair<String, String>>() + for (i in 0 until arr.length()) { + val obj = arr.optJSONObject(i) ?: continue + list.add(obj.optString("name") to obj.optString("id")) + } + cachedGenres = list + return list + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + val tags = mutableSetOf<MangaTag>() + + tags.add(MangaTag("Ongoing", "status_ongoing", source)) + tags.add(MangaTag("Completed", "status_completed", source)) + tags.add(MangaTag("Canceled", "status_canceled", source)) + tags.add(MangaTag("Dropped", "status_dropped", source)) + tags.add(MangaTag("Coming Soon", "status_coming_soon", source)) + tags.add(MangaTag("Mass Released", "status_mass_released", source)) + + tags.add(MangaTag("Manga", "type_manga", source)) + tags.add(MangaTag("Manhua", "type_manhua", source)) + tags.add(MangaTag("Manhwa", "type_manhwa", source)) + tags.add(MangaTag("Russian", "type_russian", source)) + tags.add(MangaTag("Spanish", "type_spanish", source)) + + genres.forEach { (name, id) -> + tags.add(MangaTag(name, "genre_$id", source)) + } + + return MangaListFilterOptions( + availableTags = tags, + availableStates = emptySet(), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val status = filter.tags.firstOrNull { it.key.startsWith("status_") }?.key?.removePrefix("status_") + val type = filter.tags.firstOrNull { it.key.startsWith("type_") }?.key?.removePrefix("type_") + val genreIds = filter.tags.filter { it.key.startsWith("genre_") }.map { it.key.removePrefix("genre_") } + + val (orderBy, orderDir) = when (order) { + SortOrder.POPULARITY -> "totalViews" to "desc" + SortOrder.UPDATED -> "lastChapterAddedAt" to "desc" + SortOrder.ADDED -> "createdAt" to "desc" + SortOrder.ALPHABETICAL -> "postTitle" to "asc" + else -> "totalViews" to "desc" + } + + val baseHttpUrl = "$apiBaseUrl/api/query".toHttpUrlOrNull() ?: return emptyList() + val urlBuilder = baseHttpUrl.newBuilder().apply { + addQueryParameter("page", page.toString()) + addQueryParameter("perPage", pageSize.toString()) + if (query.isNotEmpty()) addQueryParameter("searchTerm", query) + if (!status.isNullOrEmpty()) addQueryParameter("seriesStatus", status.uppercase()) + if (!type.isNullOrEmpty()) addQueryParameter("seriesType", type.uppercase()) + if (genreIds.isNotEmpty()) addQueryParameter("genreIds", genreIds.joinToString(",")) + addQueryParameter("orderBy", orderBy) + addQueryParameter("orderDirection", orderDir) + } + + val (mangas, _) = fetchSearchPageWithFallback(urlBuilder, page) + return mangas + } + + private suspend fun fetchSearchPageWithFallback( + initialUrlBuilder: HttpUrl.Builder, + startPage: Int + ): Pair<List<Manga>, Boolean> { + var currentPage = startPage + while (true) { + val url = initialUrlBuilder + .setQueryParameter("page", currentPage.toString()) + .build() + .toString() + + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val posts = json.optJSONArray("posts") ?: JSONArray() + val totalCount = json.optInt("totalCount", 0) + + val mangas = mutableListOf<Manga>() + for (i in 0 until posts.length()) { + val obj = posts.optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + mangas.add(obj.toManga()) + } + + val hasNext = totalCount > (currentPage * pageSize) + + if (mangas.isNotEmpty() || !hasNext) { + return mangas to hasNext + } + currentPage++ + } + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + val slug = getString("slug") + val title = getString("postTitle") + val cover = optString("featuredImage", "") + return Manga( + id = generateUid("$slug#$id"), + url = "$slug#$id", + publicUrl = "$baseUrl/series/$slug", + title = title, + coverUrl = cover.ifEmpty { null }, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val (slug, id) = manga.url.split("#") + val detailsUrl = "$apiBaseUrl/api/post?postSlug=$slug" + val json = webClient.httpGet(detailsUrl, getRequestHeaders()).parseJson() + val data = json.optJSONObject("post") ?: return manga + if (data.optBoolean("isNovel", false)) throw Exception("Novels not supported") + + val title = data.optString("postTitle", manga.title) + val cover = data.optString("featuredImage", "") + val desc = data.optString("postContent", "") + val altTitles = data.optString("alternativeTitles", "") + val author = data.optString("author", "") + val artist = data.optString("artist", "") + val seriesType = data.optString("seriesType", "") + val seriesStatus = data.optString("seriesStatus", "") + val genresArr = data.optJSONArray("genres") ?: JSONArray() + val chaptersArr = data.optJSONArray("chapters") ?: JSONArray() + val totalChapters = json.optInt("totalChapterCount", chaptersArr.length()) + + val state = when (seriesStatus.uppercase()) { + "ONGOING", "COMING_SOON", "MASS_RELEASED" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "CANCELLED", "DROPPED" -> MangaState.ABANDONED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (seriesType.isNotEmpty()) { + tags.add(MangaTag(seriesType.lowercase(), seriesType, source)) + } + for (i in 0 until genresArr.length()) { + val g = genresArr.optJSONObject(i) ?: continue + val name = g.optString("name", "") + if (name.isNotEmpty()) tags.add(MangaTag(name.lowercase(), name, source)) + } + + val plainDesc = if (desc.isNotEmpty()) { + org.jsoup.Jsoup.parse(desc.replace("\n", "<br>")).text() + } else "" + val fullDesc = buildString { + append(plainDesc) + if (altTitles.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Alternative Names: $altTitles") + } + }.trim() + + val chapters = if (totalChapters > chaptersArr.length()) { + fetchChaptersFromApi(id.toInt(), slug) + } else { + parseChaptersFromArray(chaptersArr, slug) + } + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = fullDesc.takeIf { it.isNotEmpty() }, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChaptersFromApi(postId: Int, slug: String): List<MangaChapter> { + val url = "$apiBaseUrl/api/chapters?postId=$postId" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val chaptersArr = json.optJSONObject("post")?.optJSONArray("chapters") ?: JSONArray() + return parseChaptersFromArray(chaptersArr, slug) + } + + private fun parseChaptersFromArray(arr: JSONArray, mangaSlug: String): List<MangaChapter> { + val list = mutableListOf<MangaChapter>() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + if (!ch.optBoolean("isAccessible", false)) continue + val id = ch.getInt("id") + val number = ch.optString("number", "") + val title = ch.optString("title", "") + val slug = ch.optString("slug", "") + val dateStr = ch.optString("createdAt", "") + val uploadDate = try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + + val cleanTitle = if (title == "null" || title.isEmpty()) null else title + val fullTitle = buildString { + append("Chapter $number") + if (cleanTitle != null) append(" - $cleanTitle") + } + + list.add( + MangaChapter( + id = generateUid("$mangaSlug#$id"), + title = fullTitle, + number = number.toFloatOrNull() ?: 0f, + volume = 0, + url = "/series/$mangaSlug/$slug#$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("#") + if (parts.size != 2) return emptyList() + val chapterId = parts[1].toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/chapter?chapterId=$chapterId", getRequestHeaders()).parseJson() + val chData = json.optJSONObject("chapter") ?: return emptyList() + if (chData.optBoolean("isShortLinkLocked")) throw Exception("Chapter locked (short link)") + if (chData.optBoolean("isLockedByCoins")) throw Exception("Chapter locked (coins required)") + if (chData.optBoolean("isPermanentlyLocked")) throw Exception("Chapter permanently locked") + + val imagesArr = chData.optJSONArray("images") ?: return emptyList() + val pages = mutableListOf<JSONObject>() + for (i in 0 until imagesArr.length()) { + pages.add(imagesArr.optJSONObject(i) ?: continue) + } + pages.sortBy { it.optInt("order", Int.MAX_VALUE) } + + return pages.mapIndexed { _, imgObj -> + val imgUrl = imgObj.optString("url", "").replace(" ", "%20") + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val (_, id) = seed.url.split("#") + val postId = id.toIntOrNull() ?: return emptyList() + val json = webClient.httpGet("$apiBaseUrl/api/recommendations?postId=$postId&limit=25", getRequestHeaders()).parseJson() + val recs = json.optJSONArray("recommendations") ?: return emptyList() + val mangas = mutableListOf<Manga>() + for (i in 0 until recs.length()) { + val obj = recs.optJSONObject(i) ?: continue + if (obj.optBoolean("isNovel", false)) continue + mangas.add(obj.toManga()) + } + return mangas + } +} diff --git a/src/main/kotlin/tsuki/site/en/VyManga.kt b/src/main/kotlin/tsuki/site/en/VyManga.kt new file mode 100644 index 00000000..1ba8d066 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/VyManga.kt @@ -0,0 +1,240 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.WordSet + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.selectLast +import tsuki.util.src +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet + +@MangaSourceParser("VYMANGA", "VyManga", "en") +internal class VyManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VYMANGA, pageSize = 36) { + + override val configKeyDomain: ConfigKey.Domain = ConfigKey.Domain("mangavyvy.net") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(domain) + append("/search?search_po=0&q=") + + filter.query?.let { + append(filter.query.urlEncoded()) + } + + append("&author_po=0&author=") + + append("&completed=") + filter.states.oneOrThrowIfMany()?.let { + append( + when (it) { + MangaState.ONGOING -> "0" + MangaState.FINISHED -> "1" + else -> "" + }, + ) + } + + append("&sort=") + when (order) { + SortOrder.POPULARITY -> append("viewed&sort_type=desc") + SortOrder.POPULARITY_ASC -> append("viewed&sort_type=asc") + SortOrder.RATING -> append("scored&sort_type=desc") + SortOrder.RATING_ASC -> append("scored&sort_type=asc") + SortOrder.NEWEST -> append("created_at&sort_type=desc") + SortOrder.NEWEST_ASC -> append("created_at&sort_type=asc") + SortOrder.UPDATED -> append("updated_at&sort_type=desc") + SortOrder.UPDATED_ASC -> append("updated_at&sort_type=asc") + else -> append("updated_at&sort_type=desc") + } + + filter.tags.forEach { tag -> + append("&genre[]=") + append(tag.key) + } + + filter.tagsExclude.forEach { tagsExclude -> + append("&exclude_genre[]=") + append(tagsExclude.key) + } + + append("&page=") + append(page.toString()) + } + val doc = webClient.httpGet(url).parseHtml() + return doc.select(".comic-item").map { div -> + val href = div.selectFirstOrThrow("a").attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = div.selectFirst(".comic-image img")?.src(), + title = div.selectFirst(".comic-title")?.text().orEmpty(), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + source = source, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/search").parseHtml() + + val tags = mutableSetOf<MangaTag>() + doc.select(".check-genre div div:has(.checkbox-genre)").forEach { element -> + val label = element.selectFirst("label")?.text()?.trim() + val input = element.selectFirst(".checkbox-genre") + val key = input?.attr("data-value")?.trim() + + if (!label.isNullOrBlank() && !key.isNullOrBlank()) { + tags += MangaTag( + title = label, + key = key, + source = source, + ) + } + } + return tags + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val simpleDateFormat = SimpleDateFormat("MMM dd, yyy", sourceLocale) + return manga.copy( + tags = doc.select("div.col-md-7 p a[href*=genre]").mapToSet { a -> + MangaTag( + key = a.attr("href").substringAfterLast('/'), + title = a.text(), + source = source, + ) + }, + description = doc.selectFirst("div.summary p.content")?.text().orEmpty(), + state = when (doc.selectLast("div.col-md-7 p:contains(Status) span")?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + }, + rating = doc.selectFirst("div.col-md-7 p:contains(Rating)")?.text()?.substringAfterLast(':') + ?.substringBefore('/')?.toFloat() ?: RATING_UNKNOWN, + chapters = doc.select("div.list div.list-group a").mapChapters(reversed = true) { i, a -> + val url = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(url), + title = a.selectFirst("span")?.textOrNull(), + number = i + 1f, + volume = 0, + url = url, + scanlator = null, + uploadDate = parseChapterDate( + simpleDateFormat, + a.selectFirst("p")?.text(), + ), + branch = null, + source = source, + ) + }, + ) + } + + private fun parseChapterDate(dateFormat: DateFormat, date: String?): Long { + val d = date?.lowercase() ?: return 0 + return when { + d.endsWith(" ago") -> parseRelativeDate(date) + else -> dateFormat.parseSafe(date) + } + } + + private fun parseRelativeDate(date: String): Long { + val number = Regex("""(\d+)""").find(date)?.value?.toIntOrNull() ?: return 0 + val cal = Calendar.getInstance() + return when { + WordSet("second").anyWordIn(date) -> cal.apply { add(Calendar.SECOND, -number) }.timeInMillis + WordSet("minute", "minutes").anyWordIn(date) -> cal.apply { add(Calendar.MINUTE, -number) }.timeInMillis + WordSet("hour", "hours").anyWordIn(date) -> cal.apply { add(Calendar.HOUR, -number) }.timeInMillis + WordSet("day", "days").anyWordIn(date) -> cal.apply { add(Calendar.DAY_OF_MONTH, -number) }.timeInMillis + WordSet("week", "weeks").anyWordIn(date) -> cal.apply { add(Calendar.WEEK_OF_YEAR, -number) }.timeInMillis + WordSet("month", "months").anyWordIn(date) -> cal.apply { add(Calendar.MONTH, -number) }.timeInMillis + WordSet("year").anyWordIn(date) -> cal.apply { add(Calendar.YEAR, -number) }.timeInMillis + else -> 0 + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("img.d-block").map { img -> + val url = img.requireSrc() + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt b/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt new file mode 100644 index 00000000..90858dbf --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/WebtoonXYZ.kt @@ -0,0 +1,27 @@ +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaParserSource + +@MangaSourceParser("WEBTOONXYZ", "WebtoonXYZ", "en") +internal class WebtoonXYZ(context: MangaLoaderContext): + MadaraParser(context, MangaParserSource.WEBTOONXYZ, "webtoon.xyz", 20) { + + override val datePattern = "dd MMMM yyyy" + + override val listUrl = "webtoons/" + + private val thumbnailRegex = Regex("-\\d+x\\d+(\\.[a-zA-Z]+)$") + + override fun parseMangaList(doc: org.jsoup.nodes.Document): List<Manga> { + return super.parseMangaList(doc).map { manga -> + manga.copy( + coverUrl = manga.coverUrl?.replace(thumbnailRegex, "$1") + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt b/src/main/kotlin/tsuki/site/en/WeebCentral.kt similarity index 87% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt rename to src/main/kotlin/tsuki/site/en/WeebCentral.kt index 145e51ba..d09a8abf 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/WeebCentral.kt +++ b/src/main/kotlin/tsuki/site/en/WeebCentral.kt @@ -1,4 +1,36 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.AbstractMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.attrAsAbsoluteUrlOrNull +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.mapChapters +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.selectFirstOrThrow import kotlinx.coroutines.async import kotlinx.coroutines.coroutineScope @@ -6,35 +38,6 @@ import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl import org.jsoup.nodes.Document import org.jsoup.nodes.Element -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.AbstractMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.LinkResolver -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.attrAsAbsoluteUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.getCookies -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale @@ -375,4 +378,4 @@ internal class WeebCentral(context: MangaLoaderContext) : AbstractMangaParser(co return resolver.resolveManga(this, mangaId) } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt b/src/main/kotlin/tsuki/site/en/ZinManga.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt rename to src/main/kotlin/tsuki/site/en/ZinManga.kt index 98e8a4e3..ee58ca94 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/ZinManga.kt +++ b/src/main/kotlin/tsuki/site/en/ZinManga.kt @@ -1,14 +1,21 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en +package tsuki.site.en + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* import java.text.SimpleDateFormat @MangaSourceParser("ZINMANGA", "ZinManga.net", "en") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt b/src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt rename to src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt index 0236ac51..188deb6f 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/MangaBats.kt +++ b/src/main/kotlin/tsuki/site/en/mangabox/MangaBats.kt @@ -1,46 +1,46 @@ @file:Suppress("DEPRECATION") -package org.koitharu.kotatsu.parsers.site.kotatsu.en.mangabox +package tsuki.site.en.mangabox + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.parsers.MangaboxParser + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase import okhttp3.Interceptor import okhttp3.Response import org.json.JSONObject import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.parsers.MangaboxParser -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toTitleCase import java.text.SimpleDateFormat import java.util.EnumSet import java.util.LinkedHashSet import java.util.Locale import java.util.TimeZone -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient import kotlinx.coroutines.async import kotlinx.coroutines.awaitAll import kotlinx.coroutines.coroutineScope -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.mapToSet @MangaSourceParser("MANGABAT", "MangaBats", "en") internal class Mangabats(context: MangaLoaderContext) : @@ -118,19 +118,19 @@ internal class Mangabats(context: MangaLoaderContext) : override val searchQueryCapabilities: MangaSearchQueryCapabilities get() = MangaSearchQueryCapabilities( SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class), + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class), isMultiple = false, ), SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), isMultiple = false, isExclusive = true, ), SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), isMultiple = false, ), ) @@ -140,18 +140,18 @@ internal class Mangabats(context: MangaLoaderContext) : page: Int ): List<Manga> { - val titleQuery = query.criteria.filterIsInstance<Match<*>>() - .firstOrNull { it.field == TITLE_NAME } + val titleQuery = query.criteria.filterIsInstance<QueryCriteria.Match<*>>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } ?.value as? String - val tagCriterion = query.criteria.filterIsInstance<Include<*>>() - .firstOrNull { it.field == TAG } + val tagCriterion = query.criteria.filterIsInstance<QueryCriteria.Include<*>>() + .firstOrNull { it.field == SearchableField.TAG } val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key ?: tagCriterion?.values?.firstOrNull()?.toString() - val state = query.criteria.filterIsInstance<Include<*>>() - .firstOrNull { it.field == STATE } + val state = query.criteria.filterIsInstance<QueryCriteria.Include<*>>() + .firstOrNull { it.field == SearchableField.STATE } ?.values ?.firstOrNull() as? MangaState @@ -389,4 +389,4 @@ internal class Mangabats(context: MangaLoaderContext) : private const val CHAPTER_LIST_TAKE = 1000 } -} +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt b/src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt similarity index 82% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt rename to src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt index 186cd712..4349a1ee 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/mangabox/ManganatoGG.kt +++ b/src/main/kotlin/tsuki/site/en/mangabox/ManganatoGG.kt @@ -1,392 +1,392 @@ -@file:Suppress("DEPRECATION") - -package org.koitharu.kotatsu.parsers.site.kotatsu.en.mangabox - -import okhttp3.Interceptor -import okhttp3.Response -import org.json.JSONObject -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQuery -import org.koitharu.kotatsu.parsers.model.search.MangaSearchQueryCapabilities -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Include -import org.koitharu.kotatsu.parsers.model.search.QueryCriteria.Match -import org.koitharu.kotatsu.parsers.model.search.SearchCapability -import org.koitharu.kotatsu.parsers.model.search.SearchableField.STATE -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TAG -import org.koitharu.kotatsu.parsers.model.search.SearchableField.TITLE_NAME -import org.koitharu.kotatsu.parsers.parsers.MangaboxParser -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.nullIfEmpty -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJson -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.toTitleCase -import java.text.SimpleDateFormat -import java.util.EnumSet -import java.util.LinkedHashSet -import java.util.Locale -import java.util.TimeZone -import org.koitharu.kotatsu.parsers.network.OkHttpWebClient -import kotlinx.coroutines.async -import kotlinx.coroutines.awaitAll -import kotlinx.coroutines.coroutineScope -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.mapToSet - -@MangaSourceParser("MANGANATOGG", "Manganato.gg", "en") -internal class ManganatoGG(context: MangaLoaderContext) : - MangaboxParser(context, MangaParserSource.MANGANATOGG) { - - override val configKeyDomain = ConfigKey.Domain("manganato.gg") - - override val webClient = OkHttpWebClient( - context.httpClient.newBuilder() - .build(), - source, - ) - - - override val listUrl = "/genre/all" - override val searchUrl = "/search/story" - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - ) - override suspend fun getDetails(manga: Manga): Manga = coroutineScope { - val fullUrl = manga.url.toAbsoluteUrl(domain) - val doc = webClient.httpGet(fullUrl).parseHtml() - - val chaptersDeferred = async { getChapters(doc) } - - val stateText = doc.findInfo("Status")?.lowercase() - - val state = when { - stateText?.contains("ongoing") == true -> MangaState.ONGOING - stateText?.contains("completed") == true || - stateText?.contains("finish") == true -> MangaState.FINISHED - else -> null - } - - val alt = doc.findInfo("Alternative") - - manga.copy( - tags = doc.body() - .select(selectTag) - .mapToSet { a -> - val href = a.attr("href") - - MangaTag( - key = href.substringAfterLast("category=") - .substringBefore("&") - .takeIf { it != href } - ?: href.removeSuffix("/") - .substringAfterLast("/"), - - title = a.text().toTitleCase(), - source = source, - ) - }, - - description = "If empty Chapters, reload Manga!", - altTitles = setOfNotNull(alt), - state = state, - chapters = chaptersDeferred.await() - ) - } - - private fun Document.findInfo(label: String): String? { - return select("ul.manga-info-text li") - .firstOrNull { it.text().contains(label, ignoreCase = true) } - ?.text() - ?.substringAfter(":") - ?.trim() - ?.nullIfEmpty() - } - - @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) - override val searchQueryCapabilities: MangaSearchQueryCapabilities - get() = MangaSearchQueryCapabilities( - SearchCapability( - field = TAG, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - SearchCapability( - field = TITLE_NAME, - criteriaTypes = setOf(Match::class), - isMultiple = false, - isExclusive = true, - ), - SearchCapability( - field = STATE, - criteriaTypes = setOf(Include::class), - isMultiple = false, - ), - ) - - override suspend fun getListPage( - query: MangaSearchQuery, - page: Int - ): List<Manga> { - - val titleQuery = query.criteria.filterIsInstance<Match<*>>() - .firstOrNull { it.field == TITLE_NAME } - ?.value as? String - - val tagCriterion = query.criteria.filterIsInstance<Include<*>>() - .firstOrNull { it.field == TAG } - - val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key - ?: tagCriterion?.values?.firstOrNull()?.toString() - - val state = query.criteria.filterIsInstance<Include<*>>() - .firstOrNull { it.field == STATE } - ?.values - ?.firstOrNull() as? MangaState - - val url = if (!titleQuery.isNullOrBlank()) { - - "https://$domain/search/story/${ - normalizeSearchQuery(titleQuery).replace('_', '-') - }?page=$page" - - } else { - - val base = if (!tagKey.isNullOrBlank()) { - "https://$domain/genre/$tagKey" - } else { - "https://$domain/genre/all" - } - - val filter = when { - state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 - state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 - - state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 - state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 - - query.order == SortOrder.POPULARITY -> 7 - query.order == SortOrder.NEWEST -> 1 - - else -> 4 // all + latest - } - - "$base?filter=$filter&page=$page" - } - - val doc = webClient.httpGet(url).parseHtml() - return parseSearchResults(doc) - } - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - val doc = webClient.httpGet("https://$domain").parseHtml() - val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") - .drop(1) - - val result = LinkedHashSet<MangaTag>() - - for (a in tags) { - val key = a.attr("href") - .removeSuffix("/") - .substringAfter("/genre/") - .substringBefore("?") - .nullIfEmpty() - ?: continue - - if (result.any { it.key == key }) { - continue - } - - val title = a.text() - .replace(" Manga", "") - .toTitleCase(sourceLocale) - - result.add( - MangaTag( - key = key, - title = title, - source = source, - ), - ) - } - - return result - } - - override suspend fun getChapters(doc: Document): List<MangaChapter> { - val slug = doc.location() - .substringAfter("/manga/", "") - .substringBefore("/") - .trim() - .ifEmpty { return super.getChapters(doc) } - - val apiChapters = runCatching { - fetchChaptersApi(slug) - }.getOrDefault(emptyList()) - - return apiChapters.ifEmpty { - super.getChapters(doc) - } - } - - private suspend fun fetchChaptersApi(slug: String): List<MangaChapter> { - - suspend fun request(offset: Int): JSONObject { - val apiUrl = - "https://$domain/api/manga/$slug/chapters" + - "?limit=$CHAPTER_LIST_TAKE&offset=$offset" - - return webClient.httpGet(apiUrl).parseJson() - } - - fun extractChapters(json: JSONObject): List<JSONObject> { - val array = json - .optJSONObject("data") - ?.optJSONArray("chapters") - ?: return emptyList() - - return buildList { - for (i in 0 until array.length()) { - array.optJSONObject(i)?.let(::add) - } - } - } - // first request - val first = request(0) - - val chapters = mutableListOf<JSONObject>() - chapters += extractChapters(first) - - val pagination = - first.optJSONObject("data") - ?.optJSONObject("pagination") - - val hasMore = - pagination?.optBoolean("has_more", false) ?: false - - if (!hasMore) { - return mapChapters(slug, chapters) - } - - val total = - pagination.optInt("total", 0) - - val offsets = - (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) - .toList() - // parallel loading - coroutineScope { - val results = offsets.map { offset -> - async { - request(offset) - } - }.awaitAll() - results.forEach { - chapters += extractChapters(it) - } - } - return mapChapters(slug, chapters) - } - - private fun mapChapters( - slug: String, - chapters: List<JSONObject> - ): List<MangaChapter> { - - return chapters.mapNotNull { chapter -> - val chapterSlug = - chapter.optString("chapter_slug") - .nullIfEmpty() - ?: return@mapNotNull null - - val chapterName = - chapter.optString("chapter_name") - .nullIfEmpty() - ?: "Chapter" - - val number = - chapter.optString("chapter_num") - .toFloatOrNull() - ?: 0f - - val url = "/manga/$slug/$chapterSlug" - - MangaChapter( - id = generateUid(url), - title = chapterName, - number = number, - volume = 0, - url = url, - uploadDate = parseApiDate( - chapter.optString("updated_at") - ), - source = source, - scanlator = null, - branch = null, - ) - }.sortedBy { - it.number - } - } - - - private fun parseApiDate(date: String?): Long { - if (date.isNullOrBlank()) { - return 0L - } - - val formats = listOf( - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), - SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), - ).onEach { - it.timeZone = TimeZone.getTimeZone("UTC") - } - - for (format in formats) { - val parsed = format.parseSafe(date) - if (parsed != 0L) { - return parsed - } - } - - return 0L - } - - @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .set("Referer", "https://www.mangabats.com/") - .build() - - override fun intercept(chain: Interceptor.Chain): Response { - val request = chain.request() - - if (request.method == "GET" || request.method == "POST") { - val newRequest = request.newBuilder() - .removeHeader("Content-Encoding") - .removeHeader("Accept-Encoding") - .build() - - return chain.proceed(newRequest) - } - - return chain.proceed(request) - } - - companion object { - - private const val CHAPTER_LIST_TAKE = 1000 - } -} +@file:Suppress("DEPRECATION") + +package tsuki.site.en.mangabox + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.search.MangaSearchQuery +import tsuki.model.search.MangaSearchQueryCapabilities +import tsuki.model.search.QueryCriteria +import tsuki.model.search.SearchCapability +import tsuki.model.search.SearchableField +import tsuki.parsers.MangaboxParser + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.LinkedHashSet +import java.util.Locale +import java.util.TimeZone +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope + +@MangaSourceParser("MANGANATOGG", "Manganato.gg", "en") +internal class ManganatoGG(context: MangaLoaderContext) : + MangaboxParser(context, MangaParserSource.MANGANATOGG) { + + override val configKeyDomain = ConfigKey.Domain("manganato.gg") + + override val webClient = OkHttpWebClient( + context.httpClient.newBuilder() + .build(), + source, + ) + + + override val listUrl = "/genre/all" + override val searchUrl = "/search/story" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + ) + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val chaptersDeferred = async { getChapters(doc) } + + val stateText = doc.findInfo("Status")?.lowercase() + + val state = when { + stateText?.contains("ongoing") == true -> MangaState.ONGOING + stateText?.contains("completed") == true || + stateText?.contains("finish") == true -> MangaState.FINISHED + else -> null + } + + val alt = doc.findInfo("Alternative") + + manga.copy( + tags = doc.body() + .select(selectTag) + .mapToSet { a -> + val href = a.attr("href") + + MangaTag( + key = href.substringAfterLast("category=") + .substringBefore("&") + .takeIf { it != href } + ?: href.removeSuffix("/") + .substringAfterLast("/"), + + title = a.text().toTitleCase(), + source = source, + ) + }, + + description = "If empty Chapters, reload Manga!", + altTitles = setOfNotNull(alt), + state = state, + chapters = chaptersDeferred.await() + ) + } + + private fun Document.findInfo(label: String): String? { + return select("ul.manga-info-text li") + .firstOrNull { it.text().contains(label, ignoreCase = true) } + ?.text() + ?.substringAfter(":") + ?.trim() + ?.nullIfEmpty() + } + + @Deprecated("Use availableSortOrders instead", ReplaceWith("availableSortOrders")) + override val searchQueryCapabilities: MangaSearchQueryCapabilities + get() = MangaSearchQueryCapabilities( + SearchCapability( + field = SearchableField.TAG, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + SearchCapability( + field = SearchableField.TITLE_NAME, + criteriaTypes = setOf(QueryCriteria.Match::class), + isMultiple = false, + isExclusive = true, + ), + SearchCapability( + field = SearchableField.STATE, + criteriaTypes = setOf(QueryCriteria.Include::class), + isMultiple = false, + ), + ) + + override suspend fun getListPage( + query: MangaSearchQuery, + page: Int + ): List<Manga> { + + val titleQuery = query.criteria.filterIsInstance<QueryCriteria.Match<*>>() + .firstOrNull { it.field == SearchableField.TITLE_NAME } + ?.value as? String + + val tagCriterion = query.criteria.filterIsInstance<QueryCriteria.Include<*>>() + .firstOrNull { it.field == SearchableField.TAG } + + val tagKey = (tagCriterion?.values?.firstOrNull() as? MangaTag)?.key + ?: tagCriterion?.values?.firstOrNull()?.toString() + + val state = query.criteria.filterIsInstance<QueryCriteria.Include<*>>() + .firstOrNull { it.field == SearchableField.STATE } + ?.values + ?.firstOrNull() as? MangaState + + val url = if (!titleQuery.isNullOrBlank()) { + + "https://$domain/search/story/${ + normalizeSearchQuery(titleQuery).replace('_', '-') + }?page=$page" + + } else { + + val base = if (!tagKey.isNullOrBlank()) { + "https://$domain/genre/$tagKey" + } else { + "https://$domain/genre/all" + } + + val filter = when { + state == MangaState.ONGOING && query.order == SortOrder.NEWEST -> 3 + state == MangaState.ONGOING && query.order == SortOrder.UPDATED -> 6 + + state == MangaState.FINISHED && query.order == SortOrder.NEWEST -> 2 + state == MangaState.FINISHED && query.order == SortOrder.UPDATED -> 5 + + query.order == SortOrder.POPULARITY -> 7 + query.order == SortOrder.NEWEST -> 1 + + else -> 4 // all + latest + } + + "$base?filter=$filter&page=$page" + } + + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResults(doc) + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain").parseHtml() + val tags = doc.select("td a[href*='/genre/'], div.panel-genres-list a[href*='/genre/']") + .drop(1) + + val result = LinkedHashSet<MangaTag>() + + for (a in tags) { + val key = a.attr("href") + .removeSuffix("/") + .substringAfter("/genre/") + .substringBefore("?") + .nullIfEmpty() + ?: continue + + if (result.any { it.key == key }) { + continue + } + + val title = a.text() + .replace(" Manga", "") + .toTitleCase(sourceLocale) + + result.add( + MangaTag( + key = key, + title = title, + source = source, + ), + ) + } + + return result + } + + override suspend fun getChapters(doc: Document): List<MangaChapter> { + val slug = doc.location() + .substringAfter("/manga/", "") + .substringBefore("/") + .trim() + .ifEmpty { return super.getChapters(doc) } + + val apiChapters = runCatching { + fetchChaptersApi(slug) + }.getOrDefault(emptyList()) + + return apiChapters.ifEmpty { + super.getChapters(doc) + } + } + + private suspend fun fetchChaptersApi(slug: String): List<MangaChapter> { + + suspend fun request(offset: Int): JSONObject { + val apiUrl = + "https://$domain/api/manga/$slug/chapters" + + "?limit=$CHAPTER_LIST_TAKE&offset=$offset" + + return webClient.httpGet(apiUrl).parseJson() + } + + fun extractChapters(json: JSONObject): List<JSONObject> { + val array = json + .optJSONObject("data") + ?.optJSONArray("chapters") + ?: return emptyList() + + return buildList { + for (i in 0 until array.length()) { + array.optJSONObject(i)?.let(::add) + } + } + } + // first request + val first = request(0) + + val chapters = mutableListOf<JSONObject>() + chapters += extractChapters(first) + + val pagination = + first.optJSONObject("data") + ?.optJSONObject("pagination") + + val hasMore = + pagination?.optBoolean("has_more", false) ?: false + + if (!hasMore) { + return mapChapters(slug, chapters) + } + + val total = + pagination.optInt("total", 0) + + val offsets = + (CHAPTER_LIST_TAKE until total step CHAPTER_LIST_TAKE) + .toList() + // parallel loading + coroutineScope { + val results = offsets.map { offset -> + async { + request(offset) + } + }.awaitAll() + results.forEach { + chapters += extractChapters(it) + } + } + return mapChapters(slug, chapters) + } + + private fun mapChapters( + slug: String, + chapters: List<JSONObject> + ): List<MangaChapter> { + + return chapters.mapNotNull { chapter -> + val chapterSlug = + chapter.optString("chapter_slug") + .nullIfEmpty() + ?: return@mapNotNull null + + val chapterName = + chapter.optString("chapter_name") + .nullIfEmpty() + ?: "Chapter" + + val number = + chapter.optString("chapter_num") + .toFloatOrNull() + ?: 0f + + val url = "/manga/$slug/$chapterSlug" + + MangaChapter( + id = generateUid(url), + title = chapterName, + number = number, + volume = 0, + url = url, + uploadDate = parseApiDate( + chapter.optString("updated_at") + ), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { + it.number + } + } + + + private fun parseApiDate(date: String?): Long { + if (date.isNullOrBlank()) { + return 0L + } + + val formats = listOf( + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US), + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US), + ).onEach { + it.timeZone = TimeZone.getTimeZone("UTC") + } + + for (format in formats) { + val parsed = format.parseSafe(date) + if (parsed != 0L) { + return parsed + } + } + + return 0L + } + + @Deprecated("Use getRequestHeaders instead", ReplaceWith("getRequestHeaders()")) + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://www.mangabats.com/") + .build() + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + + if (request.method == "GET" || request.method == "POST") { + val newRequest = request.newBuilder() + .removeHeader("Content-Encoding") + .removeHeader("Accept-Encoding") + .build() + + return chain.proceed(newRequest) + } + + return chain.proceed(request) + } + + companion object { + + private const val CHAPTER_LIST_TAKE = 1000 + } +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt b/src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt similarity index 53% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt rename to src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt index 280d6b47..ffb752e4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/AllPornComic.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/AllPornComic.kt @@ -1,10 +1,12 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser @MangaSourceParser("ALLPORNCOMIC", "AllPornComic", "en", ContentType.HENTAI) internal class AllPornComic(context: MangaLoaderContext) : diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt new file mode 100644 index 00000000..34fda989 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/ComicLand.kt @@ -0,0 +1,206 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.urlEncoded + +import okhttp3.Interceptor +import okhttp3.Response +import org.json.JSONObject + +@MangaSourceParser("COMICLAND", "ComicLand", "en", ContentType.HENTAI) +internal class ComicLand(context: MangaLoaderContext) : PagedMangaParser( + context, MangaParserSource.COMICLAND, 20) { + + override val configKeyDomain = ConfigKey.Domain("comicland.org") + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val newRequest = request.newBuilder() + .header("Referer", "https://$domain/") + .header("Origin", "https://$domain") + .build() + return chain.proceed(newRequest) + } + + override val availableSortOrders = setOf(SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = false, + isTagsExclusionSupported = false, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isYearSupported = false, + isAuthorSearchSupported = false + ) + + private val api = "https://api.comicland.org/api" + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val offset = (page - 1) * 20 + val endpoint = + if (!filter.query.isNullOrBlank()) { + "$api/comic/search?q=${filter.query.urlEncoded()}" + } else { + val category = filter.tags.firstOrNull()?.key + when (category) { + "official" -> "$api/comics/official" + "ongoing" -> "$api/comics" + "popular" -> "$api/comics/popular" + else -> "$api/comics" + } + } + + val separator = if (endpoint.contains("?")) "&" else "?" + val response = webClient.httpGet("$endpoint${separator}offset=$offset&limit=20") + val root = JSONObject(response.body.string()) + + val data = root.optJSONObject("data") + ?: return emptyList() + + val array = data.optJSONArray("list") + ?: data.optJSONArray("items") + ?: return emptyList() + + val result = ArrayList<Manga>() + for (i in 0 until array.length()) { + val comic = + array.getJSONObject(i) + val slug = + comic.optString("slug") + result.add( + Manga( + id = generateUid(slug), + title = comic.optString("title"), + url = slug, + publicUrl = "https://comicland.org/comic/$slug", + coverUrl = comic.optString("cover_url"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + tags = emptySet(), + authors = emptySet(), + state = null, + contentRating = ContentRating.ADULT, + source = source + ) + ) + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val response = webClient.httpGet("$api/comic/detail?slug=${manga.url}") + val root = JSONObject(response.body.string()) + val comic = root.optJSONObject("data") + ?: return manga + val chapters = ArrayList<MangaChapter>() + val chapterArray = comic.optJSONArray("chapters") + + if (chapterArray != null) { + for (i in 0 until chapterArray.length()) { + val chapter = chapterArray.getJSONObject(i) + + val index = chapter.optDouble("chapter_index", 0.0) + chapters.add( + MangaChapter( + id = generateUid("${comic.optString("slug")}/$index"), + title = chapter.optString("title"), + number = index.toFloat(), + volume = 0, + url = "/comic/${comic.optString("slug")}/chapter/${index.toString().removeSuffix(".0")}", + scanlator = null, + uploadDate = 0, + branch = null, + source = source + ) + ) + } + } + return manga.copy( + title = comic.optString("title", manga.title), + description = comic.optString("description", ""), + coverUrl = comic.optString("cover_url", manga.coverUrl), + authors = parseNames(comic.optJSONArray("authors")), + tags = emptySet(), + chapters = chapters + ) + } + + private fun parseNames(array: org.json.JSONArray?): Set<String> { + if (array == null) + return emptySet() + return buildSet { + for (i in 0 until array.length()) { + add( + array.getJSONObject(i) + .optString("name") + ) + } + } + } + + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val slug = chapter.url + .substringAfter("/comic/") + .substringBefore("/chapter/") + val index = chapter.url + .substringAfter("/chapter/") + val response = webClient.httpGet("$api/chapter/pages_by_index?slug=$slug&index=$index") + val root = JSONObject(response.body.string()) + val data = root.optJSONObject("data") + ?: return emptyList() + val pages = data.optJSONArray("pages") + ?: return emptyList() + return buildList { + for (i in 0 until pages.length()) { + val image = pages.getString(i) + add( + MangaPage( + id = generateUid(image), + url = image, + preview = null, + source = source + ) + ) + } + } + } + + private val categories = mapOf( + "recommended" to "/comics", + "official" to "/comics/official", + "ongoing" to "/comics?status=ongoing", + "popular" to "/comics/popular" + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = categories.map { (key, _) -> + MangaTag( + title = key.replaceFirstChar { it.uppercase() }, + key = key, + source = source + ) + }.toSet(), + availableContentRating = emptySet() + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt similarity index 85% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt rename to src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt index 1366a933..a4eb82b4 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiNexus.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiNexus.kt @@ -1,40 +1,43 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryAdultsParser +import tsuki.config.ConfigKey + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseSafe +import tsuki.util.urlDecode +import tsuki.util.urlEncoded -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.GalleryAdultsParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.parseSafe -import org.koitharu.kotatsu.parsers.util.urlDecode -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.Base64 -import org.koitharu.kotatsu.parsers.config.ConfigKey +import org.json.JSONArray +import org.jsoup.nodes.Document private const val SERVER_PNG = "png" // old method uses now image_fallback private const val SERVER_WEBP = "webp" // new decryption method for .webp/.avif format -@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", type = ContentType.HENTAI) +@MangaSourceParser("HENTAINEXUS", "HentaiNexus", "en", ContentType.HENTAI) internal class HentaiNexus(context: MangaLoaderContext) : GalleryAdultsParser(context, MangaParserSource.HENTAINEXUS, "hentainexus.com", 30) { override val selectGallery = "div.container div.columns div.column" @@ -212,34 +215,27 @@ internal class HentaiNexus(context: MangaLoaderContext) : if (mangaPages.isEmpty() || !chapter.url.contains(mangaInternalId)) { mangaPages = getPagesInternal(chapter.url) mangaInternalId = chapter.url.split("/").last() - - // preload image urls val reader = mangaPages.firstOrNull()?.url?.substringBefore("#") if (reader != null && !pageCache.containsKey(reader)) { pageCache[reader] = getPageUrlInternal(reader) } } - return mangaPages } override suspend fun getPageUrl(page: MangaPage): String { val readerUrl = page.url.substringBefore("#") - val pages = pageCache[readerUrl] ?: getPageUrlInternal(readerUrl).also { pageCache[readerUrl] = it } - val index = page.url.substringAfter("#").toInt() - 1 - return pages[index] } private suspend fun getPagesInternal(chapterUrl: String, document: Document? = null): List<MangaPage> { val document = document ?: webClient.httpGet(chapterUrl.toAbsoluteUrl(domain)).parseHtml() val readUrl = document.selectFirstOrThrow(selectReadUrl).attr("href") - return document.select(selectTotalPage).mapIndexed { index, element -> val url = "$readUrl#${index + 1}" MangaPage( @@ -312,15 +308,9 @@ internal class HentaiNexus(context: MangaLoaderContext) : private fun normalizeImageUrl(url: String): String { return when { url.startsWith("http") -> url - - url.startsWith("//") -> - "https:$url" - - url.startsWith("/") -> - "https://$domain$url" - - else -> - "https://$domain/$url" + url.startsWith("//") -> "https:$url" + url.startsWith("/") -> "https://$domain$url" + else -> "https://$domain/$url" } } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt similarity index 65% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt rename to src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt index 891e56eb..72b65da2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HentaiXComic.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HentaiXComic.kt @@ -1,12 +1,15 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import tsuki.util.parseHtml +import tsuki.util.toTitleCase @MangaSourceParser("HENTAIXCOMIC", "HentaiXComic", "en", ContentType.HENTAI) internal class HentaixComic(context: MangaLoaderContext) : @@ -25,4 +28,4 @@ internal class HentaixComic(context: MangaLoaderContext) : } return set } -} \ No newline at end of file +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt similarity index 81% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt rename to src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt index 730a153e..8d85090c 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/HeyToon.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/HeyToon.kt @@ -1,37 +1,40 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser import okhttp3.Headers import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.json.getStringOrNull -import org.koitharu.kotatsu.parsers.util.json.mapJSON -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.parseJsonArray -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toRelativeUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseJsonArray +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON + import java.util.Calendar import java.util.EnumSet @@ -286,8 +289,8 @@ internal class HeyToon(context: MangaLoaderContext) : set(Calendar.SECOND, 0) set(Calendar.MILLISECOND, 0) }.timeInMillis - } catch (e: Exception) { + } catch (_: Exception) { 0 } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt b/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt new file mode 100644 index 00000000..d3212cfd --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Hiperdex.kt @@ -0,0 +1,19 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import okhttp3.Headers + +@MangaSourceParser("HIPERDEX", "Hiperdex", "en", ContentType.HENTAI) +internal class Hiperdex(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.HIPERDEX, "hiperdex.com") { + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("x-cfg-auth", "yceqt7qgu004") + .build() +} \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt similarity index 82% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt rename to src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt index db897ad0..a8e28e68 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/MadaraDex.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/MadaraDex.kt @@ -1,21 +1,30 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.CommonHeaders +import tsuki.network.UserAgents +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.insertCookies +import tsuki.util.nullIfEmpty +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl import okhttp3.HttpUrl.Companion.toHttpUrl import okhttp3.Interceptor import okhttp3.Response import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.exception.ParseException -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.network.CommonHeaders -import org.koitharu.kotatsu.parsers.network.UserAgents -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* private const val F_URL = "fullUrl=" diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt new file mode 100644 index 00000000..2153e4c9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manga18.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.Manga18Parser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGA18", "Manga18", "en", ContentType.HENTAI) +internal class Manga18(context: MangaLoaderContext) : + Manga18Parser(context, MangaParserSource.MANGA18, "manga18.club") diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt new file mode 100644 index 00000000..d31cd509 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manga18fx.kt @@ -0,0 +1,210 @@ +package tsuki.site.en.nsfw + + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.MangaPage +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat + +/** + * Not a real Madara website, it just imitates it good. + */ + +@MangaSourceParser("MANGA18FX", "Manga18fx", "en", ContentType.HENTAI) +internal class Manga18fx(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGA18FX, "manga18fx.com") { + + override val datePattern = "dd MMM yy" + override val selectDesc = ".dsct" + + override val selectDate = "span.chapter-time" + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = false, + isMultipleTagsSupported = false, + ) + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val scraped = doc.select(".header-bottom li a").mapNotNull { a -> + val href = a.attr("href") + val url = if (href.startsWith("http")) href else "https://$domain/$href" + val name = a.text().trim().ifBlank { return@mapNotNull null } + MangaTag(name, url, source) + }.toSet() + + val hardcoded = listOf( + MangaTag("Manhwa", "https://$domain/manga-genre/manhwa", source), + MangaTag("Manhua", "https://$domain/manga-genre/manhua", source), + MangaTag("Raw", "https://$domain/manga-genre/raw", source), + ) + + return scraped + hardcoded + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + if (query.isEmpty() && filter.tags.isNotEmpty()) { + val genreUrl = filter.tags.first().key + val doc = webClient.httpGet(genreUrl).parseHtml() + return parseLatestOrSearch(doc) + } + if (query.isNotEmpty()) { + val url = "https://$domain/search".toHttpUrl().newBuilder() + .addQueryParameter("q", query) + .addQueryParameter("page", page.toString()) + .build() + val doc = webClient.httpGet(url).parseHtml() + return parseLatestOrSearch(doc) + } + + return when (order) { + SortOrder.POPULARITY -> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + parsePopular(doc) + } + else -> { + val url = "https://$domain/page/$page" + val doc = webClient.httpGet(url).parseHtml() + parseLatestOrSearch(doc) + } + } + } + + private fun parsePopular(doc: Document): List<Manga> { + val block = doc.selectFirst(".trending-block") ?: return emptyList() + return block.select("a").map { a -> mangaFromElement(a) } + } + + private fun parseLatestOrSearch(doc: Document): List<Manga> { + return doc.select(".bsx-item").mapNotNull { item -> + val a = item.selectFirst("a") ?: return@mapNotNull null + mangaFromElement(a) + } + } + + private fun mangaFromElement(a: Element): Manga { + val href = a.attrAsRelativeUrl("href").removeSuffix("/") + val title = a.attr("title").ifBlank { a.text() } + val img = a.selectFirst("img") + val cover = img?.attr("data-src").orEmpty().ifBlank { img?.attr("src").orEmpty() } + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> = parseChapters(doc) + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = parseChapters(document) + + private fun parseChapters(doc: Document): List<MangaChapter> { + val container = doc.selectFirst(".row-content-chapter") ?: return emptyList() + val dateFormat = SimpleDateFormat(datePattern, sourceLocale) + + return container.children().mapNotNull { child -> + val a = child.selectFirst("a") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href") + val name = a.text().trim() + val dateText = child.selectFirst("span.chapter-time")?.text() + val number = name.extractChapterNumber() + + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = parseChapterDate(dateFormat, dateText), + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + var images = doc.select("div.page-break img") + if (images.isEmpty()) { + images = doc.select("div.read-content img, div.reading-content img") + } + return images.mapIndexed { index, img -> + val url = img.attr("data-src") + .ifBlank { img.attr("src") } + .ifBlank { throw ParseException("Image URL missing for page $index", fullUrl) } + + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val fullUrl = seed.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + return doc.select(".related-items .item").mapNotNull { item -> + val a = item.selectFirst("a[href*=/manga/]") ?: return@mapNotNull null + val href = a.attrAsRelativeUrl("href").removeSuffix("/") + val title = item.selectFirst("h5 a")?.text()?.trim() + ?: a.attr("title").ifBlank { a.text() } + val img = item.selectFirst("img") + val cover = img?.attr("data-src").orEmpty().ifBlank { img?.attr("src").orEmpty() } + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt new file mode 100644 index 00000000..909a9764 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/MangaDistrict.kt @@ -0,0 +1,94 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.MangaListFilterCapabilities + +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.toTitleCase + +import org.jsoup.HttpStatusException +import org.jsoup.nodes.Document + +@MangaSourceParser("MANGADISTRICT", "MangaDistrict", "en", ContentType.HENTAI) +internal class MangaDistrict(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGADISTRICT, "mangadistrict.com", pageSize = 30) { + + override val tagPrefix = "publication-genre/" + override val withoutAjax = true + override val datePattern = "MMMM d, yyyy" + override val stylePage = "?style=list" + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy( + isMultipleTagsSupported = false, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (filter.tags.isEmpty()) { + return super.getListPage(page, order, filter) + } + + val pages = page + 1 + val genreSlug = filter.tags.first().key + val sortParam = when (order) { + SortOrder.POPULARITY -> "views" + SortOrder.UPDATED -> "latest" + SortOrder.NEWEST -> "new-manga" + SortOrder.ALPHABETICAL -> "alphabet" + SortOrder.RATING -> "rating" + SortOrder.RELEVANCE -> "" + else -> "" + } + + val url = buildString { + append("https://") + append(domain) + append("/$tagPrefix$genreSlug/") + if (pages > 1) { + append("page/") + append(pages) + append("/") + } + append("?m_orderby=") + append(sortParam) + } + + val html = try { + webClient.httpGet(url).parseHtml() + } catch (e: HttpStatusException) { + throw ParseException("Failed to load page: ${e.statusCode}", url) + } + return parseMangaList(html) + } + + override suspend fun getChapters(manga: Manga, doc: Document): List<MangaChapter> { + return super.getChapters(manga, doc) + .sortedWith(compareBy<MangaChapter> { it.number }.thenBy { it.title }) + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val doc = webClient.httpGet("https://$domain/").parseHtml() + val elements = doc.select("div.genres_wrap ul li a") + return elements.mapNotNullToSet { a -> + val href = a.attr("href").removeSuffix("/").substringAfterLast(tagPrefix, "") + if (href.isBlank()) return@mapNotNullToSet null + MangaTag( + key = href, + title = a.text().trim().toTitleCase(), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt similarity index 74% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt rename to src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt index c2b4e8ae..e2883830 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18CC.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18CC.kt @@ -1,33 +1,36 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.host +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.host -import org.koitharu.kotatsu.parsers.util.mapNotNullToSet -import org.koitharu.kotatsu.parsers.util.oneOrThrowIfMany -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectFirstOrThrow -import org.koitharu.kotatsu.parsers.util.src -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.toTitleCase -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.util.HashSet import java.util.Locale @@ -91,15 +94,14 @@ internal class Manhwa18CC(context: MangaLoaderContext) : ) override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query val url = buildString { append("https://") append(domain) when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("/search?q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) append("&page=") append(page.toString()) } diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt similarity index 59% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt rename to src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt index c8ac5f66..ad005d59 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa18Com.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa18Com.kt @@ -1,18 +1,41 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* import java.text.SimpleDateFormat -import java.util.* +import java.util.EnumSet +import java.util.Locale -@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", type = ContentType.HENTAI) +@MangaSourceParser("MANHWA18", "Manhwa18.com", "en", ContentType.HENTAI) internal class Manhwa18Com(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANHWA18, pageSize = 18) { @@ -28,8 +51,13 @@ internal class Manhwa18Com(context: MangaLoaderContext) : .build() override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, SortOrder.POPULARITY, SortOrder.ALPHABETICAL, - SortOrder.NEWEST, SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.POPULARITY, // most viewed + SortOrder.RELEVANCE, // most liked + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.ADDED, // new manga + SortOrder.RATING, ) override val filterCapabilities = MangaListFilterCapabilities( @@ -39,8 +67,6 @@ internal class Manhwa18Com(context: MangaLoaderContext) : isSearchWithFiltersSupported = true, ) - private val chapterDateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) - private val chapterNumberRegex = Regex("""(?:chapter|ch|ep)\s*\.?\s*(\d+(?:\.\d+)?)""", RegexOption.IGNORE_CASE) private val numericRegex = Regex("""\d+""") @@ -73,28 +99,12 @@ internal class Manhwa18Com(context: MangaLoaderContext) : }.toSet() } - private fun extractCoverUrl(doc: org.jsoup.nodes.Document, fallback: String = ""): String { - doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content")?.let { return it } - doc.selectFirst(".lazy-bg")?.attrAsAbsoluteUrl("data-bg")?.let { return it } - val styleUrl = doc.selectFirst(".img-in-ratio")?.attr("style") - ?: doc.selectFirst(".au-cover-art")?.attr("style") - if (styleUrl != null) { - val url = styleUrl.substringAfter("url('").substringBefore("')") - .ifEmpty { styleUrl.substringAfter("url(\"").substringBefore("\"") } - if (url.isNotEmpty()) { - return if (url.startsWith("http")) url else url.toAbsoluteUrl(domain) - } - } - return fallback - } - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } val url = buildString { append("https://$domain/tim-kiem?page=$page") - if (query != null) { + if (filter.query != null) { append("&q=") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } append("&accept_genres=") if (filter.tags.isNotEmpty()) { @@ -108,9 +118,11 @@ internal class Manhwa18Com(context: MangaLoaderContext) : append( when (order) { SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" SortOrder.POPULARITY -> "top" + SortOrder.RELEVANCE -> "like" SortOrder.UPDATED -> "update" - SortOrder.NEWEST -> "new" + SortOrder.ADDED -> "new" SortOrder.RATING -> "like" else -> "update" } @@ -151,59 +163,88 @@ internal class Manhwa18Com(context: MangaLoaderContext) : }.filterNotNull() } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst("h1.au-info-title")?.text()?.trim() ?: manga.title - val coverUrl = extractCoverUrl(doc, manga.coverUrl.orEmpty()) - val description = doc.selectFirst(".au-syn-body.summary-content")?.text()?.trim() + val title = doc.selectFirst("span.series-name a")?.text()?.trim() ?: manga.title + + val coverUrl = doc.selectFirst("meta[property=og:image]")?.attrAsAbsoluteUrl("content") + ?: doc.selectFirst(".img-in-ratio")?.attr("style")?.let { style -> + val url = style.substringAfter("url('").substringBefore("')") + .ifEmpty { style.substringAfter("url(\"").substringBefore("\"") } + if (url.isNotEmpty()) url.toAbsoluteUrl(domain) else null + } ?: manga.coverUrl.orEmpty() - val statusText = doc.select(".au-stat") - .find { it.selectFirst(".k")?.text()?.equals("Status", ignoreCase = true) == true } - ?.selectFirst(".v")?.text()?.trim() + val descElement = doc.selectFirst(".summary-content") + val description = descElement?.let { + it.selectFirst(".summary-more")?.remove() + it.text().trim() + } + + val statusText = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Status:", ignoreCase = true) == true } + ?.selectFirst(".info-value a")?.text()?.trim() val state = when (statusText?.lowercase()) { - "ongoing" -> MangaState.ONGOING + "on going" -> MangaState.ONGOING "completed" -> MangaState.FINISHED "on hold" -> MangaState.PAUSED else -> null } - val tags = doc.select(".au-genres a.au-genre").mapNotNull { a -> + val altTitle = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Other name:", ignoreCase = true) == true } + ?.selectFirst(".info-value")?.text()?.trim() + ?.removeSurrounding("(", ")") + + val genreItem = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Genre:", ignoreCase = true) == true } + val tags = genreItem?.select("a[href*='/genre/']")?.mapNotNull { a -> val slug = a.attr("href").substringAfter("/genre/").substringBefore("/").ifBlank { null } ?: return@mapNotNull null - MangaTag(a.text().trim(), slug, source) - }.toSet() + val name = a.text().trim() + if (slug.isNotEmpty() && name.isNotEmpty()) MangaTag(name, slug, source) else null + }?.toSet() ?: emptySet() + + val author = doc.select(".info-item") + .firstOrNull { it.selectFirst(".info-name")?.text()?.equals("Author:", ignoreCase = true) == true } + ?.selectFirst(".info-value a")?.text()?.trim() val chapters = parseChapters(doc) - val fullManga = manga.copy( + val result = manga.copy( title = title, + altTitles = if (altTitle != null) setOf(altTitle) else emptySet(), coverUrl = coverUrl, description = description, state = state, tags = tags, - authors = emptySet(), + authors = if (author != null) setOf(author) else emptySet(), chapters = chapters, ) - - detailsCache[manga.url] = fullManga - return fullManga + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result } private fun parseChapters(doc: org.jsoup.nodes.Document): List<MangaChapter> { - return doc.select(".au-chgrid a.au-chtile").mapNotNull { a -> + val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ENGLISH) + return doc.select("ul.list-chapters.at-series > a").mapNotNull { a -> val chUrl = a.attrAsAbsoluteUrl("href").toRelativeUrl(domain) - val chTitle = a.selectFirst(".au-chtile-num")?.text()?.trim() ?: return@mapNotNull null - val dateStr = a.selectFirst(".au-chtile-date")?.text()?.substringAfter("· ")?.trim() - val uploadDate = dateStr?.let { - runCatching { chapterDateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + val chTitle = a.selectFirst(".chapter-name")?.text()?.trim() ?: return@mapNotNull null + val dateTimeStr = a.selectFirst(".chapter-time")?.text()?.trim() + val uploadDate = dateTimeStr?.substringAfter(" - ")?.let { + runCatching { dateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L val number = chapterNumberRegex.find(chTitle)?.groupValues?.getOrNull(1)?.toFloatOrNull() ?: numericRegex.find(chUrl)?.value?.toFloatOrNull() diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt similarity index 79% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt rename to src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt index 89aaa5e3..8cca2cce 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/en/hentais/Manhwa210.kt +++ b/src/main/kotlin/tsuki/site/en/nsfw/Manhwa210.kt @@ -1,42 +1,45 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.en.hentais +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.removeSuffix +import tsuki.util.requireSrc +import tsuki.util.selectLast +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.ContentRating -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.Manga -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaListFilter -import org.koitharu.kotatsu.parsers.model.MangaListFilterCapabilities -import org.koitharu.kotatsu.parsers.model.MangaListFilterOptions -import org.koitharu.kotatsu.parsers.model.MangaPage -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.model.MangaState -import org.koitharu.kotatsu.parsers.model.MangaTag -import org.koitharu.kotatsu.parsers.model.RATING_UNKNOWN -import org.koitharu.kotatsu.parsers.model.SortOrder -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrl -import org.koitharu.kotatsu.parsers.util.attrOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapToSet -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml -import org.koitharu.kotatsu.parsers.util.removeSuffix -import org.koitharu.kotatsu.parsers.util.requireSrc -import org.koitharu.kotatsu.parsers.util.selectLast -import org.koitharu.kotatsu.parsers.util.textOrNull -import org.koitharu.kotatsu.parsers.util.toAbsoluteUrl -import org.koitharu.kotatsu.parsers.util.urlEncoded import java.text.SimpleDateFormat import java.util.EnumSet import java.util.Locale -@MangaSourceParser("MANHWA210", "Manhwa210", "en", type = ContentType.HENTAI) +@MangaSourceParser("MANHWA210", "Manhwa210", "en", ContentType.HENTAI) internal class Manhwa210(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.MANHWA210, 60) { @@ -91,7 +94,8 @@ internal class Manhwa210(context: MangaLoaderContext) : } } - @get:Synchronized + private val detailsCacheLock = Any() + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 } @@ -99,12 +103,14 @@ internal class Manhwa210(context: MangaLoaderContext) : private val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.ENGLISH) override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() val author = root.selectFirst("div.mt-2:contains(Artist) span a")?.textOrNull() - val details = manga.copy( + val result = manga.copy( altTitles = setOfNotNull(root.selectLast("div.grow div:contains(Alt name) span")?.textOrNull()), state = when (root.selectFirst("div.mt-2:contains(Status) span.text-blue-500")?.text()) { "Ongoing" -> MangaState.ONGOING @@ -140,9 +146,10 @@ internal class Manhwa210(context: MangaLoaderContext) : } .sortedBy { it.number } ) - - detailsCache[manga.url] = details - return details + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result } private fun extractChapterNumber(title: String): Float { diff --git a/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt new file mode 100644 index 00000000..2184b76e --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/ManhwaRead.kt @@ -0,0 +1,311 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.util.Base64 +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.nio.charset.StandardCharsets +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANHWAREAD", "ManhwaRead", "en", ContentType.HENTAI) +internal class ManhwaRead(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANHWAREAD, pageSize = 12) { + + override val configKeyDomain = ConfigKey.Domain("manhwaread.com") + private val baseUrl = "https://$domain" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .build() + + private val dateFormat = SimpleDateFormat("dd/MM/yyyy", Locale.ROOT) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, // A-Z + SortOrder.RATING, // rating + SortOrder.POPULARITY, // all time + SortOrder.POPULARITY_TODAY, // daily + SortOrder.POPULARITY_WEEK, // weekly + SortOrder.POPULARITY_MONTH, // monthly + SortOrder.POPULARITY_YEAR, // yearly + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + companion object { + val GENRE_ID_MAP = mapOf( + "Action" to "650", "Adventure" to "645", "Comedy" to "536", + "Drama" to "530", "Ecchi" to "537", "Fantasy" to "646", + "Hentai" to "531", "Horror" to "590", "Isekai" to "2735", + "Mahou Shoujo" to "2696", "Mystery" to "626", "Psychological" to "591", + "Romance" to "538", "Sci-Fi" to "688", "Slice of Life" to "532", + "Sports" to "677", "Supernatural" to "544", "Thriller" to "580" + ) + val allTags: Set<MangaTag> = run { + val genreTags = GENRE_ID_MAP.keys.map { name -> + MangaTag(name, "genre:${GENRE_ID_MAP[name]}", MangaParserSource.MANHWAREAD) + } + genreTags.toSet() + } + } + + private val tags by lazy { allTags } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = tags, + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortBy = when (order) { + SortOrder.UPDATED -> "release" + SortOrder.NEWEST -> "new" +// SortOrder.ALPHABETICAL -> "alphabet" +// SortOrder.RATING -> "rating" +// SortOrder.POPULARITY -> "all_top" +// SortOrder.POPULARITY_TODAY -> "daily_top" +// SortOrder.POPULARITY_WEEK -> "weekly_top" +// SortOrder.POPULARITY_MONTH -> "monthly_top" +// SortOrder.POPULARITY_YEAR -> "yearly_top" + // doenst work idk why? search resulsts are different too + else -> "release" + } + + val url = buildString { + append(baseUrl) + if (page > 1) { + append("/page/") + append(page) + append("/") + } + + val params = linkedMapOf<String, String?>() + params["s"] = filter.query ?: "" + params["sortby"] = sortBy + + if (!filter.query.isNullOrBlank()) { + params["keyword_mode"] = "contain" + params["s_mode"] = "AND" + } + + filter.states.oneOrThrowIfMany()?.let { state -> + params["status"] = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "on-hold" + MangaState.ABANDONED -> "canceled" + else -> null + } + } + + append(params.toQueryString()) + + val includedGenres = mutableListOf<String>() + val includedTags = mutableListOf<String>() + val excludedTags = mutableListOf<String>() + + filter.tags.forEach { tag -> + val key = tag.key + when { + key.startsWith("genre:") -> includedGenres.add(key.removePrefix("genre:")) + key.startsWith("tag:") -> includedTags.add(key.removePrefix("tag:")) + } + } + filter.tagsExclude.forEach { tag -> + val key = tag.key + when { + key.startsWith("tag:") -> excludedTags.add(key.removePrefix("tag:")) + } + } + + includedGenres.forEach { append("&genres[]=$it") } + includedTags.forEach { append("&including[]=$it") } + excludedTags.forEach { append("&excluding[]=$it") } + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select(".main-container .manga-item").mapNotNull { card -> + val link = card.selectFirst("a.manga-item__link") ?: return@mapNotNull null + val href = link.absUrl("href") + val title = link.text().trim() + val cover = card.selectFirst(".manga-item__img img")?.absUrl("src") + Manga( + id = generateUid(href), + url = href.removePrefix(baseUrl), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = cover, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + @get:Synchronized + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override suspend fun getDetails(manga: Manga): Manga { + detailsCache[manga.url]?.let { return it } + + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst("#mangaSummary .manga-titles h1")?.text() ?: manga.title + val cover = doc.selectFirst("head meta[property=og:image]")?.absUrl("content") ?: manga.coverUrl + + val author = doc.select("#mangaSummary .text-primary:contains(Author:) + .flex a span:first-child").text().trim() + val artist = doc.select("#mangaSummary .text-primary:contains(Artist:) + .flex a span:first-child").text().trim() + val publisher = doc.select("#mangaSummary .text-primary:contains(Publisher:) + .flex a span:first-child").text().trim() + + val statusText = doc.selectFirst("#mangaSummary .manga-status")?.attr("data-status") + val state = when (statusText) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "canceled" -> MangaState.ABANDONED + "on-hold" -> MangaState.PAUSED + else -> null + } + + val genres = doc.select("#mangaSummary .manga-genres a").map { it.text() } + val tags = doc.select("#mangaSummary .text-primary:contains(Tags:) + .flex a span:first-child").map { it.text() } + val allTags = (genres + tags).map { name -> + val genreId = GENRE_ID_MAP[name] + if (genreId != null) MangaTag(name, "genre:$genreId", source) + else MangaTag(name, "tag:$name", source) + }.toSet() + + val descBuilder = StringBuilder() + if (publisher.isNotEmpty()) descBuilder.append("Publisher: $publisher\n") + doc.selectFirst("#mangaDesc > .manga-desc__content")?.text()?.let { descBuilder.append("\n$it") } + doc.selectFirst("#mangaSummary .manga-titles h2")?.text()?.split("|")?.joinToString("\n") { "- " + it.trim() }?.let { altTitles -> + if (altTitles.isNotEmpty()) descBuilder.append("\n\nAlternative titles:\n$altTitles") + } + + val chapters = parseChapters(doc) + + val result = manga.copy( + title = title, + coverUrl = cover, + authors = setOfNotNull(author.takeIf { it.isNotEmpty() }, artist.takeIf { it.isNotEmpty() }), + state = state, + tags = allTags, + description = descBuilder.toString().trim(), + chapters = chapters, + ) + + detailsCache[manga.url] = result + return result + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + return doc.select("#chaptersList > a.chapter-item").mapNotNull { el -> + val href = el.absUrl("href") + val name = el.selectFirst("span.chapter-item__name")?.text() ?: return@mapNotNull null + val dateStr = el.selectFirst("span.chapter-item__date")?.text() + val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L + MangaChapter( + id = generateUid(href), + title = name, + number = 0f, + volume = 0, + url = href.removePrefix(baseUrl), + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + val script = doc.selectFirst("script:containsData(chapterData)") + ?: throw Exception("Chapter data not found") + val dataMatch = CHAPTER_DATA_REGEX.find(script.data()) ?: throw Exception("Chapter data regex failed") + val jsonStr = dataMatch.groupValues[1] + val chapterData = JSONObject(jsonStr) + val base = chapterData.getString("base") + val encodedData = chapterData.getString("data") + val decodedBytes = Base64.getDecoder().decode(encodedData) + val decodedStr = String(decodedBytes, StandardCharsets.UTF_8) + val imagesArray = JSONArray(decodedStr) + + return (0 until imagesArray.length()).map { i -> + val imgObj = imagesArray.getJSONObject(i) + val imgSrc = imgObj.getString("src") + val imgUrl = "$base/$imgSrc" + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private fun SimpleDateFormat.parseSafe(date: String): Long { + return try { parse(date)?.time ?: 0L } catch (_: Exception) { 0L } + } + + private fun Map<String, String?>.toQueryString(): String { + val sb = StringBuilder() + for ((key, value) in this) { + if (!value.isNullOrEmpty()) { + if (sb.isNotEmpty()) sb.append("&") + sb.append(key).append("=").append(value.urlEncoded()) + } + } + return if (sb.isNotEmpty()) "?$sb" else "" + } + private val CHAPTER_DATA_REGEX = Regex("""var\s+chapterData\s*=\s*(\{.*\})""") +} diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt b/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt new file mode 100644 index 00000000..0c0b284d --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Milftoon.kt @@ -0,0 +1,15 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("MILFTOON", "MilfToon", "en", ContentType.HENTAI) +internal class MilfToon(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MILFTOON, "milftoon.xxx", 20) { + override val withoutAjax = true + override val listUrl = "comics/" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt b/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt new file mode 100644 index 00000000..29029202 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/NovelCrow.kt @@ -0,0 +1,17 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("NOVELCROW", "NovelCrow", "en", ContentType.HENTAI) +internal class Novelcrow(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.NOVELCROW, "novelcrow.com", 24) { + + override val withoutAjax = true + override val tagPrefix = "genres-2/" + override val listUrl = "" +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt new file mode 100644 index 00000000..20860b12 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/OmegaScan.kt @@ -0,0 +1,12 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HeanCmsParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("OMEGASCANS", "OmegaScans", "en", ContentType.HENTAI) +internal class OmegaScans(context: MangaLoaderContext) : + HeanCmsParser(context, MangaParserSource.OMEGASCANS, "omegascans.org") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt new file mode 100644 index 00000000..242b3dd5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/TempleScan.kt @@ -0,0 +1,368 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.generateUid +import tsuki.util.json.extractNextJs +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.extractChapterNumber + +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Response +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("TEMPLESCAN", "Temple Scan", "en", ContentType.HENTAI) +internal class TempleScan(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.TEMPLESCAN, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("templetoons.com") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Origin", "https://$domain") + .set("User-Agent", config[userAgentKey]) + return builder.build() + } + + private val rscHeaders get() = getRequestHeaders().newBuilder() + .set("rsc", "1") + .build() + + @Volatile + private var seriesCache: List<SeriesItem>? = null + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (seriesCache == null) { + val response = webClient.httpGet("https://$domain/comics".toHttpUrl(), rscHeaders) + seriesCache = parseBrowseSeries(response) + } + + val allSeries = seriesCache ?: return emptyList() + + val statusFilter = filter.states.firstOrNull() + val filtered = allSeries.filter { series -> + val query = filter.query?.trim()?.takeIf(String::isNotBlank) + val matchesQuery = if (query != null) { + series.title.contains(query, ignoreCase = true) || + (series.alternativeNames?.contains(query, ignoreCase = true) == true) + } else true + val matchesStatus = statusFilter == null || series.status == stateToApi(statusFilter) + matchesQuery && matchesStatus + } + + val sorted = when (order) { + SortOrder.UPDATED -> filtered.sortedByDescending { it.updated } + SortOrder.NEWEST -> filtered.sortedByDescending { it.created } + SortOrder.POPULARITY -> filtered.sortedByDescending { it.views } + SortOrder.ALPHABETICAL -> filtered.sortedBy { it.title } + else -> filtered + } + + val startIndex = (page - 1) * pageSize + val endIndex = minOf(startIndex + pageSize, sorted.size) + if (startIndex >= sorted.size) return emptyList() + + return sorted.subList(startIndex, endIndex).map { item -> + Manga( + id = generateUid("/comic/${item.slug}"), + url = "/comic/${item.slug}", + publicUrl = "https://$domain/comic/${item.slug}", + title = item.title, + altTitles = emptySet(), + coverUrl = item.thumbnail?.toAbsoluteUrl(domain), + authors = emptySet(), + state = when (item.status) { + "Ongoing" -> MangaState.ONGOING + "Hiatus" -> MangaState.PAUSED + "Completed" -> MangaState.FINISHED + "Canceled", "Dropped" -> MangaState.ABANDONED + else -> null + }, + contentRating = null, + tags = emptySet(), + rating = RATING_UNKNOWN, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removePrefix("/comic/") + val response = webClient.httpGet("https://$domain/comic/$slug".toHttpUrl(), rscHeaders) + val details = parseSeriesDetails(response) + + return manga.copy( + title = details.title, + altTitles = details.alternativeNames?.let { setOf(it) } ?: emptySet(), + coverUrl = details.thumbnail?.toAbsoluteUrl(domain), + description = details.description?.let { desc -> + val cleanDesc = if (desc.contains("#")) desc.substringBefore("#") + .replace(TRAILING_WORD_REGEX, "").trim() + else desc + Jsoup.parse(cleanDesc).text() + }?.trim().orEmpty(), + authors = details.author?.let { setOf(it) } ?: emptySet(), + state = when (details.status) { + "Ongoing" -> MangaState.ONGOING + "Hiatus" -> MangaState.PAUSED + "Completed" -> MangaState.FINISHED + "Canceled", "Dropped" -> MangaState.ABANDONED + else -> null + }, + contentRating = if (details.adult) ContentRating.ADULT else ContentRating.SAFE, + tags = buildSet { + details.badge?.let { add(MangaTag(it.lowercase(), it, source)) } + details.year?.let { add(MangaTag(it, it, source)) } + if (details.adult) add(MangaTag("adult", "Adult", source)) + details.tags?.forEach { tag -> + add(MangaTag(tag.tag.name.lowercase(), tag.tag.name, source)) + } + details.description?.let { desc -> + TAG_REGEX.findAll(desc).forEach { match -> + val tagName = match.groupValues[1] + add(MangaTag(tagName.lowercase(), tagName, source)) + } + } + }, + chapters = getChapterList(slug), + ) + } + + private suspend fun getChapterList(mangaSlug: String): List<MangaChapter> { + val response = webClient.httpGet("https://$domain/comic/$mangaSlug".toHttpUrl(), rscHeaders) + val chapterData = parseChapterList(response) ?: return emptyList() + + return chapterData.seasons.flatMap { season -> + season.chapters.filter { it.price == 0 }.map { chapter -> + MangaChapter( + id = generateUid("$mangaSlug/${chapter.slug}"), + title = chapter.name, + number = chapter.name.extractChapterNumber(), + url = "/comic/$mangaSlug/${chapter.slug}", + uploadDate = chapter.created, + source = source, + volume = 0, + scanlator = null, + branch = null, + ) + } + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val response = webClient.httpGet("https://$domain${chapter.url}".toHttpUrl(), rscHeaders) + val pagesData = parsePagesList(response) + return pagesData.pages.mapIndexed { _, url -> + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(domain), + preview = null, + source = source, + ) + } + } + + private fun parseBrowseSeries(response: Response): List<SeriesItem> { + val jsonArray: JSONArray = response.extractNextJs { + it is JSONArray && it.length() > 0 && it.optJSONObject(0)?.has("series_slug") == true + } as? JSONArray + ?: throw RuntimeException("Browse series array not found") + + return (0 until jsonArray.length()).map { i -> + jsonArray.getJSONObject(i).toSeriesItem() + } + } + + private fun parseSeriesDetails(response: Response): SeriesDetails { + val obj: JSONObject = response.extractNextJs { + it is JSONObject && it.has("series_slug") && it.has("title") + } as? JSONObject + ?: throw RuntimeException("Series details not found") + return obj.toSeriesDetails() + } + + private fun parseChapterList(response: Response): ChapterList? { + val obj: JSONObject? = response.extractNextJs { + it is JSONObject && it.has("Season") + } as? JSONObject + return obj?.toChapterList() + } + + private fun parsePagesList(response: Response): PagesList { + val obj: JSONObject = response.extractNextJs { + it is JSONObject && it.has("pages") + } as? JSONObject + ?: throw RuntimeException("Pages list not found") + return obj.toPagesList() + } + + private fun JSONObject.toSeriesItem(): SeriesItem { + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + return SeriesItem( + slug = getString("series_slug"), + title = getString("title"), + alternativeNames = optString("alternative_names", null), + thumbnail = optString("thumbnail", null), + status = optString("status", null), + updated = dateFormat.parseSafe(optString("update_chapter", null)), + created = dateFormat.parseSafe(optString("created_at", null)), + views = optLong("total_views", 0), + ) + } + + private fun JSONObject.toSeriesDetails(): SeriesDetails { + return SeriesDetails( + slug = getString("series_slug"), + title = getString("title"), + thumbnail = optString("thumbnail", null), + author = optString("author", null), + studio = optString("studio", null), + year = optString("release_year", null), + alternativeNames = optString("alternative_names", null), + adult = optBoolean("adult", false), + badge = optString("badge", null), + status = optString("status", null), + description = optString("description", null), + tags = optJSONArray("tag_series")?.let { arr -> + (0 until arr.length()).map { i -> + val tagObj = arr.getJSONObject(i).getJSONObject("tag") + TagWrapper(Tag(tagObj.getString("name"))) + } + }, + ) + } + + private fun JSONObject.toChapterList(): ChapterList { + val seasonsArray = getJSONArray("Season") + val seasons = (0 until seasonsArray.length()).map { i -> + val seasonObj = seasonsArray.getJSONObject(i) + val chaptersArray = seasonObj.getJSONArray("Chapter") + val chapters = (0 until chaptersArray.length()).map { j -> + val chap = chaptersArray.getJSONObject(j) + ChapterItem( + name = chap.getString("chapter_name"), + title = chap.optString("chapter_title", null), + slug = chap.getString("chapter_slug"), + price = chap.getInt("price"), + created = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ENGLISH) + .parseSafe(chap.optString("created_at", null)), + ) + } + Season(chapters) + } + return ChapterList(seasons) + } + + private fun JSONObject.toPagesList(): PagesList { + val pagesArray = getJSONArray("pages") + val pages = (0 until pagesArray.length()).map { pagesArray.getString(it) } + return PagesList(pages) + } + + private fun stateToApi(state: MangaState): String = when (state) { + MangaState.ONGOING -> "Ongoing" + MangaState.FINISHED -> "Completed" + MangaState.PAUSED -> "Hiatus" + MangaState.ABANDONED -> "Canceled" + else -> "" + } + + data class SeriesItem( + val slug: String, + val title: String, + val alternativeNames: String?, + val thumbnail: String?, + val status: String?, + val updated: Long, + val created: Long, + val views: Long, + ) + + data class SeriesDetails( + val slug: String, + val title: String, + val thumbnail: String?, + val author: String?, + val studio: String?, + val year: String?, + val alternativeNames: String?, + val adult: Boolean, + val badge: String?, + val status: String?, + val description: String?, + val tags: List<TagWrapper>?, + ) + + data class TagWrapper(val tag: Tag) + data class Tag(val name: String) + + data class ChapterList(val seasons: List<Season>) + data class Season(val chapters: List<ChapterItem>) + data class ChapterItem( + val name: String, + val title: String?, + val slug: String, + val price: Int, + val created: Long, + ) + + data class PagesList(val pages: List<String>) + + companion object { + private val TAG_REGEX = Regex("""(?i)#(\w+)""") + private val TRAILING_WORD_REGEX = Regex("""[\w\s]+:?\s*$""") + } +} diff --git a/src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt b/src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt new file mode 100644 index 00000000..ea7b085a --- /dev/null +++ b/src/main/kotlin/tsuki/site/en/nsfw/Toonily.kt @@ -0,0 +1,15 @@ +package tsuki.site.en.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("TOONILY", "Toonily", "en", ContentType.HENTAI) +internal class Toonily(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.TOONILY, "toonily.com") { + override val listUrl = "webtoon/" + override val datePattern = "MMMM dd, yyyy" +} diff --git a/src/main/kotlin/tsuki/site/es/LectorManga.kt b/src/main/kotlin/tsuki/site/es/LectorManga.kt new file mode 100644 index 00000000..8b66c6d1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/LectorManga.kt @@ -0,0 +1,229 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import java.util.EnumSet + +// Mirror of zerocomics.net? + +@MangaSourceParser("LECTORMANGA", "Lector Manga", "es") +internal class LectorManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.LECTORMANGA, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("lectormangass.net") + private val baseUrl = "https://$domain" + private val apiBase = "https://api.zerocomics.net/api" + private val siteHeader = "lectormanga" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + private val apiHeaders: Headers + get() = Headers.Builder() + .add("Accept", "application/json") + .add("X-Site", siteHeader) + .add("User-Agent", config[userAgentKey]) + .build() + + override suspend fun getFilterOptions() = MangaListFilterOptions() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val (sortParam, orderParam) = when (order) { + SortOrder.UPDATED -> "" to "" + SortOrder.RATING -> "rating" to "desc" + SortOrder.ALPHABETICAL -> "name" to "asc" + else -> "" to "" + } + + val url = "$apiBase/series".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("sort", sortParam) + .addQueryParameter("order", orderParam) + + if (filter.query != null) { + url.addQueryParameter("q", filter.query) + } + + val json = webClient.httpGet(url.build(), apiHeaders).parseJson() + val data = json.optJSONArray("data") ?: return emptyList() + + return (0 until data.length()).map { i -> + val obj = data.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getString("titulo") + val cover = obj.optString("portada", "") + Manga( + id = generateUid(slug), + url = slug, + publicUrl = "$baseUrl/comics/$slug", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url + val doc = webClient.httpGet("$baseUrl/comics/$slug").parseHtml() + + val title = doc.selectFirst("h1.info-title")?.text()?.trim() ?: manga.title + val cover = doc.selectFirst(".cover-image img")?.attr("abs:src") + ?: doc.selectFirst("meta[property='og:image']")?.attr("content") + ?: manga.coverUrl + + val description = doc.selectFirst(".info-desc-text")?.text()?.trim() + + val genres = doc.select(".genre-chips-wrap a.v-chip") + .map { it.text().trim() } + .filter { it.isNotBlank() } + .toSet() + + val author = doc.selectFirst("meta[name=author]")?.attr("content") + val artist = doc.selectFirst("meta[name=artist]")?.attr("content") + + val statusText = doc.selectFirst(".status-chip span")?.text()?.trim() + val state = when (statusText?.lowercase()) { + "en emisión" -> MangaState.ONGOING + "finalizado" -> MangaState.FINISHED + "pausado" -> MangaState.PAUSED + "abandonado" -> MangaState.ABANDONED + else -> null + } + + val chapters = parseChapters(doc) + + val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() + val type = doc.selectFirst("meta[name=type]")?.attr("content") + if (!type.isNullOrBlank()) tagSet += MangaTag(type, type, source) + + return manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + description = description, + authors = setOfNotNull(author, artist).ifEmpty { emptySet() }, + tags = tagSet, + state = state, + chapters = chapters, + contentRating = ContentRating.SAFE, + ) + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + val items = doc.select("div[data-chapter-num]") + return items.mapNotNull { item -> + val link = item.selectFirst("a") ?: return@mapNotNull null + val href = link.attr("href").removePrefix("/") // e.g., "comics/.../capitulo-20" + + val chapterSlug = href.substringAfterLast("/") // "capitulo-20" + val number = chapterSlug.substringAfter("capitulo-", "").toFloatOrNull() ?: 0f + + val titleDiv = link.selectFirst("div.d-flex.justify-space-between > div:first-child") + val title = titleDiv?.text()?.trim() ?: "Capítulo ${number.toInt()}" + + val dateDiv = link.selectFirst("div.text--disabled div.d-flex > div:first-child") + val dateText = dateDiv?.text()?.trim() + val uploadDate = parseRelativeDate(dateText) + + MangaChapter( + id = generateUid("/$href"), + url = "/$href", + title = title, + number = number, + volume = 0, + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val readerUrl = "$baseUrl/${chapter.url.removePrefix("/")}" + val doc = webClient.httpGet(readerUrl).parseHtml() + + val images = doc.select("div.reader-images img, img.reader-page, img[src*='bmcdn.my.id']") + if (images.isNotEmpty()) { + return images.map { img -> + val url = img.attr("abs:src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + return doc.select("img").map { img -> + val url = img.attr("abs:src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun parseRelativeDate(text: String?): Long { + if (text.isNullOrBlank()) return 0L + val clean = text.trim().removePrefix("hace ").trim() + val parts = clean.split(" ") + if (parts.size != 2) return 0L + val amount = parts[0].toIntOrNull() ?: return 0L + val unit = parts[1].lowercase() + val now = System.currentTimeMillis() + val millis = when (unit) { + "h" -> amount * 60 * 60 * 1000L + "min", "minuto", "minutos" -> amount * 60 * 1000L + "día", "días", "dia", "dias" -> amount * 24 * 60 * 60 * 1000L + "sem", "semanas", "semana" -> amount * 7 * 24 * 60 * 60 * 1000L + "mes", "meses" -> amount * 30 * 24 * 60 * 60 * 1000L + "año", "años", "ano", "anos" -> amount * 365 * 24 * 60 * 60 * 1000L + else -> 0L + } + return if (millis > 0) now - millis else 0L + } +} diff --git a/src/main/kotlin/tsuki/site/es/MangaOni.kt b/src/main/kotlin/tsuki/site/es/MangaOni.kt new file mode 100644 index 00000000..08ad9124 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/MangaOni.kt @@ -0,0 +1,360 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import java.nio.charset.Charset +import java.text.SimpleDateFormat +import java.util.Base64 +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAONI", "MangaOni", "es") +internal class MangaOni(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAONI, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("manga-oni.com") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd HH:mm:ss", Locale.US) + + private val hideNsfwKey = ConfigKey.ShowSuspiciousContent(false) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(hideNsfwKey) + } + + override fun getRequestHeaders(): okhttp3.Headers { + val builder = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Origin", "https://$domain") + .set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8") + .set("Accept-Language", "es-ES,es;q=0.9,en;q=0.8") + .set("User-agent", config[userAgentKey]) + return builder.build() + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, // visitas + SortOrder.UPDATED, // recientes + SortOrder.ALPHABETICAL, // nombre + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (_, name, id) -> + MangaTag(key = id, title = name, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, // "En desarrollo" -> 1 + MangaState.FINISHED, // "Completo" -> 0 + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, // 0 + ContentType.MANHWA, // 1 + ContentType.OTHER, // "One Shot" 2 + ContentType.MANHUA, // 3 + ContentType.NOVEL, // 4 + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + if (query.isNotEmpty()) { + return searchManga(query, page) + } + + val url = when (order) { + SortOrder.UPDATED -> "https://$domain/recientes?p=$page" + else -> buildDirectoryUrl(page, order, filter) + } + + val doc = webClient.httpGet(url).parseHtml() + return when (order) { + SortOrder.UPDATED -> parseLatestListing(doc) + else -> parseDirectoryListing(doc) + } + } + + private fun buildDirectoryUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return StringBuilder("https://$domain/directorio?").apply { + append("genero=") + append(filter.tags.firstOrNull()?.key ?: "false") + + append("&estado=") + append( + filter.states.firstOrNull()?.let { + when (it) { + MangaState.ONGOING -> "1" + MangaState.FINISHED -> "0" + else -> null + } + } ?: "false" + ) + + append("&tipo=") + append( + filter.types.firstOrNull()?.let { + when (it) { + ContentType.MANGA -> "0" + ContentType.MANHWA -> "1" + ContentType.OTHER -> "2" + ContentType.MANHUA -> "3" + ContentType.NOVEL -> "4" + else -> null + } + } ?: "false" + ) + + append("&adulto=") + if (config[hideNsfwKey]) { + append("0") + } else { + when (filter.contentRating.oneOrThrowIfMany()) { + ContentRating.ADULT -> append("1") + ContentRating.SAFE -> append("0") + else -> append("false") + } + } + + append("&filtro=") + append( + when (order) { + SortOrder.POPULARITY -> "visitas" + SortOrder.ALPHABETICAL -> "nombre" + else -> "visitas" + } + ) + append("&orden=") + append(if (order == SortOrder.ALPHABETICAL) "asc" else "desc") + + append("&p=$page") + }.toString() + } + + private suspend fun searchManga(query: String, page: Int): List<Manga> { + val url = "https://$domain/buscar?q=${query.urlEncoded()}&p=$page" + val doc = webClient.httpGet(url).parseHtml() + return parseSearchResult(doc) + } + + private fun parseLatestListing(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("div._1bJU3").map { element -> + val a = element.selectFirst("a._2dU-m") ?: return@map null + val href = a.attrAsRelativeUrl("href") + val cover = a.selectFirst("img")?.attr("data-src") ?: a.selectFirst("img")?.attr("src") + val title = element.selectFirst("a[data-test=latest-update-name]")?.text() ?: return@map null + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover?.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + private fun parseDirectoryListing(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("#article-div a").map { element -> + val href = element.attrAsRelativeUrl("href") + val cover = element.select("img").attr("abs:data-src").ifEmpty { + element.select("img").attr("abs:src") + } + val title = element.select("div:eq(1)").text() + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private fun parseSearchResult(doc: org.jsoup.nodes.Document): List<Manga> { + return doc.select("#article-div > div").map { element -> + val a = element.selectFirst("a") ?: return@map null + val href = a.attrAsRelativeUrl("href") + val title = a.text().ifEmpty { element.select("a").lastOrNull()?.text() } ?: return@map null + val cover = element.select("img").attr("abs:src") + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = cover.ifEmpty { null }, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + }.filterNotNull() + } + + // ───────────────────────── DETAILS ──────────────────────────── + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.publicUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.select("img[src*=cover]").attr("abs:src") + val description = doc.select("div#sinopsis").lastOrNull()?.ownText() + val author = doc.select("div#info-i").text().let { info -> + if (info.contains("Autor", true)) { + info.substringAfter("Autor:").substringBefore("Fecha:").trim() + } else null + } ?: "N/A" + + val genres = doc.select("div#categ a").eachText() + val tags = genres.map { g -> MangaTag(key = g.lowercase(), title = g, source = source) }.toSet() + + val status = when (doc.selectFirst("strong:contains(Estado) + span")?.text()) { + "En desarrollo" -> MangaState.ONGOING + "Finalizado" -> MangaState.FINISHED + else -> null + } + + val chapters = doc.select("div#c_list a").mapIndexed { _, element -> + val chHref = element.attrAsRelativeUrl("href") + val name = element.text() + val chNum = element.select("span").attr("data-num").toFloatOrNull() ?: -1f + val dateStr = element.select("span").attr("datetime") + val uploadDate = dateStr?.let { dateFormat.parseSafe(it) } ?: 0L + MangaChapter( + id = generateUid(chHref), + url = chHref, + title = name, + number = chNum, + volume = 0, + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + coverUrl = cover.ifEmpty { manga.coverUrl }, + description = description, + authors = setOf(author), + tags = tags, + state = status, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val script = doc.selectFirst("script:containsData(unicap)") + ?.data() ?: throw Exception("No se encontró unicap") + val encoded = script.substringAfter("'").substringBefore("'") + val decoded = Base64.getDecoder().decode(encoded) + .toString(Charset.defaultCharset()) + val path = decoded.substringBefore("||") + val filesStr = decoded.substringAfter("[").substringBefore("]") + val files = filesStr.split(",").map { it.trim().removeSurrounding("\"") } + return files.mapIndexed { _, file -> + val imageUrl = path + file + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time }.getOrDefault(0L) } ?: 0L + } + + companion object { + private val GENRES = listOf( + Triple("Todos", "Todos", "false"), + Triple("Comedia", "Comedia", "1"), + Triple("Drama", "Drama", "2"), + Triple("Acción", "Acción", "3"), + Triple("Escolar", "Escolar", "4"), + Triple("Romance", "Romance", "5"), + Triple("Ecchi", "Ecchi", "6"), + Triple("Aventura", "Aventura", "7"), + Triple("ShÅnen", "ShÅnen", "8"), + Triple("ShÅjo", "ShÅjo", "9"), + Triple("Deportes", "Deportes", "10"), + Triple("Psicológico", "Psicológico", "11"), + Triple("Fantasía", "Fantasía", "12"), + Triple("Mecha", "Mecha", "13"), + Triple("Gore", "Gore", "14"), + Triple("Yaoi", "Yaoi", "15"), + Triple("Yuri", "Yuri", "16"), + Triple("Misterio", "Misterio", "17"), + Triple("Sobrenatural", "Sobrenatural", "18"), + Triple("Seinen", "Seinen", "19"), + Triple("Ficción", "Ficción", "20"), + Triple("Harem", "Harem", "21"), + Triple("Webtoon", "Webtoon", "25"), + Triple("Histórico", "Histórico", "27"), + Triple("Músical", "Músical", "30"), + Triple("Ciencia ficción", "Ciencia ficción", "31"), + Triple("ShÅjo-ai", "ShÅjo-ai", "32"), + Triple("Josei", "Josei", "33"), + Triple("Magia", "Magia", "34"), + Triple("Artes Marciales", "Artes Marciales", "35"), + Triple("Horror", "Horror", "36"), + Triple("Demonios", "Demonios", "37"), + Triple("Supervivencia", "Supervivencia", "38"), + Triple("Recuentos de la vida", "Recuentos de la vida", "39"), + Triple("ShÅnen ai", "ShÅnen ai", "40"), + Triple("Militar", "Militar", "41"), + Triple("Eroge", "Eroge", "42"), + Triple("Isekai", "Isekai", "43"), + ) + } +} diff --git a/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt new file mode 100644 index 00000000..5ae2c923 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/OlympusScanlation.kt @@ -0,0 +1,266 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.TimeUnit + +@MangaSourceParser("OLYMPUSSCANLATION", "Olympus Scanlation", "es") +internal class OlympusScanlation(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.OLYMPUSSCANLATION, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("olympusxyz.com") + private val baseUrl = "https://$domain" + private val apiBaseUrl = "https://panel.olympusxyz.com" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept", "application/json") + .set("X-Requested-With", "XMLHttpRequest") + .build() + + @Volatile + private var seriesList: List<MangaDto> = emptyList() + @Volatile + private var lastFetchTime: Long = 0L + private val seriesMutex = Mutex() + private val slugMap = ConcurrentHashMap<String, String>() + + private suspend fun fetchSeriesListIfNeeded() { + val now = System.currentTimeMillis() + if (seriesList.isNotEmpty() && (now - lastFetchTime) < TimeUnit.HOURS.toMillis(1)) return + seriesMutex.withLock { + if (seriesList.isNotEmpty() && (now - lastFetchTime) < TimeUnit.HOURS.toMillis(1)) return + val response = webClient.httpGet("$baseUrl/api/series/list", getRequestHeaders()) + val json = response.parseJson() + val dataArray = json.optJSONArray("data") ?: return + val comics = mutableListOf<MangaDto>() + for (i in 0 until dataArray.length()) { + val obj = dataArray.optJSONObject(i) ?: continue + if (obj.optString("type") != "comic") continue + val id = obj.getInt("id") + val name = obj.optString("name", "") + val slug = obj.optString("slug", "") + val cover = obj.optString("cover", "") + val summary = obj.optString("summary", "") + val statusId = obj.optJSONObject("status")?.optInt("id") ?: -1 + val genres = mutableListOf<String>() + obj.optJSONArray("genres")?.let { arr -> + for (j in 0 until arr.length()) { + arr.optJSONObject(j)?.optString("name", "")?.takeIf { it.isNotEmpty() }?.let(genres::add) + } + } + comics.add(MangaDto(id, name, slug, cover, summary, statusId, genres)) + slugMap[id.toString()] = slug + } + seriesList = comics + lastFetchTime = now + } + } + + private data class MangaDto( + val id: Int, val name: String, val slug: String, val cover: String, + val summary: String, val statusId: Int, val genres: List<String> + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of( + MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED, MangaState.ABANDONED, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + fetchSeriesListIfNeeded() + + val query = filter.query?.trim().orEmpty() + + if (query.isNotEmpty()) { + val filtered = seriesList.filter { it.name.contains(query, ignoreCase = true) } + val start = (page - 1) * pageSize + if (start >= filtered.size) return emptyList() + val end = (start + pageSize).coerceAtMost(filtered.size) + return filtered.subList(start, end).map { it.toManga() } + } + + val apiUrl = when (order) { + SortOrder.POPULARITY -> "$baseUrl/api/rankings?page=$page&period=total_ranking" + SortOrder.UPDATED -> "$baseUrl/api/new-chapters?page=$page" + else -> "$baseUrl/api/rankings?page=$page&period=total_ranking" + } + + val json = webClient.httpGet(apiUrl, getRequestHeaders()).parseJson() + val dataArray = json.optJSONArray("data") ?: return emptyList() + val mangas = mutableListOf<Manga>() + for (i in 0 until dataArray.length()) { + val obj = dataArray.optJSONObject(i) ?: continue + if (obj.optString("type") != "comic") continue + val id = obj.getInt("id") + val name = obj.optString("name", "") + val slug = obj.optString("slug", "") + val cover = obj.optString("cover", "") + slugMap[id.toString()] = slug + mangas.add(MangaDto(id, name, slug, cover, "", -1, emptyList()).toManga()) + } + return mangas + } + + private fun MangaDto.toManga() = Manga( + id = generateUid(id.toString()), + url = id.toString(), + publicUrl = "$baseUrl/series/comic-$slug", + title = name, + coverUrl = cover.takeIf { it.isNotEmpty() }?.let { + if (it.startsWith("http")) it else "$baseUrl/$it" + } ?: "", + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + + override suspend fun getDetails(manga: Manga): Manga { + fetchSeriesListIfNeeded() + val mangaId = manga.url + val slug = slugMap[mangaId] ?: return manga + + val url = "$baseUrl/api/series/$slug?type=comic" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val data = json.optJSONObject("data") ?: return manga + + val title = data.optString("name", manga.title) + val cover = data.optString("cover", manga.coverUrl) + val summary = data.optString("summary", "") + val statusId = data.optJSONObject("status")?.optInt("id") ?: -1 + val state = when (statusId) { + 1 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.FINISHED + 5 -> MangaState.ABANDONED + else -> null + } + val genres = mutableSetOf<MangaTag>() + data.optJSONArray("genres")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.optString("name", "")?.takeIf { it.isNotEmpty() }?.let { + genres.add(MangaTag(it.lowercase(), it, source)) + } + } + } + val chapters = fetchChapters(mangaId, slug) + + return manga.copy( + title = title, + coverUrl = cover?.takeIf { it.isNotEmpty() }?.let { + if (it.startsWith("http")) it else "$baseUrl/$it" + } ?: manga.coverUrl, + description = summary, + tags = genres, + state = state, + chapters = chapters, + ) + } + + private suspend fun fetchChapters(mangaId: String, slug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + var page = 1 + var total = Int.MAX_VALUE + while (chapters.size < total) { + val url = "$apiBaseUrl/api/series/$slug/chapters?page=$page&direction=desc&type=comic" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val dataArray = json.optJSONArray("data") ?: break + total = json.optJSONObject("meta")?.optInt("total", dataArray.length()) ?: dataArray.length() + for (i in 0 until dataArray.length()) { + val ch = dataArray.optJSONObject(i) ?: continue + val id = ch.getInt("id") + val name = ch.optString("name", "") + val publishedAt = ch.optString("published_at", "") + val uploadDate = try { dateFormat.parse(publishedAt)?.time ?: 0L } catch (_: Exception) { 0L } + chapters.add( + MangaChapter( + id = generateUid("$mangaId/$id"), + title = "Capitulo $name", + number = name.toFloatOrNull() ?: 0f, + volume = 0, + url = "$mangaId/$id", + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + page++ + } + return chapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("/") + if (parts.size != 2) return emptyList() + val mangaId = parts[0] + val chapterId = parts[1] + val slug = slugMap[mangaId] ?: return emptyList() + + val url = "$baseUrl/api/capitulo/comic-$slug/$chapterId" + val json = webClient.httpGet(url, getRequestHeaders()).parseJson() + val pagesArray = json.optJSONObject("chapter")?.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).map { i -> + val imgUrl = pagesArray.getString(i) + MangaPage(id = generateUid(imgUrl), url = imgUrl, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/tsuki/site/es/ShadowManga.kt b/src/main/kotlin/tsuki/site/es/ShadowManga.kt new file mode 100644 index 00000000..37c1a842 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/ShadowManga.kt @@ -0,0 +1,322 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import kotlinx.coroutines.delay +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import kotlin.time.Duration.Companion.milliseconds + +@MangaSourceParser("SHADOWMANGA", "Shadow Manga", "es") +internal class ShadowManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SHADOWMANGA, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("shademanga.com") + private val baseApi = "https://$domain/api/series-locales" + + private val showAdultKey = ConfigKey.ShowSuspiciousContent(false) + + private val cdnHosts = listOf("media.shademanga.com", "cdn.shademanga.com") + private val fallbackPrefix = "/api/media/" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS", sourceLocale) + + private val rateLimiter = Mutex() + override val webClient by lazy { + tsuki.network.OkHttpWebClient( + context.httpClient.newBuilder() + .addInterceptor { chain -> + if (chain.request().url.host == domain) { + runBlocking { rateLimiter.withLock { delay(333.milliseconds) } } + } + chain.proceed(chain.request()) + } + .build(), + source + ) + } + + override fun getRequestHeaders() = Headers.Builder() + .add("Referer", "https://$domain/") + .build() + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + keys.add(showAdultKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + ) + + @Volatile private var genreCache: Set<MangaTag>? = null + private val genreMutex = Mutex() + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchGenres(), + ) + + private suspend fun fetchGenres(): Set<MangaTag> { + genreCache?.let { return it } + return genreMutex.withLock { + genreCache?.let { return it } + try { + val bodyString = webClient.httpGet("$baseApi/tags", getRequestHeaders()).body?.string() + ?: return@withLock emptySet() + val arr = JSONArray(bodyString) + val tags = LinkedHashSet<MangaTag>(arr.length()) + for (i in 0 until arr.length()) { + val name = arr.optString(i, "").trim() + if (name.isNotEmpty()) tags.add(MangaTag(title = name, key = name, source = source)) + } + genreCache = tags + tags + } catch (index: Exception) { + throw Exception(index) + } + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val adultParam = if (config[showAdultKey]) "true" else "false" + if (page > 1) return emptyList() + + val query = filter.query?.trim().orEmpty() + val includedTags = filter.tags.map { it.key } + val excludedTags = filter.tagsExclude.map { it.key } + + val url: String + val isSearch = query.isNotEmpty() || includedTags.isNotEmpty() || excludedTags.isNotEmpty() + + when { + isSearch -> { + val builder = "$baseApi/search-candidates".toHttpUrl().newBuilder() + .addQueryParameter("q", query) + .addQueryParameter("includeAdult", adultParam) + .addQueryParameter("showSinPortada", "false") + .addQueryParameter("take", "120") + includedTags.forEach { builder.addQueryParameter("tags", it) } + url = builder.build().toString() + } + order == SortOrder.POPULARITY -> url = "$baseApi/popular" + else -> url = "$baseApi/novedades" + } + + val bodyString = webClient.httpGet(url, getRequestHeaders()).body?.string() + ?: return emptyList() + val rawJson = runCatching { JSONArray(bodyString) }.getOrNull() + ?: return emptyList() + + val allSeries = if (!isSearch) { + val isGenreWrapped = rawJson.length() > 0 && rawJson.optJSONObject(0)?.has("genero") == true + if (isGenreWrapped) { + val seen = HashSet<Int>() + val seriesList = mutableListOf<JSONObject>() + for (i in 0 until rawJson.length()) { + val group = rawJson.optJSONObject(i) ?: continue + val seriesArr = group.optJSONArray("series") ?: continue + for (j in 0 until seriesArr.length()) { + seriesArr.optJSONObject(j)?.let { series -> + val id = series.optInt("id", -1) + if (id > 0 && seen.add(id)) seriesList.add(series) + } + } + } + seriesList + } else { + (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } + } + } else { + (0 until rawJson.length()).mapNotNull { rawJson.optJSONObject(it) } + } + + val filtered = if (excludedTags.isNotEmpty() || includedTags.isNotEmpty()) { + allSeries.filter { series -> + val genres = series.optString("generos", "").split(",").map { it.trim() } + val includeOk = includedTags.isEmpty() || genres.any { it in includedTags } + val excludeOk = excludedTags.isEmpty() || genres.none { it in excludedTags } + includeOk && excludeOk + } + } else allSeries + + val sorted = if (isSearch) { + filtered.sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.optString("titulo", "") }) + } else filtered + + return sorted.map { parseMangaFromSeries(it) } + } + + private fun parseMangaFromSeries(obj: JSONObject): Manga { + val id = obj.optInt("id") + val title = obj.optString("titulo", "") + val cover = obj.optString("portadaUrl", "").takeIf { it.isNotBlank() } + return Manga( + id = generateUid(id.toString()), + title = title, + altTitles = emptySet(), + url = id.toString(), + publicUrl = "https://$domain/serie/local/$id", + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val id = manga.url + val json = webClient.httpGet("$baseApi/$id", getRequestHeaders()).parseJson() + + val title = json.optString("titulo", manga.title) + val cover = json.optString("portadaUrl", "").takeIf { it.isNotBlank() } ?: manga.coverUrl + val description = json.optString("descripcion", "").takeIf { it.isNotBlank() } + val author = json.optString("autor", "").takeIf { it.isNotBlank() } + val statusText = json.optString("estado", "") + val state = when (statusText.lowercase()) { + "en curso" -> MangaState.ONGOING + "completado" -> MangaState.FINISHED + "pausada" -> MangaState.PAUSED + else -> null + } + val genres = json.optString("generos", "").split(",").map { it.trim() } + .filter { it.isNotEmpty() } + .map { MangaTag(title = it, key = it, source = source) } + .toSet() + + val chapters = json.optJSONArray("capitulos")?.let { parseChapters(it, id) } ?: emptyList() + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = setOfNotNull(author), + state = state, + tags = genres, + chapters = chapters, + ) + } + + private fun parseChapters(arr: JSONArray, mangaId: String): List<MangaChapter> { + val list = mutableListOf<MangaChapter>() + for (i in 0 until arr.length()) { + val ch = arr.optJSONObject(i) ?: continue + val chId = ch.optInt("id") + val number = ch.optDouble("numeroCapitulo", 0.0).toFloat() + val title = ch.optString("titulo", "").trim().takeIf { it.isNotEmpty() } + val dateStr = ch.optString("fechaSubida", "") + val uploadDate = runCatching { dateFormat.parse(dateStr)?.time ?: 0L }.getOrDefault(0L) + + val displayNumber = if (number == number.toInt().toFloat()) number.toInt().toString() else number.toString() + val chapterTitle = buildString { + append("Cap. ") + append(displayNumber) + if (title != null) append(" - $title") + } + + list.add( + MangaChapter( + id = generateUid("$mangaId/$chId"), + title = chapterTitle, + url = "$mangaId/$chId", + number = number, + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return list.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split('/') + if (parts.size != 2) return emptyList() + val mangaId = parts[0] + val chapterId = parts[1] + + val json = webClient.httpGet("$baseApi/$mangaId/capitulos/$chapterId/paginas", getRequestHeaders()).parseJson() + val pagesArr = json.optJSONArray("paginas") ?: return emptyList() + + return (0 until pagesArr.length()).map { i -> + val originalUrl = pagesArr.optString(i) + MangaPage( + id = generateUid(originalUrl), + url = originalUrl, + preview = null, + source = source + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val originalUrl = page.url + val response = webClient.httpGet(originalUrl, getRequestHeaders()) + if (response.isSuccessful) { + response.close() + return originalUrl + } + response.close() + // CDN fallback + val url = originalUrl.toHttpUrl() + val key = when { + url.encodedPath.startsWith(fallbackPrefix) -> url.encodedPath.removePrefix(fallbackPrefix) + else -> url.encodedPath.removePrefix("/") + } + // Try other CDN hosts + for (host in cdnHosts) { + if (host == url.host) continue + val cdnUrl = "https://$host/$key" + val cdnResponse = webClient.httpGet(cdnUrl, getRequestHeaders()) + if (cdnResponse.isSuccessful) { + cdnResponse.close() + return cdnUrl + } + cdnResponse.close() + } + // Fallback to main domain API + val fallbackUrl = "https://$domain$fallbackPrefix$key" + return fallbackUrl + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/tsuki/site/es/Submanhwa.kt b/src/main/kotlin/tsuki/site/es/Submanhwa.kt new file mode 100644 index 00000000..75346a29 --- /dev/null +++ b/src/main/kotlin/tsuki/site/es/Submanhwa.kt @@ -0,0 +1,193 @@ +package tsuki.site.es + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("SUBMANHWA", "Submanhwa", "es") +internal class Submanhwa(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SUBMANHWA, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("submanhwa.com") + + private val dateFormat = SimpleDateFormat("dd MMM. yyyy", Locale.ENGLISH) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Accept-Language", "es-PE,es;q=0.9,en-US;q=0.8,en;q=0.7") + .set("Referer", "https://$domain/") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + val url = if (query.isNotEmpty()) { + "https://$domain/filterList?page=$page&sortBy=views&asc=false&alpha=${query.urlEncoded()}" + } else when (order) { + SortOrder.POPULARITY -> "https://$domain/filterList?page=$page&sortBy=views&asc=false" + SortOrder.UPDATED -> "https://$domain" // latest updates + else -> "https://$domain/filterList?page=$page&sortBy=views&asc=false" + } + + val doc = webClient.httpGet(url).parseHtml() + + val elements = if (order == SortOrder.UPDATED && query.isEmpty()) { + doc.select("div[class^=manga-item]") + } else { + doc.select(".series-card") + } + + return elements.mapNotNull { element -> + val a = element.selectFirst("a") ?: return@mapNotNull null + val href = a.absUrl("href") + val title = element.selectFirst("h3[class^=manga-title] a, .series-title")?.text() ?: a.text() + val coverUrl = element.selectFirst("img")?.absUrl("src") + + Manga( + id = generateUid(href), + url = href, + publicUrl = href, + title = title, + coverUrl = coverUrl, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst(".manga-title-centered")?.text() ?: manga.title + val coverUrl = doc.selectFirst("img")?.absUrl("src") ?: manga.coverUrl + val description = doc.selectFirst("h5:contains(Resumen) + p")?.text() + + val box = doc.selectFirst(".main-content > .boxed-modern") + + val state = when (box?.selectFirst(".detail-label:contains(Estado) + .detail-value span")?.text()?.lowercase()) { + "completa" -> MangaState.FINISHED + "en curso" -> MangaState.ONGOING + else -> null + } + + val author = box?.selectFirst(".detail-label:contains(Autor) + .detail-value a")?.text() + val artist = box?.selectFirst(".detail-label:contains(Artist) + .detail-value a")?.text() + val genres = box?.select(".detail-label:contains(Categor) + .detail-value a") + ?.map { MangaTag(it.text().lowercase(), it.text(), source) } + .orEmpty() + .toSet() + + val chapters = parseChapters(doc) + + return manga.copy( + title = title, + coverUrl = coverUrl, + description = description, + authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet(), + tags = genres, + state = state, + chapters = chapters, + ) + } + + private fun parseChapters(doc: Document): List<MangaChapter> { + return doc.select(".chapters-grid [class^=chapter-card]").mapNotNull { element -> + val a = element.selectFirst("a.chapter-link") ?: return@mapNotNull null + val name = a.text() + val href = a.absUrl("href") + + val dateText = element.selectFirst("span:has(i.glyphicon-time)")?.text() + ?: element.selectFirst(".chapter-preview-meta > span")?.text() + val uploadDate = dateText?.let { parseDate(it) } ?: 0L + + val number = Regex("""\d+(\.\d+)?""").find(name)?.value?.toFloatOrNull() ?: 0f + + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + }.sortedBy { it.number } + } + + private fun parseDate(text: String): Long { + return try { + dateFormat.parse(text.trim())?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("#all img").mapIndexed { idx, img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: "" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + private fun Element.imgAttr(): String { + if (hasAttr("data-src")) return absUrl("data-src") + if (hasAttr("data-lazy-src")) return absUrl("data-lazy-src") + return absUrl("src") + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/tsuki/site/fr/AnimeSama.kt b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt new file mode 100644 index 00000000..d90afe9e --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/AnimeSama.kt @@ -0,0 +1,347 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.network.OkHttpWebClient + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson + +import kotlinx.coroutines.Dispatchers +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.HttpUrl +import java.util.EnumSet +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.withContext +import java.net.URLEncoder + +/** + * I dont like this website. + * Todo: fix cover image + */ + +@MangaSourceParser("ANIMESAMA", "AnimeSama", "fr") +internal class AnimeSama(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ANIMESAMA, pageSize = 48) { + + override val configKeyDomain = ConfigKey.Domain("anime-sama.to") + private val baseUrl = "https://$domain" + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://cdn.jsdelivr.net/gh/Anime-Sama/IMG@img/autres/logo.png", 2600, null), + ), + domain, + ) + } + + override val webClient by lazy { + OkHttpWebClient(context.httpClient.newBuilder().build(), source) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "$baseUrl/") + .set("Accept-Language", "fr-FR") + .build() + + private var cachedGenres: Set<MangaTag>? = null + + private suspend fun fetchGenres(): Set<MangaTag> { + if (cachedGenres != null) return cachedGenres!! + val doc = webClient.httpGet("$baseUrl/catalogue", getRequestHeaders()).parseHtml() + val tags = doc.select("#list_genres #genreList label").mapNotNull { label -> + val input = label.selectFirst("input[name=genre[]]") ?: return@mapNotNull null + val value = input.attr("value") + val name = label.selectFirst("span")?.text() ?: return@mapNotNull null + MangaTag(name, value, source) + }.toSet() + cachedGenres = tags + return tags + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, // catalogue listing + SortOrder.UPDATED // latest additions on homepage + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = fetchGenres(), + availableStates = emptySet() + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + + if (query.startsWith("http")) { + val url = query.toHttpUrlOrNull() + if (url != null && url.host == domain && url.pathSegments.contains("catalogue")) { + return fetchMangaFromUrl(url) + } + } + + if (query.isNotEmpty()) { + return fetchSearch(query, filter.tags.map { it.key }, page) + } + return when (order) { + SortOrder.UPDATED -> fetchLatest() + else -> fetchSearch("", filter.tags.map { it.key }, page) // POPULARITY or any other + } + } + + private suspend fun fetchMangaFromUrl(url: HttpUrl): List<Manga> { + val path = url.encodedPath.removeSuffix("/").substringBefore("/scan") + val mangaUrl = url.newBuilder().encodedPath("$path/").build() + val doc = webClient.httpGet(mangaUrl.toString(), getRequestHeaders()).parseHtml() + val manga = parseMangaDetails(doc, mangaUrl.toString()) + return listOfNotNull(manga) + } + + private suspend fun fetchLatest(): List<Manga> { + val doc = webClient.httpGet(baseUrl, getRequestHeaders()).parseHtml() + return doc.select("div#containerAjoutsScans > div").mapNotNull { el -> + parseCardElement(el, removeSuffix = "scan/vf/") + } + } + + private suspend fun fetchSearch(query: String, genreIds: List<String>, page: Int): List<Manga> { + val urlBuilder = "$baseUrl/catalogue".toHttpUrl().newBuilder() + .addQueryParameter("type[]", "Scans") + .addQueryParameter("page", page.toString()) + if (query.isNotEmpty()) urlBuilder.addQueryParameter("search", query) + genreIds.forEach { urlBuilder.addQueryParameter("genre[]", it) } + val doc = webClient.httpGet(urlBuilder.build().toString(), getRequestHeaders()).parseHtml() + return doc.select("div#list_catalog > div").mapNotNull { el -> parseCardElement(el) } + } + + private fun proxyImage(originalUrl: String?, width: Int = 480): String? { + if (originalUrl.isNullOrBlank()) return null + val encoded = URLEncoder.encode(originalUrl, "UTF-8") + return "https://wsrv.nl/?url=$encoded&w=$width" + } + + private fun parseCardElement(el: Element, removeSuffix: String = ""): Manga? { + val a = el.selectFirst("a") ?: return null + val rawUrl = a.absUrl("href") + val url = if (removeSuffix.isNotEmpty()) rawUrl.removeSuffix(removeSuffix) else rawUrl + val title = el.selectFirst("h2.card-title")?.text() ?: return null + val cover = proxyImage(el.selectFirst("img")?.absUrl("src")) + return Manga( + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url, getRequestHeaders()).parseHtml() + val detailedManga = parseMangaDetails(doc, manga.url) ?: manga + val chapters = fetchChapters(doc, manga.url) + return detailedManga.copy(chapters = chapters) + } + + private fun parseMangaDetails(doc: Document, url: String): Manga? { + val title = doc.selectFirst("div.my-2 h1")?.text() ?: return null + val cover = proxyImage(doc.selectFirst("img#coverOeuvre")?.absUrl("src")) + val author = doc.selectFirst("span.info-lbl:contains(Créateur) + span.info-val")?.text() + val statusText = doc.selectFirst("span.info-lbl:contains(État) + span.info-val")?.text() + val state = when (statusText?.lowercase()) { + "en cours" -> MangaState.ONGOING + "terminé" -> MangaState.FINISHED + else -> null + } + val description = doc.selectFirst("p#synopsisText")?.text() + val genres = doc.select("span.genre-pill").map { it.text() }.toSet() + val tags = genres.map { MangaTag(it.lowercase(), it, source) }.toSet() + + return Manga( + id = generateUid(url), + url = url, + publicUrl = url, + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = tags, + state = state, + authors = setOfNotNull(author), + description = description, + source = source + ) + } + + private suspend fun fetchChapters(mangaDoc: Document, mangaUrl: String): List<MangaChapter> { + val scriptContent = mangaDoc.select("script:containsData(panneauScan(\"nom\", \"url\"))").toString() + val lines = scriptContent.split(";").drop(1) + val scanPanels = lines.mapNotNull { line -> + SCAN_PANEL_REGEX.find(line)?.destructured?.let { (name, url) -> + if (!url.contains("va")) name to url else null + } + } + if (scanPanels.isEmpty()) return emptyList() + + val chapterLists = coroutineScope { + scanPanels.map { (scanTitle, scanUrl) -> + async { fetchScanGroupChapters(mangaUrl, scanTitle, scanUrl) } + }.awaitAll() + } + + val seen = mutableSetOf<String>() + val allChapters = mutableListOf<MangaChapter>() + for (list in chapterLists) { + for (ch in list) { + if (seen.add(ch.title.toString())) allChapters.add(ch) + } + } + + return allChapters + .sortedBy { ch -> ch.url.toHttpUrlOrNull()?.queryParameter("id")?.toIntOrNull() ?: 0 } + .sortedBy { it.number } + } + + private suspend fun fetchScanGroupChapters(mangaUrl: String, scanTitle: String, scanUrl: String): List<MangaChapter> { + val subUrl = mangaUrl.trimEnd('/') + "/" + scanUrl.trim('/') + val subDoc = webClient.httpGet(subUrl, getRequestHeaders()).parseHtml() + + var title: String? = null + val mainPageLink = subDoc.selectFirst("a:has(#imgOeuvre.grayscale)")?.absUrl("href") + if (!mainPageLink.isNullOrEmpty()) { + val mainDoc = webClient.httpGet(mainPageLink, getRequestHeaders()).parseHtml() + title = mainDoc.getWorkTitle() + } + if (title.isNullOrBlank()) title = subDoc.getWorkTitle() + if (title.isBlank()) return emptyList() + + val scanlator = scanTitle.replace(SCANS_REGEX, "").trim() + val chapterApiUrl = "$baseUrl/s2/scans/get_nb_chap_et_img.php".toHttpUrl().newBuilder() + .addQueryParameter("oeuvre", title).build() + val apiResponse = webClient.httpGet(chapterApiUrl.toString(), getRequestHeaders()).parseJson() + val imageCounts = mutableMapOf<String, Int>() + for (key in apiResponse.keys()) { + imageCounts[key] = apiResponse.optInt(key, 0) + } + + val html = subDoc.html() + val chapterList = mutableListOf<MangaChapter>() + var chapterDelay = 0 + + if (html.contains("resetListe()")) { + val commands = html.split(";") + for (command in commands) { + when { + CREATE_LIST_REGEX.find(command) != null -> { + val data = CREATE_LIST_REGEX.find(command)!!.groupValues[1].split(",") + val start = data[0].trim().toInt() + val end = data[1].trim().toInt() + for (i in start..end) { + chapterList.add(buildChapter(i.toString(), chapterList.size + 1, title, scanlator, chapterApiUrl)) + } + } + NEW_SP_REGEX.find(command) != null -> { + val chapterName = NEW_SP_REGEX.find(command)!!.groupValues[1] + chapterList.add(buildChapter(chapterName, chapterList.size + 1, title, scanlator, chapterApiUrl)) + chapterDelay++ + } + } + } + } + + val totalChapters = imageCounts.size + for (i in chapterList.size until totalChapters) { + val id = (chapterList.size + 1 - chapterDelay).toString() + chapterList.add(buildChapter(id, chapterList.size + 1, title, scanlator, chapterApiUrl)) + } + return chapterList + } + + private fun buildChapter(chapterName: String, id: Int, title: String, scanlator: String, baseChapterUrl: HttpUrl): MangaChapter { + val chapterUrl = baseChapterUrl.newBuilder() + .addQueryParameter("id", id.toString()) + .addQueryParameter("title", title) + .build() + return MangaChapter( + id = generateUid(chapterUrl.toString()), + title = "Chapitre $chapterName", + number = id.toFloat(), + volume = 0, + url = chapterUrl.toString(), + uploadDate = 0, + scanlator = scanlator, + branch = null, + source = source + ) + } + + private fun Document.getWorkTitle(): String = selectFirst("#titreOeuvre")?.textNodes()?.firstOrNull()?.wholeText ?: "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = chapter.url.toHttpUrlOrNull() ?: return emptyList() + val chapterId = url.queryParameter("id") ?: return emptyList() + val rawTitle = url.queryParameter("title") ?: return emptyList() + val encodedTitle = withContext(Dispatchers.IO) { + URLEncoder.encode(rawTitle, "UTF-8") + }.replace("+", "%20") + val apiUrl = "$baseUrl/s2/scans/get_nb_chap_et_img.php".toHttpUrl().newBuilder() + .addQueryParameter("oeuvre", rawTitle) + .build() + val json = webClient.httpGet(apiUrl.toString(), getRequestHeaders()).parseJson() + val imageCount = json.optInt(chapterId, 0) + + return (1..imageCount).map { index -> + MangaPage( + id = generateUid("$rawTitle/$chapterId/$index"), + url = "$baseUrl/s2/scans/$encodedTitle/$chapterId/$index.jpg", + preview = null, + source = source + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + companion object { + private val SCAN_PANEL_REGEX = """panneauScan\("(.+?)", "(.+?)"\)""".toRegex() + private val CREATE_LIST_REGEX = """creerListe\((\d+,\s*\d+)\)""".toRegex() + private val NEW_SP_REGEX = """newSP\((\d+(\.\d+)?|"(.*?)")\)""".toRegex() + private val SCANS_REGEX = """(Scans|\(|\))""".toRegex() + } +} diff --git a/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt b/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt new file mode 100644 index 00000000..cbe74576 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/MangaScantrad.kt @@ -0,0 +1,13 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGASCANTRAD", "Manga-Scantrad", "fr") +internal class MangaScantrad(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MANGASCANTRAD, "manga-scantrad.io") { + override val datePattern = "d MMMM yyyy" +} diff --git a/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt new file mode 100644 index 00000000..9a5982a7 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/MangasOriginesFr.kt @@ -0,0 +1,14 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.OriginesParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGASORIGINESFR", "Mangas Origines", "fr") +class MangasOriginesFr(context: MangaLoaderContext) : + OriginesParser(context, MangaParserSource.MANGASORIGINESFR, "mangas-origines.fr") { + override val mangaPath = "oeuvre" + override val legacyMangaPaths = setOf("catalogues") +} diff --git a/src/main/kotlin/tsuki/site/fr/RimuScans.kt b/src/main/kotlin/tsuki/site/fr/RimuScans.kt new file mode 100644 index 00000000..583b5792 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/RimuScans.kt @@ -0,0 +1,334 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.json.extractNextJs + +import kotlinx.coroutines.coroutineScope +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("RIMUSCANS", "Rimu Scans", "fr") +class RimuScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.RIMUSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("rimuscan.fr") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.FRENCH).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isSearchWithFiltersSupported = true, + ) + + private val genresCache = mutableListOf<String>() + private var genresLoaded = false + + private suspend fun loadGenres() { + if (genresLoaded) return + runCatching { + val json = webClient.httpGet("https://$domain/api/admin/genres").parseJson() + val arr = json.optJSONArray("genres") ?: return + genresCache.clear() + for (i in 0 until arr.length()) { + genresCache.add(arr.getString(i)) + } + genresLoaded = true + } + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = run { + loadGenres() + genresCache.map { MangaTag(key = it, title = it, source = source) }.toSet()}, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val json = webClient.httpGet(url).parseJson() + return parseSeriesList(json) + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return "https://$domain/api/series".toHttpUrl().newBuilder().apply { + val query = filter.query + if (!query.isNullOrEmpty()) { + addQueryParameter("search", query) + } else { + when (order) { + SortOrder.POPULARITY -> addQueryParameter("sort", "popular") + SortOrder.RATING -> addQueryParameter("sort", "rating") + SortOrder.ALPHABETICAL -> addQueryParameter("sort", "az") + SortOrder.UPDATED -> addQueryParameter("sort", "updated") + SortOrder.NEWEST -> addQueryParameter("sort", "new") + else -> addQueryParameter("sort", "updated") + } + if (filter.types.isNotEmpty()) { + val typeParam = filter.types.mapNotNull { type -> + when (type) { + ContentType.MANHWA -> "webtoon" + ContentType.MANGA -> "manga" + else -> null + } + }.joinToString(",") + if (typeParam.isNotEmpty()) addQueryParameter("types", typeParam) + } + if (filter.states.isNotEmpty()) { + val statusParam = filter.states.mapNotNull { state -> + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> null + } + }.joinToString(",") + if (statusParam.isNotEmpty()) addQueryParameter("status", statusParam) + } + if (filter.tags.isNotEmpty()) { + addQueryParameter("genres", filter.tags.joinToString(",") { it.key }) + } + } + addQueryParameter("page", page.toString()) + }.build().toString() + } + + private fun parseSeriesList(json: JSONObject): List<Manga> { + val seriesArr = json.optJSONArray("series") ?: return emptyList() + return (0 until seriesArr.length()).mapNotNull { i -> + val obj = seriesArr.getJSONObject(i) + val slug = obj.optString("slug").takeIf { it.isNotEmpty() } + val title = obj.optString("title").takeIf { it.isNotEmpty() } + val cover = obj.optString("cover_url").takeIf { it.isNotEmpty() } + if (slug == null || title == null) { + null + } else { + Manga( + id = generateUid(slug), + title = title, + altTitles = emptySet(), + contentRating = null, + url = "/manga/$slug", + publicUrl = "https://$domain/manga/$slug", + coverUrl = cover?.toAbsoluteUrl(), + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl()).parseHtml() + val details = parseDetailsFromLd(doc, manga) + val chapters = extractChapters(doc, manga.url.substringAfterLast("/")) + details.copy( + chapters = chapters, + ) + } + + private fun parseDetailsFromLd(doc: Document, fallback: Manga): Manga { + val ldScripts = doc.select("script[type=application/ld+json]") + for (script in ldScripts) { + val data = script.data() + if (data.contains("\"ComicSeries\"")) { + val json = JSONObject(data) + val name = json.optString("name").ifEmpty { fallback.title } + val description = json.optString("description") + val image = json.optString("image") + val author = json.optJSONObject("author")?.optString("name") + val illustrator = json.optJSONObject("illustrator")?.optString("name") + val genres = json.optJSONArray("genre")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + val altNames = json.optJSONArray("alternateName")?.let { arr -> + (0 until arr.length()).map { arr.getString(it) } + } ?: emptyList() + + val h1 = doc.selectFirst("h1") + val badges = h1?.previousElementSibling()?.select("span")?.map { it.text().trim() } ?: emptyList() + val typeLabel = badges.getOrNull(0) + val statusLabel = badges.getOrNull(1) + val state = when (statusLabel?.lowercase()) { + "en cours", "ongoing" -> MangaState.ONGOING + "terminé", "termine", "completed" -> MangaState.FINISHED + "en pause", "hiatus", "on hiatus" -> MangaState.PAUSED + "annulé", "annule", "abandonné", "abandonne", "cancelled" -> MangaState.ABANDONED + else -> null + } + val contentType = when (typeLabel?.lowercase()) { + "webtoon", "manhwa" -> ContentType.MANHWA + "manhua" -> ContentType.MANHUA + "manga" -> ContentType.MANGA + else -> null + } + val allGenres = buildSet { + if (contentType != null) add(contentType.toString().lowercase()) + genres.forEach { add(it) } + } + val tags = allGenres.map { MangaTag(key = it.lowercase(), title = it, source = source) }.toSet() + val fullDescription = buildString { + if (description.isNotEmpty()) append(description) + if (altNames.isNotEmpty()) { + if (isNotEmpty()) append("\n\n") + append("Titres alternatifs : ") + append(altNames.joinToString(", ")) + } + }.ifEmpty { null } + + return fallback.copy( + title = name, + description = fullDescription, + coverUrl = image.toAbsoluteUrl(), + authors = setOfNotNull(author, illustrator).filter { it.isNotEmpty() }.toSet(), + tags = tags, + state = state, + ) + } + } + return fallback + } + + private fun extractChapters(doc: Document, mangaSlug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + val seenNumbers = mutableSetOf<Double>() + + doc.extractNextJs { element -> + if (element is JSONObject && element.has("number") && element.has("type")) { + runCatching { + val number = element.optDouble("number", Double.NaN) + if (!number.isNaN() && seenNumbers.add(number)) { + val type = element.optString("type", "NORMAL") + val title = element.optString("title", "").trim() + val releaseDate = element.optString("releaseDate").ifEmpty { null } + val numberString = number.toString().substringBefore(".0") + val chapterName = buildString { + append("Chapitre $numberString") + if (title.isNotEmpty() && title != "Chapitre $numberString") { + if (title.contains("Chapitre", true) || title.contains("Chapter", true)) { + clear() + append(title) + } else { + append(" : $title") + } + } + }.let { if (type.equals("PREMIUM", true)) "🔒 $it" else it } + val date = releaseDate?.let { dateFormat.parse(it)?.time } ?: 0L + chapters.add( + MangaChapter( + id = generateUid("$mangaSlug-$number"), + url = "/read/$mangaSlug/$numberString", + title = chapterName, + number = number.toFloat(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + ) + } + } + } + false + } + + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl()).parseHtml() + val chapterNumber = chapter.number.toDouble() + val chapterObjects = mutableListOf<JSONObject>() + + doc.extractNextJs { element -> + if (element is JSONObject && element.has("number") && element.has("images")) { + chapterObjects.add(element) + } + false + } + + val chapterObj = chapterObjects.firstOrNull { + it.optDouble("number") == chapterNumber && (it.optJSONArray("images")?.length() ?: 0) > 0 + } ?: chapterObjects.firstOrNull { it.optDouble("number") == chapterNumber } + ?: return emptyList() + + if (chapterObj.optString("type").equals("PREMIUM", true)) { + throw ParseException("Ce chapitre est premium. Lisez-le sur le site.", chapter.url.toAbsoluteUrl()) + } + + val imagesArr = chapterObj.optJSONArray("images") ?: return emptyList() + val images = mutableListOf<Pair<Int, String>>() + for (i in 0 until imagesArr.length()) { + val imgObj = imagesArr.getJSONObject(i) + val order = imgObj.optInt("order") + val url = imgObj.optString("url").takeIf { it.isNotEmpty() } ?: continue + images.add(order to url) + } + return images.sortedBy { it.first }.mapIndexed { _, (_, url) -> + MangaPage( + id = generateUid(url), + url = url.toAbsoluteUrl(), + preview = null, + source = source, + ) + } + } + + private fun okhttp3.Response.parseJson(): JSONObject { + val body = body?.string() ?: throw ParseException("Empty response body", request.url.toString()) + return JSONObject(body) + } + + private fun String.toAbsoluteUrl(): String = when { + startsWith("http") -> this + startsWith("/") -> "https://$domain$this" + else -> "https://$domain/$this" + } +} diff --git a/src/main/kotlin/tsuki/site/fr/SushiScan.kt b/src/main/kotlin/tsuki/site/fr/SushiScan.kt new file mode 100644 index 00000000..d0b3f738 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/SushiScan.kt @@ -0,0 +1,114 @@ +package tsuki.site.fr + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SUSHISCAN", "Sushi-Scan", "fr") +internal class SushiScan(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SUSHISCAN, "sushiscan.net") { + + override val mangaDirectory = "catalogue" + override val dateFormat = SimpleDateFormat("dd MMMM yyyy", Locale.FRENCH) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/$mangaDirectory/") + .build() + + override fun parseStatus(text: String?): MangaState? { + val status = text?.lowercase(Locale.FRENCH) ?: return null + return when { + status.contains("en cours") -> MangaState.ONGOING + status.contains("terminé") -> MangaState.FINISHED + status.contains("abandonné") -> MangaState.ABANDONED + status.contains("en pause") -> MangaState.PAUSED + else -> null + } + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (!filter.query.isNullOrBlank()) { + val url = "https://$domain/page/$page?s=${filter.query.urlEncoded()}" + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + return super.getListPage(page, order, filter) + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val base = super.getDetails(manga) + + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val table = doc.selectFirst("table.infotable") + val hasTable = table != null + val authors = if (hasTable) { + val authorCell = table.selectFirst("tr:has(td:contains(Auteur)) td:last-child") + val artistCell = table.selectFirst("tr:has(td:contains(Artiste)) td:last-child") + listOfNotNull( + authorCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" }, + artistCell?.text()?.trim()?.takeIf { it.isNotBlank() && it != "n/a" && it != "N/A" } + ).toSet() + } else { + doc.select(".author, .artist, .fmed span, .tsinfo .imptdt:contains(Auteur) i, .spe span:contains(Auteur) a") + .mapNotNull { it.text().trim().takeIf { t -> t.isNotBlank() && t != "n/a" && t != "N/A" } } + .toSet() + } + + val statusText = if (hasTable) { + table.selectFirst("tr:has(td:contains(Statut)) td:last-child")?.text() + } else { + doc.select(".imptdt:contains(Statut) i, .tsinfo .imptdt:contains(Statut) a, .fmed b:contains(Statut)+span span") + .text() + .ifEmpty { + doc.selectFirst("div.post-content_item:contains(Statut) div.summary-content")?.text() + } + } + val state = parseStatus(statusText) + + base.copy( + authors = authors, + state = state, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + val scriptContent = doc.selectFirst("script:containsData(ts_reader)")?.data() + if (scriptContent != null) { + val jsonString = scriptContent.substringAfter("ts_reader.run(").substringBefore(");") + val json = JSONObject(jsonString) + val sources = json.optJSONArray("sources") + if (sources != null && sources.length() > 0) { + val firstSource = sources.getJSONObject(0) + val images = firstSource.optJSONArray("images") + if (images != null) { + return (0 until images.length()).map { i -> + val url = images.getString(i).replace("http://", "https://") + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + } + } + return super.getPages(chapter) + } +} diff --git a/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt b/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt new file mode 100644 index 00000000..2c0b7284 --- /dev/null +++ b/src/main/kotlin/tsuki/site/fr/nsfw/HentaiOrigines.kt @@ -0,0 +1,86 @@ +package tsuki.site.fr.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.OriginesParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("HENTAIORIGINES", "Hentai Origines", "fr", ContentType.HENTAI) +class HentaiOrigines(context: MangaLoaderContext) : + OriginesParser(context, MangaParserSource.HENTAIORIGINES, "hentai-origines.com") { + + override val mangaPath = "manga" + + override val origins = listOf( + "Doujinshi" to "doujinshi", + "Pornhwa" to "pornhwa", + "Pornhua" to "pornhua", + "Hentai" to "hentai", + ) + + override val genres = listOf( + "Action" to "action", + "Alien" to "alien", + "Alpha" to "alpha", + "Amitié" to "amitie", + "Art martiaux" to "art-martiaux", + "Aventure" to "aventure", + "Belle-mère" to "belle-mere", + "Boy's Love" to "yaoi", + "Campus" to "campus", + "Comédie" to "comedie", + "Domination" to "domination", + "Drame" to "drame", + "Démon" to "demon", + "Ecchi" to "ecchi", + "Fantasy" to "fantasy", + "Futanari" to "futanari", + "Furry" to "furry", + "Fétichisme" to "fetichisme", + "Gallerie" to "gallerie", + "Gangster" to "gangster", + "Gofast" to "gofast", + "Gore" to "gore", + "Guideverse" to "guideverse", + "Hardcore" to "hardcore", + "Harem" to "harem", + "Historique" to "historique", + "Horreur" to "horreur", + "Humiliation" to "humiliation", + "Inceste" to "inceste", + "Isekai" to "isekai", + "Josei" to "josei", + "Loli" to "loli", + "Love" to "love", + "Magie" to "magie", + "Mature" to "mature", + "Milf" to "milf", + "Mini-série" to "mini-serie", + "Monsters girls" to "monsters-girls", + "Ntr" to "ntr", + "Office" to "office", + "Omégaverse" to "omegaverse", + "Oneshot" to "oneshot", + "Parodie" to "parodie", + "Professeur" to "professeur", + "Psychologie" to "psychologie", + "Rape" to "rape", + "Romance" to "romance", + "Réincarnation" to "reincarnation", + "School life" to "school-life", + "Sci-fi" to "sci-fi", + "Shonen-ai" to "shonen-ai", + "Slice of life" to "slice-of-life", + "Smut" to "smut", + "Soft" to "soft", + "Sport" to "sport", + "Surnaturel" to "surnaturel", + "Tomgirl" to "tomgirl", + "Tragédie" to "tragedie", + "Triangle amoureux" to "triangle-amoureux", + "Uncensored" to "uncensored", + "Yuri" to "yuri", + ) +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt b/src/main/kotlin/tsuki/site/id/AinzScans.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt rename to src/main/kotlin/tsuki/site/id/AinzScans.kt index cf97ba11..8f804cdb 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/AinzScans.kt +++ b/src/main/kotlin/tsuki/site/id/AinzScans.kt @@ -1,301 +1,325 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import okhttp3.HttpUrl.Companion.toHttpUrlOrNull -import org.json.JSONArray -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.Locale -import java.util.EnumSet -import java.util.TimeZone -import kotlinx.coroutines.sync.Mutex -import kotlinx.coroutines.sync.withLock - -@MangaSourceParser("AINZSCANS", "AinzScans", "id") -internal class AinzScans(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.AINZSCANS, pageSize = 20) { - - override val configKeyDomain = ConfigKey.Domain("v2.ainzscans01.com") - - private val apiBase = "https://api.ainzscans01.com/api" - - override val sourceLocale: Locale = Locale.US - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.NEWEST, - SortOrder.RATING, - SortOrder.ALPHABETICAL, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isMultipleTagsSupported = true, - isTagsExclusionSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = getOrFetchGenres().values.toSet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), - availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), - ) - - @Volatile - private var genresCache: Map<String, MangaTag>? = null - private val genresMutex = Mutex() - - private suspend fun getOrFetchGenres(): Map<String, MangaTag> { - genresCache?.let { return it } - return genresMutex.withLock { - genresCache ?: run { - val jsonStr = webClient.httpGet("$apiBase/genres").body?.string().orEmpty() - val map = mutableMapOf<String, MangaTag>() - try { - val array = JSONArray(jsonStr) - for (i in 0 until array.length()) { - val obj = array.getJSONObject(i) - val name = obj.getString("name") - val slug = obj.getString("slug") - map[slug] = MangaTag(title = name, key = slug, source = source) - } - } catch (_: Exception) { } - genresCache = map - map - } - } - } - - @get:Synchronized - private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { - override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 - } - - override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .build() - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = buildString { - append(apiBase) - append("/search?type=COMIC") - append("&limit=").append(pageSize) - append("&page=").append(page) - - filter.query?.takeIf { it.isNotBlank() }?.let { - append("&q=").append(it.urlEncoded()) - } - append("&sort=") - append( - when (order) { - SortOrder.UPDATED -> "latest" - SortOrder.POPULARITY -> "views" - SortOrder.NEWEST -> "new" - SortOrder.RATING -> "rate" - SortOrder.ALPHABETICAL -> "az" - else -> "latest" - } - ) - filter.states.oneOrThrowIfMany()?.let { state -> - append("&status=") - append( - when (state) { - MangaState.ONGOING -> "ONGOING" - MangaState.FINISHED -> "COMPLETED" - MangaState.PAUSED -> "HIATUS" - else -> "" - } - ) - } - filter.tags.firstOrNull()?.let { tag -> - append("&genre=").append(tag.key) - } - filter.types.oneOrThrowIfMany()?.let { type -> - append("&comic_type=") - append( - when (type) { - ContentType.MANGA -> "MANGA" - ContentType.MANHWA -> "MANHWA" - ContentType.MANHUA -> "MANHUA" - else -> "" - } - ) - } - } - - val json = webClient.httpGet(url).body?.string().orEmpty() - return parseListJson(json) - } - - private fun parseListJson(json: String): List<Manga> { - val result = mutableListOf<Manga>() - try { - val obj = JSONObject(json) - val dataArray = obj.optJSONArray("data") ?: return emptyList() - for (i in 0 until dataArray.length()) { - val item = dataArray.getJSONObject(i) - val slug = item.optString("slug").ifEmpty { continue } - val relativeUrl = "/comic/$slug" - val title = item.optString("title").ifEmpty { slug } - val cover = item.optString("poster_image_url").nullIfEmpty() - result.add( - Manga( - id = generateUid(relativeUrl), - url = relativeUrl, - publicUrl = "https://$domain$relativeUrl", - title = title, - coverUrl = cover, - rating = RATING_UNKNOWN, - contentRating = ContentRating.SAFE, - tags = emptySet(), - state = null, - authors = emptySet(), - altTitles = emptySet(), - source = source, - ) - ) - } - } catch (_: Exception) { - } - return result - } - - override suspend fun getDetails(manga: Manga): Manga { - detailsCache[manga.url]?.let { return it } - - val slug = manga.url.substringAfter("/comic/") - val json = webClient.httpGet("$apiBase/series/comic/$slug").body?.string().orEmpty() - val result = parseDetailsJson(json, manga) - - detailsCache[manga.url] = result - return result - } - - private fun parseDetailsJson(json: String, manga: Manga): Manga { - try { - val obj = JSONObject(json) - val slug = obj.optString("slug") - val relativeUrl = "/comic/$slug" - - val title = obj.optString("title").ifEmpty { manga.title } - val desc = obj.optString("synopsis").nullIfEmpty()?.let { org.jsoup.Jsoup.parse(it).text() } - val rating = obj.optString("rating_average").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN - val state = when (obj.optString("comic_status").uppercase()) { - "ONGOING" -> MangaState.ONGOING - "COMPLETED" -> MangaState.FINISHED - "HIATUS" -> MangaState.PAUSED - else -> null - } - val author = obj.optString("author_name").takeIf { it.isNotEmpty() } - val artist = obj.optString("artist_name").takeIf { it.isNotEmpty() } - val genres = obj.optJSONArray("genres")?.let { arr -> - (0 until arr.length()).mapNotNull { i -> - val g = arr.getJSONObject(i) - val gSlug = g.optString("slug") - val gName = g.optString("name") - if (gSlug.isNotEmpty() && gName.isNotEmpty()) MangaTag(gName, gSlug, source) else null - }.toSet() - } ?: emptySet() - - val chapters = parseUnitsJson(obj.optJSONArray("units"), slug) - - return manga.copy( - title = title, - url = relativeUrl, - publicUrl = "https://$domain$relativeUrl", - description = desc, - rating = rating, - state = state, - authors = setOfNotNull(author, artist), - tags = genres, - chapters = chapters, - ) - } catch (_: Exception) { - return manga - } - } - - private fun parseUnitsJson(array: JSONArray?, seriesSlug: String): List<MangaChapter> { - if (array == null) return emptyList() - val chapters = mutableListOf<MangaChapter>() - for (i in 0 until array.length()) { - val obj = array.getJSONObject(i) - val chSlug = obj.optString("slug").ifEmpty { continue } - val number = obj.optString("number") - val date = obj.optString("created_at") - val uploadDate = parseDate(date) - chapters.add( - MangaChapter( - id = generateUid("/comic/$seriesSlug/chapter/$chSlug"), - url = "/comic/$seriesSlug/chapter/$chSlug", - title = "Chapter ${number.removeSuffix(".00")}", - number = number.toFloatOrNull() ?: (chapters.size + 1).toFloat(), - volume = 0, - uploadDate = uploadDate, - scanlator = null, - branch = null, - source = source, - ) - ) - } - return chapters.reversed() - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val slug = chapter.url.substringAfter("/comic/").substringBefore("/chapter/") - val chSlug = chapter.url.substringAfterLast("/") - val json = webClient.httpGet("$apiBase/series/comic/$slug/chapter/$chSlug").body?.string().orEmpty() - try { - val obj = JSONObject(json) - val chapterObj = obj.optJSONObject("chapter") ?: return emptyList() - val pagesArray = chapterObj.optJSONArray("pages") ?: return emptyList() - return (0 until pagesArray.length()).mapNotNull { i -> - val pageObj = pagesArray.getJSONObject(i) - val rawUrl = pageObj.optString("image_url").nullIfEmpty() ?: return@mapNotNull null - val fixedUrl = fixImageUrl(rawUrl) - MangaPage(id = generateUid(fixedUrl), url = fixedUrl, preview = null, source = source) - } - } catch (_: Exception) { - return emptyList() - } - } - - private fun fixImageUrl(url: String): String { - var fixed = if (url.startsWith("http")) url else "https://api.ainzscans01.com$url" - - if (fixed.contains("googleusercontent.com") || fixed.contains("bp.blogspot.com")) { - fixed = fixed.replace(Regex("""=[swh]\d+[^/?]*($|\?)""", RegexOption.IGNORE_CASE), "=s0$1") - .replace(Regex("""/[swh]\d+[^/]*/""", RegexOption.IGNORE_CASE), "/s0/") - } - - val httpUrl = fixed.toHttpUrlOrNull() - if (httpUrl != null && httpUrl.queryParameterNames.any { it == "w" || it == "width" || it == "resize" }) { - fixed = httpUrl.newBuilder() - .removeAllQueryParameters("w") - .removeAllQueryParameters("width") - .removeAllQueryParameters("resize") - .build() - .toString() - } - return fixed - } - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - private fun parseDate(dateStr: String?): Long { - if (dateStr.isNullOrEmpty()) return 0L - return try { - dateFormat.parse(dateStr)?.time ?: 0L - } catch (_: Exception) { 0L } - } - - private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this -} +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.urlEncoded + +import okhttp3.HttpUrl.Companion.toHttpUrlOrNull +import org.json.JSONArray +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.Locale +import java.util.EnumSet +import java.util.TimeZone +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.Jsoup +import kotlin.collections.toSet + +@MangaSourceParser("AINZSCANS", "AinzScans", "id") +internal class AinzScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.AINZSCANS, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("v2.ainzscans01.com") + + private val apiBase = "https://api.ainzscans01.com/api" + + override val sourceLocale: Locale = Locale.US + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrFetchGenres().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + + @Volatile + private var genresCache: Map<String, MangaTag>? = null + private val genresMutex = Mutex() + + private suspend fun getOrFetchGenres(): Map<String, MangaTag> { + genresCache?.let { return it } + return genresMutex.withLock { + genresCache ?: run { + val jsonStr = webClient.httpGet("$apiBase/genres").body?.string().orEmpty() + val map = mutableMapOf<String, MangaTag>() + try { + val array = JSONArray(jsonStr) + for (i in 0 until array.length()) { + val obj = array.getJSONObject(i) + val name = obj.getString("name") + val slug = obj.getString("slug") + map[slug] = MangaTag(title = name, key = slug, source = source) + } + } catch (_: Exception) { } + genresCache = map + map + } + } + } + + private val detailsCacheLock = Any() + + private val detailsCache = object : LinkedHashMap<String, Manga>(16, 0.75f, true) { + override fun removeEldestEntry(eldest: MutableMap.MutableEntry<String, Manga>?): Boolean = size > 10 + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append(apiBase) + append("/search?type=COMIC") + append("&limit=").append(pageSize) + append("&page=").append(page) + + filter.query?.takeIf { it.isNotBlank() }?.let { + append("&q=").append(it.urlEncoded()) + } + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "views" + SortOrder.NEWEST -> "new" + SortOrder.RATING -> "rate" + SortOrder.ALPHABETICAL -> "az" + else -> "latest" + } + ) + filter.states.oneOrThrowIfMany()?.let { state -> + append("&status=") + append( + when (state) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "COMPLETED" + MangaState.PAUSED -> "HIATUS" + else -> "" + } + ) + } + filter.tags.firstOrNull()?.let { tag -> + append("&genre=").append(tag.key) + } + filter.types.oneOrThrowIfMany()?.let { type -> + append("&comic_type=") + append( + when (type) { + ContentType.MANGA -> "MANGA" + ContentType.MANHWA -> "MANHWA" + ContentType.MANHUA -> "MANHUA" + else -> "" + } + ) + } + } + + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseListJson(json) + } + + private fun parseListJson(json: String): List<Manga> { + val result = mutableListOf<Manga>() + try { + val obj = JSONObject(json) + val dataArray = obj.optJSONArray("data") ?: return emptyList() + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val slug = item.optString("slug").ifEmpty { continue } + val relativeUrl = "/comic/$slug" + val title = item.optString("title").ifEmpty { slug } + val cover = item.optString("poster_image_url").nullIfEmpty() + result.add( + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + title = title, + coverUrl = cover, + rating = RATING_UNKNOWN, + contentRating = ContentRating.SAFE, + tags = emptySet(), + state = null, + authors = emptySet(), + altTitles = emptySet(), + source = source, + ) + ) + } + } catch (_: Exception) { + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + synchronized(detailsCacheLock) { + detailsCache[manga.url]?.let { return it } + } + + val slug = manga.url.substringAfter("/comic/") + val json = webClient.httpGet("$apiBase/series/comic/$slug").body?.string().orEmpty() + val result = parseDetailsJson(json, manga) + + synchronized(detailsCacheLock) { + detailsCache[manga.url] = result + } + return result + } + + private fun parseDetailsJson(json: String, manga: Manga): Manga { + try { + val obj = JSONObject(json) + val slug = obj.optString("slug") + val relativeUrl = "/comic/$slug" + + val title = obj.optString("title").ifEmpty { manga.title } + val desc = obj.optString("synopsis").nullIfEmpty()?.let { Jsoup.parse(it).text() } + val rating = obj.optString("rating_average").toFloatOrNull()?.div(10f) ?: RATING_UNKNOWN + val state = when (obj.optString("comic_status").uppercase()) { + "ONGOING" -> MangaState.ONGOING + "COMPLETED" -> MangaState.FINISHED + "HIATUS" -> MangaState.PAUSED + else -> null + } + val author = obj.optString("author_name").takeIf { it.isNotEmpty() } + val artist = obj.optString("artist_name").takeIf { it.isNotEmpty() } + val genres = obj.optJSONArray("genres")?.let { arr -> + (0 until arr.length()).mapNotNull { i -> + val g = arr.getJSONObject(i) + val gSlug = g.optString("slug") + val gName = g.optString("name") + if (gSlug.isNotEmpty() && gName.isNotEmpty()) MangaTag(gName, gSlug, source) else null + }.toSet() + } ?: emptySet() + + val chapters = parseUnitsJson(obj.optJSONArray("units"), slug) + + return manga.copy( + title = title, + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + description = desc, + rating = rating, + state = state, + authors = setOfNotNull(author, artist), + tags = genres, + chapters = chapters, + ) + } catch (_: Exception) { + return manga + } + } + + private fun parseUnitsJson(array: JSONArray?, seriesSlug: String): List<MangaChapter> { + if (array == null) return emptyList() + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until array.length()) { + val obj = array.getJSONObject(i) + val chSlug = obj.optString("slug").ifEmpty { continue } + val number = obj.optString("number") + val date = obj.optString("created_at") + val uploadDate = parseDate(date) + chapters.add( + MangaChapter( + id = generateUid("/comic/$seriesSlug/chapter/$chSlug"), + url = "/comic/$seriesSlug/chapter/$chSlug", + title = "Chapter ${number.removeSuffix(".00")}", + number = number.toFloatOrNull() ?: (chapters.size + 1).toFloat(), + volume = 0, + uploadDate = uploadDate, + scanlator = null, + branch = null, + source = source, + ) + ) + } + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val slug = chapter.url.substringAfter("/comic/").substringBefore("/chapter/") + val chSlug = chapter.url.substringAfterLast("/") + val json = webClient.httpGet("$apiBase/series/comic/$slug/chapter/$chSlug").body?.string().orEmpty() + try { + val obj = JSONObject(json) + val chapterObj = obj.optJSONObject("chapter") ?: return emptyList() + val pagesArray = chapterObj.optJSONArray("pages") ?: return emptyList() + return (0 until pagesArray.length()).mapNotNull { i -> + val pageObj = pagesArray.getJSONObject(i) + val rawUrl = pageObj.optString("image_url").nullIfEmpty() ?: return@mapNotNull null + val fixedUrl = fixImageUrl(rawUrl) + MangaPage(id = generateUid(fixedUrl), url = fixedUrl, preview = null, source = source) + } + } catch (_: Exception) { + return emptyList() + } + } + + private fun fixImageUrl(url: String): String { + var fixed = if (url.startsWith("http")) url else "https://api.ainzscans01.com$url" + + if (fixed.contains("googleusercontent.com") || fixed.contains("bp.blogspot.com")) { + fixed = fixed.replace(Regex("""=[swh]\d+[^/?]*($|\?)""", RegexOption.IGNORE_CASE), "=s0$1") + .replace(Regex("""/[swh]\d+[^/]*/""", RegexOption.IGNORE_CASE), "/s0/") + } + + val httpUrl = fixed.toHttpUrlOrNull() + if (httpUrl != null && httpUrl.queryParameterNames.any { it == "w" || it == "width" || it == "resize" }) { + fixed = httpUrl.newBuilder() + .removeAllQueryParameters("w") + .removeAllQueryParameters("width") + .removeAllQueryParameters("resize") + .build() + .toString() + } + return fixed + } + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.US).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + private fun parseDate(dateStr: String?): Long { + if (dateStr.isNullOrEmpty()) return 0L + return try { + dateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { 0L } + } + + private fun String?.nullIfEmpty(): String? = if (this.isNullOrEmpty()) null else this +} diff --git a/src/main/kotlin/tsuki/site/id/Alawale.kt b/src/main/kotlin/tsuki/site/id/Alawale.kt new file mode 100644 index 00000000..b826a739 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Alawale.kt @@ -0,0 +1,319 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseRaw +import tsuki.util.urlEncoded + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("ALAWALE", "Alawale", "id") +internal class Alawale(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ALAWALE, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("alawale.net") + private val baseUrl = "https://$domain" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val doc = webClient.httpGet("$baseUrl/daftar-komik").parseHtml() + + val genreTags = doc.select("select[aria-label=Genre] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "genre:$value", title = opt.text().trim(), source = source) + }.toSet() + + val typeTags = doc.select("select[aria-label=Tipe] option") + .mapNotNull { opt -> + val value = opt.`val`().trim() + if (value.isEmpty()) null + else MangaTag(key = "type:$value", title = opt.text().trim(), source = source) + }.toSet() + + return MangaListFilterOptions( + availableTags = genreTags + typeTags, + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "chapter" + SortOrder.ALPHABETICAL -> "title" + else -> "update" + } + + val url = "$baseUrl/api/komik".toHttpUrl().newBuilder() + .addQueryParameter("sort", sort) + .addQueryParameter("limit", pageSize.toString()) + .addQueryParameter("offset", ((page - 1) * pageSize).toString()) + + if (filter.query != null) { + url.addQueryParameter("q", filter.query.urlEncoded()) + } + + filter.tags.forEach { tag -> + when { + tag.key.startsWith("genre:") -> url.addQueryParameter("genre", tag.key.removePrefix("genre:")) + tag.key.startsWith("type:") -> url.addQueryParameter("type", tag.key.removePrefix("type:")) + } + } + filter.types.forEach { ct -> + val typeParam = when (ct) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Comic" + else -> null + } + if (typeParam != null) url.addQueryParameter("type", typeParam) + } + + val json = webClient.httpGet(url.build()).parseJson() + val items = json.optJSONArray("items") ?: return emptyList() + + return (0 until items.length()).map { i -> + val obj = items.getJSONObject(i) + val slug = obj.getString("slug") + val title = obj.getString("title") + val type = obj.optString("type", "") + val coverPath = obj.optString("cover_remote", "") + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + Manga( + id = generateUid("/$slug"), + url = "/$slug", + publicUrl = "$baseUrl/$slug", + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = coverUrl.ifBlank { null }, + tags = setOfNotNull(type.takeIf { it.isNotBlank() }?.let { MangaTag(it, it, source) }), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + + val apiJson = webClient.httpGet("$baseUrl/api/komik?slugs=$slug").parseJson() + val apiItem = apiJson?.optJSONArray("items")?.optJSONObject(0) + val title = apiItem?.optString("title") ?: manga.title + val coverPath = apiItem?.optString("cover_remote") ?: "" + val coverUrl = if (coverPath.startsWith("/")) "$baseUrl$coverPath" else coverPath + + val bookScript = doc.selectFirst("script[type=\"application/ld+json\"]:containsData(Book)") + var description: String? = null + var author: String? = null + val genres = mutableListOf<String>() + + if (bookScript != null) { + val json = JSONObject(bookScript.data()) + description = json.optString("description", null) + author = json.optJSONObject("author")?.optString("name") + val genreArray = json.optJSONArray("genre") + if (genreArray != null) { + for (i in 0 until genreArray.length()) { + genres.add(genreArray.getString(i)) + } + } + } + + if (author.isNullOrBlank()) { + author = doc.select(".detail-info .kv span:contains(Author) b")?.text()?.trim() + } + + val synopsisBody = doc.selectFirst(".syn-body")?.text()?.trim() + if (!synopsisBody.isNullOrBlank() && description.isNullOrBlank()) { + description = synopsisBody + } + + if (genres.isEmpty()) { + genres.addAll(doc.select(".detail-info .genres a.chip").map { it.text() }) + } + + val tagSet = genres.map { MangaTag(it, it, source) }.toMutableSet() + val type = apiItem?.optString("type", "") + if (!type.isNullOrBlank()) { + tagSet += MangaTag(type, type, source) + } + + val chapters = parseChapters(doc, slug) + + return manga.copy( + title = title, + coverUrl = coverUrl.ifBlank { manga.coverUrl }, + description = description, + authors = setOfNotNull(author), + tags = tagSet, + chapters = chapters, + contentRating = ContentRating.SAFE, + ) + } + + private fun parseChapters(doc: Document, slug: String): List<MangaChapter> { + val chapterLinks = doc.select("div.chap-list a.chap-item") + .distinctBy { it.attr("abs:href") } + + return chapterLinks.map { link -> + val href = link.attr("href").removePrefix("/") + val chapterSlug = href.substringAfter("$slug/ch/") + val name = link.selectFirst("span")?.text()?.trim() ?: chapterSlug + val number = parseChapterNumber(chapterSlug) + + MangaChapter( + id = generateUid("/$href"), + url = "/$href", + title = name, + number = number, + volume = 0, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + } + + private fun parseChapterNumber(chapterSlug: String): Float { + val clean = chapterSlug.substringBefore("-bahasa-indonesia") + val parts = clean.split("-") + return if (parts.size == 2) { + val main = parts[0].toFloatOrNull() ?: 0f + val sub = parts[1].toFloatOrNull() ?: 0f + main + sub / 10f + } else { + clean.toFloatOrNull() ?: 0f + } + } + + // RSC payloard + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = "$baseUrl/${chapter.url.removePrefix("/")}" + val rawHtml = webClient.httpGet(fullUrl).parseRaw() + val unescapedHtml = rawHtml.replace("\\/", "/") + + val imageRegex = Regex("""https://bmcdn\.my\.id/[^\s"\\]+\.jpg""") + val allUrls = imageRegex.findAll(unescapedHtml) + .map { it.value } + .distinct() + .toList() + + if (allUrls.isNotEmpty()) { + val pages = allUrls.mapNotNull { url -> + val filename = url.substringAfterLast("/") + val dashIndex = filename.indexOf('-') + if (dashIndex < 0) return@mapNotNull null + val pageNumStr = filename.substring(0, dashIndex) + val pageNum = pageNumStr.toIntOrNull() ?: return@mapNotNull null + pageNum to url + }.sortedBy { it.first } + .map { (_, url) -> + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + if (pages.isNotEmpty()) return pages + } + + val doc = Jsoup.parse(unescapedHtml) + val pageDivs = doc.select("div.reader-page") + if (pageDivs.isEmpty()) return emptyList() + + val totalPages = pageDivs.size + val firstImageUrl = doc.head() + .select("link[rel=preload][as=image]") + .map { it.attr("abs:href") } + .firstOrNull { it.isNotBlank() } + ?: return emptyList() + + val lastSegment = firstImageUrl.substringAfterLast("/") + val dashIndex = lastSegment.indexOf('-') + if (dashIndex < 0) return emptyList() + + val hashPart = lastSegment.substring(dashIndex + 1) + val basePath = firstImageUrl.substringBeforeLast("/") + "/" + + return (0 until totalPages).map { index -> + val url = "${basePath}${index}-${hashPart}" + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val slug = seed.url.removePrefix("/") + val doc = webClient.httpGet("$baseUrl/$slug").parseHtml() + return doc.select("section.related .card").mapNotNull { card -> + val link = card.selectFirst("a") ?: return@mapNotNull null + val href = link.attr("href").removePrefix("/") + val title = link.selectFirst(".t")?.text() ?: return@mapNotNull null + val coverImg = link.selectFirst("img") + val coverUrl = coverImg?.attr("abs:src") + Manga( + id = generateUid("/$href"), + url = "/$href", + publicUrl = "$baseUrl/$href", + title = title, + coverUrl = coverUrl, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/id/Apkomik.kt b/src/main/kotlin/tsuki/site/id/Apkomik.kt new file mode 100644 index 00000000..ecfeb609 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Apkomik.kt @@ -0,0 +1,24 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("APKOMIK", "Apkomik", "id") +internal class Apkomik(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.APKOMIK, "01.apkomik.com") { + override val dateFormat = SimpleDateFormat("MMMM dd, yyyy", Locale("id")) + + override fun parseAltTitles(doc: Document, table: Element?, hasTable: Boolean): Set<String> { + return doc.select(".wd-full:contains(Alternative) span, .wd-full:contains(Alt) span") + .flatMap { splitAltTitles(it.text()) } + .toSet() + } +} diff --git a/src/main/kotlin/tsuki/site/id/Bacami.kt b/src/main/kotlin/tsuki/site/id/Bacami.kt new file mode 100644 index 00000000..40a6d11a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Bacami.kt @@ -0,0 +1,326 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import kotlinx.coroutines.coroutineScope +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("BACAMI", "Bacami", "id") +internal class Bacami(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BACAMI, 20) { + + override val configKeyDomain = ConfigKey.Domain("v1.bacami.site") + + private val dateFormat = SimpleDateFormat("dd MMMM, yyyy", Locale.ENGLISH) + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (key, title) -> + MangaTag(key = key, title = title, source = source) + }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = filter.query + ?.takeIf { it.isNotBlank() } + ?.let { q -> + val cleaned = q.replace("Bahasa Indonesia", "").trim() + "https://$domain/search/${cleaned.urlEncoded()}/page/$page/" + } + ?: buildString { + append("https://$domain/custom-search/") + + if (filter.tags.isNotEmpty()) { + append("genre/") + append(filter.tags.first().key) + append("/") + } + + filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> append("status/hot/") + MangaState.FINISHED -> append("status/tamat/") + else -> {} + } + } + + filter.types.firstOrNull()?.let { type -> + when (type) { + ContentType.MANGA -> append("type/manga/") + ContentType.MANHWA -> append("type/manhwa/") + ContentType.MANHUA -> append("type/manhua/") + else -> {} + } + } + + when (order) { + SortOrder.POPULARITY -> append("orderby/score/") + SortOrder.UPDATED -> append("orderby/latest/") + SortOrder.ALPHABETICAL -> append("orderby/name/") + else -> append("orderby/latest/") + } + + append("page/") + append(page) + append("/") + } + + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("article.genre-card").mapNotNull { element -> + val link = element.selectFirst("div.genre-cover > a") ?: return@mapNotNull null + val title = element.selectFirst("div.genre-info > a")?.text() ?: return@mapNotNull null + val href = link.attr("href") + val cover = element.selectFirst("div.genre-cover > a > img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + }?.toAbsoluteUrl(domain) + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val content = doc.selectFirst("#komik > section.manga-content") + + val title = content?.selectFirst("header > h1")?.text() ?: manga.title + val cover = content?.selectFirst("figure .image-wrap img")?.let { + it.attr("data-src").ifBlank { it.attr("src") } + }?.toAbsoluteUrl(domain) ?: manga.coverUrl.orEmpty() + + val description = content?.selectFirst("p.manga-description")?.text().orEmpty() + val altTitle = content?.selectFirst("p.manga-altname")?.text().orEmpty() + val fullDescription = if (altTitle.isNotBlank()) { + "$description\n\nAlternative Title: $altTitle".trim() + } else { + description + } + + val author = content?.selectFirst(".info-item:contains(Author) .info-value")?.text() + ?: content?.selectFirst("div > div > div:nth-child(3) > span.info-value")?.text() + ?: "" + + val tags = content?.select("nav > span > a")?.map { it.text() } + ?.map { MangaTag(key = it.lowercase(), title = it, source = source) } + ?.toSet().orEmpty() + + val state = when { + doc.selectFirst(".hot-tag, .project-tag") != null -> MangaState.ONGOING + doc.selectFirst(".tamat-tag") != null -> MangaState.FINISHED + else -> null + } + + val chapters = doc.select("ol.chapter-list > li").mapNotNull { element -> + val link = element.selectFirst("a.ch-link") ?: return@mapNotNull null + val href = link.attr("href") + val name = link.text().substringAfter("–").trim() + val date = element.selectFirst("span.ch-date")?.text() + + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = parseDate(date), + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + + manga.copy( + title = title, + coverUrl = cover, + description = fullDescription, + authors = setOfNotNull(author.takeIf { it.isNotBlank() }), + tags = tags, + state = state, + chapters = chapters, + ) + } + + private fun parseDate(date: String?): Long { + if (date.isNullOrBlank()) return 0L + return runCatching { dateFormat.parse(date)?.time ?: 0L }.getOrDefault(0L) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val scriptContent = doc.selectFirst("script:containsData(imageUrls)")?.data() + ?: return emptyList() + + val jsonString = scriptContent.substringAfter("imageUrls:").substringBefore("],").plus("]") + val jsonArray = JSONArray(jsonString) + + return (0 until jsonArray.length()).map { i -> + val url = jsonArray.getString(i) + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + companion object { + private val GENRES = listOf( + "action-2" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "apocalypse" to "Apocalypse", + "comedy" to "Comedy", + "comedy-mystery-romance-slice-of-life-supernatural" to "Comedy Mystery Romance Slice Of Life Supernatural", + "comedy-romance-slice-of-life" to "Comedy Romance Slice Of Life", + "cooking" to "Cooking", + "crime" to "Crime", + "cultivation" to "Cultivation", + "demons" to "Demons", + "doujinshi" to "Doujinshi", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "furry" to "Furry", + "game" to "Game", + "gender-bender" to "Gender Bender", + "genius" to "Genius", + "gore" to "Gore", + "harem" to "Harem", + "hentai" to "Hentai", + "historical" to "Historical", + "horror" to "Horror", + "isekai" to "Isekai", + "josei" to "Josei", + "lolicon" to "Lolicon", + "long-strip" to "Long Strip", + "love-polygon" to "Love Polygon", + "magic" to "Magic", + "magical-girl" to "Magical Girl", + "manhua" to "Manhua", + "manhwa" to "Manhwa", + "martial-art" to "Martial Art", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medical" to "Medical", + "military" to "Military", + "monster" to "Monster", + "monster-girls" to "Monster Girls", + "monsters" to "Monsters", + "music" to "Music", + "mystery" to "Mystery", + "mystery-shounen" to "Mystery Shounen", + "mythology" to "Mythology", + "one-shot" to "One Shot", + "oneshot" to "Oneshot", + "parody" to "Parody", + "philosophical" to "Philosophical", + "police" to "Police", + "post-apocalyptic" to "Post-Apocalyptic", + "psychological" to "Psychological", + "rebirth" to "Rebirth", + "reincarnation" to "Reincarnation", + "romance" to "Romance", + "romantic-subtext" to "Romantic Subtext", + "samurai" to "Samurai", + "school" to "School", + "school-life" to "School Life", + "sci-fi" to "Sci-fi", + "seinen" to "Seinen", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "shounen-ai" to "Shounen Ai", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "space" to "Space", + "sports" to "Sports", + "superhero" to "Superhero", + "supernatural" to "Supernatural", + "super-power" to "Super Power", + "survival" to "Survival", + "suspense" to "Suspense", + "system" to "System", + "team-sports" to "Team Sports", + "thriller" to "Thriller", + "time-travel" to "Time Travel", + "tragedy" to "Tragedy", + "urban" to "Urban", + "urban-fantasy" to "Urban Fantasy", + "vampire" to "Vampire", + "video-game" to "Video Game", + "villainess" to "Villainess", + "visual-arts" to "Visual Arts", + "webtoon" to "Webtoon", + "webtoons" to "Webtoons", + "wuxia" to "Wuxia", + "yaoi" to "Yaoi", + "yuri" to "Yuri", + "zombies" to "Zombies", + ) + } +} diff --git a/src/main/kotlin/tsuki/site/id/CGBUM.kt b/src/main/kotlin/tsuki/site/id/CGBUM.kt new file mode 100644 index 00000000..401162f8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/CGBUM.kt @@ -0,0 +1,306 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("CGBUM", "CGBUM", "id") +class CGBUM(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CGBUM, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("cgbum.com") + + override val availableSortOrders: Set<SortOrder> = setOf( + SortOrder.UPDATED, // latest + SortOrder.ADDED, // newest + SortOrder.ADDED_ASC, // oldest + SortOrder.POPULARITY, // views + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = GENRES.map { genre -> + MangaTag(key = genre, title = genre, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ContentType.HENTAI + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = if (!filter.query.isNullOrBlank()) { + buildSearchUrl(page, filter.query.urlEncoded()) + } else { + buildBrowseUrl(page, order, filter) + } + val doc = webClient.httpGet(url).parseHtml() + + val cards = doc.select("article.comic-card") + return if (cards.isEmpty()) emptyList() else cards.mapNotNull { parseMangaCard(it) } + } + + private fun buildSearchUrl(page: Int, query: String): String { + return buildString { + append("https://$domain/cari?q=") + append(query.urlEncoded().replace("%20", "+")) + if (page > 1) { + append("&page=") + append(page) + } + } + } + + private fun buildBrowseUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return buildString { + append("https://$domain/daftar-komik?") + append("type=") + filter.types.firstOrNull()?.let { type -> + append( + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.HENTAI -> "pornhwa" + else -> {} + } + ) + } + append("&status=") + filter.states.firstOrNull()?.let { state -> + append( + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "tamat" + else -> {} + } + ) + } + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.ADDED -> "newest" + SortOrder.ADDED_ASC -> "oldest" + SortOrder.POPULARITY -> "views" + else -> "latest" + } + ) + filter.tags.take(3).forEach { tag -> + append("&genres%5B%5D=") + append(tag.key) + } + if (page > 1) { + append("&page=") + append(page) + } + } + } + + private fun parseMangaCard(card: Element): Manga? { + val coverLink = card.selectFirst("a.comic-card-cover") ?: return null + val href = coverLink.attrAsRelativeUrl("href") + val titleEl = card.selectFirst(".comic-card-title a") + val title = titleEl?.text() ?: return null + val coverUrl = card.selectFirst("img")?.src()?.takeIf { it.isNotBlank() } + + val typeBadge = card.selectFirst(".badge-type")?.text() ?: "Unknown" + val typeTag = when (typeBadge.lowercase()) { + "manga" -> MangaTag("type:manga", "Manga", source) + "manhwa" -> MangaTag("type:manhwa", "Manhwa", source) + "manhua" -> MangaTag("type:manhua", "Manhua", source) + else -> null + } + + val isAdult = card.selectFirst(".badge-pornhwa") != null + + val statusText = card.selectFirst(".badge-status")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "tamat" -> MangaState.FINISHED + else -> null + } + + val tags = mutableSetOf<MangaTag>() + if (typeTag != null) tags.add(typeTag) + if (isAdult) tags.add(MangaTag("adult", "Adult", source)) + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + authors = emptySet(), + tags = tags, + rating = RATING_UNKNOWN, + state = state, + contentRating = if (isAdult) ContentRating.ADULT else null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.selectFirst(".comic-cover img")?.src() ?: manga.coverUrl + + val desc = doc.selectFirst(".comic-synopsis .synopsis-content")?.text() ?: "" + + val genreElements = doc.select(".comic-genres .genre-pill") + val tags = genreElements.mapNotNull { a -> + val genreName = a.text().trim() + if (genreName.isNotEmpty()) MangaTag(key = genreName, title = genreName, source = source) + else null + }.toSet() + + val statusText = doc.selectFirst(".badge-status")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "tamat" -> MangaState.FINISHED + else -> null + } + + val isAdult = doc.selectFirst(".badge-pornhwa") != null || + tags.any { it.title.equals("adult", ignoreCase = true) } + val contentRating = if (isAdult) ContentRating.ADULT else null + + val authorRow = doc.select(".meta-row").firstOrNull { it.selectFirst(".meta-label")?.text() == "Author" } + val author = authorRow?.selectFirst(".meta-value")?.text()?.trim()?.takeIf { it.isNotBlank() && it != "-" } + val authors = if (author != null) setOf(author) else emptySet() + + val chapterLinks = doc.select(".chapter-grid a.ch-grid-item") + val chapters = chapterLinks.map { a -> + val href = a.attrAsRelativeUrl("href") + val label = a.text().trim() + val number = label.extractChapterNumber() + MangaChapter( + id = generateUid(href), + url = href, + title = label, + number = number, + volume = 0, + uploadDate = 0, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + publicUrl = fullUrl, + coverUrl = cover, + description = desc, + tags = tags, + state = state, + authors = authors, + contentRating = contentRating, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val containers = doc.select(".page-container") + if (containers.isEmpty()) { + throw ParseException("No image containers found", fullUrl) + } + + return containers.mapNotNull { container -> + val url = container.attr("data-url").takeIf { it.isNotBlank() } + ?: container.selectFirst("img")?.attr("src")?.takeIf { it.isNotBlank() } + if (url == null) return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + val fullUrl = seed.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val section = doc.selectFirst("section:has(.section-header h2:contains(Baca Juga))") + ?: return emptyList() + + return section.select("article.comic-card").mapNotNull { card -> + parseMangaCard(card) + } + } + + companion object { + private val GENRES = listOf( + "adaptation", "adult", "adultmature", "age gap", "ahegao", "aliens", "anal", + "animals", "anthology", "bdsm", "beasts", "big ass", "big breast", "big breasts", + "big penis", "bisexual", "blackmail", "bloody", "blowjob", "body swap", "bodyswap", + "bondage", "business suit", "cheating", "cheating infidelity", "childhood friends", + "collar", "college life", "comedy", "condom", "contest winning", "cooking", "crime", + "cunnilingus", "curse", "dark skin", "defloration", "delinquents", "demon girl", + "demons", "double penetration", "doujinshi", "drama", "dungeons", "ecchi", "elf", + "emperor s daughter", "exhibitionism", "fantasy", "femdom", "fetish", "ffm threesome", + "filming", "fingering", "footjob", "full color", "futanari", "game", "gender bender", + "genderswap", "ghost", "ghosts", "glasses", "gore", "group", "gyaru", "handjob", + "harem", "hentai", "historical", "horns", "huge breast", "humiliation", + "inverted nipples", "isekai", "josei", "josei w", "kemomimi", "lingerie", "lolicon", + "milf", "magic", "maid", "manhua", "manhwa", "martial arts", "masturbation", + "mature", "medical", "military", "mind break", "mind control", "mmf threesome", + "monster girls", "monsters", "music", "mystery", "ntr", "nakadashi", "netorare", + "non human", "obsessive male lead", "office workers", "omegaverse", "oneshot", + "paizuri", "police", "pregnant", "psychological", "rape", "regression", + "reincarnation", "revenge", "reverse harem", "romance", "royal family", "royalty", + "runaway", "school life", "sci fi", "seinen", "seinen m", "sex toys", "shoujo ai", + "shoujo g", "shounen ai", "shounen b", "showbiz", "slice of life", "small breast", + "smut", "space", "sports", "stocking", "story arc", "super power", "supernatural", + "survival", "thriller", "time travel", "tomboy", "tower climbing", "traditional games", + "tragedy", "transmigration", "twintails", "unusual pupils", "vampires", "video games", + "villainess", "violence", "virginity", "virtual reality", "webtoon", "wuxia", + "yakuzas", "yaoi bl", "yuri gl", "zombies", "action", "adventure", "boys", "girl", + "hairy", "horror", "manga", "rofan" + ) + } +} diff --git a/src/main/kotlin/tsuki/site/id/CosmicScans.kt b/src/main/kotlin/tsuki/site/id/CosmicScans.kt new file mode 100644 index 00000000..81d6a3cb --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/CosmicScans.kt @@ -0,0 +1,429 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseHtml + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONArray +import org.json.JSONObject +import org.jsoup.Jsoup +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("COSMICSCANSID", "CosmicScans", "id") +internal class CosmicScans(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.COSMICSCANSID, pageSize = 20) { + + override val configKeyDomain = ConfigKey.Domain("02.cosmicscans.to") + private val apiUrl = "https://cdncid.csmcscns.id/v1/manga" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.NEWEST, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .set("Origin", "https://$domain") + .set("Referer", "https://$domain/") + .build() + + private val cursorCache = mutableMapOf<String, String>() + private val lastPage = mutableMapOf<String, Int>() + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = GENRES.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val endpoint = if (query.isNotBlank()) "search" else "filter" + val key = buildSearchKey(endpoint, query, order, filter) + lastPage[key] = page + + val url = "$apiUrl/$endpoint".toHttpUrl().newBuilder() + .addQueryParameter("limit", pageSize.toString()) + .apply { + if (query.isNotBlank()) { + addQueryParameter("q", query) + } + if (endpoint == "filter") { + addQueryParameter("order_by", mapSortOrder(order)) + } + if (endpoint != "search") { + if (page > 1) { + cursorCache["$key:$page"]?.takeIf { it.isNotBlank() }?.let { + addQueryParameter("after", it) + } + } + } + } + .build() + + val response = try { + webClient.httpGet(url, getRequestHeaders()) + } catch (e: Exception) { + throw ParseException("Failed to load list page $page (URL: $url): ${e.message}", url.toString(), e) + } + + val responseBody = try { + response.parseJson() + } catch (e: Exception) { + throw ParseException("Invalid JSON from list API (URL: $url): ${e.message}", url.toString(), e) + } + + val result = try { + parseMangaListResponse(responseBody, key) + } catch (e: Exception) { + throw ParseException("Failed to parse manga list (URL: $url): ${e.message}", url.toString(), e) + } + + return applyClientFilters(result, filter) + } + + private fun mapSortOrder(order: SortOrder): String = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.POPULARITY -> "popular" + SortOrder.ALPHABETICAL -> "az" + SortOrder.ALPHABETICAL_DESC -> "za" + SortOrder.NEWEST -> "added" + else -> "update" + } + + private fun buildSearchKey(endpoint: String, query: String, order: SortOrder, filter: MangaListFilter): + String = listOf( + endpoint, query, mapSortOrder(order), + filter.states.joinToString(",") { it.name }, + filter.types.joinToString(",") { it.name }, + filter.tags.joinToString(",") { it.key }, + ).joinToString(":") + + private fun parseMangaListResponse(json: JSONObject, key: String): List<Manga> { + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown API error") + throw ParseException("CosmicScans API error: $message", "https://$domain") + } + + val data = json.optJSONArray("data") + ?: throw ParseException("Missing 'data' array in list response", "https://$domain") + + val cursor = json.optJSONObject("cursor") + val page = lastPage[key] ?: 1 + + if (cursor != null) { + val nextCursor = cursor.optString("nextCursor", "") + if (nextCursor.isNotBlank()) { + cursorCache["$key:${page + 1}"] = nextCursor + } + } + + val mangas = mutableListOf<Manga>() + for (i in 0 until data.length()) { + val obj = data.getJSONObject(i) + mangas += parseMangaFromListJson(obj) + } + return mangas + } + + private fun parseMangaFromListJson(obj: JSONObject): Manga { + val title = obj.optString("title", "").ifBlank { + throw ParseException("Missing 'title' in manga entry", "https://$domain") + } + val slug = obj.optString("slug", "").ifBlank { + throw ParseException("Missing 'slug' in manga entry: $title", "https://$domain") + } + val cover = obj.optString("cover", "") + val status = obj.optString("status", "") + val genresArray = obj.optJSONArray("genres") ?: JSONArray() + val genres = (0 until genresArray.length()).map { genresArray.getString(it) } + + val state = when (status.lowercase(Locale.ROOT)) { + "ongoing" -> MangaState.ONGOING + "completed", "complete" -> MangaState.FINISHED + "hiatus", "on hiatus", "on-hold", "on hold" -> MangaState.PAUSED + else -> null + } + + return Manga( + id = generateUid("/series/$slug"), + url = "/series/$slug", + publicUrl = "https://$domain/series/$slug/", + title = title, + coverUrl = cover, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + authors = emptySet(), + state = state, + source = source, + ) + } + + private fun applyClientFilters(mangas: List<Manga>, filter: MangaListFilter): List<Manga> { + return mangas.filter { manga -> + if (filter.states.isNotEmpty() && manga.state !in filter.states) { + return@filter false + } + + if (filter.types.isNotEmpty()) { + val mangaType = manga.description + ?.substringAfter("Type: ", "") + ?.lowercase(Locale.ROOT) + val matchesType = filter.types.any { type -> + when (type) { + ContentType.MANGA -> mangaType == "manga" + ContentType.MANHWA -> mangaType == "manhwa" + ContentType.MANHUA -> mangaType == "manhua" + else -> false + } + } + if (!matchesType) return@filter false + } + + if (filter.tags.isNotEmpty()) { + val mangaTagKeys = manga.tags.map { it.key }.toSet() + val allMatch = filter.tags.all { tag -> tag.key in mangaTagKeys } + if (!allMatch) return@filter false + } + + true + } + } + + override suspend fun getDetails(manga: Manga): Manga { + suspend fun fetchDetailFromSlug(slug: String): JSONObject? { + val apiUrl = "$apiUrl/mangaDetail/$slug" + return try { + val response = webClient.httpGet(apiUrl, getRequestHeaders()) + val json = response.parseJson() + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown error") + throw ParseException("Details API error for slug '$slug': $message", apiUrl) + } + json.optJSONObject("data") + } catch (e: ParseException) { + throw e + } catch (_: Exception) { + null + } + } + + val initialSlug = manga.url + .removePrefix("/series/") + .removeSuffix("/") + .ifBlank { throw ParseException("Manga URL is blank: ${manga.url}", manga.url) } + + var data = fetchDetailFromSlug(initialSlug) + + if (data == null) { + val pageUrl = manga.publicUrl + if (pageUrl.isBlank()) { + throw ParseException("Public URL missing for manga: ${manga.title}", manga.url) + } + + val doc = try { + webClient.httpGet(pageUrl).parseHtml() + } catch (e: Exception) { + throw ParseException("Failed to load manga page for slug resolution: $pageUrl", pageUrl, e) + } + + val canonicalUrl = doc.selectFirst("link[rel=canonical]")?.attr("href") + ?: doc.selectFirst("meta[property=og:url]")?.attr("content") + ?: pageUrl + + val canonicalSlug = canonicalUrl + .substringAfterLast('/') + .substringBefore('?') + .trim() + .ifBlank { throw ParseException("Could not extract canonical slug from page: $pageUrl", pageUrl) } + + if (canonicalSlug != initialSlug) { + data = fetchDetailFromSlug(canonicalSlug) + } + } + + if (data == null) { + throw ParseException( + "Details not found for manga '${manga.title}' (tried slugs: $initialSlug)", + manga.publicUrl, + ) + } + + val title = data.optString("title", manga.title).ifBlank { manga.title } + val cover = data.optString("cover", manga.coverUrl) + val sinopsis = data.optString("sinopsis", "").takeIf { it.isNotBlank() && it != "null" } + val status = data.optString("status", "") + val author = data.optString("author", "").takeIf { it.isNotBlank() && it != "null" } + val genresArray = data.optJSONArray("genre") ?: data.optJSONArray("genres") ?: JSONArray() + val genres = (0 until genresArray.length()).map { genresArray.getString(it) } + + val state = when (status.lowercase(Locale.ROOT)) { + "ongoing" -> MangaState.ONGOING + "completed", "complete" -> MangaState.FINISHED + "hiatus", "on hiatus", "on-hold", "on hold" -> MangaState.PAUSED + else -> null + } + + val rating = data.optString("rating", "").toFloatOrNull() + val normalizedRating = if (rating != null && rating > 0f) rating / 2f else RATING_UNKNOWN + + val chaptersJson = data.optJSONArray("chapters") ?: JSONArray() + val chapters = parseChapters(chaptersJson) + + return manga.copy( + title = title, + coverUrl = cover, + description = sinopsis, + authors = setOfNotNull(author), + tags = genres.map { MangaTag(it, it.lowercase(Locale.ROOT), source) }.toSet(), + state = state, + rating = normalizedRating, + chapters = chapters, + contentRating = null, + ) + } + + private fun parseChapters(chaptersJson: JSONArray): List<MangaChapter> { + if (chaptersJson.length() == 0) { + return emptyList() + } + + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until chaptersJson.length()) { + val obj = chaptersJson.getJSONObject(i) + val slug = obj.optString("slug", "").ifBlank { continue } + val chapterNumRaw = obj.optString("chapterNum", "").ifBlank { continue } + val time = obj.optString("time", "") + + val number = Regex("""^(\d+(?:\.\d+)?)""") + .find(chapterNumRaw) + ?.groupValues + ?.get(1) + ?.toFloatOrNull() + ?: continue + + val uploadDate = runCatching { dateFormat.parse(time)?.time }.getOrNull() ?: 0L + + chapters += MangaChapter( + id = generateUid("/chapter/$slug"), + title = "Chapter $chapterNumRaw".trim(), + url = "/chapter/$slug", + number = number, + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + } + + return chapters.sortedBy { it.number } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterSlug = chapter.url.substringAfterLast('/') + if (chapterSlug.isBlank()) { + throw ParseException("Chapter URL missing slug: ${chapter.url}", chapter.url) + } + + val url = "$apiUrl/readingPage/$chapterSlug" + val response = try { + webClient.httpGet(url, getRequestHeaders()) + } catch (e: Exception) { + throw ParseException("Failed to load pages (URL: $url): ${e.message}", url, e) + } + + val json = try { + response.parseJson() + } catch (e: Exception) { + throw ParseException("Invalid JSON from reading page API (URL: $url): ${e.message}", url, e) + } + + if (!json.optBoolean("success", true)) { + val message = json.optString("message", "Unknown error") + throw ParseException("Reading page API error: $message", url) + } + + val data = json.optJSONObject("data") + ?: throw ParseException("Missing 'data' object in reading page response", url) + + val chaptersArray = data.optJSONArray("chapters") + ?: throw ParseException("Missing 'chapters' array in reading page data", url) + + if (chaptersArray.length() == 0) { + throw ParseException("No pages found in reading page data", url) + } + + val pages = mutableListOf<MangaPage>() + for (i in 0 until chaptersArray.length()) { + val html = chaptersArray.getString(i) + val img = Jsoup.parse(html).selectFirst("img") + ?: throw ParseException("Page $i does not contain an <img> tag", url) + val imageUrl = img.attr("src").takeIf { it.isNotBlank() } + ?: throw ParseException("Page $i has empty src attribute", url) + pages += MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + + return pages + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + companion object { + private val GENRES = listOf( + "Action", "Adventure", "Comedy", "Drama", "Fantasy", "Martial Arts", "Romance", + "School Life", "Shounen", "Supernatural", "System", "Thriller", "Murim", + ) + } +} diff --git a/src/main/kotlin/tsuki/site/id/Ikiru.kt b/src/main/kotlin/tsuki/site/id/Ikiru.kt new file mode 100644 index 00000000..dd6f63c1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Ikiru.kt @@ -0,0 +1,11 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.NatsuParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("IKIRU", "Ikiru", "id") +internal class Ikiru(context: MangaLoaderContext) : + NatsuParser(context, MangaParserSource.IKIRU,"07.ikiru.wtf") diff --git a/src/main/kotlin/tsuki/site/id/Kiryuu.kt b/src/main/kotlin/tsuki/site/id/Kiryuu.kt new file mode 100644 index 00000000..8bc65fa1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Kiryuu.kt @@ -0,0 +1,11 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.NatsuParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("KIRYUU", "Kiryuu", "id") +internal class Kiryuu(context: MangaLoaderContext) : + NatsuParser(context, MangaParserSource.KIRYUU, "v7.kiryuu.to") diff --git a/src/main/kotlin/tsuki/site/id/Komiku.kt b/src/main/kotlin/tsuki/site/id/Komiku.kt new file mode 100644 index 00000000..55924925 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Komiku.kt @@ -0,0 +1,287 @@ +package tsuki.site.id + +import okhttp3.Headers +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaReaderParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.EnumSet + +@MangaSourceParser("KOMIKU", "Komiku", "id") +internal class Komiku(context: MangaLoaderContext) : + MangaReaderParser(context, MangaParserSource.KOMIKU, "komiku.org", 10, 10) { + + private val apiDomain = "api.$domain" + override val datePattern = "dd/MM/yyyy" + override val selectPage = "#Baca_Komik img" + override val selectTestScript = "script:containsData(thisIsNeverFound)" + override val listUrl = "/manga/" + override val selectMangaList = "div.bge" + override val selectMangaListImg = "div.bgei img" + override val selectMangaListTitle = "div.kan h3" + override val selectChapter = "#Daftar_Chapter tr:has(td.judulseries)" + override val detailsDescriptionSelector = "#Sinopsis > p" + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + ) + + private val allGenres = listOf( + "action" to "Action", "adult" to "Adult", "adventure" to "Adventure", + "comedy" to "Comedy", "cooking" to "Cooking", "drama" to "Drama", + "ecchi" to "Ecchi", "fantasy" to "Fantasy", "gender-bender" to "Gender Bender", + "harem" to "Harem", "historical" to "Historical", "horror" to "Horror", + "isekai" to "Isekai", "josei" to "Josei", "magic" to "Magic", + "manga" to "Manga", "manhwa" to "Manhwa", "martial-arts" to "Martial Arts", + "mature" to "Mature", "mecha" to "Mecha", "medical" to "Medical", + "military" to "Military", "mystery" to "Mystery", "one-shot" to "One Shot", + "psychological" to "Psychological", "romance" to "Romance", + "school-life" to "School Life", "sci-fi" to "Sci-fi", "seinen" to "Seinen", + "shoujo" to "Shoujo", "shounen" to "Shounen", "slice-of-life" to "Slice of Life", + "sports" to "Sports", "supernatural" to "Supernatural", "tragedy" to "Tragedy", + "yaoi" to "Yaoi", "yuri" to "Yuri", + "academy" to "Academy", "adaptation" to "Adaptation", "apocalypse" to "Apocalypse", + "beasts" to "Beasts", "blacksmith" to "Blacksmith", "comic" to "Comic", + "crime" to "Crime", "crossdressing" to "Crossdressing", + "dark-fantasy" to "Dark Fantasy", "demons" to "Demons", + "doujinshi" to "Doujinshi", "entertainment" to "Entertainment", + "game" to "Game", "genderswap" to "Genderswap", "genius" to "Genius", + "ghosts" to "Ghosts", "gore" to "Gore", "gyaru" to "Gyaru", + "hentai" to "Hentai", "knight" to "Knight", "long-strip" to "Long Strip", + "magical-girls" to "Magical Girls", "mangatoon" to "Mangatoon", + "martial-art" to "Martial Art", "mc-rebirth" to "MC Rebirth", + "monster" to "Monster", "monster-girls" to "Monster Girls", + "monsters" to "Monsters", "murim" to "Murim", "music" to "Music", + "office-workers" to "Office Workers", "oneshot" to "Oneshot", + "police" to "Police", "regression" to "Regression", + "reincarnation" to "Reincarnation", "revenge" to "Revenge", + "school" to "School", "sexual-violence" to "Sexual Violence", + "shotacon" to "Shotacon", "shoujo-ai" to "Shoujo Ai", + "shounen-ai" to "Shounen Ai", "slow-life" to "Slow Life", + "smut" to "Smut", "sport" to "Sport", "strategy" to "Strategy", + "super-power" to "Super Power", "survival" to "Survival", + "sword-fight" to "Sword Fight", "sword-master" to "Sword Master", + "swormanship" to "Swormanship", "system" to "System", + "thriller" to "Thriller", "trauma" to "Trauma", "vampire" to "Vampire", + "villainess" to "Villainess", "violence" to "Violence", + "web-comic" to "Web Comic", "webtoon" to "Webtoon", + "webtoons" to "Webtoons", "xianxia" to "Xianxia", "xuanhuan" to "Xuanhuan" + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = allGenres.map { (title, key) -> MangaTag(key, title, source) }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim().orEmpty() + val url = buildString { + append("https://") + append(apiDomain) + if (query.isNotEmpty()) { + append("/?post_type=manga&s=") + append(query.urlEncoded()) + if (page > 1) { + append("&page=") + append(page) + } + } else { + append("/manga/page/") + append(page) + append("/") + val params = mutableListOf<String>() + val genres = filter.tags.map { it.key } + genres.getOrNull(0)?.let { params += "genre=$it" } + genres.getOrNull(1)?.let { params += "genre2=$it" } + val orderParam = when (order) { + SortOrder.UPDATED -> "modified" + SortOrder.NEWEST -> "date" + SortOrder.POPULARITY -> "meta_value_num" + SortOrder.ALPHABETICAL -> "title" + else -> null + } + if (orderParam != null) params += "orderby=$orderParam" + filter.types.oneOrThrowIfMany()?.let { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> null + } + }?.let { params += "tipe=$it" } + filter.states.oneOrThrowIfMany()?.let { + when (it) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "end" + else -> null + } + }?.let { params += "statusmanga=$it" } + if (params.isNotEmpty()) { + append("?") + append(params.joinToString("&")) + } + } + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + override fun parseMangaList(docs: Document): List<Manga> { + return docs.select(selectMangaList).mapNotNull { element -> + val a = element.selectFirst("div.bgei a[href*=/manga/]") + ?: element.selectFirst("a[href*=/manga/]") + ?: return@mapNotNull null + + val href = a.attr("href") + val relativeUrl = href.toRelativeUrl(domain) + + val thumbnailUrl = element.selectFirst(selectMangaListImg)?.let { img -> + img.attr("data-src").ifBlank { img.attr("src") } + }?.substringBeforeLast("?")?.toAbsoluteUrl(domain) + + val title = element.selectFirst(selectMangaListTitle)?.text() + ?: return@mapNotNull null + + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + title = title, + altTitles = emptySet(), + publicUrl = href.toAbsoluteUrl(domain), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = thumbnailUrl, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + description = element.selectFirst("div.kan p")?.text(), + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val docs = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val chapters = docs.select(selectChapter).mapChapters(reversed = true) { index, element -> + val a = element.selectFirst("td.judulseries a") ?: return@mapChapters null + val url = a.attrAsRelativeUrl("href") + val dateText = element.selectFirst("td.tanggalseries")?.text() + val date = if (dateText != null && dateText.contains("lalu")) { + parseRelativeDate(dateText) + } else { + SimpleDateFormat(datePattern, sourceLocale).parseSafe(dateText) + } + MangaChapter( + id = generateUid(url), + title = a.selectFirst("span")?.text() ?: a.text(), + url = url, + number = index + 1f, + volume = 0, + scanlator = null, + uploadDate = date, + branch = null, + source = source, + ) + } + + return parseInfo(docs, manga, chapters) + } + + override suspend fun parseInfo(docs: Document, manga: Manga, chapters: List<MangaChapter>): Manga { + val tags = docs.select("ul.genre li.genre a, table.inftable a[href*=/genre/]").mapNotNullToSet { element -> + val href = element.attr("href") + val genreKey = href.substringAfter("/genre/").substringBefore("/").ifBlank { return@mapNotNullToSet null } + val genreTitle = element.selectFirst("span[itemprop='genre']")?.text() ?: element.text() + MangaTag( + key = genreKey, + title = genreTitle.toTitleCase(sourceLocale), + source = source, + ) + } + + val statusText = docs.selectFirst("table.inftable tr > td:contains(Status) + td")?.text().orEmpty() + val state = when { + statusText.contains("Ongoing", true) -> MangaState.ONGOING + statusText.contains("Completed", true) || statusText.contains("Tamat", true) || statusText.contains("End", true) -> MangaState.FINISHED + else -> null + } + + val author = docs.selectFirst("table.inftable tr:has(td:contains(Pengarang)) td:last-child")?.text() + val altTitle = docs.selectFirst("table.inftable tr:has(td:contains(Judul Indonesia)) td:last-child")?.text() + + val thumbnail = docs.selectFirst("div.ims > img")?.let { img -> + img.attr("data-src").ifBlank { img.attr("src") } + }?.substringBeforeLast("?")?.toAbsoluteUrl(domain) + + return manga.copy( + altTitles = setOfNotNull(altTitle), + description = docs.selectFirst(detailsDescriptionSelector)?.text() ?: manga.description, + state = state ?: manga.state, + authors = setOfNotNull(author), + tags = tags, + chapters = chapters, + coverUrl = thumbnail ?: manga.coverUrl, + ) + } + + private fun parseRelativeDate(date: String): Long { + val trimmedDate = date.substringBefore(" lalu").removeSuffix("s").split(" ") + val calendar = Calendar.getInstance() + when (trimmedDate[1]) { + "jam" -> calendar.add(Calendar.HOUR_OF_DAY, -trimmedDate[0].toInt()) + "menit" -> calendar.add(Calendar.MINUTE, -trimmedDate[0].toInt()) + "detik" -> calendar.add(Calendar.SECOND, 0) + } + return calendar.timeInMillis + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select(selectPage).mapIndexed { _, img -> + val src = img.attr("abs:src") + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .set("Referer", "https://$domain/") + .set("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .build() +} diff --git a/src/main/kotlin/tsuki/site/id/LumosKomik.kt b/src/main/kotlin/tsuki/site/id/LumosKomik.kt new file mode 100644 index 00000000..65659456 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/LumosKomik.kt @@ -0,0 +1,282 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.attrAsRelativeUrl +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import java.time.Instant +import java.time.ZoneId +import java.time.ZonedDateTime +import java.time.temporal.ChronoUnit +import java.util.Base64 +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("LUMOSKOMIK", "LumosKomik", "id") +internal class LumosKomik(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.LUMOSKOMIK, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("03.lumosgg.com") + + private val baseUrl = "https://$domain" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genres = fetchGenres() + return MangaListFilterOptions( + availableTags = genres, + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + private suspend fun fetchGenres(): Set<MangaTag> { + val doc = webClient.httpGet("$baseUrl/browse").parseHtml() + return doc.select("label[data-bf-genre-name] input.bf-genre-cb[value]") + .mapNotNull { input -> + val value = input.attr("value").trim() + val name = input.parent()?.selectFirst("span.truncate")?.text()?.trim() + if (value.isNotEmpty() && !name.isNullOrEmpty()) { + MangaTag(name, value, source) + } else null + } + .toSet() + } + + /** + * When scraping they have a hidden honeypot in the listing crazy + */ + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sort = when (order) { + SortOrder.UPDATED -> "latest" + SortOrder.POPULARITY -> "popular" + SortOrder.RATING -> "rating" + SortOrder.ALPHABETICAL -> "az" + else -> "latest" + } + + val url = "$baseUrl/browse".toHttpUrl().newBuilder().apply { + if (!filter.query.isNullOrBlank()) addQueryParameter("q", filter.query) + if (sort.isNotEmpty()) addQueryParameter("sort", sort) + if (filter.states.isNotEmpty()) addQueryParameter("status", mapState(filter.states.first())) + if (filter.types.isNotEmpty()) addQueryParameter("type", mapType(filter.types.first())) + if (filter.tags.isNotEmpty()) addQueryParameter("genre", filter.tags.joinToString(",") { it.key }) + addQueryParameter("page", page.toString()) + }.build() + + val doc = webClient.httpGet(url).parseHtml() + val list = parseMangaList(doc) + return list.filterNot { it.url.contains("honeypot") } + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a[href^=\"/comic/\"]:has(img:not([src*=\"flagcdn\"]))") + .mapNotNull { element -> + val img = element.selectFirst("img") ?: return@mapNotNull null + val title = img.attr("alt").takeIf { it.isNotBlank() } ?: element.text().trim() + if (title.isBlank()) return@mapNotNull null + + val href = element.attrAsRelativeUrl("href") + Manga( + id = generateUid(href), + title = title, + url = href, + publicUrl = href.toAbsoluteUrl(domain), + coverUrl = img.attr("abs:src"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + .distinctBy { it.url } + } + + private fun mapState(state: MangaState?): String = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + else -> "" + } + + private fun mapType(type: ContentType?): String = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + + val title = doc.selectFirst("h1")?.text()?.trim() ?: manga.title + val cover = doc.selectFirst("img[src*=\"/cover\"]")?.attr("abs:src") + ?: doc.selectFirst("aside img")?.attr("abs:src") + ?: manga.coverUrl + val description = run { + val encoded = doc.selectFirst("div[data-sr]")?.attr("data-sr")?.takeIf { it.isNotBlank() } + val decoded = encoded?.let { decodeBase64(it) }?.trim()?.takeIf { it.isNotBlank() } + decoded ?: doc.selectFirst("p.text-sm")?.text()?.trim()?.takeIf { it.isNotBlank() } + } + + val genres = doc.select("a[href*=\"genre=\"]") + .asSequence() + .map { it.text().trim() } + .filter { it.isNotBlank() } + .distinct() + .toSet() + .map { name -> + MangaTag( + title = name, + key = name.lowercase(Locale.ROOT).replace(" ", "-"), + source = source, + ) + } + .toSet() + + val statusText = doc.selectFirst("main")?.text() ?: doc.text() + val state = when { + statusText.contains("completed", ignoreCase = true) || + statusText.contains("tamat", ignoreCase = true) -> MangaState.FINISHED + statusText.contains("ongoing", ignoreCase = true) || + statusText.contains("berjalan", ignoreCase = true) -> MangaState.ONGOING + statusText.contains("hiatus", ignoreCase = true) -> MangaState.PAUSED + else -> null + } + + val chapters = parseChapterList(doc) + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + tags = genres, + state = state, + chapters = chapters, + ) + } + + private fun parseChapterList(doc: Document): List<MangaChapter> { + return doc.select("a[href*=\"/read/\"][data-chapter]").mapNotNull { element -> + val href = element.attrAsRelativeUrl("href") + val name = element.selectFirst("span.text-sm, span[class*=\"font-semibold\"]")?.text()?.trim() + ?: element.text().trim() + val number = element.attr("data-chapter").trim().toFloatOrNull() + ?: name.extractChapterNumber() + val dateStr = element.selectFirst("span.tabular-nums, span[class*=\"tabular-nums\"]")?.text()?.trim() + MangaChapter( + id = generateUid(href), + title = name, + number = number, + volume = 0, + url = href, + uploadDate = parseRelativeDate(dateStr), + source = source, + scanlator = null, + branch = null, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select("img[src*=\"file/comic\"], img[src*=\"imgsvr\"]") + .mapNotNull { img -> + val src = img.attr("abs:src") + if (src.isBlank() || src.contains("/api/image/p/")) { + null + } else { + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + } + + private fun decodeBase64(encoded: String): String? = try { + val padded = encoded.trim().let { s -> + val rem = s.length % 4 + if (rem != 0) s + "=".repeat(4 - rem) else s + } + String(Base64.getDecoder().decode(padded), Charsets.UTF_8).trim().takeIf { it.isNotBlank() } + } catch (_: Exception) { + null + } + + private fun parseRelativeDate(dateStr: String?): Long { + if (dateStr.isNullOrBlank()) return 0L + val trimmed = dateStr.lowercase(Locale.ROOT).trim() + if (trimmed == "baru saja") return System.currentTimeMillis() + + val number = Regex("\\d+").find(trimmed)?.value?.toLongOrNull() ?: return 0L + val nowInstant = Instant.now() + val nowZoned = ZonedDateTime.now(ZoneId.systemDefault()) + + return when { + "detik" in trimmed -> nowInstant.minus(number, ChronoUnit.SECONDS).toEpochMilli() + "menit" in trimmed -> nowInstant.minus(number, ChronoUnit.MINUTES).toEpochMilli() + "jam" in trimmed -> nowInstant.minus(number, ChronoUnit.HOURS).toEpochMilli() + "hari" in trimmed -> nowInstant.minus(number, ChronoUnit.DAYS).toEpochMilli() + "minggu" in trimmed -> nowInstant.minus(number * 7, ChronoUnit.DAYS).toEpochMilli() + "bulan" in trimmed -> nowZoned.minusMonths(number).toInstant().toEpochMilli() + "tahun" in trimmed -> nowZoned.minusYears(number).toInstant().toEpochMilli() + else -> 0L + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt b/src/main/kotlin/tsuki/site/id/MGKomik.kt similarity index 81% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt rename to src/main/kotlin/tsuki/site/id/MGKomik.kt index 225bb7a3..77cf70f8 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/MGKomik.kt +++ b/src/main/kotlin/tsuki/site/id/MGKomik.kt @@ -1,178 +1,176 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id - -import okhttp3.Interceptor -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaParserAuthProvider -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.* -import androidx.collection.arraySetOf - -@MangaSourceParser("MGKOMIK", "MGKomik", "id") -internal class MGKomik(context: MangaLoaderContext) : - MadaraParser(context, MangaParserSource.MGKOMIK, "id.mgkomik.cc"), Interceptor, MangaParserAuthProvider { - - override val datePattern = "dd MMM yy" - override val tagPrefix = "genres/" - override val selectDesc = "div.description-summary div.summary__content p" - - override val withoutAjax = true - - override fun intercept(chain: Interceptor.Chain): okhttp3.Response { - val request = chain.request() - val path = request.url.encodedPath - val isAjax = path.contains("admin-ajax.php") || - path.contains("wp-json") || - path.endsWith("/ajax/chapters") - - return if (isAjax) { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .header("X-Requested-With", "XMLHttpRequest") - .header("Sec-Fetch-Dest", "empty") - .header("Sec-Fetch-Mode", "cors") - .header("Sec-Fetch-Site", "same-origin") - .header("Origin", "https://$domain") - .header("Priority", "u=1, i") - .removeHeader("Sec-Fetch-User") - .removeHeader("Upgrade-Insecure-Requests") - .build() - ) - } else { - chain.proceed( - request.newBuilder() - .header("Sec-CH-UA-Model", "\"\"") - .build() - ) - } - } - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - if (order == SortOrder.POPULARITY) { - val p = page + 1 - val url = buildString { - append("https://") - append(domain) - append("/komik") - if (p > 1) { - append("/page/") - append(p) - } - append("/?m_orderby=trending") - } - return parseMangaList(webClient.httpGet(url).parseHtml()) - } - return super.getListPage(page, order, filter) - } - - override suspend fun fetchAvailableTags(): Set<MangaTag> { - return super.fetchAvailableTags() - } - - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = availableTags(), - ) - - - private fun availableTags() = arraySetOf( - MangaTag("2 hours ago", "2-hours-ago", source), - MangaTag("7 hours ago", "7-hours-ago", source), - MangaTag("8 hours ago", "8-hours-ago", source), - MangaTag("11 hours ago", "11-hours-ago", source), - MangaTag("13 hours ago", "13-hours-ago", source), - MangaTag("15 hours ago", "15-hours-ago", source), - MangaTag("1 day ago", "1-day-ago", source), - MangaTag("2 days ago", "2-days-ago", source), - MangaTag("Action", "action", source), - MangaTag("Adaptation", "adaptation", source), - MangaTag("Adult", "adult", source), - MangaTag("Adventure", "adventure", source), - MangaTag("Animals", "animals", source), - MangaTag("Apocalypse", "apocalypse", source), - MangaTag("Comedy", "comedy", source), - MangaTag("Cooking", "cooking", source), - MangaTag("Crime", "crime", source), - MangaTag("Demon", "demon", source), - MangaTag("Demons", "demons", source), - MangaTag("Drama", "drama", source), - MangaTag("Dungeons", "dungeons", source), - MangaTag("Ecchi", "ecchi", source), - MangaTag("Fantasy", "fantasy", source), - MangaTag("Fight", "fight", source), - MangaTag("Fighting", "fighting", source), - MangaTag("Full Color", "full-color", source), - MangaTag("Game", "game", source), - MangaTag("Gender Bender", "gender-bender", source), - MangaTag("Gore", "gore", source), - MangaTag("Harem", "harem", source), - MangaTag("Historical", "historical", source), - MangaTag("Horror", "horror", source), - MangaTag("Isekai", "isekai", source), - MangaTag("Josei", "josei", source), - MangaTag("Josei(W)", "joseiw", source), - MangaTag("Kids", "kids", source), - MangaTag("Magic", "magic", source), - MangaTag("Manga", "manga", source), - MangaTag("Mangatoon", "mangatoon", source), - MangaTag("Manhua", "manhua", source), - MangaTag("Manhwa", "manhwa", source), - MangaTag("Martial Arts", "martial-arts", source), - MangaTag("Mature", "mature", source), - MangaTag("Mecha", "mecha", source), - MangaTag("Medical", "medical", source), - MangaTag("Military", "military", source), - MangaTag("Modern", "modern", source), - MangaTag("Monsters", "monsters", source), - MangaTag("Murim", "murim", source), - MangaTag("Music", "music", source), - MangaTag("Mystery", "mystery", source), - MangaTag("Office Workers", "office-workers", source), - MangaTag("One-Shot", "one-shot", source), - MangaTag("Overpowered", "overpowered", source), - MangaTag("Police", "police", source), - MangaTag("Psychological", "psychological", source), - MangaTag("Regression", "regression", source), - MangaTag("Reincarnation", "reincarnation", source), - MangaTag("Revenge", "revenge", source), - MangaTag("Reverse Harem", "reverse-harem", source), - MangaTag("Rofan", "rofan", source), - MangaTag("Romance", "romance", source), - MangaTag("School", "school", source), - MangaTag("School Life", "school-life", source), - MangaTag("Sci-fi", "sci-fi", source), - MangaTag("Seinen", "seinen", source), - MangaTag("Shoujo", "shoujo", source), - MangaTag("Shoujo Ai", "shoujoai", source), - MangaTag("Shoujo AI", "shoujo-ai", source), - MangaTag("Shoujo(G)", "shoujog", source), - MangaTag("Shounen", "shounen", source), - MangaTag("Slice of Life", "slice-of-life", source), - MangaTag("Smut", "smut", source), - MangaTag("Sports", "sports", source), - MangaTag("Super Power", "super-power", source), - MangaTag("Superhero", "superhero", source), - MangaTag("Supernatural", "supernatural", source), - MangaTag("Supranatural", "supranatural", source), - MangaTag("Survival", "survival", source), - MangaTag("System", "system", source), - MangaTag("Thriller", "thriller", source), - MangaTag("Time Travel", "time-travel", source), - MangaTag("Tragedy", "tragedy", source), - MangaTag("Transmigration", "transmigration", source), - MangaTag("Vampire", "vampire", source), - MangaTag("Villainess", "villainess", source), - MangaTag("Violence", "violence", source), - MangaTag("War", "war", source), - MangaTag("Webtoon", "webtoon", source), - MangaTag("Webtoons", "webtoons", source), - MangaTag("Wuxia", "wuxia", source), - MangaTag("Yuri", "yuri", source), - ) - -} +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.MangaParserAuthProvider +import tsuki.parsers.MadaraParser + +import tsuki.model.Manga +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.parseHtml + +import okhttp3.Interceptor +import androidx.collection.arraySetOf +import okhttp3.Response + +@MangaSourceParser("MGKOMIK", "MGKomik", "id") +internal class MGKomik(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.MGKOMIK, "id.mgkomik.cc"), Interceptor, MangaParserAuthProvider { + + override val datePattern = "dd MMM yy" + override val tagPrefix = "genres/" + override val selectDesc = "div.description-summary div.summary__content p" + + override val withoutAjax = true + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val path = request.url.encodedPath + val isAjax = path.contains("admin-ajax.php") || + path.contains("wp-json") || + path.endsWith("/ajax/chapters") + + val builder = request.newBuilder() + .header("Sec-CH-UA-Model", "\"\"") + + if (isAjax) { + builder.header("X-Requested-With", "XMLHttpRequest") + .header("Sec-Fetch-Dest", "empty") + .header("Sec-Fetch-Mode", "cors") + .header("Sec-Fetch-Site", "same-origin") + .header("Origin", "https://$domain") + .header("Priority", "u=1, i") + .removeHeader("Sec-Fetch-User") + .removeHeader("Upgrade-Insecure-Requests") + } + + return chain.proceed(builder.build()) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + if (order == SortOrder.POPULARITY) { + val p = page + 1 + val url = buildString { + append("https://") + append(domain) + append("/komik") + if (p > 1) { + append("/page/") + append(p) + } + append("/?m_orderby=trending") + } + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + return super.getListPage(page, order, filter) + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + return super.fetchAvailableTags() + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = availableTags(), + ) + + private fun availableTags() = arraySetOf( + MangaTag("2 hours ago", "2-hours-ago", source), + MangaTag("7 hours ago", "7-hours-ago", source), + MangaTag("8 hours ago", "8-hours-ago", source), + MangaTag("11 hours ago", "11-hours-ago", source), + MangaTag("13 hours ago", "13-hours-ago", source), + MangaTag("15 hours ago", "15-hours-ago", source), + MangaTag("1 day ago", "1-day-ago", source), + MangaTag("2 days ago", "2-days-ago", source), + MangaTag("Action", "action", source), + MangaTag("Adaptation", "adaptation", source), + MangaTag("Adult", "adult", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Animals", "animals", source), + MangaTag("Apocalypse", "apocalypse", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Crime", "crime", source), + MangaTag("Demon", "demon", source), + MangaTag("Demons", "demons", source), + MangaTag("Drama", "drama", source), + MangaTag("Dungeons", "dungeons", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Fight", "fight", source), + MangaTag("Fighting", "fighting", source), + MangaTag("Full Color", "full-color", source), + MangaTag("Game", "game", source), + MangaTag("Gender Bender", "gender-bender", source), + MangaTag("Gore", "gore", source), + MangaTag("Harem", "harem", source), + MangaTag("Historical", "historical", source), + MangaTag("Horror", "horror", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Josei", "josei", source), + MangaTag("Josei(W)", "joseiw", source), + MangaTag("Kids", "kids", source), + MangaTag("Magic", "magic", source), + MangaTag("Manga", "manga", source), + MangaTag("Mangatoon", "mangatoon", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Mature", "mature", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Medical", "medical", source), + MangaTag("Military", "military", source), + MangaTag("Modern", "modern", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Murim", "murim", source), + MangaTag("Music", "music", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Office Workers", "office-workers", source), + MangaTag("One-Shot", "one-shot", source), + MangaTag("Overpowered", "overpowered", source), + MangaTag("Police", "police", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Regression", "regression", source), + MangaTag("Reincarnation", "reincarnation", source), + MangaTag("Revenge", "revenge", source), + MangaTag("Reverse Harem", "reverse-harem", source), + MangaTag("Rofan", "rofan", source), + MangaTag("Romance", "romance", source), + MangaTag("School", "school", source), + MangaTag("School Life", "school-life", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Shoujo Ai", "shoujoai", source), + MangaTag("Shoujo AI", "shoujo-ai", source), + MangaTag("Shoujo(G)", "shoujog", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Smut", "smut", source), + MangaTag("Sports", "sports", source), + MangaTag("Super Power", "super-power", source), + MangaTag("Superhero", "superhero", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Supranatural", "supranatural", source), + MangaTag("Survival", "survival", source), + MangaTag("System", "system", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Time Travel", "time-travel", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Transmigration", "transmigration", source), + MangaTag("Vampire", "vampire", source), + MangaTag("Villainess", "villainess", source), + MangaTag("Violence", "violence", source), + MangaTag("War", "war", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Webtoons", "webtoons", source), + MangaTag("Wuxia", "wuxia", source), + MangaTag("Yuri", "yuri", source), + ) + +} diff --git a/src/main/kotlin/tsuki/site/id/ManhwaListID.kt b/src/main/kotlin/tsuki/site/id/ManhwaListID.kt new file mode 100644 index 00000000..e7e4a853 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/ManhwaListID.kt @@ -0,0 +1,11 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANHWALISTID", "ManhwaList ID", "id") +internal class ManhwaListID(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.MANHWALISTID, "manhwalist02.asia") diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt b/src/main/kotlin/tsuki/site/id/ReYume.kt similarity index 85% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt rename to src/main/kotlin/tsuki/site/id/ReYume.kt index 1a684f3c..8265b539 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/ReYume.kt +++ b/src/main/kotlin/tsuki/site/id/ReYume.kt @@ -1,15 +1,33 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.ZeistMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.requireSrc +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.json.asTypedList import org.json.JSONArray import org.json.JSONObject import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.parsers.ZeistMangaParser -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.asTypedList import java.text.SimpleDateFormat +import kotlin.collections.dropLast @MangaSourceParser("REYUME", "ReYume", "id") internal class ReYume(context: MangaLoaderContext) : @@ -24,20 +42,19 @@ internal class ReYume(context: MangaLoaderContext) : override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { val startIndex = maxMangaResults * (page - 1) + 1 - val query = filter.query val url = buildString { append("https://") append(domain) append("/feeds/posts/default/-/") when { - !query.isNullOrEmpty() -> { + !filter.query.isNullOrEmpty() -> { append("Series") append("?alt=json&orderby=published&max-results=") append((maxMangaResults + 1).toString()) append("&start-index=") append(startIndex.toString()) append("&q=label:Series+") - append(query.urlEncoded()) + append(filter.query.urlEncoded()) } else -> { if (filter.tags.isNotEmpty()) { @@ -122,14 +139,14 @@ internal class ReYume(context: MangaLoaderContext) : val mangaTitle = doc.selectFirst("h1#post-title")?.text().orEmpty() return json.mapIndexedNotNull { i, j -> - val name = j.getJSONObject("title").getString("\$t") + val name = j.getJSONObject("title").getString($$"$t") val chapterName = if (mangaTitle.isNotEmpty() && name.contains(mangaTitle, ignoreCase = true)) { name.replace(mangaTitle, "", ignoreCase = true).trim().trim('-').trim().takeIf { it.isNotEmpty() } ?: name } else { name } val href = j.getJSONArray("link").asTypedList<JSONObject>().first { it.getString("rel") == "alternate" }.getString("href") - val dateText = j.getJSONObject("published").getString("\$t").substringBefore("T") + val dateText = j.getJSONObject("published").getString($$"$t").substringBefore("T") val slug = mangaUrl.substringAfterLast('/') val slugChapter = href.substringAfterLast('/') if (slug == slugChapter) return@mapIndexedNotNull null @@ -174,4 +191,4 @@ internal class ReYume(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/id/Shinigami.kt b/src/main/kotlin/tsuki/site/id/Shinigami.kt new file mode 100644 index 00000000..29901a9c --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Shinigami.kt @@ -0,0 +1,298 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import okhttp3.Headers +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("SHINIGAMI", "Shinigami", "id") +internal class Shinigami(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.SHINIGAMI, 30) { + + override val configKeyDomain = ConfigKey.Domain("g.shinigami.asia") + private val apiSuffix = "api.shngm.io/v1" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + @Volatile + private var apiHeaders: Headers? = null + + private fun getApiHeaders(): Headers { + if (apiHeaders == null) { + apiHeaders = getRequestHeaders().newBuilder() + .add("Accept", "application/json") + .add("DNT", "1") + .add("Origin", "https://$domain") + .add("Sec-GPC", "1") + .removeAll("X-Requested-With") + .build() + } + return apiHeaders!! + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("referer", "https://$domain/") + .add("sec-fetch-dest", "empty") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.NEWEST, + SortOrder.NEWEST_ASC, + SortOrder.RATING, + SortOrder.RATING_ASC, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = fetchTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED, MangaState.PAUSED), + availableContentTypes = EnumSet.of(ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA), + ) + } + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildString { + append("https://") + append(apiSuffix) + append("/manga/list") + append("?page=").append(page) + append("&page_size=").append(pageSize) + + append("&sort=") + append( + when (order) { + SortOrder.POPULARITY, SortOrder.POPULARITY_ASC -> "popularity" + SortOrder.NEWEST, SortOrder.NEWEST_ASC -> "latest" + SortOrder.RATING, SortOrder.RATING_ASC -> "rating" + else -> "latest" + } + ) + append("&sort_order=") + append( + when (order) { + SortOrder.POPULARITY_ASC, SortOrder.NEWEST_ASC, SortOrder.RATING_ASC -> "asc" + else -> "desc" + } + ) + + filter.states.oneOrThrowIfMany()?.let { + append("&status=") + append( + when (it) { + MangaState.FINISHED -> "completed" + MangaState.ONGOING -> "ongoing" + MangaState.PAUSED -> "hiatus" + else -> "" + } + ) + } + + if (filter.types.isNotEmpty()) { + append("&format=") + append( + filter.types.joinToString(",") { + when (it) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + } + ) + } + + if (filter.tags.isNotEmpty()) { + append("&genre_include=") + filter.tags.joinTo(this, separator = ",") { it.key } + append("&genre_include_mode=and") + } + + if (filter.tagsExclude.isNotEmpty()) { + append("&genre_exclude=") + filter.tagsExclude.joinTo(this, separator = ",") { it.key } + append("&genre_exclude_mode=and") + } + + if (!filter.author.isNullOrEmpty()) { + append("&author=").append(filter.author.urlEncoded()) + } + + if (!filter.query.isNullOrEmpty()) { + append("&q=").append(filter.query.urlEncoded()) + } + } + + val json = webClient.httpGet(url, getApiHeaders()).parseJson() + val data = json.getJSONArray("data") + + return data.mapJSON { jo -> + val id = jo.getString("manga_id") + Manga( + id = generateUid(id), + url = "/manga/detail/$id", + publicUrl = "https://$domain/series/$id", + title = jo.getString("title"), + altTitles = setOfNotNull(jo.getStringOrNull("alternative_title")), + coverUrl = jo.getStringOrNull("cover_image_url"), + largeCoverUrl = jo.getStringOrNull("cover_portrait_url"), + authors = jo.optJSONObject("taxonomy") + ?.optJSONArray("Author") + ?.mapJSONToSet { it.getString("name") } + .orEmpty(), + tags = jo.optJSONObject("taxonomy") + ?.optJSONArray("Genre") + ?.mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name").toTitleCase(sourceLocale), + source = source, + ) + }.orEmpty(), + state = when (jo.getIntOrDefault("status", 0)) { + 1 -> MangaState.ONGOING + 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED + else -> null + }, + description = jo.getStringOrNull("description"), + contentRating = null, + source = source, + rating = RATING_UNKNOWN, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val json = webClient.httpGet("https://$apiSuffix${manga.url}", getApiHeaders()).parseJson() + val jo = json.getJSONObject("data") + + return manga.copy( + tags = jo.optJSONObject("taxonomy") + ?.optJSONArray("Genre") + ?.mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name"), + source = source, + ) + } ?: manga.tags, + authors = jo.optJSONObject("taxonomy") + ?.optJSONArray("Author") + ?.mapJSONToSet { it.getString("name") } + ?: manga.authors, + state = when (jo.getIntOrDefault("status", 0)) { + 1 -> MangaState.ONGOING + 2 -> MangaState.FINISHED + 3 -> MangaState.PAUSED + else -> null + }, + description = jo.getStringOrNull("description"), + largeCoverUrl = jo.getStringOrNull("cover_portrait_url"), + chapters = getChapters(jo.getString("manga_id")), + ) + } + + private suspend fun getChapters(mangaId: String): List<MangaChapter> { + val url = "https://$apiSuffix/chapter/$mangaId/list?page_size=3000" + val json = webClient.httpGet(url, getApiHeaders()).parseJson() + val data = json.getJSONArray("data") + + return data.mapJSON { jo -> + val chapterId = jo.getString("chapter_id") + val number = jo.getDouble("chapter_number") + + MangaChapter( + id = generateUid(chapterId), + title = "Chapter ${number.toFloat().toString().removeSuffix(".0")}", + number = number.toFloat(), + url = "/chapter/detail/$chapterId", + scanlator = null, + uploadDate = jo.getString("release_date").parseDate(), + branch = null, + source = source, + volume = 0, + ) + }.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val json = webClient.httpGet("https://$apiSuffix${chapter.url}", getApiHeaders()).parseJson() + val data = json.getJSONObject("data") + val chapterData = data.getJSONObject("chapter") + val baseUrl = data.getString("base_url") + val basePath = chapterData.getString("path") + val pagesArray = chapterData.getJSONArray("data") + + return List(pagesArray.length()) { i -> + val imgExt = pagesArray.getString(i) + val imageUrl = "$baseUrl$basePath$imgExt" + + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchTags(): Set<MangaTag> { + val json = webClient.httpGet("https://$apiSuffix/genre/list", getApiHeaders()).parseJson() + return json.getJSONArray("data").mapJSONToSet { + MangaTag( + key = it.getString("slug"), + title = it.getString("name").toTitleCase(sourceLocale), + source = source, + ) + } + } + + private fun String.parseDate(): Long { + return try { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'", Locale.US) + .apply { timeZone = TimeZone.getTimeZone("UTC") } + .parse(this)?.time ?: 0L + } catch (_: Exception) { 0L } + } +} diff --git a/src/main/kotlin/tsuki/site/id/Siikomik.kt b/src/main/kotlin/tsuki/site/id/Siikomik.kt new file mode 100644 index 00000000..acf8956c --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Siikomik.kt @@ -0,0 +1,24 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource + +import org.jsoup.nodes.Element + +@MangaSourceParser("SIIKOMIK", "Siikomik", "id") +internal class Siikomik(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.SIIKOMIK, "siikomik.id") { + + override val listUrl = "komik/" + + override fun transformChapterName(element: Element, name: String): String { + return if (element.hasClass("premium") || element.hasClass("premium-block")) { + "🔒 $name" + } else { + name + } + } +} diff --git a/src/main/kotlin/tsuki/site/id/TheManga.kt b/src/main/kotlin/tsuki/site/id/TheManga.kt new file mode 100644 index 00000000..2de842c2 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/TheManga.kt @@ -0,0 +1,338 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.RATING_UNKNOWN +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.MangaState +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.toAbsoluteUrl +import tsuki.util.urlEncoded +import tsuki.util.parseHtml +import tsuki.util.extractChapterNumber + +import org.jsoup.nodes.Document +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("THEMANGA", "TheManga", "id") +internal class TheManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.THEMANGA, 24) { + + override val configKeyDomain = ConfigKey.Domain("themanga.site") + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ssXXX", Locale.ROOT) + + override val availableSortOrders: EnumSet<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = false, + isYearSupported = true, + isAuthorSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (key, title) -> + MangaTag(key = key, title = title, source = source) + }.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + val noFilters = filter.query.isNullOrBlank() && + filter.tags.isEmpty() && + filter.states.isEmpty() && + filter.types.isEmpty() && + filter.year == 0 && + filter.author.isNullOrBlank() + + if (noFilters) { + return when (order) { + SortOrder.POPULARITY -> "https://$domain/?q=&sort=popular&page=$page" + SortOrder.UPDATED -> "https://$domain/?q=&sort=latest_update&page=$page" + else -> "https://$domain/explore?q=&page=$page" + } + } + + return buildString { + append("https://$domain/explore?") + val params = mutableListOf<String>() + + filter.query?.takeIf { it.isNotBlank() }?.let { + params.add("q=${it.urlEncoded()}") + } + + params.add("page=$page") + + filter.states.firstOrNull()?.let { state -> + when (state) { + MangaState.ONGOING -> params.add("status=ongoing") + MangaState.FINISHED -> params.add("status=completed") + else -> {} + } + } + + filter.tags.firstOrNull()?.let { tag -> + params.add("genre=${tag.key.urlEncoded()}") + } + + filter.types.firstOrNull()?.let { type -> + when (type) { + ContentType.MANGA -> params.add("type=manga") + ContentType.MANHWA -> params.add("type=manhwa") + ContentType.MANHUA -> params.add("type=manhua") + else -> {} + } + } + + if (filter.year != 0) { + params.add("year=${filter.year}") + } + + filter.author?.takeIf { it.isNotBlank() }?.let { + params.add("author=${it.urlEncoded()}") + } + + append(params.joinToString("&")) + } + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a.card, a.manga-card").mapNotNull { element -> + val href = element.attr("href") + val title = element.selectFirst(".card-title")?.text() + ?: return@mapNotNull null + val cover = element.selectFirst(".card-cover img, .cover img")?.absUrl("src") + + Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + authors = emptySet(), + coverUrl = cover, + rating = RATING_UNKNOWN, + tags = emptySet(), + state = null, + contentRating = null, + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst(".hero-title")?.text() ?: manga.title + val cover = doc.selectFirst(".hero-cover img")?.absUrl("src") ?: manga.coverUrl.orEmpty() + val description = doc.selectFirst(".synopsis-text")?.text().orEmpty() + + val author = doc.selectFirst(".meta-item-label:matchesOwn(^Author$) + .meta-item-value")?.text() + val artist = doc.selectFirst(".meta-item-label:matchesOwn(^Artist$) + .meta-item-value")?.text() + val authors = setOfNotNull(author, artist).filter { it.isNotBlank() }.toSet() + + val tags = doc.select(".meta-pill-row .meta-pill").map { it.text() } + .map { MangaTag(key = it.lowercase(), title = it, source = source) } + .toSet() + + val typeMeta = doc.selectFirst(".meta-item-label:matchesOwn(^Type$) + .meta-item-value")?.text() + val allTags = tags.toMutableSet().apply { + typeMeta?.takeIf { it.isNotBlank() }?.let { + add(MangaTag(key = it.lowercase(), title = it, source = source)) + } + } + + val state = when (doc.selectFirst(".hero-status-badge")?.text()?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val chapterUrl = "$fullUrl?all=1" + val chapterDoc = webClient.httpGet(chapterUrl).parseHtml() + val chapters = chapterDoc.select(".chapter-row").mapNotNull { element -> + val href = element.attr("data-href") + val name = element.selectFirst(".chapter-title")?.text() + ?: return@mapNotNull null + val dateText = element.selectFirst("[data-local-time]")?.attr("data-local-time") + val date = dateText?.let { + runCatching { dateFormat.parse(it)?.time ?: 0L }.getOrDefault(0L) + } ?: 0L + + MangaChapter( + id = generateUid(href), + url = href, + title = name, + number = name.extractChapterNumber(), + volume = 0, + uploadDate = date, + scanlator = null, + branch = null, + source = source, + ) + }.reversed() + + return manga.copy( + title = title, + coverUrl = cover, + description = description, + authors = authors, + tags = allTags, + state = state, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = if (!chapter.url.contains("/chapter/")) { + // Support old Madara URLs + val segments = chapter.url.split("/").filter { it.isNotBlank() } + if (segments.size >= 2) { + val mangaSlug = segments[0] + val number = segments[1].removePrefix("chapter-").replace("-", ".") + val dotIndex = number.indexOf('.') + val formatted = if (dotIndex >= 0) { + number.padEnd(dotIndex + 3, '0') + } else { + "$number.00" + } + "https://$domain/manga/$mangaSlug/chapter/$formatted" + } else { + chapter.url.toAbsoluteUrl(domain) + } + } else { + chapter.url.toAbsoluteUrl(domain) + } + + val doc = webClient.httpGet(fullUrl).parseHtml() + return doc.select("img.page-img").mapIndexed { _, img -> + val url = img.absUrl("src") + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + companion object { + private val GENRES = listOf( + "4-koma" to "4-Koma", + "action" to "Action", + "adult" to "Adult", + "adventure" to "Adventure", + "aliens" to "Aliens", + "animals" to "Animals", + "anthology" to "Anthology", + "comedy" to "Comedy", + "cooking" to "Cooking", + "crime" to "Crime", + "crossdressing" to "Crossdressing", + "delinquents" to "Delinquents", + "demon" to "Demon", + "demons" to "Demons", + "drama" to "Drama", + "ecchi" to "Ecchi", + "fantasy" to "Fantasy", + "game" to "Game", + "gender-bender" to "Gender Bender", + "genderswap" to "Genderswap", + "ghosts" to "Ghosts", + "gore" to "Gore", + "gyaru" to "Gyaru", + "harem" to "Harem", + "historical" to "Historical", + "horror" to "Horror", + "incest" to "Incest", + "isekai" to "Isekai", + "josei" to "Josei", + "loli" to "Loli", + "mafia" to "Mafia", + "magic" to "Magic", + "magical-girls" to "Magical Girls", + "martial-art" to "Martial Art", + "martial-arts" to "Martial Arts", + "mature" to "Mature", + "mecha" to "Mecha", + "medical" to "Medical", + "military" to "Military", + "monster" to "Monster", + "monster-girls" to "Monster Girls", + "monsters" to "Monsters", + "music" to "Music", + "mystery" to "Mystery", + "ninja" to "Ninja", + "office-workers" to "Office Workers", + "oneshot" to "Oneshot", + "philosophical" to "Philosophical", + "police" to "Police", + "psychological" to "Psychological", + "regression" to "Regression", + "reincarnation" to "Reincarnation", + "reverse-harem" to "Reverse Harem", + "romance" to "Romance", + "samurai" to "Samurai", + "school" to "School", + "school-life" to "School Life", + "sci-fi" to "Sci-Fi", + "seinen" to "Seinen", + "sexual-violence" to "Sexual Violence", + "shotacon" to "Shotacon", + "shoujo" to "Shoujo", + "shoujo-ai" to "Shoujo Ai", + "shounen" to "Shounen", + "slice-of-life" to "Slice of Life", + "smut" to "Smut", + "sports" to "Sports", + "super-power" to "Super Power", + "supernatural" to "Supernatural", + "survival" to "Survival", + "suspense" to "Suspense", + "system" to "System", + "thriller" to "Thriller", + "time-travel" to "Time Travel", + "tragedy" to "Tragedy", + "urban" to "Urban", + "vampire" to "Vampire", + "video-games" to "Video Games", + "villainess" to "Villainess", + "virtual-reality" to "Virtual Reality", + "web-comic" to "Web Comic", + "webtoons" to "Webtoons", + "yuri" to "Yuri", + "zombies" to "Zombies", + ) + } +} diff --git a/src/main/kotlin/tsuki/site/id/VoraToon.kt b/src/main/kotlin/tsuki/site/id/VoraToon.kt new file mode 100644 index 00000000..13288505 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/VoraToon.kt @@ -0,0 +1,406 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany + +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.json.JSONObject +import java.net.URLEncoder +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("VORATOON", "VoraToon", "id") +internal class VoraToon(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.VORATOON, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("v1.voratoon.com") + + private val apiBase = "https://$domain/backend" + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept", "application/json") + .add("Accept-language", "en-US,en;q=0.9,id;q=0.8") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.POPULARITY_ASC, + SortOrder.UPDATED, + SortOrder.UPDATED_ASC, + SortOrder.ALPHABETICAL, + SortOrder.ALPHABETICAL_DESC, + SortOrder.RATING, + SortOrder.RATING_ASC, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + val genreMap = fetchGenreMap() + return MangaListFilterOptions( + availableTags = genreMap.values.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val sortParam = when (order) { + SortOrder.POPULARITY -> "popularity" to "desc" + SortOrder.POPULARITY_ASC -> "popularity" to "asc" + SortOrder.UPDATED -> "latest" to "desc" + SortOrder.UPDATED_ASC -> "updated" to "asc" + SortOrder.ALPHABETICAL -> "alphabetical" to "asc" + SortOrder.ALPHABETICAL_DESC -> "alphabetical" to "desc" + SortOrder.RATING -> "rating" to "desc" + SortOrder.RATING_ASC -> "rating" to "asc" + else -> null + } + + val url = buildString { + append(apiBase) + append("/series?includeMeta=true") + append("&take=").append(pageSize) + append("&page=").append(page) + + sortParam?.let { (field, dir) -> + append("&sort=").append(field) + append("&sortOrder=").append(dir) + } + + if (!filter.query.isNullOrEmpty()) { + val q = URLEncoder.encode(filter.query, "UTF-8") + append("&filter=title=like=\"$q\",nativeTitle=like=\"$q\"") + } + + filter.states.oneOrThrowIfMany()?.let { state -> + append("&status=") + append( + when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "canceled" + else -> "" + }, + ) + } + + filter.types.oneOrThrowIfMany()?.let { type -> + append("&format=") + append( + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + }, + ) + } + + if (filter.tags.isNotEmpty()) { + append("&genreIds=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + val json = webClient.httpGet(url).body?.string().orEmpty() + return parseSeriesList(json) + } + + private fun parseSeriesList(json: String): List<Manga> { + val result = mutableListOf<Manga>() + try { + val root = JSONObject(json) + val dataArray = root.getJSONArray("data") + for (i in 0 until dataArray.length()) { + val item = dataArray.getJSONObject(i) + val data = item.getJSONObject("data") + val slug = data.getString("slug") + val relativeUrl = "/series/$slug" + + result.add( + Manga( + id = generateUid(relativeUrl), + url = relativeUrl, + publicUrl = "https://$domain$relativeUrl", + title = data.getString("title"), + altTitles = emptySet(), + rating = data.toRating(), + contentRating = null, + coverUrl = data.nonNullString("coverImage"), + tags = emptySet(), + state = data.toMangaState(), + authors = setOfNotNull(data.nonNullString("author")), + source = source, + ), + ) + } + } catch (_: Exception) { + } + return result + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.removeSuffix("/").substringAfterLast("/") + + val seriesUrl = "$apiBase/series/$slug" + val json = webClient.httpGet(seriesUrl).body?.string().orEmpty() + + return try { + val root = JSONObject(json) + val data = root.getJSONObject("data") + val seriesData = data.getJSONObject("data") + + val title = seriesData.getString("title") + val nativeTitle = seriesData.nonNullString("nativeTitle") + val synopsis = seriesData.optString("synopsis", "") + val author = seriesData.nonNullString("author") + val coverImage = seriesData.optString("coverImage", "") + val backgroundImage = seriesData.nonNullString("backgroundImage") + val format = seriesData.nonNullString("format")?.lowercase(Locale.ROOT) + + val genres = mutableSetOf<MangaTag>() + seriesData.optJSONArray("genres")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.toGenreTagOrNull()?.let(genres::add) + } + } + + val releaseYear = seriesData.nonNullString("releaseDate") + val extraTags = buildSet { + releaseYear?.let { add(MangaTag(it, it, source)) } + format?.let { + add(MangaTag(it.replaceFirstChar { c -> c.uppercase() }, it, source)) + } + } + + val chaptersUrl = "$apiBase/series/$slug/chapters" + val chaptersJson = webClient.httpGet(chaptersUrl).body?.string().orEmpty() + val chapters = parseChapters(chaptersJson, slug) + + manga.copy( + title = title, + altTitles = setOfNotNull(nativeTitle), + description = synopsis, + state = seriesData.toMangaState(), + authors = setOfNotNull(author), + tags = genres + extraTags, + coverUrl = coverImage, + largeCoverUrl = backgroundImage ?: coverImage, + rating = seriesData.toRating(), + chapters = chapters, + ) + } catch (_: Exception) { + manga + } + } + + private fun parseChapters(json: String, slug: String): List<MangaChapter> { + val chapters = mutableListOf<MangaChapter>() + try { + val root = JSONObject(json) + val chaptersArray = root.getJSONArray("data") ?: return emptyList() + + for (i in 0 until chaptersArray.length()) { + val ch = chaptersArray.getJSONObject(i) + val chapterData = ch.optJSONObject("data") + + val chapterIndex = chapterData?.optDouble("index", 0.0) ?: 0.0 + val formattedIndex = chapterIndex.toString().removeSuffix(".0") + + val chapterUrl = "/series/$slug/chapter/$formattedIndex" + + val title = chapterData?.nonNullString("title") + val titleText = if (title != null) { + "Chapter $formattedIndex: $title" + } else { + "Chapter $formattedIndex" + } + + val uploadDate = parseChapterDate(ch.optString("createdAt", "")) + + chapters.add( + MangaChapter( + id = generateUid(chapterUrl), + title = titleText, + url = chapterUrl, + number = chapterIndex.toFloat(), + volume = 0, + scanlator = null, + uploadDate = uploadDate, + branch = null, + source = source, + ) + ) + } + } catch (index: Exception) { + throw Exception("Error parsing chapters.", index) + } + return chapters.reversed() + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val parts = chapter.url.split("/") + if (parts.size < 5) return emptyList() + val slug = parts[2] + val chapterIndex = parts[4] + val url = "$apiBase/series/$slug/chapters/$chapterIndex" + + val json = webClient.httpGet(url).body?.string().orEmpty() + try { + val root = JSONObject(json) + val data = root.optJSONObject("data") ?: return emptyList() + + val imagesArray = data.optJSONObject("data")?.optJSONArray("images") + if (imagesArray != null) { + return (0 until imagesArray.length()).mapNotNull { i -> + val imgUrl = imagesArray.getString(i) + if (imgUrl.isBlank()) { + null + } else { + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ) + } + } + } + + val dataImages = data.optJSONObject("dataImages") + if (dataImages != null) { + val pages = mutableListOf<MangaPage>() + val sortedKeys = dataImages.keys().asSequence().toList() + .sortedBy { it.toIntOrNull() ?: Int.MAX_VALUE } + + for (key in sortedKeys) { + val imgUrl = dataImages.getString(key) + if (imgUrl.isNotBlank()) { + pages.add( + MangaPage( + id = generateUid(imgUrl), + url = imgUrl, + preview = null, + source = source, + ), + ) + } + } + return pages + } + + return emptyList() + } catch (_: Exception) { + return emptyList() + } + } + + @Volatile + private var genreCache: Map<String, MangaTag>? = null + private val genreMutex = Mutex() + + private suspend fun fetchGenreMap(): Map<String, MangaTag> { + return genreMutex.withLock { + genreCache ?: fetchGenreMapInternal().also { genreCache = it } + } + } + + private suspend fun fetchGenreMapInternal(): Map<String, MangaTag> { + val url = "$apiBase/genres" + val jsonStr = webClient.httpGet(url).body?.string().orEmpty() + val map = mutableMapOf<String, MangaTag>() + + try { + val dataArray = JSONObject(jsonStr).getJSONArray("data") + for (i in 0 until dataArray.length()) { + dataArray.getJSONObject(i).toGenreTagOrNull()?.let { + map[it.title] = it + } + } + } catch (_: Exception) { + } + return map + } + + private fun parseChapterDate(dateStr: String): Long { + if (dateStr.isBlank()) return 0L + return try { + isoDateFormat.parse(dateStr)?.time ?: 0L + } catch (_: Exception) { + 0L + } + } + + private fun JSONObject.nonNullString(key: String): String? { + return optString(key).takeIf { it.isNotBlank() && it != "null" } + } + + private fun JSONObject.toMangaState(): MangaState? { + val status = nonNullString("status")?.lowercase(Locale.ROOT) ?: return null + return when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "canceled" -> MangaState.ABANDONED + else -> null + } + } + + private fun JSONObject.toRating(): Float { + val raw = optDouble("rating", 0.0) + return if (raw > 0.0) (raw / 10.0).toFloat() else RATING_UNKNOWN + } + + private fun JSONObject.toGenreTagOrNull(): MangaTag? { + val name = optJSONObject("data")?.optString("name").orEmpty().trim() + val id = optInt("id", -1) + if (name.isEmpty() || id == -1) return null + return MangaTag(title = name, key = id.toString(), source = source) + } + + companion object { + private val isoDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX", Locale.ROOT) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt b/src/main/kotlin/tsuki/site/id/Westmanga.kt similarity index 82% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt rename to src/main/kotlin/tsuki/site/id/Westmanga.kt index 93aa0fae..26e0faa2 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/Westmanga.kt +++ b/src/main/kotlin/tsuki/site/id/Westmanga.kt @@ -1,17 +1,40 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.Favicon +import tsuki.model.Favicons + +import tsuki.util.generateUid +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseJson +import tsuki.util.json.getLongOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet import kotlinx.coroutines.sync.Mutex import kotlinx.coroutines.sync.withLock import okhttp3.Headers +import okhttp3.HttpUrl import okhttp3.HttpUrl.Companion.toHttpUrl -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import org.koitharu.kotatsu.parsers.util.json.* -import java.util.* +import org.jsoup.Jsoup +import java.util.EnumSet import javax.crypto.Mac import javax.crypto.spec.SecretKeySpec @@ -19,11 +42,20 @@ import javax.crypto.spec.SecretKeySpec internal class WestmangaParser(context: MangaLoaderContext) : PagedMangaParser(context, MangaParserSource.WESTMANGA, pageSize = 20) { - override val configKeyDomain = ConfigKey.Domain("v1.westmanga.cc") + override val configKeyDomain = ConfigKey.Domain("westmanga.co") private val apiDomain: String get() = "data.mantweh.online" + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/favicon.ico", 48, null), + ), + domain, + ) + } + private val accessKey = "WM_WEB_FRONT_END" private val secretKey = "xxxoidj" @@ -139,18 +171,20 @@ internal class WestmangaParser(context: MangaLoaderContext) : data.optJSONArray("genres")?.let { genres -> for (i in 0 until genres.length()) { val genre = genres.getJSONObject(i) - add(MangaTag( - title = genre.getString("name"), - key = genre.getInt("id").toString(), - source = source, - )) + add( + MangaTag( + title = genre.getString("name"), + key = genre.getInt("id").toString(), + source = source, + ) + ) } } } val description = buildString { data.getStringOrNull("sinopsis")?.let { synopsis -> - append(org.jsoup.Jsoup.parseBodyFragment(synopsis).wholeText().trim()) + append(Jsoup.parseBodyFragment(synopsis).wholeText().trim()) } data.getStringOrNull("alternative_name")?.let { alt -> append("\n\nAlternative Name: ") @@ -235,7 +269,7 @@ internal class WestmangaParser(context: MangaLoaderContext) : return@withLock genres } - private fun createApiHeaders(url: okhttp3.HttpUrl): Headers { + private fun createApiHeaders(url: HttpUrl): Headers { val timestamp = (System.currentTimeMillis() / 1000).toString() val message = "wm-api-request" val key = timestamp + "GET" + url.encodedPath + accessKey + secretKey diff --git a/src/main/kotlin/tsuki/site/id/Wurmz.kt b/src/main/kotlin/tsuki/site/id/Wurmz.kt new file mode 100644 index 00000000..7dd29c24 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/Wurmz.kt @@ -0,0 +1,277 @@ +package tsuki.site.id + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.extractChapterNumber +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.src +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl +import tsuki.util.urlEncoded + +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("WURMZ", "Wurmz", "id") +class Wurmz(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.WURMZ, pageSize = 30) { + + override val configKeyDomain = ConfigKey.Domain("wurmz.net") + + override val availableSortOrders: Set<SortOrder> = setOf( + SortOrder.UPDATED, + SortOrder.ADDED, + SortOrder.ADDED_ASC, + SortOrder.POPULARITY_TODAY, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + SortOrder.POPULARITY, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = false, + isSearchWithFiltersSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = GENRES.map { genre -> + MangaTag(key = genre, title = genre, source = source) + }.toSet(), + availableStates = EnumSet.of( + MangaState.ONGOING, + MangaState.FINISHED, + MangaState.PAUSED, + MangaState.ABANDONED, + ), + availableContentTypes = EnumSet.of( + ContentType.MANGA, + ContentType.MANHWA, + ContentType.MANHUA, + ), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildListUrl(page, order, filter) + val doc = webClient.httpGet(url).parseHtml() + + val cards = doc.select("article.comic-card") + if (cards.isEmpty()) { + return emptyList() + } + return cards.mapNotNull { parseMangaCard(it) } + } + + private fun buildListUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return buildString { + append("https://") + append(domain) + append("/semua-komik") + + val params = mutableMapOf<String, String>() + + filter.query?.takeIf { it.isNotEmpty() }?.let { + params["q"] = it.replace("Bahasa Indonesia", "").trim() + } + + filter.types.firstOrNull()?.let { type -> + params["type"] = when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + else -> "" + } + } + + filter.states.firstOrNull()?.let { state -> + params["status"] = when (state) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "tamat" + MangaState.PAUSED -> "hiatus" + MangaState.ABANDONED -> "drop" + else -> "" + } + } + + filter.tags.take(3).forEach { tag -> + params["genres[]"] = tag.key + } + + params["sort"] = when (order) { + SortOrder.UPDATED -> "update" + SortOrder.ADDED -> "new" + SortOrder.ADDED_ASC -> "old" + SortOrder.POPULARITY_TODAY -> "popular_today" + SortOrder.POPULARITY_WEEK -> "popular_7d" + SortOrder.POPULARITY_MONTH -> "popular_30d" + SortOrder.POPULARITY -> "popular_all" + else -> "update" + } + + if (page > 1) { + params["page"] = page.toString() + } + + if (params.isNotEmpty()) { + append("?") + append(params.entries.joinToString("&") { (key, value) -> + val encodedKey = if (key.endsWith("[]")) key else key.urlEncoded() + "$encodedKey=${value.urlEncoded()}" + }) + } + } + } + + private fun parseMangaCard(card: Element): Manga? { + val link = card.selectFirst("a[href]") ?: return null + val href = link.attrAsRelativeUrl("href") + val titleEl = card.selectFirst("h2.comic-title") + val title = titleEl?.text() ?: return null + val coverEl = card.selectFirst("img") + val coverUrl = coverEl?.src() + + return Manga( + id = generateUid(href), + url = href, + publicUrl = href.toAbsoluteUrl(domain), + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + authors = emptySet(), + tags = emptySet(), + rating = RATING_UNKNOWN, + state = null, + contentRating = null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val fullUrl = manga.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val title = doc.selectFirst("h1")?.text() ?: manga.title + val cover = doc.selectFirst(".cover-frame img")?.src() ?: manga.coverUrl + + val desc = doc.selectFirst(".mt-5 p.text-muted")?.text() ?: "" + + val genreElements = doc.select(".flex.flex-wrap.gap-1.5 .chip") + val tags = genreElements.mapNotNull { a -> + val genreName = a.text() + MangaTag(key = genreName, title = genreName, source = source) + }.toSet() + + val statusText = doc.selectFirst(".status-badge")?.text() + val state = when (statusText?.lowercase()) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "hiatus" -> MangaState.PAUSED + "cancelled" -> MangaState.ABANDONED + else -> null + } + + val authorText = doc.selectFirst(".text-xs.text-muted.mt-2.5")?.text() + val authors = authorText?.substringAfter("Oleh ")?.substringBefore(" ·")?.split('|') + ?.map { it.trim() } + ?.filter { it.isNotEmpty() && it != "-?" } + ?.toSet() ?: emptySet() + + val altText = doc.selectFirst(".text-sm.text-faint.mt-1")?.text() + val altTitles = altText?.split("·")?.map { it.trim() }?.toSet() ?: emptySet() + + val chapterLinks = doc.select(".chap-cell") + val chapters = chapterLinks.map { a -> + val href = a.attrAsRelativeUrl("href") + val label = a.text().trim() + val number = label.extractChapterNumber() + MangaChapter( + id = generateUid(href), + url = href, + title = label, + number = number, + volume = 0, + uploadDate = 0, + source = source, + scanlator = null, + branch = null, + ) + }.sortedBy { it.number } + + return manga.copy( + title = title, + publicUrl = fullUrl, + coverUrl = cover, + description = desc, + tags = tags, + state = state, + authors = authors, + altTitles = altTitles, + chapters = chapters, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val fullUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(fullUrl).parseHtml() + + val imgElements = doc.select(".reader-page img") + if (imgElements.isEmpty()) { + val lazyImgs = doc.select("img[data-src]") + if (lazyImgs.isNotEmpty()) { + return lazyImgs.map { img -> + val url = img.attr("data-src").toRelativeUrl(domain) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + throw ParseException("No images found on chapter page", fullUrl) + } + + return imgElements.map { img -> + val url = img.attr("src").ifEmpty { img.attr("data-src") }.toRelativeUrl(domain) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + companion object { + private val GENRES = listOf( + "Fantasy", "Romance", "Drama", "Action", "Comedy", "Shounen", "Adventure", "Shoujo", + "School", "Slice of Life", "Seinen", "Supernatural", "Historical", "Josei", "Isekai", + "Webtoons", "Martial Art", "Harem", "Ecchi", "Reincarnation", "Magic", "Mystery", + "Psychological", "Yaoi", "Mature", "Horror", "Tragedy", "Shounen Ai", "Sci-fi", + "Demons", "Adaptation", "Gender Bender", "Game", "Yuri", "Shoujo Ai", "Thriller", + "Sport", "Cooking", "Crime", "Gore", "Royal Family", "Super Power", "Regression", + "Military", "Royalty", "Vampire", "Office Workers", "Transmigration", "Medical", + "Time Travel", "Childhood Friends", "Music", "Monsters", "Revenge", "College Life", + "Villainess", "Kids", "Mecha", "One Shot", "Police", "Omegaverse", "Girls", + "Reverse Harem", "Animals", "Ghosts", "Project", "Age Gap", "Showbiz", "Violence", + "Boys", "Shotacon", "Survival", "Beasts", "Bloody", "Crossdressing", "Delinguents", + "Dungeons", "Gyaru", "Non-human", "Post-Apocalyptic", "Zombies", "Doujinshi", + "18+", "Bodyswap", "NTR", "Yakuzas", "BDSM", "Lolicon", "Philosophical", "System", + "Aliens", "Hentai", "Anthology", "Fetish", "Incest", "Virtual Reality", "Dementia", + "Nakadashi", "Samurai", "4-Koma", "Cheating", "Femdom", "Mafia", "Milf", "Ninja", + "Cunnilingus", "Guideverse", "Infidelity", "Parodi", "Reverse Isekai", "Villain" + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt similarity index 87% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt rename to src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt index 625f0b21..1820086e 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/id/hentais/BacaKomikpics.kt +++ b/src/main/kotlin/tsuki/site/id/nsfw/BacaKomikpics.kt @@ -1,184 +1,203 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.id.hentais - -import okhttp3.Headers -import org.json.JSONArray -import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.util.* - -@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) -internal class BacaKomikpics(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { // increased page size slightly for performance - - override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") - - override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") - - override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { - super.onCreateConfig(keys) - keys.add(userAgentKey) - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") - .build() - - override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = false, // weird handling of website - isMultipleTagsSupported = false, - ) - - override suspend fun getFilterOptions() = MangaListFilterOptions( - availableTags = emptySet(), - ) - - override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { - val url = when (order) { - SortOrder.POPULARITY -> "https://$domain/komik-populer/" - SortOrder.UPDATED -> { - if (page == 0) "https://$domain/?s=&post_type=komik_series" - else "https://$domain/page/${page + 1}/?s&post_type=komik_series" - } - else -> "https://$domain/komik-populer/" - } - val doc = webClient.httpGet(url).parseHtml() - return parseMangaList(doc) - } - - private fun parseMangaList(doc: Document): List<Manga> { - return doc.select("a.card").mapNotNull { card -> - val href = card.attrAsAbsoluteUrl("href") - val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null - val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() - val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() - val status = card.selectFirst(".status-label")?.text()?.trim() - Manga( - id = generateUid(href), - url = href.toRelativeUrl(domain), - publicUrl = href, - title = title, - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = ContentRating.ADULT, - coverUrl = cover ?: "", - tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), - state = when (status) { - "Ongoing" -> MangaState.ONGOING - "Completed" -> MangaState.FINISHED - "Hiatus" -> MangaState.PAUSED - else -> null - }, - authors = emptySet(), - source = source, - ) - } - } - - override suspend fun getDetails(manga: Manga): Manga { - val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() - val title = doc.selectFirst(".post-title h1")?.text()?.trim() - ?: doc.selectFirst("h1.entry-title")?.text()?.trim() - ?: manga.title - - val description = doc.selectFirst(".sinopsis")?.text()?.trim() - ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" - - val cover = doc.selectFirst(".series-thumb img")?.attr("src") - ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl - - val tags = doc.select("a.genre-link").mapNotNull { a -> - val genre = a.text().trim() - val key = a.attr("href").substringAfter("/genre/").substringBefore("/") - if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null - }.toSet() - - val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() - ?.substringAfter("Status :")?.trim() - val state = when { - statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING - statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED - statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED - else -> null - } - - val chapters = extractChapters(doc) - - return manga.copy( - title = title, - description = description, - coverUrl = cover, - tags = tags, - state = state, - authors = emptySet(), - chapters = chapters, - ) - } - - private fun extractChapters(doc: Document): List<MangaChapter> { - val script = doc.select("script").find { it.data().contains("chapterData") } - ?: return emptyList() - val data = script.data() - - val prefixIndex = data.indexOf("chapterData") - if (prefixIndex == -1) return emptyList() - - val windowStart = (prefixIndex - 500).coerceAtLeast(0) - val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) - val window = data.substring(windowStart, windowEnd) - - val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) - val match = arrayRegex.find(window) ?: return emptyList() - val jsonStr = match.groupValues[1].replace("\\/", "/") - - return try { - val jsonArray = JSONArray(jsonStr) - (0 until jsonArray.length()).mapNotNull { i -> - val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null - val chTitle = obj.optString("title", "") - val chUrl = obj.optString("url", "") - val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) - .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) - MangaChapter( - id = generateUid(chUrl), - title = chTitle, - number = number, - volume = 0, - url = chUrl.toRelativeUrl(domain), - scanlator = null, - uploadDate = 0L, - branch = null, - source = source, - ) - } - } catch (e: Exception) { - e.printStackTrace() - emptyList() - } - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() - return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> - var src = img.attr("src") - if (src.isBlank()) src = img.attr("data-src") - if (src.isBlank()) return@mapNotNull null - MangaPage( - id = generateUid(src), - url = src, - preview = null, - source = source, - ) - } - } - - private fun String.nullIfEmpty() = ifBlank { null } -} +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.model.ContentType + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.toAbsoluteUrl +import tsuki.util.toRelativeUrl + +import okhttp3.Headers +import org.json.JSONArray +import org.jsoup.nodes.Document +import java.util.EnumSet + +@MangaSourceParser("BACAKOMIKPICS", "BacaKomik.pics", "id", ContentType.HENTAI) +internal class BacaKomikpics(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.BACAKOMIKPICS, 24) { + + override val configKeyDomain = ConfigKey.Domain("bacakomik.pics") + + override val userAgentKey = ConfigKey.UserAgent("Mozilla/5.0 (Linux; Android 13; SM-S908B) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Mobile Safari/537.36") + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .add("Accept-Language", "id-ID,id;q=0.9,en-US;q=0.8,en;q=0.7") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.POPULARITY, SortOrder.UPDATED) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = false, + isMultipleTagsSupported = false, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = emptySet(), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = when (order) { + SortOrder.POPULARITY -> "https://$domain/komik-populer/" + SortOrder.UPDATED -> { + if (page == 0) "https://$domain/?s=&post_type=komik_series" + else "https://$domain/page/${page + 1}/?s&post_type=komik_series" + } + else -> "https://$domain/komik-populer/" + } + val doc = webClient.httpGet(url).parseHtml() + return parseMangaList(doc) + } + + private fun parseMangaList(doc: Document): List<Manga> { + return doc.select("a.card").mapNotNull { card -> + val href = card.attrAsAbsoluteUrl("href") + val title = card.selectFirst(".card-title")?.text()?.trim() ?: return@mapNotNull null + val cover = card.selectFirst("img")?.attr("src")?.nullIfEmpty() + val type = card.classNames().find { it.startsWith("card-") }?.removePrefix("card-")?.uppercase() + val status = card.selectFirst(".status-label")?.text()?.trim() + Manga( + id = generateUid(href), + url = href.toRelativeUrl(domain), + publicUrl = href, + title = title, + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = ContentRating.ADULT, + coverUrl = cover ?: "", + tags = setOfNotNull(type?.let { MangaTag(it, it, source) }), + state = when (status) { + "Ongoing" -> MangaState.ONGOING + "Completed" -> MangaState.FINISHED + "Hiatus" -> MangaState.PAUSED + else -> null + }, + authors = emptySet(), + source = source, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val title = doc.selectFirst(".post-title h1")?.text()?.trim() + ?: doc.selectFirst("h1.entry-title")?.text()?.trim() + ?: manga.title + + val description = doc.selectFirst(".sinopsis")?.text()?.trim() + ?: doc.selectFirst(".entry-content")?.text()?.trim() ?: "" + + val cover = doc.selectFirst(".series-thumb img")?.attr("src") + ?: doc.selectFirst("img.wp-post-image")?.attr("src") ?: manga.coverUrl + + val tags = doc.select("a.genre-link").mapNotNull { a -> + val genre = a.text().trim() + val key = a.attr("href").substringAfter("/genre/").substringBefore("/") + if (genre.isNotEmpty() && key.isNotEmpty()) MangaTag(genre, key, source) else null + }.toSet() + + val statusText = doc.select(".sinopsis strong:contains(Status)").first()?.parent()?.text() + ?.substringAfter("Status :")?.trim() + val state = when { + statusText != null && "ongoing" in statusText.lowercase() -> MangaState.ONGOING + statusText != null && "completed" in statusText.lowercase() -> MangaState.FINISHED + statusText != null && "hiatus" in statusText.lowercase() -> MangaState.PAUSED + else -> null + } + + val chapters = extractChapters(doc) + + return manga.copy( + title = title, + description = description, + coverUrl = cover, + tags = tags, + state = state, + authors = emptySet(), + chapters = chapters, + ) + } + + private fun extractChapters(doc: Document): List<MangaChapter> { + val script = doc.select("script").find { it.data().contains("chapterData") } + ?: return emptyList() + val data = script.data() + + val prefixIndex = data.indexOf("chapterData") + if (prefixIndex == -1) return emptyList() + + val windowStart = (prefixIndex - 500).coerceAtLeast(0) + val windowEnd = (prefixIndex + 50_000).coerceAtMost(data.length) + val window = data.substring(windowStart, windowEnd) + + val arrayRegex = Regex("""chapterData\s*=\s*(\[[^\]]*\])\s*;""", RegexOption.DOT_MATCHES_ALL) + val match = arrayRegex.find(window) ?: return emptyList() + val jsonStr = match.groupValues[1].replace("\\/", "/") + + return try { + val jsonArray = JSONArray(jsonStr) + (0 until jsonArray.length()).mapNotNull { i -> + val obj = jsonArray.optJSONObject(i) ?: return@mapNotNull null + val chTitle = obj.optString("title", "") + val chUrl = obj.optString("url", "") + val number = Regex("""CHAPTER\s*(\d+)""", RegexOption.IGNORE_CASE) + .find(chTitle)?.groupValues?.get(1)?.toFloatOrNull() ?: (i + 1f) + MangaChapter( + id = generateUid(chUrl), + title = chTitle, + number = number, + volume = 0, + url = chUrl.toRelativeUrl(domain), + scanlator = null, + uploadDate = 0L, + branch = null, + source = source, + ) + } + } catch (e: Exception) { + e.printStackTrace() + emptyList() + } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)).parseHtml() + return doc.select(".viewer-komik img, .img-wrapper img").mapNotNull { img -> + var src = img.attr("src") + if (src.isBlank()) src = img.attr("data-src") + if (src.isBlank()) return@mapNotNull null + MangaPage( + id = generateUid(src), + url = src, + preview = null, + source = source, + ) + } + } + + private fun String.nullIfEmpty() = ifBlank { null } +} diff --git a/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt b/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt new file mode 100644 index 00000000..b9a4ae7e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/DojingNet.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("DOJINGNET", "Dojing.net", "id", ContentType.HENTAI) +internal class DojingNet(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.DOJINGNET, "dojing.net") diff --git a/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt new file mode 100644 index 00000000..55aab75b --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/DoujinDesu.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.DoujinDesuParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("DOUJINDESU", "DoujinDesu", "id", ContentType.HENTAI) +internal class DoujinDesu(context: MangaLoaderContext) : + DoujinDesuParser(context, MangaParserSource.DOUJINDESU) diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt b/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt new file mode 100644 index 00000000..7b24b607 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Kanzenin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("KANZENIN", "Kanzenin", "id", ContentType.HENTAI) +internal class Kanzenin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KANZENIN, "kanzenin.info") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt b/src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt new file mode 100644 index 00000000..c2137725 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/KomikDewasaART.kt @@ -0,0 +1,15 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("KOMIKDEWASAART", "Komik Dewasa.art", "id", ContentType.HENTAI) +internal class KomikDewasaART(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KOMIKDEWASAART, "komikdewasa.art") { + override val mangaDirectory = "komik" + override val chapterListSelector = "div.bxcl li:not(:has(a[data-bs-target='#lockedChapterModal']))" +} diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt b/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt new file mode 100644 index 00000000..dc618062 --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Komiktap.kt @@ -0,0 +1,104 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.MangaParserAuthProvider +import tsuki.parsers.MangaThemesia +import tsuki.exception.ParseException + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl + +import okhttp3.Interceptor +import okhttp3.MediaType.Companion.toMediaTypeOrNull +import okhttp3.Response +import okhttp3.ResponseBody.Companion.asResponseBody +import org.json.JSONArray +import org.json.JSONObject +import java.util.Base64 + +@MangaSourceParser("KOMIKTAP", "Komiktap", "id", ContentType.HENTAI) +internal class Komiktap(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.KOMIKTAP, "komiktap.info", pageSize = 25), MangaParserAuthProvider, Interceptor { + + private val selectScript = "script[src^='data:text/javascript;base64,']" + private val selectTestScript = "script:containsData(ts_reader.run)" + private val selectPage = "div#readerarea img" + + override val authUrl: String get() = "https://$domain/" + + override suspend fun isAuthorized(): Boolean { + val response = webClient.httpGet("https://$domain/") + return response.headers["x-sucuri-cache"] != null || + response.headers["x-sucuri-id"] == null + } + + override suspend fun getUsername(): String = "" + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterUrl = chapter.url.toAbsoluteUrl(domain) + val doc = webClient.httpGet(chapterUrl).parseHtml() + + val test = doc.select(selectTestScript) + if (test.isEmpty()) { + return doc.select(selectPage).mapNotNull { img -> + val url = img.imgAttr().takeIf { it.isNotBlank() } ?: return@mapNotNull null + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + val images = try { + val base64Script = doc.select(selectScript) + var decode = "" + for (elem in base64Script) { + val src = elem.attr("src") + if (src.startsWith("data:text/javascript;base64,")) { + decode = Base64.getDecoder() + .decode(src.removePrefix("data:text/javascript;base64,")) + .toString(Charsets.UTF_8) + if (decode.startsWith("ts_reader.run")) break + } + } + + if (decode.startsWith("ts_reader.run")) { + extractImagesFromScript(decode) + } else { + val scriptElem = doc.selectFirstOrThrow(selectTestScript) + extractImagesFromScript(scriptElem.data()) + } + } catch (e: Exception) { + throw ParseException("Failed to extract images: ${e.message}", chapterUrl) + } + + return (0 until images.length()).map { i -> + val url = images.getString(i) + MangaPage(id = generateUid(url), url = url, preview = null, source = source) + } + } + + private fun extractImagesFromScript(scriptBody: String): JSONArray { + val json = scriptBody.substringAfter('(').substringBeforeLast(')') + return JSONObject(json) + .getJSONArray("sources") + .getJSONObject(0) + .getJSONArray("images") + } + + override fun intercept(chain: Interceptor.Chain): Response { + val request = chain.request() + val response = chain.proceed(request) + val mime = response.headers["Content-Type"] + if (response.isSuccessful && mime == "application/octet-stream") { + val newBody = response.body.source().asResponseBody("image/jpeg".toMediaTypeOrNull()!!) + return response.newBuilder().body(newBody).build() + } + return response + } +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt b/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt new file mode 100644 index 00000000..2c35927a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/MangaSusuku.kt @@ -0,0 +1,18 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("MANGASUSUKU", "MangaSusuku", "id", ContentType.HENTAI) +internal class MangaSusuku(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.MANGASUSUKU, "mangasusuku.com") { + override val mangaDirectory = "komik" + override val dateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH) +} \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt b/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt new file mode 100644 index 00000000..b456847e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Sasangeyou.kt @@ -0,0 +1,17 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import java.text.SimpleDateFormat +import java.util.Locale + +@MangaSourceParser("SASANGEYOUNET", "Sasangeyou.net", "id", ContentType.HENTAI) +internal class Sasangeyou(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SASANGEYOUNET, "sasangeyou.net") { + override val dateFormat = SimpleDateFormat("MM/dd/yyyy", Locale("id")) +} diff --git a/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt b/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt new file mode 100644 index 00000000..f567453e --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/SekteDoujin.kt @@ -0,0 +1,12 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MangaThemesia + +import tsuki.model.MangaParserSource +import tsuki.model.ContentType + +@MangaSourceParser("SEKTEDOUJIN", "SekteDoujin", "id", ContentType.HENTAI) +internal class SekteDoujin(context: MangaLoaderContext) : + MangaThemesia(context, MangaParserSource.SEKTEDOUJIN, "sektedoujin.cc") \ No newline at end of file diff --git a/src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt b/src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt new file mode 100644 index 00000000..31eca85a --- /dev/null +++ b/src/main/kotlin/tsuki/site/id/nsfw/Yurilab.kt @@ -0,0 +1,188 @@ +package tsuki.site.id.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.attrOrNull +import tsuki.util.generateUid +import tsuki.util.mapNotNullToSet +import tsuki.util.parseHtml +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.text.SimpleDateFormat +import java.util.Calendar +import java.util.Locale + +@MangaSourceParser("YURILAB", "YuriLab", "id", ContentType.HENTAI) +internal class YuriLab(context: MangaLoaderContext) : + MadaraParser(context, MangaParserSource.YURILAB, "yurilab.top", pageSize = 30) { + + override val sourceLocale: Locale = Locale.ENGLISH + override val withoutAjax = true + + override val filterCapabilities: MangaListFilterCapabilities + get() = super.filterCapabilities.copy(isMultipleTagsSupported = false) + + override fun parseMangaList(doc: Document): List<Manga> { + return super.parseMangaList(doc).map { manga -> + manga.copy(coverUrl = manga.coverUrl?.replace(Regex("""-\d+x\d+(?=\.\w+$)"""), "")) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val author = doc.selectFirst(".author-content a, .manga-author a")?.textOrNull() + return super.getDetails(manga).copy( + authors = setOfNotNull(author).ifEmpty { manga.authors } + ) + } + + override suspend fun fetchAvailableTags(): Set<MangaTag> { + val url = "https://$domain/?s=&post_type=wp-manga" + val docs = webClient.httpGet(url).parseHtml() + val genreLinks = docs.select(".genres-filter .dropdown-menu a[href*='genre=']") + return genreLinks.mapNotNullToSet { el -> + val href = el.attrOrNull("href") ?: return@mapNotNullToSet null + val match = Regex("""genre=([^&]+)""").find(href) + val key = match?.groupValues?.get(1) ?: return@mapNotNullToSet null + val title = el.textOrNull()?.trim()?.toTitleCase(sourceLocale) ?: return@mapNotNullToSet null + MangaTag( + title = title, + key = key, + source = source, + ) + } + } + + override val selectGenre = ".genres-content a[href*='genre'], .tags-content a[href*='tag']" + + override suspend fun createMangaTag(a: Element): MangaTag? { + val href = a.attrOrNull("href") ?: return null + val tagKey = extractTagKey(href) ?: return null + val title = a.textOrNull()?.trim() ?: return null + return MangaTag( + title = title, + key = tagKey, + source = source, + ) + } + + private fun extractTagKey(href: String): String? { + val genreMatch = Regex("""genre=([^&/?]+)""").find(href) + if (genreMatch != null) return genreMatch.groupValues[1] + val pattern = Regex("""series-genre/([^/?]+)|series-tag/([^/?]+)""", RegexOption.IGNORE_CASE) + return pattern.find(href)?.groupValues?.getOrNull(1)?.takeIf { it.isNotEmpty() } + ?: pattern.find(href)?.groupValues?.getOrNull(2)?.takeIf { it.isNotEmpty() } + } + + override val selectChapter = "ul.version-chap li.wp-manga-chapter" + + override fun transformChapterName(element: Element, name: String): String { + return if (element.hasClass("premium") || element.hasClass("premium-block")) { + "🔒 ${name.trim()}" + } else { + name.trim() + } + } + + private fun String?.applyChapterNumber(index: Int): String { + val base = this ?: "Chapter" + return if (base == "Chapter" || base == "🔒 Chapter") { + base.replace("Chapter", "Chapter ${index + 1}") + } else { + base + } + } + + override suspend fun loadChapters(mangaUrl: String, document: Document): List<MangaChapter> = coroutineScope { + val allChapters = mutableListOf<MangaChapter>() + var page = 1 + val batchSize = 5 + val dateFormat = SimpleDateFormat("d MMMM yyyy", sourceLocale) + + while (true) { + val deferreds = (page until page + batchSize).map { currentPage -> + async { + try { + val ajaxUrl = mangaUrl.toAbsoluteUrl(domain).removeSuffix("/") + "/ajax/chapters/?t=$currentPage" + val ajaxDocs = webClient.httpPost( + ajaxUrl.toHttpUrl(), + emptyMap(), + Headers.Builder().add("X-Requested-With", "XMLHttpRequest").build(), + ).parseHtml() + + val lis = ajaxDocs.select(selectChapter) + lis.mapNotNull { li -> + val a = li.selectFirst("a") ?: return@mapNotNull null + val rawHref = a.attrAsRelativeUrl("href") + + val baseName = a.ownText().ifEmpty { null } ?: a.selectFirst("p")?.textOrNull() + ?: "Chapter" + + val finalName = transformChapterName(li, baseName) + + var dateText = li.selectFirst("a.c-new-tag")?.attr("title") ?: li.selectFirst(selectDate)?.text() + if (dateText != null && !dateText.contains("ago", true) && !dateText.contains(Regex("""\d{4}"""))) { + val year = Calendar.getInstance().get(Calendar.YEAR) + dateText = "$dateText $year" + } + + MangaChapter( + id = generateUid(rawHref + finalName), + url = rawHref, + title = finalName, + number = 0f, + volume = 0, + branch = null, + uploadDate = parseChapterDate(dateFormat, dateText), + scanlator = null, + source = source, + ) + } + } catch (_: Exception) { + emptyList() + } + } + } + + val batches = deferreds.awaitAll() + var emptyBatch = false + for (pageChapters in batches) { + if (pageChapters.isEmpty()) { + emptyBatch = true + break + } + allChapters.addAll(pageChapters) + } + + if (emptyBatch) { + break + } + page += batchSize + } + allChapters.reversed().mapIndexed { index, chapter -> + chapter.copy( + title = chapter.title.applyChapterNumber(index), + number = (index + 1).toFloat() + ) + } + } +} diff --git a/src/main/kotlin/tsuki/site/pt/HiperToon.kt b/src/main/kotlin/tsuki/site/pt/HiperToon.kt new file mode 100644 index 00000000..f709796e --- /dev/null +++ b/src/main/kotlin/tsuki/site/pt/HiperToon.kt @@ -0,0 +1,11 @@ +package tsuki.site.pt + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.MangaParserSource + +@MangaSourceParser("HIPERTOON", "Hipertoon", "pt") +internal class HiperToon(context: MangaLoaderContext) : + HiperParser(context, MangaParserSource.HIPERTOON, "hipertoon.com") \ No newline at end of file diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt b/src/main/kotlin/tsuki/site/pt/Roxinha.kt similarity index 89% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt rename to src/main/kotlin/tsuki/site/pt/Roxinha.kt index e2737907..82efa58b 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/Roxinha.kt +++ b/src/main/kotlin/tsuki/site/pt/Roxinha.kt @@ -1,218 +1,235 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt - -import okhttp3.Headers -import okhttp3.HttpUrl.Companion.toHttpUrl -import org.json.JSONObject -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.config.ConfigKey -import org.koitharu.kotatsu.parsers.core.PagedMangaParser -import org.koitharu.kotatsu.parsers.model.* -import org.koitharu.kotatsu.parsers.util.* -import java.text.SimpleDateFormat -import java.util.* - -@MangaSourceParser("ROXINHA", "Roxinha", "pt") -internal class Roxinha(context: MangaLoaderContext) : - PagedMangaParser(context, MangaParserSource.ROXINHA, pageSize = 24) { - - override val configKeyDomain = ConfigKey.Domain("roxinha.online") - - override val availableSortOrders: Set<SortOrder> = EnumSet.of( - SortOrder.ALPHABETICAL, - SortOrder.UPDATED, - SortOrder.POPULARITY, - SortOrder.RATING, - ) - - override val filterCapabilities = MangaListFilterCapabilities( - isSearchSupported = true, - isSearchWithFiltersSupported = true, - isMultipleTagsSupported = false, - - ) - - private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { - timeZone = TimeZone.getTimeZone("UTC") - } - - override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() - .add("Referer", "https://$domain/") - .add("Origin", "https://$domain") - .build() - - override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( - availableTags = emptySet(), - availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), - availableContentTypes = EnumSet.of( - ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS // Webtoon - ), - ) - - override suspend fun getListPage( - page: Int, - order: SortOrder, - filter: MangaListFilter, - ): List<Manga> { - val offset = (page - 1) * pageSize - val url = "https://$domain/api/manga/search/advanced".toHttpUrl().newBuilder().apply { - addQueryParameter("limit", pageSize.toString()) - addQueryParameter("offset", offset.toString()) - addQueryParameter("mode", "default") - - if (!filter.query.isNullOrBlank()) { - addQueryParameter("q", filter.query!!) - } - - val (sortField, sortOrder) = when (order) { - SortOrder.ALPHABETICAL -> "title" to "ASC" - SortOrder.UPDATED -> "updatedAt" to "DESC" - SortOrder.POPULARITY -> "views" to "DESC" - SortOrder.RATING -> "avgRating" to "DESC" - else -> "title" to "ASC" - } - addQueryParameter("sort", sortField) - addQueryParameter("order", sortOrder) - - if (filter.states.isNotEmpty()) { - val status = when (filter.states.first()) { - MangaState.ONGOING -> "ongoing" - MangaState.FINISHED -> "completed" - else -> null - } - status?.let { addQueryParameter("status", it) } - } - - if (filter.types.isNotEmpty()) { - val type = when (filter.types.first()) { - ContentType.MANGA -> "Manga" - ContentType.MANHWA -> "Manhwa" - ContentType.MANHUA -> "Manhua" - ContentType.COMICS -> "Webtoon" - else -> null - } - type?.let { addQueryParameter("type", it) } - } - }.build() - - val json = webClient.httpGet(url).parseJson() - val mangasArray = json.optJSONArray("mangas") ?: return emptyList() - - val list = mutableListOf<Manga>() - for (i in 0 until mangasArray.length()) { - val item = mangasArray.getJSONObject(i) - list.add(item.toManga()) - } - return list - } - - private fun JSONObject.toManga(): Manga { - val id = getInt("id") - return Manga( - id = generateUid(id.toString()), - url = id.toString(), - publicUrl = "https://$domain/manga/$id", - title = getString("title"), - altTitles = emptySet(), - rating = RATING_UNKNOWN, - contentRating = null, - coverUrl = optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } ?: "", - tags = emptySet(), - state = null, - authors = emptySet(), - source = source, - ) - } - - override suspend fun getDetails(manga: Manga): Manga { - val id = manga.url - val json = webClient.httpGet("https://$domain/api/manga/$id").parseJson() - - val title = json.getString("title") - val cover = json.optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } - val author = json.optString("author", null).nullIfEmpty() - val description = json.optString("description", null).nullIfEmpty() - val status = json.optString("status", null) - val genresRaw = json.optString("genres", null) - val tags = genresRaw?.split(",")?.mapNotNull { it -> it.trim().takeIf { it.isNotEmpty() } }?.map { - MangaTag(it, it.lowercase().replace(" ", "_"), source) - }.orEmpty().toSet() - - val state = when (status) { - "ongoing" -> MangaState.ONGOING - "completed" -> MangaState.FINISHED - else -> null - } - - val chaptersArray = json.optJSONArray("chapters") - val chapters = chaptersArray?.let { arr -> - (0 until arr.length()).map { i -> arr.getJSONObject(i).toChapter() } - } ?: emptyList() - - return manga.copy( - title = title, - coverUrl = cover ?: manga.coverUrl, - largeCoverUrl = cover ?: manga.largeCoverUrl, - authors = setOfNotNull(author), - description = description, - state = state, - tags = tags, - chapters = chapters, - ) - } - - private fun JSONObject.toChapter(): MangaChapter { - val id = getInt("id") - val number = optDouble("chapterNumber", Double.NaN).let { if (it.isNaN()) -1f else it.toFloat() } - val rawTitle = optString("title", null) - val createdAt = optString("createdAt", null) - - val title = if (!rawTitle.isNullOrBlank() && rawTitle.toDoubleOrNull() == null) { - rawTitle - } else { - "Capítulo ${number.toString().removeSuffix(".0")}" - } - - return MangaChapter( - id = generateUid(id.toString()), - title = title, - number = number, - volume = 0, - url = id.toString(), // chapter ID - uploadDate = createdAt?.let { dateFormat.parseSafe(it) } ?: 0L, - source = source, - scanlator = null, - branch = null, - ) - } - - override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { - val chapterId = chapter.url - // access ticket - val ticketJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId/access").parseJson() - val ticket = ticketJson.getString("ticket") - - // fetch chapter pages with ticket header - val pagesHeaders = getRequestHeaders().newBuilder() - .add("x-chapter-access", ticket) - .build() - val pagesJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId", pagesHeaders).parseJson() - val pagesArray = pagesJson.optJSONArray("pages") ?: return emptyList() - - return (0 until pagesArray.length()).map { i -> - val url = pagesArray.getString(i) - val fullUrl = if (url.startsWith("http")) url else "https://$domain$url" - MangaPage( - id = generateUid(fullUrl), - url = fullUrl, - preview = null, - source = source, - ) - } - } - - override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() - - private fun String.nullIfEmpty() = ifBlank { null } - private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() -} +package tsuki.site.pt + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("ROXINHA", "Roxinha", "pt") +internal class Roxinha(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ROXINHA, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("roxinha.online") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.ALPHABETICAL, + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = false, + + ) + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override suspend fun getFilterOptions(): MangaListFilterOptions = MangaListFilterOptions( + availableTags = emptySet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS // Webtoon + ), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + val offset = (page - 1) * pageSize + val url = "https://$domain/api/manga/search/advanced".toHttpUrl().newBuilder().apply { + addQueryParameter("limit", pageSize.toString()) + addQueryParameter("offset", offset.toString()) + addQueryParameter("mode", "default") + + if (!filter.query.isNullOrBlank()) { + addQueryParameter("q", filter.query!!) + } + + val (sortField, sortOrder) = when (order) { + SortOrder.ALPHABETICAL -> "title" to "ASC" + SortOrder.UPDATED -> "updatedAt" to "DESC" + SortOrder.POPULARITY -> "views" to "DESC" + SortOrder.RATING -> "avgRating" to "DESC" + else -> "title" to "ASC" + } + addQueryParameter("sort", sortField) + addQueryParameter("order", sortOrder) + + if (filter.states.isNotEmpty()) { + val status = when (filter.states.first()) { + MangaState.ONGOING -> "ongoing" + MangaState.FINISHED -> "completed" + else -> null + } + status?.let { addQueryParameter("status", it) } + } + + if (filter.types.isNotEmpty()) { + val type = when (filter.types.first()) { + ContentType.MANGA -> "Manga" + ContentType.MANHWA -> "Manhwa" + ContentType.MANHUA -> "Manhua" + ContentType.COMICS -> "Webtoon" + else -> null + } + type?.let { addQueryParameter("type", it) } + } + }.build() + + val json = webClient.httpGet(url).parseJson() + val mangasArray = json.optJSONArray("mangas") ?: return emptyList() + + val list = mutableListOf<Manga>() + for (i in 0 until mangasArray.length()) { + val item = mangasArray.getJSONObject(i) + list.add(item.toManga()) + } + return list + } + + private fun JSONObject.toManga(): Manga { + val id = getInt("id") + return Manga( + id = generateUid(id.toString()), + url = id.toString(), + publicUrl = "https://$domain/manga/$id", + title = getString("title"), + altTitles = emptySet(), + rating = RATING_UNKNOWN, + contentRating = null, + coverUrl = optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } ?: "", + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val id = manga.url + val json = webClient.httpGet("https://$domain/api/manga/$id").parseJson() + + val title = json.getString("title") + val cover = json.optString("cover", null)?.let { if (it.startsWith("http")) it else "https://$domain$it" } + val author = json.optString("author", null).nullIfEmpty() + val description = json.optString("description", null).nullIfEmpty() + val status = json.optString("status", null) + val genresRaw = json.optString("genres", null) + val tags = genresRaw?.split(",")?.mapNotNull { it -> it.trim().takeIf { it.isNotEmpty() } }?.map { + MangaTag(it, it.lowercase().replace(" ", "_"), source) + }.orEmpty().toSet() + + val state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + else -> null + } + + val chaptersArray = json.optJSONArray("chapters") + val chapters = chaptersArray?.let { arr -> + (0 until arr.length()).map { i -> arr.getJSONObject(i).toChapter() } + } ?: emptyList() + + return manga.copy( + title = title, + coverUrl = cover ?: manga.coverUrl, + largeCoverUrl = cover ?: manga.largeCoverUrl, + authors = setOfNotNull(author), + description = description, + state = state, + tags = tags, + chapters = chapters, + ) + } + + private fun JSONObject.toChapter(): MangaChapter { + val id = getInt("id") + val number = optDouble("chapterNumber", Double.NaN).let { if (it.isNaN()) -1f else it.toFloat() } + val rawTitle = optString("title", null) + val createdAt = optString("createdAt", null) + + val title = if (!rawTitle.isNullOrBlank() && rawTitle.toDoubleOrNull() == null) { + rawTitle + } else { + "Capítulo ${number.toString().removeSuffix(".0")}" + } + + return MangaChapter( + id = generateUid(id.toString()), + title = title, + number = number, + volume = 0, + url = id.toString(), // chapter ID + uploadDate = createdAt?.let { dateFormat.parseSafe(it) } ?: 0L, + source = source, + scanlator = null, + branch = null, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url + // access ticket + val ticketJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId/access").parseJson() + val ticket = ticketJson.getString("ticket") + + // fetch chapter pages with ticket header + val pagesHeaders = getRequestHeaders().newBuilder() + .add("x-chapter-access", ticket) + .build() + val pagesJson = webClient.httpGet("https://$domain/api/manga/chapter/$chapterId", pagesHeaders).parseJson() + val pagesArray = pagesJson.optJSONArray("pages") ?: return emptyList() + + return (0 until pagesArray.length()).map { i -> + val url = pagesArray.getString(i) + val fullUrl = if (url.startsWith("http")) url else "https://$domain$url" + MangaPage( + id = generateUid(fullUrl), + url = fullUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun String.nullIfEmpty() = ifBlank { null } + private fun SimpleDateFormat.parseSafe(date: String): Long? = runCatching { parse(date)?.time }.getOrNull() +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt b/src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt similarity index 51% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt rename to src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt index 492eea9b..58fd52ac 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/pt/hentais/LerHentais.kt +++ b/src/main/kotlin/tsuki/site/pt/nsfw/LerHentais.kt @@ -1,11 +1,13 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.pt.hentais +package tsuki.site.pt.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.HiperParser + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource import okhttp3.Headers -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.HiperParser @MangaSourceParser("LERHENTAIS", "LerHentais", "pt", ContentType.HENTAI) internal class LerHentais(context: MangaLoaderContext) : @@ -14,4 +16,4 @@ internal class LerHentais(context: MangaLoaderContext) : override fun getRequestHeaders(): Headers = super.getRequestHeaders().newBuilder() .set("x-flux-node", "G2ZsDdWhUwdU82Vw") .build() -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/ru/AComics.kt b/src/main/kotlin/tsuki/site/ru/AComics.kt new file mode 100644 index 00000000..666651d5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/AComics.kt @@ -0,0 +1,212 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.attrAsAbsoluteUrl +import tsuki.util.generateUid +import tsuki.util.insertCookies +import tsuki.util.mapNotNullToSet +import tsuki.util.oneOrThrowIfMany +import tsuki.util.parseHtml +import tsuki.util.requireSrc +import tsuki.util.selectFirstOrThrow +import tsuki.util.textOrNull +import tsuki.util.toAbsoluteUrl + +import androidx.collection.ArrayMap +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import org.jsoup.nodes.Document +import java.util.EnumSet +import kotlin.collections.toSet + +@MangaSourceParser("ACOMICS", "AComics", "ru", ContentType.COMICS) +internal class AComics(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ACOMICS, pageSize = 10) { + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.ALPHABETICAL, + SortOrder.POPULARITY, + ) + + override val configKeyDomain = ConfigKey.Domain("acomics.ru") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + context.cookieJar.insertCookies(domain, "ageRestrict=18") + } + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = getOrCreateTagMap().values.toSet(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + + override suspend fun getListPage( + page: Int, + order: SortOrder, + filter: MangaListFilter, + ): List<Manga> { + val url = buildString { + append("https://") + append(domain) + when { + !filter.query.isNullOrEmpty() -> { + if (page > 0) { + return emptyList() + } + append("/search?keyword=") + append(filter.query) + } + + else -> { + append("/comics?ratings[]=1&ratings[]=2&ratings[]=3&ratings[]=4&ratings[]=5&ratings[]=6&skip=") + append(page * 10) + append("&sort=") + append( + when (order) { + SortOrder.UPDATED -> "last_update" + SortOrder.ALPHABETICAL -> "serial_name" + SortOrder.POPULARITY -> "subscr_count" + else -> "last_update" + }, + ) + + if (filter.tags.isNotEmpty()) { + append("&categories=") + append(filter.tags.joinToString(separator = ",") { it.key }) + } + + if (filter.states.isNotEmpty()) { + append("&updatable=") + append( + filter.states.oneOrThrowIfMany().let { + when (it) { + MangaState.ONGOING -> "yes" + MangaState.FINISHED -> "no" + else -> "0" + } + }, + ) + } + } + } + } + + return parseMangaList(webClient.httpGet(url).parseHtml()) + } + + private fun parseMangaList(docs: Document): List<Manga> { + return docs.select("section.serial-card").map { + val a = it.selectFirstOrThrow("a.cover") + val url = a.attrAsAbsoluteUrl("href") + "/about" + Manga( + id = generateUid(url), + url = url, + title = it.selectFirstOrThrow("h2.title a").text(), + altTitles = emptySet(), + publicUrl = url, + rating = RATING_UNKNOWN, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = a.selectFirstOrThrow("img").attrAsAbsoluteUrl("data-real-src"), + tags = emptySet(), + state = null, + authors = emptySet(), + source = source, + ) + } + } + + private var tagCache: ArrayMap<String, MangaTag>? = null + private val mutex = Mutex() + + private suspend fun getOrCreateTagMap(): Map<String, MangaTag> = mutex.withLock { + tagCache?.let { return@withLock it } + val tagMap = ArrayMap<String, MangaTag>() + val tagElements = webClient.httpGet("https://$domain/comics") + .parseHtml() + .select("form.catalog-filters-form fieldset.categories label") + for (el in tagElements) { + val name = el.ownText().trim() + val key = el.selectFirstOrThrow("input[name='categories[]']").attr("value") + tagMap[name] = MangaTag( + title = name, + key = key, + source = source, + ) + } + tagCache = tagMap + return@withLock tagMap + } + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val tagMap = getOrCreateTagMap() + val tags = doc.select("p.serial-about-badges .category").mapNotNullToSet { tagMap[it.text()] } + val author = doc.selectFirst("p.serial-about-authors a")?.textOrNull() + return manga.copy( + tags = tags, + description = doc.selectFirst("section.serial-about-text p")?.text(), + authors = setOfNotNull(author), + chapters = listOf( + MangaChapter( + id = manga.id, + title = manga.title, + number = 1f, + volume = 0, + url = manga.url.replace("/about", "/"), + scanlator = null, + uploadDate = 0, + branch = null, + source = source, + ), + ), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val doc = webClient.httpGet(chapter.url + "1").parseHtml() + val totalPages = doc.selectFirstOrThrow("h1.reader-issue-title span.number") + .text() + .substringAfterLast('/') + .toInt() + return (1..totalPages).map { + val url = chapter.url + it + MangaPage( + id = generateUid(url), + url = url, + preview = null, + source = source, + ) + } + } + + override suspend fun getPageUrl(page: MangaPage): String { + val doc = webClient.httpGet(page.url.toAbsoluteUrl(domain)).parseHtml() + return doc.selectFirstOrThrow("section.reader-issue img.issue").requireSrc() + } +} diff --git a/src/main/kotlin/tsuki/site/ru/AstraManga.kt b/src/main/kotlin/tsuki/site/ru/AstraManga.kt new file mode 100644 index 00000000..ae3edac1 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/AstraManga.kt @@ -0,0 +1,367 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.ContentType +import tsuki.model.Favicon +import tsuki.model.Favicons +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.parseJson + +import okhttp3.HttpUrl.Companion.toHttpUrl +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("ASTRAMANGA", "AstraManga", "ru") +internal class AstraManga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.ASTRAMANGA, pageSize = 24) { + + override val configKeyDomain = ConfigKey.Domain("astramanga.org") + + override suspend fun getFavicons(): Favicons { + return Favicons( + listOf( + Favicon("https://$domain/favicon.ico", 256, null), + ), + domain, + ) + } + + private val apiUrl = "https://api.$domain/api/v1" + private val mediaUrl = "https://$domain/media" + + private val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("UTC") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + override fun getRequestHeaders() = super.getRequestHeaders().newBuilder() + .add("Referer", "https://$domain/") + .add("Origin", "https://$domain") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.POPULARITY, + SortOrder.UPDATED, + SortOrder.NEWEST, + SortOrder.RATING, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities = MangaListFilterCapabilities( + isSearchSupported = true, + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isYearRangeSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = GENRES.map { (name, id) -> MangaTag(name, id, source) }.toSet(), + availableContentTypes = EnumSet.of( + ContentType.MANGA, ContentType.MANHWA, ContentType.MANHUA, ContentType.COMICS + ), + ) + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = buildSearchUrl(page, order, filter) + val json = webClient.httpGet(url).parseJson() + val data = json.optJSONObject("data") ?: return emptyList() + val titles = data.optJSONArray("titles") ?: return emptyList() + return (0 until titles.length()).map { i -> + parseSearchManga(titles.getJSONObject(i)) + } + } + + private fun buildSearchUrl(page: Int, order: SortOrder, filter: MangaListFilter): String { + return "$apiUrl/search".toHttpUrl().newBuilder() + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", "30") + .apply { + filter.query?.trim()?.takeIf { it.isNotEmpty() }?.let { + addQueryParameter("q", it) + } + + val sortParam = when (order) { + SortOrder.POPULARITY -> "-popularity" + SortOrder.UPDATED -> "-updated_at" + SortOrder.NEWEST -> "-created_at" + SortOrder.RATING -> "-rating" + SortOrder.ALPHABETICAL -> "name" + else -> "-popularity" + } + addQueryParameter("sort", sortParam) + + filter.types.mapNotNull { type -> + when (type) { + ContentType.MANGA -> "manga" + ContentType.MANHWA -> "manhwa" + ContentType.MANHUA -> "manhua" + ContentType.COMICS -> "comics" + else -> null + } + }.takeIf { it.isNotEmpty() }?.let { + addQueryParameter("types", it.joinToString(",")) + } + + filter.tags.map { it.key } + .takeIf { it.isNotEmpty() } + ?.let { addQueryParameter("genres", it.joinToString(",")) } + + filter.tagsExclude.map { it.key } + .takeIf { it.isNotEmpty() } + ?.let { addQueryParameter("exclude_genres", it.joinToString(",")) } + + if (filter.yearFrom > 0) addQueryParameter("year_from", filter.yearFrom.toString()) + if (filter.yearTo > 0) addQueryParameter("year_to", filter.yearTo.toString()) + } + .build() + .toString() + } + + private fun parseSearchManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("name") + val coverUrl = resolveCoverUrl(obj) + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = emptySet(), + coverUrl = coverUrl, + largeCoverUrl = null, + description = null, + tags = emptySet(), + authors = emptySet(), + state = null, + rating = RATING_UNKNOWN, + contentRating = null, + source = source, + ) + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url + val titleUrl = "$apiUrl/titles/$slug" + val json = webClient.httpGet(titleUrl).parseJson() + val data = json.getJSONObject("data") + val updated = parseDetailManga(data) + + val titleId = data.getInt("id") + val branchesUrl = "$apiUrl/titles/$titleId/branches" + val branchesJson = webClient.httpGet(branchesUrl).parseJson() + val branches = branchesJson.getJSONObject("data").getJSONArray("branches") + var branchId = -1 + var maxChapters = 0 + for (i in 0 until branches.length()) { + val branch = branches.getJSONObject(i) + val isMain = branch.optBoolean("is_main", false) + val count = branch.optInt("count_chapters", 0) + if (isMain || count > maxChapters) { + branchId = branch.getInt("id") + maxChapters = count + } + } + if (branchId == -1) return updated.copy(chapters = emptyList()) + + val chaptersUrl = "$apiUrl/branches/$branchId/chapters" + val chaptersJson = webClient.httpGet( + chaptersUrl.toHttpUrl().newBuilder() + .addQueryParameter("size", maxChapters.coerceAtLeast(100).toString()) + .build() + ).parseJson() + val items = chaptersJson.getJSONObject("data").getJSONArray("items") + val chapters = mutableListOf<MangaChapter>() + for (i in 0 until items.length()) { + val ch = items.getJSONObject(i) + val chId = ch.getLong("id") + val number = ch.getDouble("number").toFloat() + val volume = ch.optInt("volume_number", 0) + val name = ch.optString("name", null) + val publishedAt = ch.optString("published_at", null) + val uploadDate = dateFormat.parseSafe(publishedAt) + val title = buildString { + if (volume > 0) append("Том $volume ") + append("Глава ${number.toString().removeSuffix(".0")}") + if (!name.isNullOrBlank()) append(" — $name") + } + chapters.add( + MangaChapter( + id = generateUid(chId.toString()), + title = title, + number = number, + volume = volume, + url = "$slug/${number.toString().removeSuffix(".0")}/$chId", + uploadDate = uploadDate, + source = source, + scanlator = null, + branch = null, + ) + ) + } + return updated.copy(chapters = chapters.sortedBy { it.number }) + } + + private fun parseDetailManga(obj: JSONObject): Manga { + val slug = obj.getString("slug") + val title = obj.getString("name") + val coverUrl = resolveCoverUrl(obj) + val description = obj.optString("description", null) + val status = obj.optString("status", null) + val state = when (status) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "paused" -> MangaState.PAUSED + "frozen", "discontinued" -> MangaState.ABANDONED + else -> null + } + + val rawRating = obj.optDouble("rating", -1.0) + val rating = if (rawRating >= 0.0) (rawRating / 10.0).toFloat() else RATING_UNKNOWN + + val genresArray = obj.optJSONArray("genres") + val tagsArray = obj.optJSONArray("tags") + val allTags = mutableSetOf<MangaTag>() + genresArray?.let { arr -> + for (i in 0 until arr.length()) { + val genre = arr.getJSONObject(i) + val id = genre.getString("id") + val name = genre.getString("name") + allTags.add(MangaTag(id, name, source)) + } + } + tagsArray?.let { arr -> + for (i in 0 until arr.length()) { + val tag = arr.getJSONObject(i) + val id = tag.getString("id") + val name = tag.getString("name") + if (allTags.none { it.title == name }) { + allTags.add(MangaTag(id, name, source)) + } + } + } + + val authors = mutableSetOf<String>() + obj.optJSONObject("publishing_house")?.optString("name")?.let { authors.add(it) } + obj.optJSONArray("publishers")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optJSONObject(i)?.optString("name")?.let { authors.add(it) } + } + } + + val altTitles = mutableSetOf<String>() + obj.optString("secondary_name", null)?.let { altTitles.add(it) } + obj.optJSONArray("alternative_names")?.let { arr -> + for (i in 0 until arr.length()) { + arr.optString(i)?.let { altTitles.add(it) } + } + } + + val isNsfw = obj.optBoolean("is_erotic", false) || + obj.optBoolean("is_yaoi", false) || + obj.optBoolean("is_lgbt", false) + val contentRating = if (isNsfw) ContentRating.ADULT else ContentRating.SAFE + + return Manga( + id = generateUid(slug), + url = slug, + publicUrl = "https://$domain/manga/$slug", + title = title, + altTitles = altTitles, + coverUrl = coverUrl, + largeCoverUrl = null, + description = description, + tags = allTags, + authors = authors, + state = state, + rating = rating, + contentRating = contentRating, + source = source, + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val chapterId = chapter.url.substringAfterLast("/") + val url = "$apiUrl/chapters/$chapterId/pages" + val json = webClient.httpGet(url).parseJson() + val pages = json.getJSONObject("data").getJSONArray("pages") + return (0 until pages.length()).map { i -> + val imageUrl = pages.getJSONObject(i).getString("image_url") + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + private fun resolveCoverUrl(obj: JSONObject): String? { + val coverVersions = obj.optJSONObject("cover_versions") + coverVersions?.optString("mid")?.takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + coverVersions?.optString("high")?.takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + obj.optString("cover_image").takeIf { it.isNotBlank() }?.let { return resolveUrl(it) } + return null + } + + private fun resolveUrl(path: String): String { + return if (path.startsWith("http://") || path.startsWith("https://")) { + path + } else { + "$mediaUrl/$path" + } + } + + private fun SimpleDateFormat.parseSafe(date: String?): Long { + return date?.let { runCatching { parse(it)?.time ?: 0L }.getOrDefault(0L) } ?: 0L + } + + companion object { + private val GENRES = mapOf( + "Ðниме" to "1", "ÐнтиутопиÑ" to "2", "ÐпокалиптичеÑкий" to "3", "Ðрт" to "4", + "Безумие" to "5", "Боевик" to "6", "Боевые иÑкуÑÑтва" to "7", "ВеÑтерн" to "8", + "Военное" to "9", "Выживание" to "10", "Гарем" to "11", "ГероичеÑкое Ñ„Ñнтези" to "12", + "Гуро" to "13", "ГÑг-юмор" to "14", "Детектив" to "15", "ДетÑкое" to "16", + "ДзёÑей" to "17", "Драма" to "18", "Завоевание мира" to "19", "ИÑекай" to "20", + "ИÑкуÑÑтво" to "21", "ИÑторичеÑкий" to "22", "Киберпанк" to "23", "Кодомо" to "24", + "КомедиÑ" to "25", "КоÑмоÑ" to "26", "Криминал / ПреÑтупники" to "27", + "КулинариÑ" to "28", "КультивациÑ" to "29", "ЛитреÑ" to "30", + "Махо-Ñёдзё" to "31", "Меха" to "32", "МиÑтика" to "33", "МифологиÑ" to "34", + "Мурим" to "35", "ÐÐ°ÑƒÑ‡Ð½Ð°Ñ Ñ„Ð°Ð½Ñ‚Ð°Ñтика" to "36", "Обратный Гарем" to "37", + "ОмегаверÑ" to "38", "ПародиÑ" to "39", "ПовÑедневноÑÑ‚ÑŒ" to "40", + "ПоÑтапокалипÑиÑ" to "41", "ПриключениÑ" to "42", "ПÑихологиÑ" to "43", + "РегреÑÑиÑ" to "44", "РиÑование" to "45", "Романтика" to "46", + "СамурайÑкий боевик" to "47", "СверхъеÑтеÑтвенное" to "48", "Сёдзе" to "49", + "Сёнен" to "50", "Спорт" to "51", "Средневековье" to "52", "Стимпанк" to "53", + "СÑйнÑн" to "54", "СÑньÑÑ" to "55", "ТрагедиÑ" to "56", "Триллер" to "57", + "УжаÑÑ‹" to "58", "ФантаÑтика" to "59", "ФилоÑофиÑ" to "60", "ФÑнтези" to "61", + "Ð¨ÐºÐ¾Ð»ÑŒÐ½Ð°Ñ Ð¶Ð¸Ð·Ð½ÑŒ" to "62", "Экшен" to "63", "Элементы юмора" to "64", + "Юмор" to "65", "ÐžÐ±Ñ€Ð°Ð·Ð¾Ð²Ð°Ñ‚ÐµÐ»ÑŒÐ½Ð°Ñ Ð»Ð¸Ñ‚ÐµÑ€Ð°Ñ‚ÑƒÑ€Ð°" to "131" + ) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/DesuMe.kt b/src/main/kotlin/tsuki/site/ru/DesuMe.kt new file mode 100644 index 00000000..a7ef5ff9 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/DesuMe.kt @@ -0,0 +1,282 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.network.UserAgents +import tsuki.core.PagedMangaParser + +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.LinkResolver +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.parseHtml +import tsuki.util.parseJson +import tsuki.util.selectFirstOrThrow +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase + +import okhttp3.Headers +import okhttp3.HttpUrl +import org.jsoup.nodes.Document +import org.jsoup.nodes.Element +import java.util.EnumSet + +@MangaSourceParser("DESUME", "Desu", "ru") +internal class DesuMe(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.DESUME, 20) { + + override val configKeyDomain = ConfigKey.Domain("desu.uno") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.ALPHABETICAL, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalogDocument().parseDesuTags(), + ) + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add("User-Agent", UserAgents.KOTATSU) + .build() + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + if (query != null) { + if (page != searchPaginator.firstPage) return emptyList() + val document = webClient.httpPost( + "https://$domain/manga/search/", + mapOf("q" to query), + ).parseHtml() + return document.parseDesuSearchResults().map(::parseSearchManga) + } + + val url = buildCatalogUrl(page, order, filter) + return webClient.httpGet(url) + .parseHtml() + .parseDesuCatalogItems() + .map(::parseCatalogManga) + } + + override suspend fun getDetails(manga: Manga): Manga { + val root = webClient.httpGet(manga.url.toAbsoluteUrl(domain)) + .parseHtml() + .selectFirstOrThrow("#animeView") + val title = root.selectFirstOrThrow("meta[itemprop=headline]").attr("content") + val alternativeTitles = buildSet { + root.selectFirst("meta[itemprop=alternativeHeadline]") + ?.attr("content") + ?.takeIf { it.isNotBlank() } + ?.let(::add) + root.selectFirst(".alternativeHeadline") + ?.text() + ?.split(',') + ?.map(String::trim) + ?.filterTo(this) { it.isNotEmpty() } + } + val mangaId = root.selectFirst("ul.chlist[data-manga_id]") + ?.attr("data-manga_id") + ?.toLongOrNull() + ?: error("Cannot find Desu manga id") + val chapters = root.select("ul.chlist > li").map { item -> + val link = item.selectFirstOrThrow("h4 > a") + val chapterUrl = link.attrAsRelativeUrl("href") + val match = DESU_CHAPTER_PATH.find(chapterUrl) + ?: error("Unsupported Desu chapter URL: $chapterUrl") + val chapterId = item.selectFirst("[data-chapters_id]") + ?.attr("data-chapters_id") + ?.toLongOrNull() + ?: error("Cannot find Desu chapter id") + MangaChapter( + id = generateUid(chapterId), + url = "/api/manga/$mangaId/chapters/$chapterId", + source = source, + number = match.groupValues[2].toFloat(), + volume = match.groupValues[1].toInt(), + title = link.selectFirst(".title")?.text()?.removePrefix("- ")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = 0L, + branch = null, + ) + } + + return manga.copy( + title = title, + altTitles = alternativeTitles, + publicUrl = root.selectFirstOrThrow("link[itemprop=url]").attr("href"), + largeCoverUrl = root.selectFirstOrThrow("img[itemprop=image]").attr("src"), + description = root.selectFirst("#description .russian")?.html(), + tags = root.select("a[itemprop=genre]").mapTo(linkedSetOf()) { link -> + MangaTag( + key = link.attr("href").substringAfter("genres="), + title = link.text().removePrefix("# ").toTitleCase(), + source = source, + ) + }, + state = root.selectFirst(".b-anime_status_tag")?.classNames()?.toDesuState(), + rating = root.selectFirst("meta[itemprop=ratingValue]") + ?.attr("content") + ?.toFloatOrNull() + ?.div(10f) + ?: RATING_UNKNOWN, + chapters = chapters.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val pages = webClient.httpGet(chapter.url.toAbsoluteUrl(domain)) + .parseJson() + .getJSONObject("chapter") + .getJSONArray("pages") + + return (0 until pages.length()).map { index -> + val imageUrl = pages.getJSONObject(index).getString("url") + MangaPage( + id = generateUid(imageUrl), + preview = null, + source = chapter.source, + url = imageUrl, + ) + } + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val document = webClient.httpGet(link).parseHtml() + val root = document.selectFirst("#animeView") ?: return null + val title = root.selectFirst("meta[itemprop=headline]")?.attr("content") ?: return null + val url = root.selectFirst("link[itemprop=url]")?.attr("href") ?: return null + return resolver.resolveManga(this, id = generateUid(url), url = url, title = title) + } + + private suspend fun fetchCatalogDocument(): Document = webClient + .httpGet("https://$domain/manga/") + .parseHtml() + + private fun buildCatalogUrl(page: Int, order: SortOrder, filter: MangaListFilter): String = buildString { + append("https://") + append(domain) + append("/manga/?page=") + append(page) + val sortKey = when (order) { + SortOrder.UPDATED -> "updated" + SortOrder.POPULARITY -> "popular" + SortOrder.NEWEST -> "id" + SortOrder.ALPHABETICAL -> "name" + else -> error("Unsupported Desu sort order: $order") + } + if (sortKey != "updated") { + append("&order_by=") + append(sortKey) + } + if (filter.tags.isNotEmpty()) { + append("&genres=") + append(filter.tags.joinToString(",") { it.key }) + } + } + + private fun parseCatalogManga(item: Element): Manga { + val link = item.selectFirstOrThrow("a.animeTitle") + val url = link.attrAsRelativeUrl("href") + val cover = item.selectFirstOrThrow("a.avatar .img") + .attr("style") + .let { style -> DESU_BACKGROUND_IMAGE.find(style)?.groupValues?.get(1) } + ?: error("Cannot find Desu catalog cover") + val russianTitle = item.selectFirstOrThrow(".dimmed.oTitle [itemprop=title]").text() + val originalTitle = link.text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = cover, + largeCoverUrl = null, + state = null, + rating = item.parseDesuCatalogRating(), + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } + + private fun parseSearchManga(item: Element): Manga { + val url = item.attrAsRelativeUrl("href") + val russianTitle = item.selectFirstOrThrow(".itemSubTitle").text() + val originalTitle = item.selectFirstOrThrow(".itemTitle").text() + return Manga( + id = generateUid(url), + url = url, + publicUrl = url.toAbsoluteUrl(domain), + source = source, + title = russianTitle, + altTitles = setOf(originalTitle), + coverUrl = item.selectFirstOrThrow("img").attr("src"), + largeCoverUrl = null, + state = null, + rating = RATING_UNKNOWN, + contentRating = null, + tags = emptySet(), + authors = emptySet(), + description = null, + ) + } +} + +internal fun Document.parseDesuCatalogItems(): List<Element> = + select("ol.memberList > li.memberListItem") + +internal fun Document.parseDesuSearchResults(): List<Element> = + select("ol.section > li > a[href^=manga/]") + +internal fun Document.parseDesuTags(): Set<MangaTag> = + select("#animeFilter input[data-genre-id][data-genre-slug][data-genre-name]").mapTo(linkedSetOf()) { input -> + MangaTag( + "${input.attr("data-genre-id")}-${input.attr("data-genre-slug")}", + input.attr("data-genre-name").toTitleCase(), + MangaParserSource.DESUME, + ) + } + +private fun Element.parseDesuCatalogRating(): Float { + val rating = select(".animeInfo dl").asSequence() + .flatMap { it.children().asSequence() } + .windowed(2) + .firstOrNull { it[0].tagName() == "dt" && it[0].text() == "Рейтинг:" && it[1].tagName() == "dd" } + ?.get(1) + ?.text() + ?.toFloatOrNull() + return rating?.div(10f) ?: RATING_UNKNOWN +} + +private fun Set<String>.toDesuState(): MangaState? = when { + "ongoing" in this -> MangaState.ONGOING + "released" in this -> MangaState.FINISHED + "anons" in this -> MangaState.UPCOMING + isEmpty() -> null + else -> error("Unsupported Desu status classes: $this") +} + +private val DESU_BACKGROUND_IMAGE = Regex("url\\(['\"]?([^'\")]+)") +private val DESU_CHAPTER_PATH = Regex("/vol(\\d+)/ch([0-9.]+)/") diff --git a/src/main/kotlin/tsuki/site/ru/MangaChan.kt b/src/main/kotlin/tsuki/site/ru/MangaChan.kt new file mode 100644 index 00000000..0f7704f5 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaChan.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.ChanParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGACHAN", "Манга-Ñ‚Ñн", "ru") +internal class MangaChan(context: MangaLoaderContext) : + ChanParser(context, MangaParserSource.MANGACHAN) { + override val configKeyDomain = ConfigKey.Domain("manga-chan.me") +} diff --git a/src/main/kotlin/tsuki/site/ru/MangaLib.kt b/src/main/kotlin/tsuki/site/ru/MangaLib.kt new file mode 100644 index 00000000..b7526e02 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaLib.kt @@ -0,0 +1,25 @@ +package tsuki.site.ru + +import tsuki.parsers.LibSocialParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.exception.AuthRequiredException +import tsuki.exception.NotFoundException + +import tsuki.model.MangaChapter +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource + +@MangaSourceParser("MANGALIB", "MangaLib", "ru") +internal class MangaLib(context: MangaLoaderContext, ) : + LibSocialParser(context, MangaParserSource.MANGALIB, 1, "mangalib.org") { + + override val configKeyDomain = ConfigKey.Domain("mangalib.org", "mangalib.me") + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> = try { + super.getPages(chapter) + } catch (e: NotFoundException) { + throw AuthRequiredException(source, e) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/MangaWtf.kt b/src/main/kotlin/tsuki/site/ru/MangaWtf.kt new file mode 100644 index 00000000..6c035da8 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/MangaWtf.kt @@ -0,0 +1,286 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.json.asTypedList +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapNotNullToSet +import tsuki.util.nl2br +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder + +import androidx.collection.ArrayMap +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("MANGAWTF", "MangaWtf", "ru") +internal class MangaWtf(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.MANGAWTF, pageSize = 20) { + + override val availableSortOrders: Set<SortOrder> = + EnumSet.of( + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.UPDATED, + SortOrder.NEWEST, + ) + + override val configKeyDomain = ConfigKey.Domain("inkstory.net") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isTagsExclusionSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.of( + MangaState.UPCOMING, + MangaState.PAUSED, + MangaState.ONGOING, + MangaState.FINISHED, + ), + availableContentRating = EnumSet.allOf(ContentRating::class.java), + ) + + init { + paginator.firstPage = 0 + searchPaginator.firstPage = 0 + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addQueryParameter("page", page.toString()) + .addQueryParameter("size", pageSize.toString()) + .addQueryParameter("type", "COMIC") + when { + filter.query.isNullOrEmpty() -> { + url.addQueryParameter( + "sort", + when (order) { + SortOrder.UPDATED -> "updatedAt,desc" + SortOrder.POPULARITY -> "viewsCount,desc" + SortOrder.RATING -> "likesCount,desc" + SortOrder.NEWEST -> "createdAt,desc" + else -> throw IllegalArgumentException("Unsupported $order") + }, + ) + if (filter.tags.isNotEmpty()) { + url.addQueryParameter("labelsInclude", filter.tags.joinToString(",") { it.key }) + } + if (filter.tagsExclude.isNotEmpty()) { + url.addQueryParameter("labelsExclude", filter.tagsExclude.joinToString(",") { it.key }) + } + if (filter.states.isNotEmpty()) { + url.addQueryParameter( + "status", + filter.states.joinToString(",") { + when (it) { + MangaState.ONGOING -> "ONGOING" + MangaState.FINISHED -> "DONE" + MangaState.ABANDONED -> "" + MangaState.PAUSED -> "FROZEN" + MangaState.UPCOMING -> "ANNOUNCE" + else -> throw IllegalArgumentException("$it not supported") + } + }, + ) + } + if (filter.contentRating.isNotEmpty()) { + url.addQueryParameter( + "contentStatus", + filter.contentRating.joinToString(",") { + when (it) { + ContentRating.SAFE -> "SAFE" + ContentRating.SUGGESTIVE -> "UNSAFE,EROTIC" + ContentRating.ADULT -> "PORNOGRAPHIC" + } + }, + ) + } + } + + else -> { + url.addQueryParameter("search", filter.query) + } + } + val ja = webClient.httpGet(url.build()).parseJsonArray() + return ja.mapJSON { jo -> jo.toManga() } + } + + override suspend fun getDetails(manga: Manga): Manga = + coroutineScope { + val chaptersDeferred = async { getChapters(manga.url) } + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("books") + .addPathSegment(manga.url) + val jo = webClient.httpGet(url.build()).parseJson() + val isNsfwSource = jo.getStringOrNull("contentStatus").isNsfw() + Manga( + id = generateUid(jo.getString("id")), + title = jo.getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(jo.getJSONObject("name").getStringOrNull("en")), + url = jo.getString("id"), + publicUrl = "https://$domain/manga/${jo.getString("slug")}", + rating = jo.getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = jo.getString("poster"), + tags = jo.getJSONArray("labels").mapJSONToSet { it.toMangaTag() }, + state = jo.getStringOrNull("status")?.toMangaState(), + authors = jo.getJSONArray("relations").asTypedList<JSONObject>().mapNotNullToSet { + if (it.getStringOrNull("type") == "AUTHOR") { + it.getJSONObject("publisher").getStringOrNull("name") + } else { + null + } + }, + source = source, + largeCoverUrl = null, + description = jo.getString("description").nl2br(), + chapters = chaptersDeferred.await(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addPathSegment(chapter.url) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("pages").mapJSON { jo -> + MangaPage( + id = generateUid(jo.getString("id")), + url = jo.getString("image"), + preview = null, + source = source, + ) + } + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val url = urlBuilder("api").addPathSegment("v2").addPathSegment("labels") + val json = webClient.httpGet(url.build()).parseJsonArray() + return json.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + } + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> = emptyList() + + override suspend fun getPageUrl(page: MangaPage): String = page.url + + private suspend fun getChapters(mangaId: String): List<MangaChapter> { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("chapters") + .addQueryParameter("bookId", mangaId) + val ja = webClient.httpGet(url.build()).parseJsonArray() + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSSSS'Z'", Locale.ROOT) + val branches = ArrayMap<String, String>() + return ja + .mapJSON { jo -> + val number = jo.getFloatOrDefault("number", 0f) + val volume = jo.getIntOrDefault("volume", 0) + val branchId = jo.getString("branchId") + MangaChapter( + id = generateUid(jo.getString("id")), + title = jo.getStringOrNull("name"), + number = number, + volume = volume, + url = jo.getString("id"), + scanlator = null, + uploadDate = dateFormat.parseSafe(jo.getString("createdAt")), + branch = branches.getOrPut(branchId) { getBranchName(branchId) }, + source = source, + ) + }.reversed() + } + + private suspend fun getBranchName(id: String): String? { + val url = + urlBuilder("api") + .addPathSegment("v2") + .addPathSegment("branches") + .addPathSegment(id) + val json = webClient.httpGet(url.build()).parseJson() + return json.getJSONArray("publishers").mapJSONToSet { it.getStringOrNull("name") }.firstOrNull() + } + + private fun String.toMangaState() = + when (this.uppercase(Locale.ROOT)) { + "DONE" -> MangaState.FINISHED + "ONGOING" -> MangaState.ONGOING + "FROZEN" -> MangaState.PAUSED + "ANNOUNCE" -> MangaState.UPCOMING + else -> null + } + + private fun String?.isNsfw() = + this.equals("EROTIC", ignoreCase = true) || + this.equals("PORNOGRAPHIC", ignoreCase = true) + + private fun JSONObject.toMangaTag() = + MangaTag( + title = getString("name").toTitleCase(sourceLocale), + key = getString("slug"), + source = source, + ) + + private fun JSONObject.toManga(): Manga { + val isNsfwSource = getStringOrNull("contentStatus").isNsfw() + return Manga( + id = generateUid(getString("id")), + title = getJSONObject("name").getString("ru"), + altTitles = setOfNotNull(getJSONObject("name").getStringOrNull("en")), + url = getString("id"), + publicUrl = "https://$domain/manga/${getString("slug")}", + rating = getFloatOrDefault("averageRating", -10f) / 10f, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + coverUrl = getString("poster"), + tags = setOf(), + state = getStringOrNull("status")?.toMangaState(), + authors = emptySet(), + source = source, + ) + } +} diff --git a/src/main/kotlin/tsuki/site/ru/Readmanga.kt b/src/main/kotlin/tsuki/site/ru/Readmanga.kt new file mode 100644 index 00000000..db3dca70 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Readmanga.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("READMANGA_RU", "ReadManga", "ru") +internal class Readmanga(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.READMANGA_RU, 1) { + override val configKeyDomain = ConfigKey.Domain("a.zazaza.me", "3.readmanga.ru") +} diff --git a/src/main/kotlin/tsuki/site/ru/Remanga.kt b/src/main/kotlin/tsuki/site/ru/Remanga.kt new file mode 100644 index 00000000..ed0b2d47 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Remanga.kt @@ -0,0 +1,410 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaParserAuthProvider +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser +import tsuki.exception.ContentUnavailableException +import tsuki.exception.ParseException + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.SortOrder +import tsuki.model.YEAR_UNKNOWN + +import tsuki.util.LinkResolver +import tsuki.util.copyCookies +import tsuki.util.generateUid +import tsuki.util.getCookies +import tsuki.util.getDomain +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getIntOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONTo +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapChapters +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode +import tsuki.util.find + +import okhttp3.Headers +import okhttp3.HttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okhttp3.internal.closeQuietly +import org.json.JSONArray +import org.json.JSONObject +import java.text.DateFormat +import java.text.SimpleDateFormat +import java.util.Date +import java.util.EnumSet +import java.util.Locale + +private const val PAGE_SIZE = 20 +private const val TOO_MANY_REQUESTS = 429 + +@MangaSourceParser("REMANGA", "Реманга", "ru") +internal class Remanga(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.REMANGA, PAGE_SIZE), MangaParserAuthProvider, Interceptor { + + private val baseHeaders: Headers + get() = Headers.Builder() + .add("User-Agent", config[userAgentKey]) + .build() + + override fun getRequestHeaders() = getApiHeaders() + + override val configKeyDomain = ConfigKey.Domain("remanga.org", "реманга.орг") + + override val authUrl: String + get() = "https://${domain}" + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.RATING, + SortOrder.NEWEST, + ) + + override suspend fun isAuthorized(): Boolean { + return context.cookieJar.getCookies(domain).any { + it.name == "user" + } + } + + private val regexLastUrlPath = Regex("/[^/]+/?$") + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + isYearRangeSupported = true, + isTagsExclusionSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchAvailableTags(), + availableStates = EnumSet.allOf(MangaState::class.java), + availableContentRating = EnumSet.of(ContentRating.SAFE, ContentRating.SUGGESTIVE), + ) + + override fun intercept(chain: Interceptor.Chain): Response { + val response = chain.proceed(chain.request()) + if (response.code == TOO_MANY_REQUESTS) { + response.closeQuietly() + Thread.sleep(1000) + return chain.proceed(chain.request().newBuilder().build()) + } + return response + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + copyCookies() + val urlBuilder = urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("search") + .addQueryParameter("page", page.toString()) + .addQueryParameter("count", PAGE_SIZE.toString()) + .addQueryParameter("ordering", getSortKey(order)) + if (!filter.query.isNullOrEmpty()) { + urlBuilder.addQueryParameter("query", filter.query) + } else { + urlBuilder.addPathSegment("catalog") + } + for (tag in filter.tags) { + urlBuilder.addQueryParameter("genres", tag.key) + } + for (tag in filter.tagsExclude) { + urlBuilder.addQueryParameter("exclude_genres", tag.key) + } + if (filter.yearFrom != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_gte", filter.yearFrom.toString()) + } + if (filter.yearTo != YEAR_UNKNOWN) { + urlBuilder.addQueryParameter("issue_year_lte", filter.yearFrom.toString()) + } + for (age in filter.contentRating) { + when (age) { + ContentRating.SAFE -> urlBuilder.addQueryParameter("age_limit", "0") + ContentRating.SUGGESTIVE -> { + urlBuilder.addQueryParameter("age_limit", "1") + urlBuilder.addQueryParameter("age_limit", "2") + } + + else -> Unit + } + } + for (state in filter.states) { + urlBuilder.addQueryParameter( + "status", + when (state) { + MangaState.ONGOING -> "2" + MangaState.FINISHED -> "1" + MangaState.ABANDONED -> "4" + MangaState.PAUSED -> "3" + MangaState.UPCOMING -> "5" + MangaState.RESTRICTED -> "6" + }, + ) + } + val content = webClient.httpGet(urlBuilder.build()).parseJson() + .getJSONArray("results") + return content.mapJSON { jo -> + parseManga(jo) + } + } + + override suspend fun getDetails(manga: Manga): Manga { + val slug = manga.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${manga.url}", manga.publicUrl) + return getDetails(slug, manga.publicUrl) + } + + override suspend fun resolveLink(resolver: LinkResolver, link: HttpUrl): Manga? { + val slug = link.pathSegments.getOrNull(1) ?: return super.resolveLink(resolver, link) + return getDetails(slug, link.toString()) + } + + override suspend fun getRelatedManga(seed: Manga): List<Manga> { + copyCookies() + val slug = seed.url.find(regexLastUrlPath)?.removePrefix("/") + ?: throw ParseException("Cannot obtain slug from ${seed.url}", seed.publicUrl) + val json = webClient.httpGet( + // https://api.remanga.org/api/v2/titles/the_beginning_after_the_end/relations/ + urlBuilder(subdomain = "api") + .addPathSegment("api") + .addPathSegment("v2") + .addPathSegment("titles") + .addPathSegment(slug) + .addPathSegment("relations") + .build(), + ).parseJson() + .optJSONArray("titles") ?: return emptyList() + return json.mapJSON { jo -> parseManga(jo) } + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val content = webClient.httpGet(chapter.url.toAbsoluteUrl(getDomain(this, "api"))) + .parseJson() + val pages = content.optJSONArray("pages") + if (pages == null) { + val pubDate = content.getStringOrNull("pub_date")?.let { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US).parseSafe(it) + } + if (pubDate != null && pubDate > System.currentTimeMillis()) { + val at = SimpleDateFormat.getDateInstance(DateFormat.LONG).format(Date(pubDate)) + throw ContentUnavailableException("Глава Ñтанет доÑтупной $at") + } else { + throw ContentUnavailableException("Глава недоÑтупна") + } + } + val result = ArrayList<MangaPage>(pages.length()) + for (i in 0 until pages.length()) { + when (val item = pages.get(i)) { + is JSONObject -> result += parsePage(item) + is JSONArray -> item.mapJSONTo(result) { parsePage(it) } + else -> throw ParseException("Unknown json item $item", chapter.url) + } + } + return result + } + + private suspend fun fetchAvailableTags(): Set<MangaTag> { + val content = webClient.httpGet("https://api.$domain/api/forms/titles/?get=genres") + .parseJson().getJSONObject("content").getJSONArray("genres") + return content.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(), + key = jo.getInt("id").toString(), + source = source, + ) + } + } + + override suspend fun getUsername(): String { + val jo = webClient.httpGet( + url = "https://api.${domain}/api/users/current/", + ).parseJson() + return jo.getJSONObject("content").getString("username") + } + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(userAgentKey) + } + + private suspend fun getDetails(slug: String, publicUrl: String): Manga { + copyCookies() + val jo = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/$slug/", + ).parseJson() + jo.optJSONObject("detail")?.getStringOrNull("message")?.let { msg -> + throw ParseException(msg, publicUrl) + } + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + val branches = jo.getJSONArray("branches").mapJSONToSet { + it.getLong("id") to it.optJSONObject("publishers")?.getStringOrNull("name") + } + val dateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.US) + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = jo.getStringOrNull("another_name")?.split(" / ") + ?.toSet().orEmpty() + setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = when (jo.optJSONObject("age_limit")?.getIntOrDefault("id", -1)) { + 0 -> ContentRating.SAFE + 1, 2 -> ContentRating.SUGGESTIVE + else -> null + }, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + description = jo.getStringOrNull("description"), + chapters = branches.flatMap { (branchId, branchName) -> + grabChapters(branchId).mapChapters(reversed = true) { _, cjo -> + if ( + cjo.getBooleanOrDefault("is_paid", false) && + !cjo.getBooleanOrDefault("is_bought", false) + ) { + return@mapChapters null + } + val id = cjo.getLong("id") + val name = cjo.getStringOrNull("name")?.toTitleCase(Locale.ROOT) + val publishers = cjo.optJSONArray("publishers") + MangaChapter( + id = generateUid(id), + url = "/api/v2/titles/chapters/$id/", + number = cjo.getFloatOrDefault("chapter", 0f), + volume = cjo.getIntOrDefault("tome", 0), + title = name, + uploadDate = dateFormat.parseSafe(cjo.getStringOrNull("upload_date")), + scanlator = publishers?.optJSONObject(0)?.getStringOrNull("name"), + source = source, + branch = branchName, + ) + } + }, + source = source, + ) + } + + private fun parseManga(jo: JSONObject): Manga { + val url = "/manga/${jo.getString("dir")}" + val cover = jo.getJSONObject("cover") + return Manga( + id = generateUid(url), + url = url, + publicUrl = "https://$domain$url/main", + title = jo.getString("main_name"), + altTitles = setOfNotNull(jo.getStringOrNull("secondary_name")), + rating = jo.getFloatOrDefault("avg_rating", -10f) / 10f, + coverUrl = cover.getStringOrNull("mid")?.toAbsoluteUrl("api.$domain"), + largeCoverUrl = cover.getStringOrNull("high")?.toAbsoluteUrl("api.$domain"), + authors = emptySet(), + contentRating = null, + state = when (jo.optJSONObject("status")?.getIntOrDefault("id", -1) ?: -1) { + 1 -> MangaState.FINISHED + 2 -> MangaState.ONGOING + 3 -> MangaState.PAUSED + 4 -> MangaState.ABANDONED + 5 -> MangaState.UPCOMING + 6 -> MangaState.RESTRICTED + else -> null + }, + tags = jo.optJSONArray("genres")?.mapJSONToSet { g -> + MangaTag( + title = g.getString("name").toTitleCase(sourceLocale), + key = g.getInt("id").toString(), + source = source, + ) + }.orEmpty(), + source = source, + ) + } + + private fun getApiHeaders(): Headers { + val userCookie = context.cookieJar.getCookies(domain).find { + it.name == "user" + } ?: return baseHeaders + val jo = JSONObject(userCookie.value.urlDecode()) + val accessToken = jo.getStringOrNull("access_token") ?: return baseHeaders + return baseHeaders.newBuilder().add("authorization", "bearer $accessToken").build() + } + + private fun copyCookies() { + val domain = domain + context.cookieJar.copyCookies(domain, "api.$domain") + } + + private fun getSortKey(order: SortOrder?) = when (order) { + SortOrder.UPDATED -> "-chapter_date" + SortOrder.POPULARITY -> "-rating" + SortOrder.RATING -> "-votes" + SortOrder.NEWEST -> "-id" + else -> "-chapter_date" + } + + private fun parsePage(jo: JSONObject) = MangaPage( + // id = generateUid(jo.getLong("id")), 19.01.2024 page id is gone + id = generateUid(jo.getString("link")), + url = jo.getString("link"), + preview = null, + source = source, + ) + + private suspend fun grabChapters(branchId: Long): List<JSONObject> { + val result = ArrayList<JSONObject>(100) + var page = 1 + while (true) { + val content = webClient.httpGet( + url = "https://api.$domain/api/v2/titles/chapters/?branch_id=$branchId&page=$page&count=500", + ).parseJson().getJSONArray("results") + val len = content.length() + if (len == 0) { + break + } + result.ensureCapacity(result.size + len) + for (i in 0 until len) { + result.add(content.getJSONObject(i)) + } + page++ + } + return result + } +} diff --git a/src/main/kotlin/tsuki/site/ru/SeiManga.kt b/src/main/kotlin/tsuki/site/ru/SeiManga.kt new file mode 100644 index 00000000..eb5dfd7a --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/SeiManga.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("SEIMANGA", "SeiManga", "ru") +internal class SeiManga(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.SEIMANGA, 21) { + override val configKeyDomain = ConfigKey.Domain("1.seimanga.me", "seimanga.me") +} diff --git a/src/main/kotlin/tsuki/site/ru/SelfManga.kt b/src/main/kotlin/tsuki/site/ru/SelfManga.kt new file mode 100644 index 00000000..92ed3dc3 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/SelfManga.kt @@ -0,0 +1,15 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +@MangaSourceParser("SELFMANGA", "SelfManga", "ru", ContentType.OTHER) +internal class SelfManga(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.SELFMANGA, 3) { + override val configKeyDomain = ConfigKey.Domain("1.selfmanga.live") +} diff --git a/src/main/kotlin/tsuki/site/ru/Usagi.kt b/src/main/kotlin/tsuki/site/ru/Usagi.kt new file mode 100644 index 00000000..e463f179 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/Usagi.kt @@ -0,0 +1,14 @@ +package tsuki.site.ru + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.MangaParserSource + +@MangaSourceParser("USAGI", "Usagi", "ru") +internal class Usagi(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.USAGI, 1) { + override val configKeyDomain = ConfigKey.Domain("web.usagi.one") +} diff --git a/src/main/kotlin/tsuki/site/ru/WaManga.kt b/src/main/kotlin/tsuki/site/ru/WaManga.kt new file mode 100644 index 00000000..12398472 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/WaManga.kt @@ -0,0 +1,179 @@ +package tsuki.site.ru + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.core.SinglePageMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder + +import tsuki.util.json.asTypedList +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.generateUid +import tsuki.util.parseJson +import tsuki.util.parseJsonArray +import tsuki.util.parseSafe + +import org.json.JSONObject +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale + +@MangaSourceParser("WAMANGA", "WaManga", "ru") +internal class WaManga(context: MangaLoaderContext) : + SinglePageMangaParser(context, MangaParserSource.WAMANGA) { + + override val configKeyDomain = ConfigKey.Domain("wamanga.ru") + + override val availableSortOrders: Set<SortOrder> = EnumSet.of(SortOrder.ALPHABETICAL) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isMultipleTagsSupported = true, + isSearchSupported = true, + ) + + override suspend fun getFilterOptions() = MangaListFilterOptions( + availableTags = fetchCatalog() + .flatMapTo(linkedSetOf()) { it.parseWaMangaTags() }, + ) + + override suspend fun getList(order: SortOrder, filter: MangaListFilter): List<Manga> { + val query = filter.query?.trim()?.takeIf { it.isNotEmpty() } + val requiredGenres = filter.tags.mapTo(mutableSetOf()) { it.key } + + return fetchCatalog() + .asSequence() + .filter { item -> + query == null || item.parseWaMangaSearchText().any { value -> + value.contains(query, ignoreCase = true) + } + } + .filter { item -> + requiredGenres.isEmpty() || item.parseWaMangaGenres().containsAll(requiredGenres) + } + .map(::parseManga) + .sortedWith(compareBy(String.CASE_INSENSITIVE_ORDER) { it.title }) + .toList() + } + + override suspend fun getDetails(manga: Manga): Manga { + val details = webClient.httpGet(apiUrl("manga/${manga.url}")).parseJson() + val chapters = webClient.httpGet(apiUrl("manga/${manga.url}/chapters")) + .parseJsonArray() + .mapJSON(::parseChapter) + .sortedByDescending { it.number } + + return parseManga(details).copy( + id = manga.id, + url = manga.url, + chapters = chapters.reversed(), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + return webClient.httpGet(apiUrl("chapters/${chapter.url}")) + .parseJson() + .getJSONArray("files") + .asTypedList<JSONObject>() + .sortedBy { it.getString("position").toDouble() } + .map { file -> + val imageUrl = file.getString("diskFile").toWaMangaAbsoluteUrl(domain) + MangaPage( + id = generateUid(imageUrl), + url = imageUrl, + preview = null, + source = source, + ) + } + } + + private suspend fun fetchCatalog(): List<JSONObject> = webClient + .httpGet(apiUrl("manga")) + .parseJsonArray() + .asTypedList() + + private fun parseManga(item: JSONObject): Manga { + val id = item.getString("id") + val slug = item.getString("slug") + return Manga( + id = generateUid(id), + url = id, + title = item.getString("title"), + altTitles = item.parseWaMangaAlternativeTitles(), + publicUrl = "https://$domain/manga/$slug", + rating = RATING_UNKNOWN, + coverUrl = item.getString("coverUrl").toWaMangaAbsoluteUrl(domain), + largeCoverUrl = item.getString("imageUrl").toWaMangaAbsoluteUrl(domain), + tags = item.parseWaMangaTags(), + state = item.getString("statusTitle").toWaMangaState(), + authors = item.getJSONArray("authors").asTypedList<String>().toSet(), + source = source, + contentRating = if (item.getBoolean("isAdult")) ContentRating.ADULT else ContentRating.SAFE, + description = item.getStringOrNull("description"), + ) + } + + private fun parseChapter(item: JSONObject): MangaChapter { + val id = item.getString("id") + return MangaChapter( + id = generateUid(id), + url = id, + source = source, + number = item.getDouble("position").toFloat(), + volume = 0, + title = item.getStringOrNull("title")?.takeIf { it.isNotBlank() }, + scanlator = null, + uploadDate = WA_MANGA_DATE_FORMAT.get().parseSafe(item.getString("createdAt")), + branch = null, + ) + } + + private fun apiUrl(path: String): String = "https://$domain/api/v1/$path" +} + +internal fun JSONObject.parseWaMangaAlternativeTitles(): Set<String> = buildSet { + getStringOrNull("titleEnglish")?.takeIf { it.isNotBlank() }?.let(::add) + getJSONArray("alternateTitles").asTypedList<String>().filterTo(this) { it.isNotBlank() } +} + +internal fun JSONObject.parseWaMangaGenres(): Set<String> = + getJSONArray("genres").asTypedList<String>().toSet() + +internal fun JSONObject.parseWaMangaSearchText(): Set<String> = buildSet { + add(getString("title")) + addAll(parseWaMangaAlternativeTitles()) +} + +private fun JSONObject.parseWaMangaTags(): Set<MangaTag> = parseWaMangaGenres().mapTo(linkedSetOf()) { genre -> + MangaTag(genre, genre, MangaParserSource.WAMANGA) +} + +private fun String.toWaMangaState(): MangaState = when (this) { + "ongoing" -> MangaState.ONGOING + "completed" -> MangaState.FINISHED + "abandoned" -> MangaState.ABANDONED + else -> error("Unsupported WaManga status: $this") +} + +private fun String.toWaMangaAbsoluteUrl(domain: String): String = when { + startsWith("https://") || startsWith("http://") -> this + startsWith('/') -> "https://$domain$this" + else -> error("Unsupported WaManga asset URL: $this") +} + +private val WA_MANGA_DATE_FORMAT = ThreadLocal.withInitial { + SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSX", Locale.US) +} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt new file mode 100644 index 00000000..c0caf7be --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/nsfw/AllHentai.kt @@ -0,0 +1,47 @@ +package tsuki.site.ru.nsfw + +import tsuki.parsers.GroupleParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.exception.AuthRequiredException + +import tsuki.model.ContentType +import tsuki.model.MangaParserSource + +import tsuki.util.getCookies +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.urlEncoded + +@MangaSourceParser("ALLHENTAI", "AllHentai", "ru", ContentType.HENTAI) +internal class AllHentai(context: MangaLoaderContext) : + GroupleParser(context, MangaParserSource.ALLHENTAI, 1) { + + override val configKeyDomain = ConfigKey.Domain( + "20.allhen.online", + "z.ahen.me", + "24.allhen.online", + "2023.allhen.online", + ) + + override suspend fun isAuthorized(): Boolean = + super.isAuthorized() || context.cookieJar.getCookies(domain).any { it.name == "remember_me" } + + override val authUrl: String + get() { + val targetUri = "https://$domain/".urlEncoded() + return "https://qawa.org/internal/auth/login?targetUri=$targetUri&siteId=1" + } + + override suspend fun getUsername(): String { + val root = webClient.httpGet("https://qawa.org/").parseHtml().body() + val element = root.selectFirst("img.user-avatar") ?: throw AuthRequiredException(source) + val res = element.parent()?.text() + return if (res.isNullOrEmpty()) { + root.parseFailed("Cannot find username") + } else { + res + } + } +} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt b/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt new file mode 100644 index 00000000..d45a8754 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/nsfw/HenChan.kt @@ -0,0 +1,104 @@ +package tsuki.site.ru.nsfw + +import tsuki.parsers.ChanParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaParserSource +import tsuki.model.MangaTag +import tsuki.model.SortOrder + +import tsuki.util.generateUid +import tsuki.util.mapToSet +import tsuki.util.parseFailed +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.toAbsoluteUrl +import tsuki.util.toTitleCase +import tsuki.util.urlBuilder +import tsuki.util.urlDecode + +import java.util.EnumSet +import okhttp3.HttpUrl + +@MangaSourceParser("HENCHAN", "Хентай-Ñ‚Ñн", "ru", ContentType.HENTAI) +internal class HenChan(context: MangaLoaderContext) : + ChanParser(context, MangaParserSource.HENCHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "x5.h-chan.me", + "xxxx.henchan.pro", + "xxl.hentaichan.live", + "xxx.henchan.pro", + "y.hentaichan.live", + "xx.hentaichan.live", + "x.henchan.pro", + "hentaichan.live", + ) + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.NEWEST, + SortOrder.POPULARITY, + SortOrder.RATING, + ) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + val readLink = manga.url.replace("manga", "online") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + tags = root.selectFirst("div.sidetags")?.select("li.sidetag")?.mapToSet { + val a = it.children().last() ?: doc.parseFailed("Invalid tag") + MangaTag( + title = a.text().toTitleCase(), + key = a.attr("href").substringAfterLast('/').urlDecode(), + source = source, + ) + } ?: manga.tags, + chapters = listOf( + MangaChapter( + id = generateUid(readLink), + url = readLink, + source = source, + number = 0f, + volume = 0, + uploadDate = 0L, + title = null, + scanlator = null, + branch = null, + ), + ), + ) + } + + override fun buildUrl(offset: Int, order: SortOrder, filter: MangaListFilter): HttpUrl = when { + filter.query.isNullOrEmpty() && filter.tags.isEmpty() && filter.tagsExclude.isEmpty() -> { + val builder = urlBuilder().addQueryParameter("offset", offset.toString()) + when (order) { + SortOrder.POPULARITY -> { + builder.addPathSegment("mostviews") + builder.addQueryParameter("sort", "manga") + } + SortOrder.RATING -> { + builder.addPathSegment("mostfavorites") + builder.addQueryParameter("sort", "manga") + } + else -> { + builder.addPathSegment("manga") + builder.addPathSegment("newest") + } + } + builder.build() + } + else -> { + super.buildUrl(offset, order, filter) + } + } +} diff --git a/src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt b/src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt new file mode 100644 index 00000000..ccae5233 --- /dev/null +++ b/src/main/kotlin/tsuki/site/ru/nsfw/YaoiChan.kt @@ -0,0 +1,59 @@ +package tsuki.site.ru.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.config.ConfigKey +import tsuki.parsers.ChanParser + +import tsuki.model.ContentType +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource +import tsuki.model.SortOrder + +import tsuki.util.attrAsRelativeUrl +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseHtml +import tsuki.util.requireElementById +import tsuki.util.toAbsoluteUrl + +@MangaSourceParser("YAOICHAN", "Яой-Ñ‚Ñн", "ru", ContentType.HENTAI) +internal class YaoiChan(context: MangaLoaderContext) : ChanParser(context, MangaParserSource.YAOICHAN) { + + override val configKeyDomain = ConfigKey.Domain( + "v9.yaoi-chan.me", + "v10.yaoi-chan.me", + "v3.yaoi-chan.me", + "v2.yaoi-chan.me", + "v1.yaoi-chan.me", + "yaoi-chan.me", + ) + + override val availableSortOrders: Set<SortOrder> = setOf(SortOrder.NEWEST) + + override suspend fun getDetails(manga: Manga): Manga { + val doc = webClient.httpGet(manga.url.toAbsoluteUrl(domain)).parseHtml() + val root = doc.body().requireElementById("dle-content") + return manga.copy( + description = root.getElementById("description")?.html()?.substringBeforeLast("<div"), + largeCoverUrl = root.getElementById("cover")?.absUrl("src"), + chapters = root.select("table.table_cha").flatMap { table -> + table.select("div.manga") + }.mapNotNull { it.selectFirst("a") }.mapChapters(reversed = true) { i, a -> + val href = a.attrAsRelativeUrl("href") + MangaChapter( + id = generateUid(href), + title = a.text(), + number = i + 1f, + volume = 0, + url = href, + uploadDate = 0L, + source = source, + scanlator = null, + branch = null, + ) + }, + ) + } +} diff --git a/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt b/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt new file mode 100644 index 00000000..d500d822 --- /dev/null +++ b/src/main/kotlin/tsuki/site/vi/CuuTruyenParser.kt @@ -0,0 +1,507 @@ +package tsuki.site.vi + +import tsuki.ErrorMessages +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.bitmap.Bitmap +import tsuki.bitmap.Rect +import tsuki.config.ConfigKey +import tsuki.core.PagedMangaParser + +import tsuki.model.ContentRating +import tsuki.model.Manga +import tsuki.model.MangaChapter +import tsuki.model.MangaListFilter +import tsuki.model.MangaListFilterCapabilities +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaPage +import tsuki.model.MangaParserSource +import tsuki.model.MangaState +import tsuki.model.MangaTag +import tsuki.model.RATING_UNKNOWN +import tsuki.model.SortOrder +import tsuki.network.CloudFlareHelper +import tsuki.network.CommonHeaders +import tsuki.network.UserAgents + +import tsuki.util.copy +import tsuki.util.generateUid +import tsuki.util.json.getBooleanOrDefault +import tsuki.util.json.getFloatOrDefault +import tsuki.util.json.getStringOrNull +import tsuki.util.json.mapJSON +import tsuki.util.json.mapJSONToSet +import tsuki.util.mapChapters +import tsuki.util.nullIfEmpty +import tsuki.util.parseJson +import tsuki.util.parseSafe +import tsuki.util.toTitleCase +import tsuki.util.urlEncoded + +import androidx.collection.arraySetOf +import kotlinx.coroutines.async +import kotlinx.coroutines.coroutineScope +import okhttp3.Headers +import okhttp3.HttpUrl.Companion.toHttpUrl +import okhttp3.Interceptor +import okhttp3.Response +import okio.IOException +import java.text.SimpleDateFormat +import java.util.EnumSet +import java.util.Locale +import java.util.TimeZone + +@MangaSourceParser("CUUTRUYEN", "Cứu Truyện", "vi") +internal class CuuTruyenParser(context: MangaLoaderContext) : + PagedMangaParser(context, MangaParserSource.CUUTRUYEN, 24) { + + private val apiSuffix = "/api/v2" + override val userAgentKey = ConfigKey.UserAgent(UserAgents.KOTATSU) + + override val configKeyDomain = ConfigKey.Domain("cuutruyen.net", "hetcuutruyen.net") + + private val preferredServerKey = ConfigKey.PreferredImageServer( + presetValues = mapOf( + DESKTOP_COVER to "Ảnh bìa chất lượng cao (Desktop)", + MOBILE_COVER to "Ảnh bìa chất lượng thấp (Mobile)", + ), + defaultValue = DESKTOP_COVER, + ) + + override fun onCreateConfig(keys: MutableCollection<ConfigKey<*>>) { + super.onCreateConfig(keys) + keys.add(preferredServerKey) + keys.remove(userAgentKey) + } + + override fun getRequestHeaders(): Headers = Headers.Builder() + .add(CommonHeaders.ORIGIN, "https://$domain") + .add(CommonHeaders.REFERER, "https://$domain") + .build() + + override val availableSortOrders: Set<SortOrder> = EnumSet.of( + SortOrder.UPDATED, + SortOrder.POPULARITY, + SortOrder.NEWEST, + SortOrder.POPULARITY_WEEK, + SortOrder.POPULARITY_MONTH, + ) + + override val filterCapabilities: MangaListFilterCapabilities + get() = MangaListFilterCapabilities( + isSearchSupported = true, + isSearchWithFiltersSupported = true, + isMultipleTagsSupported = true, + ) + + override suspend fun getFilterOptions(): MangaListFilterOptions { + return MangaListFilterOptions( + availableTags = availableTags(), + availableStates = EnumSet.of(MangaState.ONGOING, MangaState.FINISHED), + ) + } + + override suspend fun getListPage(page: Int, order: SortOrder, filter: MangaListFilter): List<Manga> { + // Move to urlBuilder + val url = buildString { + if (!filter.query.isNullOrEmpty() || filter.tags.isNotEmpty() || filter.states.isNotEmpty()) { + append("/mangas/search?q=") + if (!filter.query.isNullOrEmpty()) { + append(filter.query.urlEncoded()) + } + + // Bug from API: Select both ONGOING + FINISHED will return empty list for all case + val state = listOf(MangaState.ONGOING, MangaState.FINISHED) + if (filter.states.containsAll(state)) { + // oneOrThrowIfMany + throw IllegalArgumentException( + ErrorMessages.FILTER_MULTIPLE_STATES_NOT_SUPPORTED + ) + } + + append("&tags=") + val tags = buildList { + addAll(filter.tags.map { "\"${space2plus(it.title.lowercase())}\"" }) + addAll( + filter.states.map { + when (it) { + MangaState.ONGOING -> "\"Ä‘ang+tiến+hành\"" + MangaState.FINISHED -> "\"đã+hoàn+thành\"" + else -> "" + } + } + ) + }.filter { it.isNotEmpty() } + append(tags.joinToString(separator = "+AND+")) + + append("&page=") + append(page.toString()) + } else { + append("/mangas") + when (order) { + SortOrder.UPDATED -> append("/recently_updated") + SortOrder.POPULARITY -> append("/top?duration=all") + SortOrder.POPULARITY_WEEK -> append("/top?duration=week") + SortOrder.POPULARITY_MONTH -> append("/top?duration=month") + SortOrder.NEWEST -> { + // clear old buildString + clear() + append("/home_a") + } + else -> append("/recently_updated") + } + + when (order) { + SortOrder.POPULARITY, SortOrder.POPULARITY_WEEK, SortOrder.POPULARITY_MONTH -> { + append("&page=") + append(page.toString()) + } + else -> { + append("?page=") + append(page.toString()) + } + } + } + + append("&per_page=") + append(pageSize) + } + + val fullUrl = "https://$domain$apiSuffix$url" + val response = webClient.httpGet(fullUrl) + + val protection = CloudFlareHelper.checkResponseForProtection(response.copy()) + if (protection != CloudFlareHelper.PROTECTION_NOT_DETECTED) { + response.close() + context.requestBrowserAction(this, fullUrl) + } + + val json = response.parseJson() + val data = json.optJSONArray("data") + ?: json.getJSONObject("data").getJSONArray("new_chapter_mangas") + ?: json.getJSONObject("data").getJSONArray("mangas") + ?: return emptyList() + + return data.mapJSON { jo -> + val author = jo.getStringOrNull("author_name") + val server = config[preferredServerKey] ?: DESKTOP_COVER + val coverUrl = if (server == MOBILE_COVER) jo.getString(MOBILE_COVER) else + jo.getString(DESKTOP_COVER) + Manga( + id = generateUid(jo.getLong("id")), + url = "$apiSuffix/mangas/${jo.getLong("id")}", + publicUrl = "https://$domain/mangas/${jo.getLong("id")}", + title = jo.getString("name"), + altTitles = emptySet(), + coverUrl = coverUrl, + largeCoverUrl = null, + authors = setOfNotNull(author), + tags = emptySet(), + state = null, + description = null, + contentRating = if (isNsfwSource) ContentRating.ADULT else null, + source = source, + rating = RATING_UNKNOWN, + ) + } + } + + override suspend fun getDetails(manga: Manga): Manga = coroutineScope { + val server = config[preferredServerKey] ?: DESKTOP_COVER + val url = "https://" + domain + manga.url + val chapters = async { + webClient.httpGet("$url/chapters").parseJson().getJSONArray("data") + } + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + val chapterDateFormat = SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSZ", Locale.ROOT).apply { + timeZone = TimeZone.getTimeZone("GMT+7") + } + val tags = json.optJSONArray("tags")?.mapJSONToSet { jo -> + MangaTag( + title = jo.getString("name").toTitleCase(sourceLocale), + key = jo.getString("slug"), + source = source, + ) + }.orEmpty() + + val state = when { + tags.any { it.key == "da-hoan-thanh" } -> MangaState.FINISHED + else -> MangaState.ONGOING // Mostly ONGOING but not marked by site owner + } + + val newTags = tags.filter { it.key != "da-hoan-thanh" && it.key != "dang-tien-hanh" }.toSet() + val author = json.optJSONObject("author")?.getStringOrNull("name")?.substringBefore(',')?.nullIfEmpty() + val title = json.getStringOrNull("name") ?: manga.title + val team = json.optJSONObject("team")?.getStringOrNull("name") + + manga.copy( + title = title, + altTitles = json.optJSONArray("titles")?.mapJSONToSet { it.getString("name") }?.minus(title).orEmpty(), + contentRating = if (json.getBooleanOrDefault("is_nsfw", false)) { + ContentRating.ADULT + } else { + ContentRating.SAFE + }, + authors = setOfNotNull(author), + description = json.getStringOrNull("full_description"), + tags = newTags, + state = state, + chapters = chapters.await().mapChapters(reversed = true) { _, jo -> + val chapterId = jo.getLong("id") + val number = jo.getFloatOrDefault("number", 0f) + val createdAt = jo.getStringOrNull("created_at") + ?.replace(Regex("([+-]\\d{2}):(\\d{2})$"), "$1$2") + MangaChapter( + id = generateUid(chapterId), + title = jo.getStringOrNull("name"), + number = number, + volume = 0, + url = "$apiSuffix/chapters/$chapterId", + scanlator = team, + uploadDate = chapterDateFormat.parseSafe(createdAt), + branch = null, + source = source, + ) + }, + largeCoverUrl = if (server == MOBILE_COVER) json.getString(MOBILE_PANORAMA) else + json.getString(DESKTOP_PANORAMA), + ) + } + + override suspend fun getPages(chapter: MangaChapter): List<MangaPage> { + val url = "https://$domain${chapter.url}" + val json = webClient.httpGet(url).parseJson().getJSONObject("data") + + return json.getJSONArray("pages").mapJSON { jo -> + val imageUrl = jo.getString("image_url").toHttpUrl().newBuilder() + val id = jo.getLong("id") + val drm = jo.getStringOrNull("drm_data") + if (!drm.isNullOrEmpty()) { + imageUrl.fragment(DRM_DATA_KEY + drm) + } + MangaPage( + id = generateUid(id), + url = imageUrl.build().toString(), + preview = null, + source = source, + ) + } + } + + override fun intercept(chain: Interceptor.Chain): Response { + var request = chain.request() + STORAGE_HOST_FALLBACK[request.url.host]?.let { host -> + request = request.newBuilder() + .url(request.url.newBuilder().host(host).build()) + .build() + } + + if (request.url.encodedPath.contains("/api/")) { + request = request.newBuilder() + .header(CLIENT_HEADER, CLIENT_VALUE) + .build() + return chain.proceed(request) + } + + val response = chain.proceed(request) + val fragment = response.request.url.fragment + + if (fragment == null || !fragment.contains(DRM_DATA_KEY)) { + return response + } + + return context.redrawImageResponse(response) { bitmap -> + val drmData = fragment.substringAfter(DRM_DATA_KEY) + unscrambleImage(bitmap, drmData) + } + } + + private fun unscrambleImage(bitmap: Bitmap, drmData: String): Bitmap { + val data = context.decodeBase64(drmData) + .decodeXorCipher(DECRYPTION_KEY) + .toString(Charsets.UTF_8) + + if (!data.startsWith("#v4|")) { + throw IOException("Invalid DRM data (does not start with expected magic bytes): $data") + } + + val result = context.createBitmap(bitmap.width, bitmap.height) + var sy = 0 + for (t in data.split('|').drop(1)) { + val (dy, height) = t.split('-').map(String::toInt) + val srcRect = Rect(0, sy, bitmap.width, sy + height) + val dstRect = Rect(0, dy, bitmap.width, dy + height) + + result.drawBitmap(bitmap, srcRect, dstRect) + sy += height + } + + return result + } + + private fun ByteArray.decodeXorCipher(key: String): ByteArray { + val k = key.toByteArray(Charsets.UTF_8) + + return this.mapIndexed { i, b -> + (b.toInt() xor k[i % k.size].toInt()).toByte() + }.toByteArray() + } + + private fun space2plus(input: String): String = input.replace(' ', '+') + + // keep old tags list, key in MangaTag is unnecessary + private fun availableTags() = arraySetOf( // big thanks to beer-psi + MangaTag("School life", "school-life", source), + MangaTag("Nsfw", "nsfw", source), + MangaTag("Monster girls", "monster-girls", source), + MangaTag("Magic", "magic", source), + MangaTag("tình yêu không được đáp lại", "tinh-yeu-khong-duoc-dap-lai", source), + MangaTag("tình yêu thuần khiết", "tinh-yeu-thuan-khiet", source), + MangaTag("Khá»a thân", "khoa-than", source), + MangaTag("Gyaru", "gyaru", source), + MangaTag("4-Koma", "4-koma", source), + MangaTag("Manga", "manga", source), + MangaTag("Thể thao", "the-thao", source), + MangaTag("Hài hÆ°á»›c", "hai-huoc", source), + MangaTag("Shounen", "shounen", source), + MangaTag("Há»c Ä‘Æ°á»ng", "hoc-duong", source), + MangaTag("Chất lượng cao", "chat-luong-cao", source), + MangaTag("Comedy", "comedy", source), + MangaTag("Action", "action", source), + MangaTag("Horror", "horror", source), + MangaTag("Sci-fi", "sci-fi", source), + MangaTag("Aliens", "aliens", source), + MangaTag("Martial Arts", "martial-arts", source), + MangaTag("Military", "military", source), + MangaTag("Monsters", "monsters", source), + MangaTag("Supernatural", "supernatural", source), + MangaTag("Web Comic", "web-comic", source), + MangaTag("Phiêu lÆ°u", "phieu-luu", source), + MangaTag("Hậu tận thế", "hau-tan-the", source), + MangaTag("Hành Ä‘á»™ng", "hanh-dong", source), + MangaTag("Sinh tồn", "sinh-ton", source), + MangaTag("Du hành thá»i gian", "du-hanh-thoi-gian", source), + MangaTag("Khoa há»c", "khoa-hoc", source), + MangaTag("Tạm ngÆ°ng", "tam-ngung", source), + MangaTag("NSFW", "nsfw", source), + MangaTag("Bạo lá»±c", "bao-luc", source), + MangaTag("Khoả thân", "khoa-than", source), + MangaTag("Bí ẩn", "bi-an", source), + MangaTag("Trinh thám", "trinh-tham", source), + MangaTag("Kinh dị", "kinh-di", source), + MangaTag("Máu me", "mau-me", source), + MangaTag("Tình dục", "tinh-duc", source), + MangaTag("Có màu", "co-mau", source), + MangaTag("Manhwa", "manhwa", source), + MangaTag("Webtoon", "webtoon", source), + MangaTag("Siêu nhiên", "sieu-nhien", source), + MangaTag("Fantasy", "fantasy", source), + MangaTag("Võ thuật", "vo-thuat", source), + MangaTag("Drama", "drama", source), + MangaTag("Hệ thống", "he-thong", source), + MangaTag("Lãng mạn", "lang-man", source), + MangaTag("Äá»i thÆ°á»ng", "doi-thuong", source), + MangaTag("Công sở", "cong-so", source), + MangaTag("Sát thủ", "sat-thu", source), + MangaTag("Phép thuật", "phep-thuat", source), + MangaTag("Tá»™i phạm", "toi-pham", source), + MangaTag("Seinen", "seinen", source), + MangaTag("Isekai", "isekai", source), + MangaTag("Chuyển sinh", "chuyen-sinh", source), + MangaTag("Harem", "harem", source), + MangaTag("Mecha", "mecha", source), + MangaTag("Trung cổ", "trung-co", source), + MangaTag("LGBT", "lgbt", source), + MangaTag("Yaoi", "yaoi", source), + MangaTag("Game", "game", source), + MangaTag("Bi kịch", "bi-kich", source), + MangaTag("Äá»™ng vật", "dong-vat", source), + MangaTag("Tâm lý", "tam-ly", source), + MangaTag("Manhua", "manhua", source), + MangaTag("Nam biến nữ", "nam-bien-nu", source), + MangaTag("Romcom", "romcom", source), + MangaTag("Award Winning", "award-winning", source), + MangaTag("Oneshot", "oneshot", source), + MangaTag("Khoa há»c viá»…n tưởng", "khoa-hoc-vien-tuong", source), + MangaTag("Dark Fantasy", "dark-fantasy", source), + MangaTag("Zombie", "zombie", source), + MangaTag("Nam x Nam", "nam-x-nam", source), + MangaTag("Giật gân", "giat-gan", source), + MangaTag("Cảnh sát", "canh-sat", source), + MangaTag("NTR", "ntr", source), + MangaTag("Cooking", "cooking", source), + MangaTag("Ẩm thá»±c", "am-thuc", source), + MangaTag("Ecchi", "ecchi", source), + MangaTag("Quái vật", "quai-vat", source), + MangaTag("Vampires", "vampires", source), + MangaTag("Nam giả nữ", "nam-gia-nu", source), + MangaTag("Yakuza", "yakuza", source), + MangaTag("Romance", "romance", source), + MangaTag("Sport", "sport", source), + MangaTag("Shoujo", "shoujo", source), + MangaTag("Ninja", "ninja", source), + MangaTag("Lịch sá»­", "lich-su", source), + MangaTag("Doujinshi", "doujinshi", source), + MangaTag("Databook", "databook", source), + MangaTag("Adventure", "adventure", source), + MangaTag("Y há»c", "y-hoc", source), + MangaTag("Miá»…n bản quyá»n", "mien-ban-quyen", source), + MangaTag("Josei", "josei", source), + MangaTag("Psychological", "psychological", source), + MangaTag("Anime", "anime", source), + MangaTag("Yuri", "yuri", source), + MangaTag("Yonkoma", "yonkoma", source), + MangaTag("Quân Ä‘á»™i", "quan-doi", source), + MangaTag("Nữ giả nam", "nu-gia-nam", source), + MangaTag("Chính trị", "chinh-tri", source), + MangaTag("Tuyển tập", "tuyen-tap", source), + MangaTag("Tu tiên", "tu-tien", source), + MangaTag("Vô CP", "vo-cp", source), + MangaTag("Xuyên không", "xuyen-khong", source), + MangaTag("Việt Nam", "viet-nam", source), + MangaTag("Việt nam", "viet-nam", source), + MangaTag("Toán há»c", "toan-hoc", source), + MangaTag("Thiếu niên", "thieu-nien", source), + MangaTag("Tình yêu", "tinh-yeu", source), + MangaTag("Chính kịch", "chinh-kich", source), + MangaTag("Ngá»t ngào", "ngot-ngao", source), + MangaTag("Wholesome", "wholesome", source), + MangaTag("Smut", "smut", source), + MangaTag("Gore", "gore", source), + MangaTag("School Life", "school-life", source), + MangaTag("Slice of Life", "slice-of-life", source), + MangaTag("Tragedy", "tragedy", source), + MangaTag("Mystery", "mystery", source), + MangaTag("Atlus", "atlus", source), + MangaTag("Sega", "sega", source), + MangaTag("RPG", "rpg", source), + MangaTag("Chuyển thể", "chuyen-the", source), + MangaTag("Historical", "historical", source), + MangaTag("Medical", "medical", source), + MangaTag("Ghosts", "ghosts", source), + MangaTag("Thriller", "thriller", source), + MangaTag("Animals", "animals", source), + MangaTag("Survival", "survival", source), + MangaTag("Samurai", "samurai", source), + MangaTag("Virtual Reality", "virtual-reality", source), + MangaTag("Video Games", "video-games", source), + MangaTag("Monster Girls", "monster-girls", source), + MangaTag("Adaption", "adaption", source), + MangaTag("Idol", "idol", source), + ) + + private companion object { + const val DRM_DATA_KEY = "drm_data=" + const val DECRYPTION_KEY = "3141592653589793" + const val MOBILE_COVER = "cover_mobile_url" + const val MOBILE_PANORAMA = "panorama_mobile_url" + const val DESKTOP_COVER = "cover_url" + const val DESKTOP_PANORAMA = "panorama_url" + const val CLIENT_HEADER = "Cuutruyen-Client" + const val CLIENT_VALUE = "OfficialWebApp-20250805" + val STORAGE_HOST_FALLBACK = mapOf( + "storage-ct.lrclib.net" to "storage-bravo.cuutruyen.net", + "storage-ct-riften.site" to "storage-charlie.cuutruyen.net", + ) + } +} diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt b/src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt similarity index 50% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt rename to src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt index 4b8eeed8..aa931403 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/TruyenTuoiTho.kt +++ b/src/main/kotlin/tsuki/site/vi/TruyenTuoiTho.kt @@ -1,9 +1,10 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.vi +package tsuki.site.vi -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.MangaParserSource @MangaSourceParser("TRUYENTUOITHO", "Truyện Tuổi ThÆ¡", "vi") internal class TruyenTuoiTho(context: MangaLoaderContext) : diff --git a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt b/src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt similarity index 66% rename from src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt rename to src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt index 04cc78c5..d0cf32c7 100644 --- a/src/main/kotlin/org/koitharu/kotatsu/parsers/site/kotatsu/vi/hentai/TruyenTranhDamMyy.kt +++ b/src/main/kotlin/tsuki/site/vi/nsfw/TruyenTranhDamMyy.kt @@ -1,17 +1,20 @@ -package org.koitharu.kotatsu.parsers.site.kotatsu.vi.hentai +package tsuki.site.vi.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.MadaraParser + +import tsuki.model.ContentType +import tsuki.model.MangaChapter +import tsuki.model.MangaParserSource + +import tsuki.util.attrAsRelativeUrlOrNull +import tsuki.util.generateUid +import tsuki.util.mapChapters +import tsuki.util.parseFailed +import tsuki.util.parseHtml import org.jsoup.nodes.Document -import org.koitharu.kotatsu.parsers.MangaLoaderContext -import org.koitharu.kotatsu.parsers.MangaSourceParser -import org.koitharu.kotatsu.parsers.model.ContentType -import org.koitharu.kotatsu.parsers.model.MangaChapter -import org.koitharu.kotatsu.parsers.model.MangaParserSource -import org.koitharu.kotatsu.parsers.parsers.MadaraParser -import org.koitharu.kotatsu.parsers.util.attrAsRelativeUrlOrNull -import org.koitharu.kotatsu.parsers.util.generateUid -import org.koitharu.kotatsu.parsers.util.mapChapters -import org.koitharu.kotatsu.parsers.util.parseFailed -import org.koitharu.kotatsu.parsers.util.parseHtml @MangaSourceParser("TRUYENTRANHDAMMYY", "Truyện Tranh Äam Mỹ", "vi", ContentType.HENTAI) internal class TruyenTranhDamMyy(context: MangaLoaderContext) : @@ -40,4 +43,4 @@ internal class TruyenTranhDamMyy(context: MangaLoaderContext) : ) } } -} \ No newline at end of file +} diff --git a/src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt b/src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt new file mode 100644 index 00000000..ef6e1072 --- /dev/null +++ b/src/main/kotlin/tsuki/site/zh/nsfw/Xiutaku.kt @@ -0,0 +1,17 @@ +package tsuki.site.zh.nsfw + +import tsuki.MangaLoaderContext +import tsuki.MangaSourceParser +import tsuki.parsers.GalleryParser + +import tsuki.model.ContentType +import tsuki.model.MangaListFilterOptions +import tsuki.model.MangaParserSource + +@MangaSourceParser("XIUTAKU", "Xiutaku", "zh", type = ContentType.OTHER) +internal class Xiutaku(context: MangaLoaderContext) : + GalleryParser(context, MangaParserSource.XIUTAKU, "xiutaku.com") { + + override suspend fun getFilterOptions(): MangaListFilterOptions = + MangaListFilterOptions(availableTags = fetchTags()) +} diff --git a/src/main/kotlin/tsuki/util/ChapterNumber.kt b/src/main/kotlin/tsuki/util/ChapterNumber.kt new file mode 100644 index 00000000..ff731e10 --- /dev/null +++ b/src/main/kotlin/tsuki/util/ChapterNumber.kt @@ -0,0 +1,13 @@ +package tsuki.util + +private val CHAPTER_REGEX = Regex("""(?i)\b(?:chapter|ch\.?|chapitre|capítulo|cap\.?|episode|ep\.?)\s*(\d+(?:\.\d+)?)""") +private val FALLBACK_REGEX = Regex("""(\d+(?:\.\d+)?)""") + +/** + * Extract from string + * Returns [Float] or 0f if no number found. + */ +fun String.extractChapterNumber(): Float { + CHAPTER_REGEX.find(this)?.groupValues?.get(1)?.toFloatOrNull()?.let { return it } + return FALLBACK_REGEX.find(this)?.value?.toFloatOrNull() ?: 0f +} diff --git a/src/main/resources/assets/i18n/messages_en.properties b/src/main/resources/assets/i18n/messages_en.properties deleted file mode 100644 index fd7fb06e..00000000 --- a/src/main/resources/assets/i18n/messages_en.properties +++ /dev/null @@ -1,27 +0,0 @@ -author_filter_title=Author -artist_filter_title=Artist -year_filter_title=Year of Released -status_filter_title=Status -status_filter_completed=Completed -status_filter_ongoing=Ongoing -status_filter_canceled=Canceled -status_filter_on_hold=On Hold -order_by_filter_title=Order By -order_by_filter_relevance=Relevance -order_by_filter_latest=Latest -order_by_filter_az=A-Z -order_by_filter_rating=Rating -order_by_filter_trending=Trending -order_by_filter_views=Most Views -order_by_filter_new=New -genre_condition_filter_title=Genre condition -genre_condition_filter_or=OR -genre_condition_filter_and=AND -adult_content_filter_title=Adult Content -adult_content_filter_all=All -adult_content_filter_none=None -adult_content_filter_only=Only -genre_filter_header=Genres filter may not work for all sources -genre_filter_title=Genres -genre_missing_warning=Press 'Reset' to attempt to show the genres -alt_names_heading=Alternative Names: diff --git a/src/main/resources/assets/i18n/messages_es.properties b/src/main/resources/assets/i18n/messages_es.properties deleted file mode 100644 index ba7132da..00000000 --- a/src/main/resources/assets/i18n/messages_es.properties +++ /dev/null @@ -1,26 +0,0 @@ -author_filter_title=Autor -artist_filter_title=Artista -year_filter_title=Año de lanzamiento -status_filter_title=Estado -status_filter_completed=Completado -status_filter_ongoing=En curso -status_filter_canceled=Cancelado -status_filter_on_hold=En pausa -order_by_filter_title=Ordenar por -order_by_filter_relevance=Relevancia -order_by_filter_latest=Reciente -order_by_filter_rating=Valoración -order_by_filter_trending=Tendencia -order_by_filter_views=Vistas -order_by_filter_new=Nuevos -genre_condition_filter_title=Condición de género -genre_condition_filter_or=O -genre_condition_filter_and=Y -adult_content_filter_title=Contenido adulto -adult_content_filter_all=Todos -adult_content_filter_none=Ninguno -adult_content_filter_only=Solo -genre_filter_header=Es posible que el filtro de géneros no funcione correctamente -genre_filter_title=Géneros -genre_missing_warning=Presione 'Restablecer' para intentar mostrar los géneros -alt_names_heading=Nombres Alternativos: \ No newline at end of file diff --git a/src/main/resources/assets/i18n/messages_pt_br.properties b/src/main/resources/assets/i18n/messages_pt_br.properties deleted file mode 100644 index 0c0a88b6..00000000 --- a/src/main/resources/assets/i18n/messages_pt_br.properties +++ /dev/null @@ -1,26 +0,0 @@ -author_filter_title=Autor -artist_filter_title=Artista -year_filter_title=Ano de lançamento -status_filter_title=Estado -status_filter_completed=Completo -status_filter_ongoing=Em andamento -status_filter_canceled=Cancelado -status_filter_on_hold=Pausado -order_by_filter_title=Ordenar por -order_by_filter_relevance=Relevância -order_by_filter_latest=Recentes -order_by_filter_rating=Avaliação -order_by_filter_trending=Tendência -order_by_filter_views=Visualizações -order_by_filter_new=Novos -genre_condition_filter_title=Operador dos gêneros -genre_condition_filter_or=OU -genre_condition_filter_and=E -adult_content_filter_title=Conteúdo adulto -adult_content_filter_all=Indiferente -adult_content_filter_none=Nenhum -adult_content_filter_only=Somente -genre_filter_header=O filtro de gêneros pode não funcionar -genre_filter_title=Gêneros -genre_missing_warning=Aperte 'Redefinir' para tentar mostrar os gêneros -alt_names_heading=Nomes alternativos: \ No newline at end of file