From b3e038a9a9bda9ee12b49175464298333f89d6bb Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 12 Oct 2026 03:18:26 +0000 Subject: [PATCH 01/49] Fix site browsing parsers and add in-app browser cookie login Browsing: - Add shared listing parser (sites/listing.rs) that merges every anchor linking to the same video, so titles come from title/aria-label/alt instead of quality badges ("1080p"), noscript markup or nothing. Fixes XVIDEOS/xHamster titles and XNXX/YouPorn cards being dropped. - RedGifs: browse via the v2 API with a cached temporary token (the site is a JS app, so HTML scraping never returned results); gif lookups are now authenticated and return direct MP4 URLs. - ThisVid: scrape search/tag listings directly (yt-dlp has no search extractor for it). - Chaturbate: use the username as the card title and strip tag markup from room subjects. - Surface yt-dlp browse errors and Reddit login requirement instead of silently returning empty pages. Cookies: - New in-app login browser (cookie_login.rs): desktop opens a separate webview window and reads cookies via cookies_for_url; Android uses a new CookieBrowserPlugin Kotlin overlay (full-screen WebView + CookieManager). Captured cookies (incl. HttpOnly) are saved as Netscape through the existing vault path. - "Log in with browser" button in Settings -> Cookies and on cookie-required site pages; cookie site picker lists all sites. - Vault: treat #HttpOnly_ lines as cookies, not comments. Co-Authored-By: Claude Opus 5.5 --- docs/cookie-import.md | 16 +- docs/sites.md | 36 +- scripts/patch-android-ytdlp.ps1 | 13 + scripts/patch-android-ytdlp.sh | 10 + scripts/validate-android-build.ps1 | 4 + .../android-overlays/CookieBrowserPlugin.kt | 193 +++++++ src-tauri/android-overlays/proguard-ytdlp.pro | 2 + src-tauri/src/commands.rs | 21 + src-tauri/src/cookie_login.rs | 507 ++++++++++++++++++ src-tauri/src/lib.rs | 5 + src-tauri/src/sites/adapters/chaturbate.rs | 8 +- src-tauri/src/sites/adapters/generic_ytdlp.rs | 73 ++- src-tauri/src/sites/adapters/pornhub.rs | 204 +------ src-tauri/src/sites/adapters/reddit.rs | 7 + src-tauri/src/sites/adapters/redgifs.rs | 88 +-- src-tauri/src/sites/extractors/redgifs.rs | 274 ++++++++-- src-tauri/src/sites/listing.rs | 399 ++++++++++++++ src-tauri/src/sites/mod.rs | 1 + src-tauri/src/state.rs | 12 + src-tauri/src/vault/mod.rs | 28 +- src/components/CookieLoginButton.tsx | 88 +++ src/lib/api/client.ts | 17 + src/lib/types.ts | 8 + src/routes/browse/$site/index.tsx | 6 +- src/routes/settings/index.tsx | 21 +- 25 files changed, 1709 insertions(+), 332 deletions(-) create mode 100644 src-tauri/android-overlays/CookieBrowserPlugin.kt create mode 100644 src-tauri/src/cookie_login.rs create mode 100644 src-tauri/src/sites/listing.rs create mode 100644 src/components/CookieLoginButton.tsx diff --git a/docs/cookie-import.md b/docs/cookie-import.md index ed17d0f..ce19e6a 100644 --- a/docs/cookie-import.md +++ b/docs/cookie-import.md @@ -2,7 +2,19 @@ ArcHive stores cookies encrypted per site for browse and yt-dlp downloads. -## Recommended: Cookie-Editor extension +## Recommended: Log in with browser (desktop + Android) + +1. In **Settings → Cookies**, select the site (or open a cookie-required site in Browse). +2. Press **Log in with browser**. ArcHive opens the site in its own browser: + - **Desktop:** a separate window. Log in, then press **Save cookies** in ArcHive (or just close the window). + - **Android:** a full-screen browser with **Cancel** / **Save cookies** buttons at the top. The system back button navigates back inside the page. +3. The captured cookies, including HttpOnly session cookies that page scripts cannot read, are converted to Netscape format and stored in the vault (on the LAN host when the app is in Remote LAN mode). + +Android does not let one app read another app's cookies (for example Chrome's), so the login has to happen inside ArcHive. Google sign-in may refuse embedded browsers; for YouTube, import cookies with Cookie-Editor instead. + +Implementation: `src-tauri/src/cookie_login.rs` (desktop window + Android bridge) and `src-tauri/android-overlays/CookieBrowserPlugin.kt` (installed by `scripts/patch-android-ytdlp.ps1` / `.sh`). + +## Cookie-Editor extension 1. Install [Cookie-Editor](https://chromewebstore.google.com/detail/cookie-editor/hlkenndednhfkekhgcdicdfddnkalmdm) (Chrome, Edge, Brave) or the Firefox equivalent. 2. Log in on the target site (PornHub, xHamster, etc.). @@ -31,6 +43,8 @@ Export or build a Netscape cookie file and paste into the Netscape textarea → | xvideos | xvideos.com | | thisvid | thisvid.com | +Lines exported as `#HttpOnly_` are treated as cookies, not comments. + ## Security - Cookies are encrypted at rest (AES-256-GCM). diff --git a/docs/sites.md b/docs/sites.md index c39dd2d..f2d7ec2 100644 --- a/docs/sites.md +++ b/docs/sites.md @@ -2,21 +2,21 @@ ## Tier A — yt-dlp (browse + download) -| Site ID | Display Name | Browse kinds | Cookies | -| --------- | ------------ | -------------------------------------------- | --------------------------- | -| youtube | YouTube | channel, search, video | Optional | -| tiktok | TikTok | channel, search, video | No | -| twitter | Twitter / X | channel, video | Recommended | -| instagram | Instagram | channel, video | Yes | -| thisvid | ThisVid | tag, search, video | Yes | -| thothub | ThotHub | tag, model, search, video | Optional (`thethothub.com`) | -| pornhub | PornHub | category, tag, search, channel, model, video | Yes | -| youporn | YouPorn | category, model, search, video | Optional | -| xnxx | XNXX | search, tag, channel, video | Optional | -| xhamster | xHamster | tag, search, channel, video | Yes | -| xvideos | XVIDEOS | tag, search, channel, video | Yes | -| reddit | Reddit | channel, search, video | OAuth for subreddits | -| eporner | Eporner | search, tag, category, video | No | +| Site ID | Display Name | Browse kinds | Cookies | +| --------- | ------------ | -------------------------------------------- | -------------------------------------------------------------- | +| youtube | YouTube | channel, search, video | Optional | +| tiktok | TikTok | channel, search, video | No | +| twitter | Twitter / X | channel, video | Recommended | +| instagram | Instagram | channel, video | Yes | +| thisvid | ThisVid | tag, search, video | Yes | +| thothub | ThotHub | tag, model, search, video | Optional (`thethothub.com`) | +| pornhub | PornHub | category, tag, search, channel, model, video | Yes | +| youporn | YouPorn | category, model, search, video | Optional | +| xnxx | XNXX | search, tag, channel, video | Optional | +| xhamster | xHamster | tag, search, channel, video | Yes | +| xvideos | XVIDEOS | tag, search, channel, video | Yes | +| reddit | Reddit | channel, search, video | Login cookies (Reddit hides listings from logged-out visitors) | +| eporner | Eporner | search, tag, category, video | No | ## Tier B — Live Cam Sites (scraping + webview bridge) @@ -27,9 +27,9 @@ ## Tier C — gallery-dl -| Site ID | Display Name | Notes | -| ------- | ------------ | ---------------------------- | -| redgifs | RedGifs | Uses gallery-dl for download | +| Site ID | Display Name | Notes | +| ------- | ------------ | -------------------------------------------------------------------- | +| redgifs | RedGifs | Browse via the RedGifs v2 API (temporary token); direct MP4 download | ## Cookie Requirements diff --git a/scripts/patch-android-ytdlp.ps1 b/scripts/patch-android-ytdlp.ps1 index cc0caf2..4da42c6 100644 --- a/scripts/patch-android-ytdlp.ps1 +++ b/scripts/patch-android-ytdlp.ps1 @@ -47,6 +47,11 @@ if (Test-Path $resumeWorkerSrc) { Copy-Item -Path $resumeWorkerSrc -Destination (Join-Path $pluginDestDir "PendingResumeWorker.kt") -Force Write-Host "Installed PendingResumeWorker.kt" } +$cookieBrowserSrc = Join-Path $Overlays "CookieBrowserPlugin.kt" +if (Test-Path $cookieBrowserSrc) { + Copy-Item -Path $cookieBrowserSrc -Destination (Join-Path $pluginDestDir "CookieBrowserPlugin.kt") -Force + Write-Host "Installed CookieBrowserPlugin.kt" +} # Ensure youtubedl-android + WorkManager dependencies $gradle = Get-Content $gradlePath -Raw @@ -125,6 +130,14 @@ if (Test-Path $proguardPath) { Write-Host "Created $proguardPath with YtDlp keep rules" } +# Projects patched before the cookie login browser existed: add its keeps. +$proguardNow = Get-Content $proguardPath -Raw +if ($proguardNow -notmatch 'CookieBrowserPlugin') { + $cookieKeeps = "-keep class com.archhive.app.CookieBrowserPlugin { *; }`r`n-keep class com.archhive.app.CookieBrowserPlugin`$OpenArgs { *; }`r`n" + Set-Content -Path $proguardPath -Value ($proguardNow.TrimEnd() + "`r`n" + $cookieKeeps) -NoNewline + Write-Host "Added CookieBrowserPlugin ProGuard keeps." +} + Write-Host "Android YtDlp overlay patch complete (youtubedl-android $YoutubeDlAndroidVersion)." # Remove stale Linux BtbN ffmpeg/ffprobe assets left from older tauri.android.conf.json diff --git a/scripts/patch-android-ytdlp.sh b/scripts/patch-android-ytdlp.sh index 9d36801..9002466 100644 --- a/scripts/patch-android-ytdlp.sh +++ b/scripts/patch-android-ytdlp.sh @@ -45,6 +45,10 @@ if [[ -f "$OVERLAYS/PendingResumeWorker.kt" ]]; then cp "$OVERLAYS/PendingResumeWorker.kt" "$PLUGIN_DEST_DIR/PendingResumeWorker.kt" echo "Installed PendingResumeWorker.kt" fi +if [[ -f "$OVERLAYS/CookieBrowserPlugin.kt" ]]; then + cp "$OVERLAYS/CookieBrowserPlugin.kt" "$PLUGIN_DEST_DIR/CookieBrowserPlugin.kt" + echo "Installed CookieBrowserPlugin.kt" +fi if ! grep -q 'youtubedl-android:library' "$GRADLE"; then python3 - "$GRADLE" "$YOUTUBE_DL_ANDROID_VERSION" "$WORK_RUNTIME_KTX_VERSION" <<'PY' @@ -109,6 +113,12 @@ else echo "Wrote ProGuard keep rules to $PROGUARD" fi +# Projects patched before the cookie login browser existed: add its keeps. +if ! grep -q 'CookieBrowserPlugin' "$PROGUARD"; then + printf '\n-keep class com.archhive.app.CookieBrowserPlugin { *; }\n-keep class com.archhive.app.CookieBrowserPlugin$OpenArgs { *; }\n' >> "$PROGUARD" + echo "Added CookieBrowserPlugin ProGuard keeps." +fi + echo "Android YtDlp overlay patch complete (youtubedl-android $YOUTUBE_DL_ANDROID_VERSION)." STALE_BINARIES="$GEN_ANDROID/app/src/main/assets/binaries" diff --git a/scripts/validate-android-build.ps1 b/scripts/validate-android-build.ps1 index ea68a5f..ad4bd28 100644 --- a/scripts/validate-android-build.ps1 +++ b/scripts/validate-android-build.ps1 @@ -36,6 +36,7 @@ $sharePlugin = Join-Path $gen "app\src\main\java\com\archhive\app\ShareIntentPlu $mainActivity = Join-Path $gen "app\src\main\java\com\archhive\app\MainActivity.kt" $fgService = Join-Path $gen "app\src\main\java\com\archhive\app\DownloadForegroundService.kt" $resumeWorker = Join-Path $gen "app\src\main\java\com\archhive\app\PendingResumeWorker.kt" +$cookieBrowser = Join-Path $gen "app\src\main\java\com\archhive\app\CookieBrowserPlugin.kt" $gradle = Join-Path $gen "app\build.gradle.kts" $proguard = Join-Path $gen "app\proguard-rules.pro" $manifest = Join-Path $gen "app\src\main\AndroidManifest.xml" @@ -56,6 +57,9 @@ if (-not (Test-Path $fgService)) { if (-not (Test-Path $resumeWorker)) { Add-Fail "Missing PendingResumeWorker.kt" } +if (-not (Test-Path $cookieBrowser)) { + Add-Fail "Missing CookieBrowserPlugin.kt (run scripts/apply-android-patches.ps1)" +} if (-not (Test-Path $gradle)) { Add-Fail "Missing app/build.gradle.kts" diff --git a/src-tauri/android-overlays/CookieBrowserPlugin.kt b/src-tauri/android-overlays/CookieBrowserPlugin.kt new file mode 100644 index 0000000..6588bc9 --- /dev/null +++ b/src-tauri/android-overlays/CookieBrowserPlugin.kt @@ -0,0 +1,193 @@ +package com.archhive.app + +import android.annotation.SuppressLint +import android.app.Activity +import android.app.Dialog +import android.graphics.Bitmap +import android.graphics.Color +import android.util.Log +import android.view.Gravity +import android.view.ViewGroup +import android.webkit.CookieManager +import android.webkit.WebChromeClient +import android.webkit.WebResourceRequest +import android.webkit.WebView +import android.webkit.WebViewClient +import android.widget.Button +import android.widget.LinearLayout +import android.widget.ProgressBar +import android.widget.TextView +import app.tauri.annotation.Command +import app.tauri.annotation.TauriPlugin +import app.tauri.plugin.Invoke +import app.tauri.plugin.JSArray +import app.tauri.plugin.JSObject +import app.tauri.plugin.Plugin + +/** + * In-app login browser used to capture site cookies on Android. + * + * Android apps cannot read Chrome's cookie jar, so ArcHive opens its own + * full-screen WebView on the site's login page. When the user taps "Save + * cookies", every cookie the WebView holds for the requested URLs, including + * HttpOnly session cookies, is read from the process-wide [CookieManager] + * and returned to Rust, which stores them in the encrypted cookie vault. + * + * Args: { url: string, cookieUrls: string[], title: string } + * Returns: { cookies: [{ url: string, header: string }] } + * Rejects with "cancelled" when the user backs out. + */ +@TauriPlugin +class CookieBrowserPlugin(private val activity: Activity) : Plugin(activity) { + + companion object { + private const val TAG = "CookieBrowserPlugin" + } + + class OpenArgs @JvmOverloads constructor( + var url: String = "", + var cookieUrls: Array = emptyArray(), + var title: String = "Log in", + ) + + @Volatile private var dialog: Dialog? = null + + @SuppressLint("SetJavaScriptEnabled") + @Command + fun open(invoke: Invoke) { + val args = invoke.parseArgs(OpenArgs::class.java) + if (!args.url.startsWith("https://") && !args.url.startsWith("http://")) { + invoke.reject("Invalid login URL") + return + } + val cookieUrls = if (args.cookieUrls.isEmpty()) arrayOf(args.url) else args.cookieUrls + + activity.runOnUiThread { + try { + dialog?.dismiss() + val cookieManager = CookieManager.getInstance() + cookieManager.setAcceptCookie(true) + + val dlg = Dialog(activity, android.R.style.Theme_Black_NoTitleBar_Fullscreen) + var settled = false + + val root = LinearLayout(activity).apply { + orientation = LinearLayout.VERTICAL + setBackgroundColor(Color.parseColor("#111114")) + } + val bar = LinearLayout(activity).apply { + orientation = LinearLayout.HORIZONTAL + gravity = Gravity.CENTER_VERTICAL + setPadding(16, 12, 16, 12) + } + val cancel = Button(activity).apply { text = "Cancel" } + val title = TextView(activity).apply { + text = args.title + setTextColor(Color.WHITE) + textSize = 15f + maxLines = 2 + setPadding(16, 0, 16, 0) + } + val save = Button(activity).apply { text = "Save cookies" } + bar.addView(cancel) + bar.addView(title, LinearLayout.LayoutParams(0, ViewGroup.LayoutParams.WRAP_CONTENT, 1f)) + bar.addView(save) + + val progress = ProgressBar(activity, null, android.R.attr.progressBarStyleHorizontal).apply { + isIndeterminate = false + max = 100 + } + + val webView = WebView(activity).apply { + settings.javaScriptEnabled = true + settings.domStorageEnabled = true + settings.databaseEnabled = true + settings.loadWithOverviewMode = true + settings.useWideViewPort = true + settings.setSupportMultipleWindows(false) + // Drop the "; wv" token: some login pages (Google, X) + // refuse embedded WebViews that advertise themselves. + settings.userAgentString = settings.userAgentString.replace("; wv", "") + webViewClient = object : WebViewClient() { + override fun shouldOverrideUrlLoading(view: WebView, request: WebResourceRequest): Boolean { + val scheme = request.url.scheme ?: return true + // Stay inside the WebView for web pages; ignore app deep links. + return scheme != "http" && scheme != "https" + } + + override fun onPageStarted(view: WebView, url: String?, favicon: Bitmap?) { + title.text = "${args.title}\n${url ?: ""}" + } + } + webChromeClient = object : WebChromeClient() { + override fun onProgressChanged(view: WebView, newProgress: Int) { + progress.progress = newProgress + progress.visibility = if (newProgress >= 100) android.view.View.GONE else android.view.View.VISIBLE + } + } + } + cookieManager.setAcceptThirdPartyCookies(webView, true) + + root.addView(bar, LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.WRAP_CONTENT)) + root.addView(progress, LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 6)) + root.addView(webView, LinearLayout.LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, 0, 1f)) + dlg.setContentView(root) + + fun finish(saveCookies: Boolean) { + if (settled) return + settled = true + try { + if (saveCookies) { + cookieManager.flush() + val list = JSArray() + for (u in cookieUrls) { + val header = cookieManager.getCookie(u) + if (!header.isNullOrBlank()) { + val entry = JSObject() + entry.put("url", u) + entry.put("header", header) + list.put(entry) + } + } + val ret = JSObject() + ret.put("cookies", list) + invoke.resolve(ret) + } else { + invoke.reject("cancelled") + } + } catch (e: Exception) { + Log.w(TAG, "cookie capture failed", e) + invoke.reject("Cookie capture failed: ${e.message}") + } finally { + webView.stopLoading() + webView.destroy() + dlg.dismiss() + dialog = null + } + } + + save.setOnClickListener { finish(true) } + cancel.setOnClickListener { finish(false) } + dlg.setOnCancelListener { finish(false) } + dlg.setOnKeyListener { _, keyCode, event -> + if (keyCode == android.view.KeyEvent.KEYCODE_BACK && + event.action == android.view.KeyEvent.ACTION_UP && + webView.canGoBack() + ) { + webView.goBack() + true + } else { + false + } + } + + dialog = dlg + dlg.show() + webView.loadUrl(args.url) + } catch (e: Exception) { + Log.e(TAG, "failed to open login browser", e) + invoke.reject("Could not open login browser: ${e.message}") + } + } + } +} diff --git a/src-tauri/android-overlays/proguard-ytdlp.pro b/src-tauri/android-overlays/proguard-ytdlp.pro index 824d6b3..5963a5c 100644 --- a/src-tauri/android-overlays/proguard-ytdlp.pro +++ b/src-tauri/android-overlays/proguard-ytdlp.pro @@ -8,6 +8,8 @@ *; } -keep class com.archhive.app.DownloadForegroundService { *; } +-keep class com.archhive.app.CookieBrowserPlugin { *; } +-keep class com.archhive.app.CookieBrowserPlugin$OpenArgs { *; } -keep class com.archhive.app.PendingResumeWorker { *; } -keepclassmembers class com.archhive.app.PendingResumeWorker { (android.content.Context, androidx.work.WorkerParameters); diff --git a/src-tauri/src/commands.rs b/src-tauri/src/commands.rs index 7a8a828..815ce87 100644 --- a/src-tauri/src/commands.rs +++ b/src-tauri/src/commands.rs @@ -395,6 +395,27 @@ pub fn save_site_cookies( map_err(state.save_site_cookies(&site_id, &cookies)) } +/// Open the in-app login browser for a site and return the cookies captured +/// once the user finishes (the frontend stores them via `save_site_cookies`, +/// which also covers Remote LAN hosts). Async: reading webview cookies from a +/// synchronous command deadlocks on Windows. +#[tauri::command] +pub async fn cookie_login( + app: AppHandle, + state: State<'_, Arc>, + site_id: String, +) -> CmdResult { + let (display_name, base_url) = map_err(state.site_login_info(&site_id))?; + let target = crate::cookie_login::login_target(&site_id, &base_url); + map_err(crate::cookie_login::capture(&app, &site_id, &display_name, &target).await) +} + +/// Desktop: save (`save = true`) or cancel the open login window. +#[tauri::command] +pub fn cookie_login_finish(app: AppHandle, save: bool) -> CmdResult<()> { + map_err(crate::cookie_login::finish(&app, save)) +} + #[tauri::command] pub fn delete_site_cookies(state: State<'_, Arc>, site_id: String) -> CmdResult<()> { map_err(state.delete_site_cookies(&site_id)) diff --git a/src-tauri/src/cookie_login.rs b/src-tauri/src/cookie_login.rs new file mode 100644 index 0000000..dcae345 --- /dev/null +++ b/src-tauri/src/cookie_login.rs @@ -0,0 +1,507 @@ +//! In-app browser login that captures a site's cookies. +//! +//! Instead of exporting cookies from an external browser, the user logs in +//! inside a browser ArcHive controls and the resulting cookies (including +//! HttpOnly session cookies that page JavaScript cannot see) are converted to +//! Netscape format for the cookie vault / yt-dlp `--cookies`. +//! +//! * Desktop: a secondary `WebviewWindow` (no IPC capability) is opened on the +//! site; closing it, or pressing "Save cookies" in the app, reads the +//! cookies via `Webview::cookies_for_url`. +//! * Android: the `CookieBrowserPlugin` Kotlin overlay shows a full-screen +//! WebView and returns `CookieManager` cookie headers when the user taps +//! "Save cookies". Other apps' cookie jars (e.g. Chrome's) are not readable +//! on Android, which is why the login happens in-app. + +use crate::error::{AppError, AppResult}; +use serde::Serialize; +use tauri::plugin::{Builder, TauriPlugin}; +use tauri::{AppHandle, Runtime}; + +/// Lifetime given to session / undated cookies so yt-dlp and the vault keep them. +const SESSION_COOKIE_TTL_SECS: i64 = 30 * 24 * 3600; +/// Fallback lifetime for header-only cookies (Android `CookieManager` hides expiry). +#[cfg_attr(not(target_os = "android"), allow(dead_code))] +const HEADER_COOKIE_TTL_SECS: i64 = 365 * 24 * 3600; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct CapturedCookie { + pub domain: String, + pub include_subdomains: bool, + pub path: String, + pub secure: bool, + pub expires: i64, + pub name: String, + pub value: String, +} + +#[derive(Debug, Clone, Serialize)] +pub struct CookieLoginResult { + pub site_id: String, + /// Netscape cookie file contents, ready for `save_site_cookies`. + pub netscape: String, + pub count: usize, +} + +/// Where the login browser starts and which URLs' cookies to collect. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct LoginTarget { + pub start_url: String, + pub cookie_urls: Vec, +} + +pub fn login_target(site_id: &str, base_url: &str) -> LoginTarget { + let base = format!("{}/", base_url.trim_end_matches('/')); + let (start, extra): (&str, &[&str]) = match site_id { + "reddit" => ( + "https://www.reddit.com/login/", + &[ + "https://www.reddit.com/", + "https://old.reddit.com/", + "https://reddit.com/", + ], + ), + "twitter" => ( + "https://x.com/i/flow/login", + &["https://x.com/", "https://twitter.com/"], + ), + "instagram" => ("https://www.instagram.com/accounts/login/", &[]), + "tiktok" => ("https://www.tiktok.com/login", &[]), + "youtube" => ( + "https://accounts.google.com/ServiceLogin?service=youtube&continue=https%3A%2F%2Fwww.youtube.com%2F", + &["https://www.youtube.com/"], + ), + "redgifs" => ("https://www.redgifs.com/login", &[]), + _ => ("", &[]), + }; + let mut cookie_urls = vec![base.clone()]; + for u in extra { + if !cookie_urls.iter().any(|c| c == u) { + cookie_urls.push((*u).to_string()); + } + } + LoginTarget { + start_url: if start.is_empty() { + base + } else { + start.to_string() + }, + cookie_urls, + } +} + +/// Netscape cookie file text. HttpOnly cookies are written as ordinary lines +/// (the `#HttpOnly_` prefix would be dropped by simple parsers). +pub fn to_netscape(cookies: &[CapturedCookie]) -> String { + let mut out = + String::from("# Netscape HTTP Cookie File\n# Captured by ArcHive login browser\n\n"); + for c in cookies { + out.push_str(&format!( + "{}\t{}\t{}\t{}\t{}\t{}\t{}\n", + c.domain, + if c.include_subdomains { + "TRUE" + } else { + "FALSE" + }, + c.path, + if c.secure { "TRUE" } else { "FALSE" }, + c.expires, + c.name, + c.value + )); + } + out +} + +/// Drop duplicates (same domain/path/name) keeping the first occurrence, and +/// cookies whose name/value would corrupt the tab-separated format. +pub fn dedupe(cookies: Vec) -> Vec { + let mut seen = std::collections::HashSet::new(); + cookies + .into_iter() + .filter(|c| { + !c.name.is_empty() + && !c.name.contains(['\t', '\n', '\r', ';', '=']) + && !c.value.contains(['\t', '\n', '\r']) + }) + .filter(|c| seen.insert((c.domain.clone(), c.path.clone(), c.name.clone()))) + .collect() +} + +/// Cookies from a `Cookie:`-style header (`a=1; b=2`) captured for `url`. +/// The header carries no domain/expiry, so cookies are scoped to the site's +/// registrable-ish host (`www.` stripped, subdomains included). +#[cfg_attr(not(target_os = "android"), allow(dead_code))] +pub fn cookies_from_header(url: &str, header: &str, now: i64) -> Vec { + let Ok(parsed) = url::Url::parse(url) else { + return Vec::new(); + }; + let Some(host) = parsed.host_str() else { + return Vec::new(); + }; + let domain = format!(".{}", host.trim_start_matches("www.")); + let secure = parsed.scheme() == "https"; + header + .split(';') + .filter_map(|pair| { + let (name, value) = pair.split_once('=')?; + let name = name.trim(); + (!name.is_empty()).then(|| CapturedCookie { + domain: domain.clone(), + include_subdomains: true, + path: "/".to_string(), + secure, + expires: now + HEADER_COOKIE_TTL_SECS, + name: name.to_string(), + value: value.trim().to_string(), + }) + }) + .collect() +} + +fn now_unix() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0) +} + +/// Open the login browser for `site_id`, wait for the user to finish, and +/// return the captured cookies as a Netscape file. +pub async fn capture( + app: &AppHandle, + site_id: &str, + display_name: &str, + target: &LoginTarget, +) -> AppResult { + let title = format!("Log in to {display_name}, then save cookies"); + let cookies = dedupe(platform::capture(app, &title, target).await?); + if cookies.is_empty() { + return Err(AppError::Site(format!( + "No cookies were captured for {display_name}. Make sure the page finished loading and you are logged in." + ))); + } + Ok(CookieLoginResult { + site_id: site_id.to_string(), + count: cookies.len(), + netscape: to_netscape(&cookies), + }) +} + +/// Desktop: finish (save=true) or cancel the open login window. +pub fn finish(app: &AppHandle, save: bool) -> AppResult<()> { + platform::finish(app, save) +} + +pub fn init() -> TauriPlugin { + Builder::new("cookie-browser") + .setup(|app, api| { + platform::setup(app, api); + Ok(()) + }) + .build() +} + +#[cfg(desktop)] +mod platform { + use super::*; + use std::sync::Mutex; + use tauri::{Manager, WebviewUrl, WebviewWindowBuilder, WindowEvent}; + use tokio::sync::mpsc; + + const LABEL: &str = "cookie-login"; + + #[derive(Default)] + pub struct DesktopLoginState { + signal: Mutex>>, + } + + pub fn setup(app: &AppHandle, _api: tauri::plugin::PluginApi) { + app.manage(DesktopLoginState::default()); + } + + fn send(app: &AppHandle, save: bool) -> bool { + app.try_state::() + .and_then(|s| s.signal.lock().ok().and_then(|g| g.clone())) + .is_some_and(|tx| tx.send(save).is_ok()) + } + + pub fn finish(app: &AppHandle, save: bool) -> AppResult<()> { + if send(app, save) { + Ok(()) + } else { + Err(AppError::Other("No login window is open.".into())) + } + } + + pub async fn capture( + app: &AppHandle, + title: &str, + target: &LoginTarget, + ) -> AppResult> { + if let Some(existing) = app.get_webview_window(LABEL) { + send(app, false); + let _ = existing.destroy(); + } + let start: url::Url = target + .start_url + .parse() + .map_err(|e| AppError::InvalidInput(format!("bad login url: {e}")))?; + + let (tx, mut rx) = mpsc::unbounded_channel::(); + let state = app + .try_state::() + .ok_or_else(|| AppError::Other("cookie-browser plugin not initialized".into()))?; + if let Ok(mut slot) = state.signal.lock() { + *slot = Some(tx.clone()); + } + + let window = WebviewWindowBuilder::new(app, LABEL, WebviewUrl::External(start)) + .title(title) + .inner_size(1024.0, 780.0) + .focused(true) + .build() + .map_err(|e| AppError::Other(format!("could not open login window: {e}")))?; + // Closing the window means "done": keep it alive until cookies are read. + window.on_window_event(move |event| { + if let WindowEvent::CloseRequested { api, .. } = event { + api.prevent_close(); + let _ = tx.send(true); + } + }); + + let save = rx.recv().await.unwrap_or(false); + if let Ok(mut slot) = state.signal.lock() { + *slot = None; + } + + let mut out = Vec::new(); + if save { + let now = now_unix(); + for u in &target.cookie_urls { + let Ok(parsed) = url::Url::parse(u) else { + continue; + }; + let host = parsed.host_str().unwrap_or_default().to_string(); + // Must run off the main thread (Windows deadlocks otherwise); + // this is an async command task, so that holds. + let cookies = match window.cookies_for_url(parsed) { + Ok(c) => c, + Err(e) => { + tracing::warn!("[cookie-login] cookies_for_url({u}) failed: {e}"); + continue; + } + }; + for c in cookies { + let (domain, include_subdomains) = match c.domain() { + Some(d) if !d.is_empty() => { + let d = if d.starts_with('.') || d == host { + d.to_string() + } else { + format!(".{d}") + }; + let sub = d.starts_with('.'); + (d, sub) + } + _ => (host.clone(), false), + }; + let expires = c + .expires_datetime() + .map(|t| t.unix_timestamp()) + .filter(|t| *t > now) + .unwrap_or(now + SESSION_COOKIE_TTL_SECS); + out.push(CapturedCookie { + domain, + include_subdomains, + path: c + .path() + .filter(|p| !p.is_empty()) + .unwrap_or("/") + .to_string(), + secure: c.secure().unwrap_or(false), + expires, + name: c.name().to_string(), + value: c.value().to_string(), + }); + } + } + } + let _ = window.destroy(); + if !save { + return Err(AppError::Other("Login cancelled.".into())); + } + Ok(out) + } +} + +#[cfg(target_os = "android")] +mod platform { + use super::*; + use serde::Deserialize; + use tauri::plugin::PluginHandle; + use tauri::Manager; + + pub struct AndroidLoginState { + handle: Option>, + } + + #[derive(Deserialize)] + struct Entry { + url: String, + header: String, + } + + #[derive(Deserialize)] + struct Response { + #[serde(default)] + cookies: Vec, + } + + pub fn setup(app: &AppHandle, api: tauri::plugin::PluginApi) { + let handle = match api.register_android_plugin("com.archhive.app", "CookieBrowserPlugin") { + Ok(h) => Some(h), + Err(e) => { + tracing::warn!("CookieBrowserPlugin registration failed: {e}"); + None + } + }; + app.manage(AndroidLoginState { handle }); + } + + pub fn finish(_app: &AppHandle, _save: bool) -> AppResult<()> { + // The native dialog has its own Save / Cancel buttons. + Ok(()) + } + + pub async fn capture( + app: &AppHandle, + title: &str, + target: &LoginTarget, + ) -> AppResult> { + let handle = app + .try_state::>() + .and_then(|s| s.handle.clone()) + .ok_or_else(|| { + AppError::Other( + "Login browser unavailable (CookieBrowserPlugin not registered). Re-run android:patches and rebuild the APK." + .into(), + ) + })?; + let payload = serde_json::json!({ + "url": target.start_url, + "cookieUrls": target.cookie_urls, + "title": title, + }); + // Blocks until the user taps Save / Cancel: keep it off the async workers. + let response: Response = tokio::task::spawn_blocking(move || { + handle.run_mobile_plugin::("open", payload) + }) + .await + .map_err(|e| AppError::Other(format!("login browser task failed: {e}")))? + .map_err(|e| { + let msg = e.to_string(); + if msg.contains("cancelled") { + AppError::Other("Login cancelled.".into()) + } else { + AppError::Other(format!("Login browser failed: {msg}")) + } + })?; + let now = now_unix(); + Ok(response + .cookies + .iter() + .flat_map(|e| cookies_from_header(&e.url, &e.header, now)) + .collect()) + } +} + +#[cfg(all(mobile, not(target_os = "android")))] +mod platform { + use super::*; + + pub fn setup(_app: &AppHandle, _api: tauri::plugin::PluginApi) {} + + pub fn finish(_app: &AppHandle, _save: bool) -> AppResult<()> { + Ok(()) + } + + pub async fn capture( + _app: &AppHandle, + _title: &str, + _target: &LoginTarget, + ) -> AppResult> { + Err(AppError::Other( + "The login browser is not available on this platform yet.".into(), + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn header_cookies_scope_to_site_domain() { + let c = cookies_from_header( + "https://www.reddit.com/", + "reddit_session=abc; token_v2=x=y", + 100, + ); + assert_eq!(c.len(), 2); + assert_eq!(c[0].domain, ".reddit.com"); + assert!(c[0].include_subdomains && c[0].secure); + assert_eq!(c[0].expires, 100 + HEADER_COOKIE_TTL_SECS); + assert_eq!(c[1].name, "token_v2"); + assert_eq!(c[1].value, "x=y"); + } + + #[test] + fn netscape_output_round_trips_through_vault_format() { + let cookies = dedupe(vec![ + CapturedCookie { + domain: ".example.com".into(), + include_subdomains: true, + path: "/".into(), + secure: true, + expires: 2_000_000_000, + name: "sid".into(), + value: "v1".into(), + }, + CapturedCookie { + domain: ".example.com".into(), + include_subdomains: true, + path: "/".into(), + secure: true, + expires: 2_000_000_000, + name: "sid".into(), + value: "dupe".into(), + }, + CapturedCookie { + domain: "example.com".into(), + include_subdomains: false, + path: "/".into(), + secure: false, + expires: 1, + name: "bad\tname".into(), + value: "x".into(), + }, + ]); + assert_eq!(cookies.len(), 1); + let text = to_netscape(&cookies); + assert!(text.starts_with("# Netscape HTTP Cookie File")); + assert!(text.contains(".example.com\tTRUE\t/\tTRUE\t2000000000\tsid\tv1\n")); + } + + #[test] + fn login_targets() { + let r = login_target("reddit", "https://old.reddit.com"); + assert_eq!(r.start_url, "https://www.reddit.com/login/"); + assert!(r + .cookie_urls + .contains(&"https://old.reddit.com/".to_string())); + let x = login_target("xvideos", "https://www.xvideos.com"); + assert_eq!(x.start_url, "https://www.xvideos.com/"); + assert_eq!(x.cookie_urls, vec!["https://www.xvideos.com/".to_string()]); + } +} diff --git a/src-tauri/src/lib.rs b/src-tauri/src/lib.rs index ad11bea..94d6a0f 100644 --- a/src-tauri/src/lib.rs +++ b/src-tauri/src/lib.rs @@ -1,4 +1,5 @@ mod commands; +mod cookie_login; mod db; mod discovery; mod downloads; @@ -179,6 +180,8 @@ pub fn run() { #[cfg(mobile)] let builder = builder.plugin(mobile::share::init()); + let builder = builder.plugin(cookie_login::init()); + let builder = builder.setup(|app| { let data_dir = app .path() @@ -288,6 +291,8 @@ pub fn run() { commands::merge_duplicates, commands::list_cookie_sites, commands::save_site_cookies, + commands::cookie_login, + commands::cookie_login_finish, commands::delete_site_cookies, commands::export_settings_backup, commands::import_settings_backup, diff --git a/src-tauri/src/sites/adapters/chaturbate.rs b/src-tauri/src/sites/adapters/chaturbate.rs index dd5d4af..b389e5f 100644 --- a/src-tauri/src/sites/adapters/chaturbate.rs +++ b/src-tauri/src/sites/adapters/chaturbate.rs @@ -209,16 +209,20 @@ struct HttpRoom { fn http_room_to_item(room: &HttpRoom) -> MediaItem { let room_url = format!("{BASE}/{}/", room.username); let embed_url = format!("{BASE}/embed/{}/", room.username); + // Room subjects embed `#x` markup; show the + // username as the card title (like Stripchat) and keep the cleaned + // subject as the description. + let subject = crate::sites::listing::strip_html_tags(&room.title); MediaItem { id: Uuid::new_v4().to_string(), - title: room.title.clone(), + title: room.username.clone(), url: room_url, thumbnail: room.thumbnail.clone(), duration: None, site_id: "chaturbate".to_string(), performers: vec![room.username.clone()], tags: vec![], - description: None, + description: (!subject.is_empty() && subject != room.username).then_some(subject), channel: Some(room.username.clone()), is_live: Some(true), viewers: room.viewers, diff --git a/src-tauri/src/sites/adapters/generic_ytdlp.rs b/src-tauri/src/sites/adapters/generic_ytdlp.rs index f65fec3..6e6eb5e 100644 --- a/src-tauri/src/sites/adapters/generic_ytdlp.rs +++ b/src-tauri/src/sites/adapters/generic_ytdlp.rs @@ -110,6 +110,58 @@ fn search_url(base: &str, site_id: &str, slug: &str) -> String { } } +/// ThisVid is a KVS tube site: listings are plain HTML (yt-dlp has no search +/// extractor for it), so scrape them like the other tube adapters. +async fn browse_thisvid(ctx: &SiteContext, query: &BrowseQuery) -> AppResult { + const BASE: &str = "https://thisvid.com"; + let slug = crate::sites::urls::path_slug(&query.slug); + let page = query.page.max(1); + let url = if query.slug.starts_with("https://thisvid.com/") { + query.slug.clone() + } else { + match query.kind { + BrowseKind::Tag | BrowseKind::Category if page > 1 => { + format!("{BASE}/tags/{slug}/{page}/") + } + BrowseKind::Tag | BrowseKind::Category => format!("{BASE}/tags/{slug}/"), + BrowseKind::Model | BrowseKind::Channel => format!("{BASE}/members/{slug}/videos/"), + _ if page > 1 => format!("{BASE}/search/{page}/?q={}", query_slug(&query.slug)), + _ => format!("{BASE}/search/?q={}", query_slug(&query.slug)), + } + }; + let items = parse_thisvid(&ctx.fetch_html(&url, "thisvid").await?); + if items.is_empty() { + return Err(AppError::Site("No videos found on ThisVid.".into())); + } + let has_more = items.len() >= 20; + Ok(BrowsePage { + items, + page: query.page, + has_more, + total: None, + ..Default::default() + }) +} + +fn parse_thisvid(html: &str) -> Vec { + const BASE: &str = "https://thisvid.com"; + let is_video = |href: &str| { + let path = href.strip_prefix(BASE).unwrap_or(href); + path.starts_with("/videos/") && path.trim_end_matches('/').len() > "/videos/".len() + }; + let no_junk = |_: &str| false; + crate::sites::listing::parse_listing( + html, + &crate::sites::listing::ListingSpec { + site_id: "thisvid", + base: BASE, + anchor_selector: "a[href*='/videos/']", + is_video_href: &is_video, + is_junk_title: &no_junk, + }, + ) +} + #[async_trait] impl SiteAdapter for GenericYtDlpAdapter { fn id(&self) -> &str { @@ -161,19 +213,30 @@ impl SiteAdapter for GenericYtDlpAdapter { } }; + if self.site_id == "thisvid" && query.kind != BrowseKind::Video { + return browse_thisvid(ctx, &query).await; + } + if matches!( query.kind, BrowseKind::Channel | BrowseKind::Model | BrowseKind::Search ) { let runner = SidecarRunner::new(ctx.app().clone()); let cookies = ctx.cookie_file_for_site(self.site_id); - let entries = match runner + let entries = runner .list_flat_playlist(&url, query.page, 24, cookies.as_deref()) .await - { - Ok(entries) => entries, - Err(_) => Vec::new(), - }; + .map_err(|e| { + let hint = if self.cookies || matches!(self.site_id, "tiktok" | "twitter") { + format!( + " {} usually needs you to be logged in: use Settings → Cookies → Log in.", + self.name + ) + } else { + String::new() + }; + AppError::Site(format!("{} browse failed: {e}.{hint}", self.name)) + })?; let items = entries .into_iter() .map(|(id, title, item_url, thumbnail)| MediaItem { diff --git a/src-tauri/src/sites/adapters/pornhub.rs b/src-tauri/src/sites/adapters/pornhub.rs index e7d52fb..39e45cd 100644 --- a/src-tauri/src/sites/adapters/pornhub.rs +++ b/src-tauri/src/sites/adapters/pornhub.rs @@ -6,7 +6,6 @@ use crate::sites::browse_fallback::ytdlp_browse_fallback; use crate::sites::urls::{path_slug, query_slug}; use crate::sites::{SiteAdapter, SiteContext}; use async_trait::async_trait; -use uuid::Uuid; macro_rules! ytdlp_tube_adapter { ($struct_name:ident, $id:expr, $name:expr, $base:expr) => { @@ -601,185 +600,25 @@ fn pornhub_category_id_for_slug(slug: &str, _orientation: BrowseOrientation) -> } fn parse_video_links(html: &str, base: &str, site_id: &str) -> AppResult> { - use scraper::{Html, Selector}; - let document = Html::parse_document(html); - let selectors: &[&str] = match site_id { - "pornhub" => &[ - ".ph-pornstar-videos-list .pcVideoListItem a[href*='view_video']", - ".pcVideoListItem a[href*='view_video']", - "a[href*='view_video.php']", - "a[href*='view_video?']", - ], - "xvideos" => &[ - // Modern XVideos IDs look like /video.abc123/slug — not /videos-i-like nav links. - "div.thumb-block a[href*='/video.']", - "div.mozaique a[href*='/video.']", - "a[href*='/video.']", - ], - "xhamster" => &[ - "a[href*='/videos/'][href*='-']", - ".thumb-list__item a[href*='/videos/']", - "a.video-thumb__image-container[href*='/videos/']", - ], - "youporn" => &[ - "a[href*='/watch/']", - ".video-box a[href*='/watch/']", - ".videoList a[href*='/watch/']", - ], - "xnxx" => &[ - "div.mozaique a[href*='/video-']", - "div.thumb-block a[href*='/video-']", - "a[href*='/video-']", - ], - _ => &[ - "a[href*='/video.']", - "a[href*='/videos/']", - "a[href*='/watch/']", - ], + let anchor_selector = match site_id { + "pornhub" => "a[href*='view_video']", + "xvideos" => "a[href*='/video.']", + "xhamster" => "a[href*='/videos/']", + "youporn" => "a[href*='/watch/']", + "xnxx" => "a[href*='/video-']", + _ => "a[href*='/video.'], a[href*='/videos/'], a[href*='/watch/']", }; - let mut items = Vec::new(); - let mut seen = std::collections::HashSet::new(); - - for sel_str in selectors { - let Ok(sel) = Selector::parse(sel_str) else { - continue; - }; - for el in document.select(&sel) { - let Some(href) = el.value().attr("href") else { - continue; - }; - if !is_site_video_href(site_id, href) { - continue; - } - let url = if href.starts_with("http") { - href.to_string() - } else if href.starts_with("//") { - format!("https:{href}") - } else { - format!("{base}{href}") - }; - if !seen.insert(url.clone()) { - continue; - } - let title = { - // 1) — works for PornHub. - let from_a_title = el - .value() - .attr("title") - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()); - // 2) — YouPorn puts the real title on the thumbnail img. - let from_img_title = from_a_title.clone().or_else(|| { - let img_sel = Selector::parse("img").ok(); - img_sel - .as_ref() - .and_then(|sel| { - el.select(sel) - .find_map(|img| img.value().attr("title").filter(|t| !t.is_empty())) - }) - .map(|s| s.to_string()) - }); - // 3) child text — YouPorn fallback. - let from_b = from_img_title.clone().or_else(|| { - let b_sel = Selector::parse("b").ok(); - b_sel - .as_ref() - .and_then(|sel| { - el.select(sel).find_map(|b| { - let t = b.text().collect::().trim().to_string(); - if !t.is_empty() && !is_junk_nav_title(&t) { - Some(t) - } else { - None - } - }) - }) - .map(|s| s.to_string()) - }); - // 4) Last resort: strip duration patterns from full text. - from_b.unwrap_or_else(|| { - crate::sites::urls::strip_duration_from_title( - el.text().collect::().trim(), - ) - }) - }; - let title = crate::sites::urls::html_unescape(&title); - if title.is_empty() || is_junk_nav_title(&title) { - continue; - } - let img_sel = Selector::parse("img").ok(); - let thumbnail = match site_id { - "pornhub" => el - .value() - .attr("data-mediumthumb") - .map(|s| s.to_string()) - .or_else(|| { - img_sel.as_ref().and_then(|sel| { - el.select(sel).find_map(|img| { - img.value() - .attr("data-src") - .or_else(|| img.value().attr("src")) - .map(|s| { - if s.starts_with("http") { - s.to_string() - } else { - format!("{base}{s}") - } - }) - }) - }) - }), - "xvideos" | "xhamster" | "youporn" | "xnxx" => img_sel.as_ref().and_then(|sel| { - el.select(sel).find_map(|img| { - img.value() - .attr("data-src") - .or_else(|| img.value().attr("data-thumb")) - .or_else(|| img.value().attr("data-original")) - .or_else(|| img.value().attr("src")) - .filter(|s| !s.starts_with("data:")) - .map(|s| { - if s.starts_with("http") || s.starts_with("//") { - if s.starts_with("//") { - format!("https:{s}") - } else { - s.to_string() - } - } else { - format!("{base}{s}") - } - }) - }) - }), - _ => None, - }; - items.push(MediaItem { - id: Uuid::new_v4().to_string(), - title, - url, - thumbnail, - duration: None, - site_id: site_id.to_string(), - performers: vec![], - tags: vec![], - description: None, - channel: None, - is_live: None, - viewers: None, - age: None, - gender: None, - stream_url: None, - embed_url: None, - }); - if items.len() >= 40 { - break; - } - } - if items.len() >= 20 { - break; - } - } - - Ok(items) + let is_video_href = |href: &str| is_site_video_href(site_id, href); + Ok(crate::sites::listing::parse_listing( + html, + &crate::sites::listing::ListingSpec { + site_id, + base, + anchor_selector, + is_video_href: &is_video_href, + is_junk_title: &is_junk_nav_title, + }, + )) } fn is_site_video_href(site_id: &str, href: &str) -> bool { @@ -794,7 +633,12 @@ fn is_site_video_href(site_id: &str, href: &str) -> bool { && !lower.contains("/profiles/") } "xhamster" => { - lower.contains("/videos/") + // Single videos live at /videos/-; hubs such as + // /creators/videos/... or /videos/best are listings. + lower + .rsplit("xhamster.com") + .next() + .is_some_and(|path| path.starts_with("/videos/")) && !lower.contains("/videos/best") && !lower.contains("/videos/newest") && !lower.contains("/categories/") diff --git a/src-tauri/src/sites/adapters/reddit.rs b/src-tauri/src/sites/adapters/reddit.rs index f31f258..3c73d8c 100644 --- a/src-tauri/src/sites/adapters/reddit.rs +++ b/src-tauri/src/sites/adapters/reddit.rs @@ -38,6 +38,13 @@ impl SiteAdapter for RedditAdapter { Err(_) => Vec::new(), }; } + if items.is_empty() { + return Err(crate::error::AppError::Site( + "No Reddit posts found. Reddit now hides listings from logged-out visitors: \ + use Settings → Cookies → Log in for Reddit, then try again." + .to_string(), + )); + } let has_more = items.len() >= 25; Ok(BrowsePage { items, diff --git a/src-tauri/src/sites/adapters/redgifs.rs b/src-tauri/src/sites/adapters/redgifs.rs index 9fd2e13..efa841a 100644 --- a/src-tauri/src/sites/adapters/redgifs.rs +++ b/src-tauri/src/sites/adapters/redgifs.rs @@ -2,7 +2,6 @@ use crate::error::{AppError, AppResult}; use crate::models::{BrowseKind, BrowsePage, BrowseQuery, DownloadPlan, DownloadTool, MediaItem}; use crate::sites::{SiteAdapter, SiteContext}; use async_trait::async_trait; -use uuid::Uuid; pub struct RedgifsAdapter; @@ -21,31 +20,41 @@ impl SiteAdapter for RedgifsAdapter { } fn supported_kinds(&self) -> Vec { - vec![BrowseKind::Tag, BrowseKind::Search, BrowseKind::Video] + vec![ + BrowseKind::Search, + BrowseKind::Tag, + BrowseKind::Model, + BrowseKind::Video, + ] } async fn browse(&self, ctx: &SiteContext, query: BrowseQuery) -> AppResult { - let url = match query.kind { - BrowseKind::Tag => format!("https://www.redgifs.com/tags/{}", query.slug), - BrowseKind::Search => format!("https://www.redgifs.com/search/{}", query.slug), - BrowseKind::Livestream => { - if query.slug.starts_with("http") { - query.slug.clone() - } else { - format!("https://www.redgifs.com/watch/{}", query.slug) - } + use crate::sites::extractors::redgifs; + const PAGE_SIZE: u32 = 40; + let slug = query.slug.trim(); + let (items, pages) = match query.kind { + BrowseKind::Video | BrowseKind::Livestream => { + let item = redgifs::extract_info(ctx, slug).await?; + (vec![item], Some(1)) + } + BrowseKind::Model | BrowseKind::Channel => { + let user = slug + .trim_end_matches('/') + .rsplit("/users/") + .next() + .unwrap_or(slug); + redgifs::user_gifs(ctx, user, query.page, PAGE_SIZE).await? } _ => { - if query.slug.starts_with("http") { - query.slug.clone() - } else { - format!("https://www.redgifs.com/watch/{}", query.slug) - } + // Empty query → trending feed; tag slugs use hyphens. + let text = slug.replace('-', " "); + redgifs::search(ctx, &text, "trending", query.page, PAGE_SIZE).await? } }; - let html = ctx.fetch_html(&url, "redgifs").await?; - let items = parse_redgifs(&html)?; - let has_more = items.len() >= 20; + if items.is_empty() { + return Err(AppError::Site("No RedGifs results for this query.".into())); + } + let has_more = pages.map_or(items.len() as u32 >= PAGE_SIZE, |p| query.page < p); Ok(BrowsePage { items, page: query.page, @@ -138,44 +147,3 @@ impl SiteAdapter for RedgifsAdapter { }) } } - -fn parse_redgifs(html: &str) -> AppResult> { - use scraper::{Html, Selector}; - let document = Html::parse_document(html); - let sel = Selector::parse("a[href*='/watch/']").unwrap(); - let mut items = Vec::new(); - for el in document.select(&sel) { - let Some(href) = el.value().attr("href") else { - continue; - }; - let url = if href.starts_with("http") { - href.to_string() - } else { - format!("https://www.redgifs.com{href}") - }; - items.push(MediaItem { - id: Uuid::new_v4().to_string(), - title: crate::sites::urls::html_unescape( - url.split('/').next_back().unwrap_or("redgif"), - ), - url, - thumbnail: None, - duration: None, - site_id: "redgifs".to_string(), - performers: vec![], - tags: vec![], - description: None, - channel: None, - is_live: None, - viewers: None, - age: None, - gender: None, - stream_url: None, - embed_url: None, - }); - if items.len() >= 30 { - break; - } - } - Ok(items) -} diff --git a/src-tauri/src/sites/extractors/redgifs.rs b/src-tauri/src/sites/extractors/redgifs.rs index 4757df6..d409bac 100644 --- a/src-tauri/src/sites/extractors/redgifs.rs +++ b/src-tauri/src/sites/extractors/redgifs.rs @@ -1,108 +1,264 @@ +//! RedGifs v2 API client. +//! +//! redgifs.com is a JavaScript single-page app: its HTML contains no listing +//! markup, so everything goes through `api.redgifs.com`. The API requires a +//! bearer token; anonymous clients get one from `/v2/auth/temporary`. The +//! token is bound to the caller's IP + User-Agent, so it is fetched with the +//! same shared `reqwest::Client` that issues the API calls and cached until +//! the API rejects it. + use crate::error::{AppError, AppResult}; use crate::models::MediaItem; use crate::sites::SiteContext; use regex::Regex; use serde_json::Value; +use std::sync::{LazyLock, Mutex}; -const REDGIFS_API: &str = "https://api.redgifs.com/v2/gifs/"; +const API: &str = "https://api.redgifs.com/v2"; +const WATCH_BASE: &str = "https://www.redgifs.com/watch/"; -pub async fn extract_info(ctx: &SiteContext, url: &str) -> AppResult { - let id = extract_gif_id(url).ok_or_else(|| { - AppError::Site("Invalid RedGifs URL. Expected https://www.redgifs.com/watch/ID".into()) - })?; +static TOKEN: LazyLock>> = LazyLock::new(|| Mutex::new(None)); +async fn fetch_token(ctx: &SiteContext) -> AppResult { let resp = ctx .client - .get(format!("{}{}", REDGIFS_API, id)) + .get(format!("{API}/auth/temporary")) .send() .await?; if !resp.status().is_success() { return Err(AppError::Site(format!( - "HTTP {} for RedGifs", + "RedGifs auth failed: HTTP {}", resp.status() ))); } - let json: Value = resp.json().await?; - let gif = json - .get("gif") - .ok_or_else(|| AppError::Site("Missing gif data in RedGifs response".into()))?; - - let title = gif - .get("title") + let token = json + .get("token") .and_then(|v| v.as_str()) - .unwrap_or("RedGifs") + .ok_or_else(|| AppError::Site("RedGifs auth response had no token".into()))? .to_string(); + if let Ok(mut slot) = TOKEN.lock() { + *slot = Some(token.clone()); + } + Ok(token) +} - let username = gif - .get("userName") - .and_then(|v| v.as_str()) - .map(|s| s.to_string()); +async fn token(ctx: &SiteContext) -> AppResult { + let cached = TOKEN.lock().ok().and_then(|slot| slot.clone()); + match cached { + Some(t) => Ok(t), + None => fetch_token(ctx).await, + } +} + +/// Authenticated GET against the API; refreshes the token once on 401/403 +/// (expired, or issued for a different IP after a network change). +async fn api_get(ctx: &SiteContext, path_and_query: &str) -> AppResult { + let url = format!("{API}{path_and_query}"); + let mut tok = token(ctx).await?; + for attempt in 0..2 { + let resp = ctx + .client + .get(&url) + .bearer_auth(&tok) + .header("Accept", "application/json") + .send() + .await?; + let status = resp.status(); + if status.is_success() { + return Ok(resp.json().await?); + } + if attempt == 0 && (status.as_u16() == 401 || status.as_u16() == 403) { + tok = fetch_token(ctx).await?; + continue; + } + return Err(AppError::Site(format!( + "RedGifs API HTTP {status} for {url}" + ))); + } + Err(AppError::Site("RedGifs API unavailable".into())) +} + +/// One page of RedGifs results as `(items, total_pages)`. +pub async fn search( + ctx: &SiteContext, + text: &str, + order: &str, + page: u32, + count: u32, +) -> AppResult<(Vec, Option)> { + let path = format!( + "/gifs/search?search_text={}&order={order}&count={count}&page={}", + crate::sites::urls::query_slug(text), + page.max(1) + ); + page_from_json(&api_get(ctx, &path).await?) +} - let duration = gif - .get("duration") - .and_then(|v| v.as_f64()) - .map(|d| d as u32); +/// One page of a creator's uploads. +pub async fn user_gifs( + ctx: &SiteContext, + user: &str, + page: u32, + count: u32, +) -> AppResult<(Vec, Option)> { + let user = crate::sites::urls::path_slug(user); + let path = format!( + "/users/{user}/search?order=new&count={count}&page={}", + page.max(1) + ); + page_from_json(&api_get(ctx, &path).await?) +} + +fn page_from_json(json: &Value) -> AppResult<(Vec, Option)> { + let gifs = json + .get("gifs") + .and_then(|v| v.as_array()) + .ok_or_else(|| AppError::Site("RedGifs response had no gifs".into()))?; + let pages = json.get("pages").and_then(|v| v.as_u64()).map(|p| p as u32); + Ok((gifs.iter().filter_map(media_item_from_gif).collect(), pages)) +} - let thumbnail = gif - .get("thumbnail") +/// Map an API gif object to a listing item (`url` is the watch page). +fn media_item_from_gif(gif: &Value) -> Option { + let id = gif.get("id")?.as_str()?.to_string(); + let username = gif + .get("userName") .and_then(|v| v.as_str()) .map(|s| s.to_string()); - - let tags = gif + let tags: Vec = gif .get("tags") .and_then(|v| v.as_array()) .map(|arr| { arr.iter() .filter_map(|t| { - t.get("text") - .and_then(|v| v.as_str()) + t.as_str() + .or_else(|| t.get("text").and_then(|v| v.as_str())) .map(|s| s.to_string()) }) - .collect::>() + .collect() }) .unwrap_or_default(); - - let video_url = gif - .get("urls") - .and_then(|u| u.get("hd")) - .and_then(|v| v.as_str()) - .or_else(|| { - gif.get("urls") - .and_then(|u| u.get("gif")) - .and_then(|v| v.as_str()) - }) - .map(|s| s.to_string()); - - Ok(MediaItem { - id: format!("redgifs_{}", id), - title, - url: video_url.unwrap_or_else(|| url.to_string()), - thumbnail, - duration, + let urls = gif.get("urls"); + let url_field = |k: &str| { + urls.and_then(|u| u.get(k)) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }; + Some(MediaItem { + title: gif_title(gif, &id, username.as_deref(), &tags), + url: format!("{WATCH_BASE}{id}"), + id: format!("redgifs_{id}"), + thumbnail: url_field("thumbnail").or_else(|| url_field("poster")), + duration: gif + .get("duration") + .and_then(|v| v.as_f64()) + .map(|d| d.round() as u32), site_id: "redgifs".to_string(), - performers: username - .as_ref() - .map(|u| vec![u.clone()]) - .unwrap_or_default(), + performers: username.iter().cloned().collect(), tags, - description: None, + description: gif + .get("description") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()), channel: username, is_live: None, viewers: None, age: None, gender: None, - stream_url: None, + // Direct MP4 — playable in-app without another resolve round trip. + stream_url: url_field("hd").or_else(|| url_field("sd")), embed_url: None, }) } +/// RedGifs items have no title field; prefer the description, then +/// "creator · tags", then the id. +fn gif_title(gif: &Value, id: &str, user: Option<&str>, tags: &[String]) -> String { + if let Some(desc) = gif + .get("description") + .and_then(|v| v.as_str()) + .and_then(|d| d.lines().map(str::trim).find(|l| !l.is_empty())) + { + return desc.chars().take(120).collect(); + } + let tag_part = tags.iter().take(3).cloned().collect::>().join(", "); + match (user, tag_part.is_empty()) { + (Some(u), false) => format!("{u} · {tag_part}"), + (Some(u), true) => format!("{u} · {id}"), + (None, false) => tag_part, + (None, true) => id.to_string(), + } +} + +pub async fn extract_info(ctx: &SiteContext, url: &str) -> AppResult { + let id = extract_gif_id(url).ok_or_else(|| { + AppError::Site("Invalid RedGifs URL. Expected https://www.redgifs.com/watch/ID".into()) + })?; + let json = api_get(ctx, &format!("/gifs/{id}")).await?; + let gif = json + .get("gif") + .ok_or_else(|| AppError::Site("Missing gif data in RedGifs response".into()))?; + media_item_from_gif(gif) + .ok_or_else(|| AppError::Site("Malformed gif data in RedGifs response".into())) +} + +/// Direct MP4 for a watch URL (`None` when the API lists no media URL). pub async fn extract_download_url(ctx: &SiteContext, url: &str) -> AppResult> { - let info = extract_info(ctx, url).await?; - Ok(Some(info.url)) + Ok(extract_info(ctx, url).await?.stream_url) +} + +pub fn extract_gif_id(url: &str) -> Option { + let re = Regex::new(r"redgifs\.com/(?:watch|ifr|i)/([A-Za-z0-9]+)").ok()?; + re.captures(url)? + .get(1) + .map(|m| m.as_str().to_ascii_lowercase()) } -fn extract_gif_id(url: &str) -> Option { - let re = Regex::new(r"redgifs\.com/watch/([A-Za-z0-9]+)").ok()?; - re.captures(url)?.get(1).map(|m| m.as_str().to_string()) +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn gif_id_from_watch_and_embed_urls() { + assert_eq!( + extract_gif_id("https://www.redgifs.com/watch/FlimsyBlueWhale#rel=x").as_deref(), + Some("flimsybluewhale") + ); + assert_eq!( + extract_gif_id("https://www.redgifs.com/ifr/abc").as_deref(), + Some("abc") + ); + assert_eq!(extract_gif_id("https://example.com/watch/abc"), None); + } + + #[test] + fn maps_search_page() { + let json: Value = serde_json::from_str( + r#"{"page":1,"pages":7,"total":300,"gifs":[ + {"id":"calmwildcat","duration":12.4,"userName":"someone","tags":["Amateur","Couple","POV","Extra"], + "urls":{"sd":"https://media.redgifs.com/Calm-mobile.mp4","hd":"https://media.redgifs.com/Calm.mp4", + "thumbnail":"https://media.redgifs.com/Calm-mobile.jpg","poster":"https://media.redgifs.com/Calm-poster.jpg"}}, + {"id":"desc","description":"\n First line\nsecond","urls":{}}, + {"nope":true} + ]}"#, + ) + .unwrap(); + let (items, pages) = page_from_json(&json).unwrap(); + assert_eq!(pages, Some(7)); + assert_eq!(items.len(), 2); + assert_eq!(items[0].url, "https://www.redgifs.com/watch/calmwildcat"); + assert_eq!(items[0].title, "someone · Amateur, Couple, POV"); + assert_eq!(items[0].duration, Some(12)); + assert_eq!( + items[0].stream_url.as_deref(), + Some("https://media.redgifs.com/Calm.mp4") + ); + assert_eq!( + items[0].thumbnail.as_deref(), + Some("https://media.redgifs.com/Calm-mobile.jpg") + ); + assert_eq!(items[1].title, "First line"); + assert_eq!(items[1].stream_url, None); + } } diff --git a/src-tauri/src/sites/listing.rs b/src-tauri/src/sites/listing.rs new file mode 100644 index 0000000..70a27af --- /dev/null +++ b/src-tauri/src/sites/listing.rs @@ -0,0 +1,399 @@ +//! Generic "video grid" parser shared by HTML-scraped tube adapters. +//! +//! Listing pages usually link each video several times: a thumbnail anchor +//! (image + duration/quality badges, no text title) and a title anchor (text +//! and/or `title=`). Taking only the first anchor per URL yields badge text +//! such as "1080p" as the title, or drops the card entirely when the first +//! anchor has no usable text. Instead, every anchor that points at the same +//! video is merged into one card, and the best title/thumbnail/duration +//! across all of them wins. + +use crate::models::MediaItem; +use regex::Regex; +use scraper::{ElementRef, Html, Node, Selector}; +use std::collections::HashMap; +use std::sync::LazyLock; +use uuid::Uuid; + +/// Upper bound on cards returned from one listing page. +pub const MAX_LISTING_ITEMS: usize = 60; + +static DURATION_RE: LazyLock = + LazyLock::new(|| Regex::new(r"\b(?:(\d{1,2}):)?(\d{1,2}):(\d{2})\b").expect("duration regex")); +static MINUTES_RE: LazyLock = + LazyLock::new(|| Regex::new(r"(?i)\b(\d{1,3})\s*min\b").expect("minutes regex")); +static EDGE_DURATION_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?:\d{1,2}:)?\d{1,2}:\d{2}\s*[-:|]?\s*|\s*[-:|]?\s*(?:(?:\d{1,2}:)?\d{1,2}:\d{2}|\d{1,3}\s*min)$") + .expect("edge duration regex") +}); +static BADGE_RE: LazyLock = LazyLock::new(|| { + Regex::new(r"(?i)^(?:\d{3,4}p(?:\s*\d{2})?|[48]k|hd|uhd|fhd|vr|new|premium|cc|\d+\s*fps|[\d\s.,:%kmb]+|views?)(?:\s*(?:cc|hd|vr))*$") + .expect("badge regex") +}); +static WS_RE: LazyLock = LazyLock::new(|| Regex::new(r"\s+").expect("ws regex")); + +/// How a site's listing page should be read. +pub struct ListingSpec<'a> { + pub site_id: &'a str, + /// Site origin used to absolutize relative hrefs / image paths. + pub base: &'a str, + /// CSS selector matching every candidate video anchor. + pub anchor_selector: &'a str, + /// Returns true when an href points at a single video page. + pub is_video_href: &'a dyn Fn(&str) -> bool, + /// Generic titles to discard (site nav links etc.). + pub is_junk_title: &'a dyn Fn(&str) -> bool, +} + +#[derive(Default)] +struct Card { + title: Option<(u8, String)>, + thumbnail: Option, + duration: Option, +} + +/// Parse a listing page into media items, merging duplicate anchors per video. +pub fn parse_listing(html: &str, spec: &ListingSpec<'_>) -> Vec { + let document = Html::parse_document(html); + let (Ok(anchor_sel), Ok(img_sel)) = ( + Selector::parse(spec.anchor_selector), + Selector::parse("img"), + ) else { + return Vec::new(); + }; + + let mut order: Vec = Vec::new(); + let mut cards: HashMap = HashMap::new(); + + for el in document.select(&anchor_sel) { + let Some(href) = el.value().attr("href") else { + continue; + }; + if !(spec.is_video_href)(href) { + continue; + } + let url = absolutize(spec.base, href.split('#').next().unwrap_or(href)); + if !cards.contains_key(&url) { + if order.len() >= MAX_LISTING_ITEMS { + continue; + } + order.push(url.clone()); + } + let card = cards.entry(url).or_default(); + + let imgs: Vec> = el.select(&img_sel).collect(); + let mut candidates: Vec<(u8, Option<&str>)> = vec![ + (4, el.value().attr("title")), + (4, el.value().attr("aria-label")), + (3, el.value().attr("data-title")), + ]; + for img in &imgs { + candidates.push((2, img.value().attr("alt"))); + candidates.push((2, img.value().attr("title"))); + } + let text = visible_text(&el); + candidates.push((1, Some(text.as_str()))); + for (score, raw) in candidates { + let Some(raw) = raw else { continue }; + if card.title.as_ref().is_some_and(|(s, _)| *s >= score) { + continue; + } + if let Some(t) = clean_listing_title(raw, spec.is_junk_title) { + card.title = Some((score, t)); + } + } + + if card.thumbnail.is_none() { + card.thumbnail = el + .value() + .attr("data-mediumthumb") + .or_else(|| el.value().attr("data-thumb")) + .and_then(|s| usable_image_url(spec.base, s)) + .or_else(|| imgs.iter().find_map(|img| image_from_img(spec.base, img))); + } + if card.duration.is_none() { + card.duration = parse_duration_text(&text); + } + } + + order + .into_iter() + .filter_map(|url| { + let card = cards.remove(&url)?; + let title = card + .title + .map(|(_, t)| t) + .or_else(|| title_from_url_slug(&url))?; + Some(MediaItem { + id: Uuid::new_v4().to_string(), + title, + url, + thumbnail: card.thumbnail, + duration: card.duration, + site_id: spec.site_id.to_string(), + performers: vec![], + tags: vec![], + description: None, + channel: None, + is_live: None, + viewers: None, + age: None, + gender: None, + stream_url: None, + embed_url: None, + }) + }) + .collect() +} + +/// Text content of an element, skipping `