diff --git a/CHANGELOG.md b/CHANGELOG.md index 8cda56a..0271b9e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Public TikTok, Vimeo, and SoundCloud metadata fetching through oEmbed, available automatically in the library, CLI, MCP server, and Python bindings. +- YouTube Shorts, live, embed, and privacy-enhanced embed URL support, with validated video IDs. + ## [0.6.0] - 2026-10-23 ### Highlights diff --git a/README.md b/README.md index a81bc4e..1aae1dc 100644 --- a/README.md +++ b/README.md @@ -30,13 +30,16 @@ fetchers match first; the default fetcher handles everything else. - `StackOverflowFetcher` - Stack Overflow and Stack Exchange question URLs - `PackageRegistryFetcher` - PyPI, crates.io, and npm package pages - `WikipediaFetcher` - Wikipedia article URLs -- `YouTubeFetcher` - YouTube watch and `youtu.be` URLs +- `YouTubeFetcher` - YouTube watch, Shorts, live, embed, and `youtu.be` URLs +- `SocialMediaFetcher` - TikTok and Vimeo videos; SoundCloud tracks and playlists - `ArXivFetcher` - arXiv abstract and PDF URLs - `HackerNewsFetcher` - Hacker News item threads - `RSSFeedFetcher` - RSS and Atom feed URLs - `DocsSiteFetcher` - docs sites with `llms.txt`/`llms-full.txt` support - `DefaultFetcher` - all remaining HTTP/HTTPS URLs with HTML conversion, streaming, timeout handling, and binary detection +See [Social media fetching](docs/social-media.md) for supported URLs and metadata limits. + ## Built-in Content Processors Content processors run after a fetcher retrieves a bounded response body. They diff --git a/crates/fetchkit/src/fetchers/mod.rs b/crates/fetchkit/src/fetchers/mod.rs index 347437d..aeb67ba 100644 --- a/crates/fetchkit/src/fetchers/mod.rs +++ b/crates/fetchkit/src/fetchers/mod.rs @@ -20,6 +20,7 @@ mod jupyter_notebook; mod package_registry; mod pubmed; mod rss_feed; +mod social_media; mod stackoverflow; mod twitter; mod wikipedia; @@ -42,6 +43,7 @@ pub use jupyter_notebook::JupyterNotebookFetcher; pub use package_registry::PackageRegistryFetcher; pub use pubmed::PubMedFetcher; pub use rss_feed::RSSFeedFetcher; +pub use social_media::SocialMediaFetcher; pub use stackoverflow::StackOverflowFetcher; pub use twitter::TwitterFetcher; pub use wikipedia::WikipediaFetcher; @@ -189,6 +191,7 @@ impl FetcherRegistry { registry.register(Box::new(PackageRegistryFetcher::new())); registry.register(Box::new(WikipediaFetcher::new())); registry.register(Box::new(YouTubeFetcher::new())); + registry.register(Box::new(SocialMediaFetcher::new())); registry.register(Box::new(ArXivFetcher::new())); registry.register(Box::new(CrossrefFetcher::new())); registry.register(Box::new(IetfRfcFetcher::new())); @@ -400,15 +403,16 @@ mod tests { assert_eq!(registry.fetchers[10].name(), "package_registry"); assert_eq!(registry.fetchers[11].name(), "wikipedia"); assert_eq!(registry.fetchers[12].name(), "youtube"); - assert_eq!(registry.fetchers[13].name(), "arxiv"); - assert_eq!(registry.fetchers[14].name(), "crossref"); - assert_eq!(registry.fetchers[15].name(), "ietf_rfc"); - assert_eq!(registry.fetchers[16].name(), "pubmed"); - assert_eq!(registry.fetchers[17].name(), "hackernews"); - assert_eq!(registry.fetchers[18].name(), "rss_feed"); - assert_eq!(registry.fetchers[19].name(), "docs_site"); - assert_eq!(registry.fetchers[20].name(), "default"); - assert_eq!(registry.fetchers.len(), 21); + assert_eq!(registry.fetchers[13].name(), "social_media"); + assert_eq!(registry.fetchers[14].name(), "arxiv"); + assert_eq!(registry.fetchers[15].name(), "crossref"); + assert_eq!(registry.fetchers[16].name(), "ietf_rfc"); + assert_eq!(registry.fetchers[17].name(), "pubmed"); + assert_eq!(registry.fetchers[18].name(), "hackernews"); + assert_eq!(registry.fetchers[19].name(), "rss_feed"); + assert_eq!(registry.fetchers[20].name(), "docs_site"); + assert_eq!(registry.fetchers[21].name(), "default"); + assert_eq!(registry.fetchers.len(), 22); } #[test] diff --git a/crates/fetchkit/src/fetchers/social_media.rs b/crates/fetchkit/src/fetchers/social_media.rs new file mode 100644 index 0000000..3082f0c --- /dev/null +++ b/crates/fetchkit/src/fetchers/social_media.rs @@ -0,0 +1,322 @@ +//! Public social media metadata through fixed provider oEmbed endpoints. +//! +//! Embed HTML is deliberately ignored: it contains executable players, not article +//! content. Only canonical public resource URLs are sent to providers; tracking +//! queries, credentials and short-link redirects are never forwarded. + +use crate::client::FetchOptions; +use crate::convert::html_to_markdown_with_base_url; +use crate::error::FetchError; +use crate::fetchers::default::{read_body_with_timeout, transport_request}; +use crate::fetchers::{validate_url_policy, Fetcher}; +use crate::types::{FetchRequest, FetchResponse, HttpMethod}; +use crate::DEFAULT_USER_AGENT; +use async_trait::async_trait; +use reqwest::header::{HeaderMap, HeaderValue, ACCEPT, USER_AGENT}; +use serde::Deserialize; +use std::time::Duration; +use url::Url; + +/// TikTok videos, Vimeo videos, and SoundCloud tracks/playlists via public oEmbed. +pub struct SocialMediaFetcher; + +impl SocialMediaFetcher { + pub fn new() -> Self { + Self + } +} + +impl Default for SocialMediaFetcher { + fn default() -> Self { + Self::new() + } +} + +struct Resource { + provider: &'static str, + endpoint: &'static str, + format: &'static str, + url: String, +} + +fn numeric(value: &str) -> bool { + !value.is_empty() && value.bytes().all(|byte| byte.is_ascii_digit()) +} + +fn slug(value: &str) -> bool { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"-_.".contains(&byte)) +} + +fn resource(url: &Url) -> Option { + if !matches!(url.scheme(), "http" | "https") { + return None; + } + let path: Vec<_> = url + .path() + .trim_end_matches('/') + .split('/') + .skip(1) + .collect(); + match (url.host_str()?, path.as_slice()) { + ("www.tiktok.com" | "tiktok.com" | "m.tiktok.com", [user, "video", id]) + if user.strip_prefix('@').is_some_and(slug) && numeric(id) => + { + Some(Resource { + provider: "TikTok", + endpoint: "https://www.tiktok.com/oembed", + format: "tiktok_video", + url: format!("https://www.tiktok.com/{user}/video/{id}"), + }) + } + ("vimeo.com" | "www.vimeo.com", [id]) | ("player.vimeo.com", ["video", id]) + if numeric(id) => + { + Some(Resource { + provider: "Vimeo", + endpoint: "https://vimeo.com/api/oembed.json", + format: "vimeo_video", + url: format!("https://vimeo.com/{id}"), + }) + } + ("soundcloud.com" | "www.soundcloud.com" | "m.soundcloud.com", parts) => { + let (user, name, playlist) = match parts { + [user, name] => (*user, *name, false), + [user, "sets", name] => (*user, *name, true), + _ => return None, + }; + if !slug(user) + || !slug(name) + || matches!( + user, + "discover" + | "stream" + | "you" + | "search" + | "charts" + | "upload" + | "settings" + | "pages" + | "terms-of-use" + | "jobs" + ) + || (!playlist + && matches!( + name, + "sets" + | "tracks" + | "albums" + | "reposts" + | "likes" + | "popular-tracks" + | "followers" + | "following" + )) + { + return None; + } + Some(Resource { + provider: "SoundCloud", + endpoint: "https://soundcloud.com/oembed", + format: if playlist { + "soundcloud_playlist" + } else { + "soundcloud_track" + }, + url: format!( + "https://soundcloud.com/{user}/{}{name}", + if playlist { "sets/" } else { "" } + ), + }) + } + _ => None, + } +} + +#[derive(Deserialize)] +struct OEmbed { + #[serde(rename = "type")] + kind: String, + title: Option, + author_name: Option, + author_url: Option, + thumbnail_url: Option, + description: Option, +} + +#[async_trait] +impl Fetcher for SocialMediaFetcher { + fn name(&self) -> &'static str { + "social_media" + } + + fn matches(&self, url: &Url) -> bool { + resource(url).is_some() + } + + async fn fetch( + &self, + request: &FetchRequest, + options: &FetchOptions, + ) -> Result { + let request = request.normalized_for_fetch()?; + let page_url = Url::parse(&request.url).map_err(|_| FetchError::InvalidUrlScheme)?; + validate_url_policy(&page_url, options)?; + let resource = resource(&page_url) + .ok_or_else(|| FetchError::FetcherError("Unsupported social media URL".into()))?; + let mut api_url = Url::parse(resource.endpoint).unwrap(); + api_url + .query_pairs_mut() + .append_pair("url", &resource.url) + .append_pair("format", "json"); + let host = api_url.host_str().unwrap().to_owned(); + let mut headers = HeaderMap::new(); + headers.insert( + USER_AGENT, + HeaderValue::from_str(options.user_agent.as_deref().unwrap_or(DEFAULT_USER_AGENT)) + .unwrap_or_else(|_| HeaderValue::from_static(DEFAULT_USER_AGENT)), + ); + headers.insert(ACCEPT, HeaderValue::from_static("application/json")); + // THREAT[TM-SSRF-010]: transport validates the complete derived URL and + // pins DNS. Redirects and URLs in provider metadata are never fetched. + let response = transport_request( + api_url, + if request.method == Some(HttpMethod::Head) { + reqwest::Method::HEAD + } else { + reqwest::Method::GET + }, + headers, + options, + Duration::from_secs(10), + &host, + 443, + ) + .await?; + let status_code = response.status; + if !(200..300).contains(&status_code) { + return Ok(FetchResponse { + url: request.url, + status_code, + error: Some(format!( + "{} oEmbed unavailable: HTTP {status_code}", + resource.provider + )), + ..Default::default() + }); + } + let mut result = FetchResponse { + url: request.url, + status_code, + content_type: Some("text/markdown".into()), + format: Some(resource.format.into()), + ..Default::default() + }; + if request.method == Some(HttpMethod::Head) { + return Ok(result); + } + let limit = options.max_body_size.unwrap_or(10 * 1024 * 1024); + let (body, incomplete) = + read_body_with_timeout(response, Duration::from_secs(30), limit).await?; + if incomplete { + return Err(FetchError::FetcherError(format!( + "Incomplete {} oEmbed metadata (body limit, timeout or stream error)", + resource.provider + ))); + } + let metadata: OEmbed = serde_json::from_slice(&body).map_err(|_| { + FetchError::FetcherError(format!("Invalid {} oEmbed metadata", resource.provider)) + })?; + if !matches!(metadata.kind.as_str(), "video" | "rich") { + return Err(FetchError::FetcherError(format!( + "Unsupported {} oEmbed type", + resource.provider + ))); + } + let mut content = render(&resource, &metadata); + let truncated = content.len() > limit; + let mut end = content.len().min(limit); + while !content.is_char_boundary(end) { + end -= 1; + } + content.truncate(end); + result.size = Some(content.len() as u64); + result.content = Some(content); + result.truncated = Some(truncated); + Ok(result) + } +} + +fn render(resource: &Resource, metadata: &OEmbed) -> String { + let title = metadata + .title + .as_deref() + .filter(|value| !value.trim().is_empty()); + let mut content = format!( + "# {}\n\n- **Platform:** {}\n- **URL:** {}\n", + title.unwrap_or(resource.provider), + resource.provider, + resource.url + ); + for (label, value) in [ + ("Author", metadata.author_name.as_deref()), + ("Author URL", metadata.author_url.as_deref()), + ("Thumbnail", metadata.thumbnail_url.as_deref()), + ] { + if let Some(value) = value.filter(|value| !value.trim().is_empty()) { + content.push_str(&format!("- **{label}:** {value}\n")); + } + } + if let Some(description) = metadata + .description + .as_deref() + .filter(|value| !value.trim().is_empty()) + { + let description = html_to_markdown_with_base_url(description, &resource.url); + if !description.trim().is_empty() { + content.push_str(&format!("\n## Description\n\n{}\n", description.trim())); + } + } + content.push_str("\n*Public metadata only; media and transcripts are not retrieved.*\n"); + content +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn matches_only_supported_resource_paths() { + let fetcher = SocialMediaFetcher::new(); + for url in [ + "https://www.tiktok.com/@scout2015/video/6718335390845095173", + "https://m.tiktok.com/@a.b/video/123/?tracking=secret", + "https://vimeo.com/123", + "https://player.vimeo.com/video/123", + "https://soundcloud.com/forss/flickermood", + "https://www.soundcloud.com/artist/sets/playlist/", + ] { + assert!(fetcher.matches(&Url::parse(url).unwrap()), "{url}"); + } + for url in [ + "https://tiktok.com/@user", + "https://tiktok.com/@user/video/nope", + "https://tiktok.com/user/video/123", + "https://vm.tiktok.com/abc", + "https://tiktok.com.evil.test/@user/video/123", + "https://vimeo.com/123/privatehash", + "https://vimeo.com/channels/staffpicks", + "https://soundcloud.com/artist", + "https://soundcloud.com/search/sounds", + "https://soundcloud.com/artist/tracks", + "https://soundcloud.com/artist/sets", + "https://soundcloud.com/artist/track/secret", + "https://soundcloud.com/artist//track", + "ftp://vimeo.com/123", + ] { + assert!(!fetcher.matches(&Url::parse(url).unwrap()), "{url}"); + } + } +} diff --git a/crates/fetchkit/src/fetchers/youtube.rs b/crates/fetchkit/src/fetchers/youtube.rs index 41bf8ee..992299f 100644 --- a/crates/fetchkit/src/fetchers/youtube.rs +++ b/crates/fetchkit/src/fetchers/youtube.rs @@ -1,7 +1,7 @@ //! YouTube video fetcher //! -//! Handles youtube.com/watch and youtu.be URLs, returning video metadata -//! and transcript text via oEmbed and noembed APIs. +//! Handles watch, Shorts, live, embed and youtu.be video URLs. Metadata uses +//! oEmbed; English captions are best-effort through the legacy timedtext API. use crate::client::FetchOptions; use crate::error::FetchError; @@ -24,7 +24,7 @@ const YOUTUBE_PORT: u16 = 443; /// YouTube video fetcher /// -/// Matches `youtube.com/watch?v={id}` and `youtu.be/{id}`, returning +/// Matches watch, Shorts, live, embed and `youtu.be` video URLs, returning /// video metadata via oEmbed and transcript when available. pub struct YouTubeFetcher; @@ -35,32 +35,36 @@ impl YouTubeFetcher { /// Extract video ID from YouTube URL fn parse_video_id(url: &Url) -> Option { - let host = url.host_str()?; - - match host { - "youtube.com" | "www.youtube.com" | "m.youtube.com" => { - // /watch?v={id} - let segments: Vec<&str> = - url.path_segments().map(|s| s.collect()).unwrap_or_default(); - if segments.first() != Some(&"watch") { - return None; - } - url.query_pairs() - .find(|(k, _)| k == "v") - .map(|(_, v)| v.to_string()) - .filter(|v| !v.is_empty()) - } - "youtu.be" => { - // /{id} - let segments: Vec<&str> = - url.path_segments().map(|s| s.collect()).unwrap_or_default(); - segments - .first() - .filter(|s| !s.is_empty()) - .map(|s| s.to_string()) - } - _ => None, + if !matches!(url.scheme(), "http" | "https") { + return None; } + let segments: Vec<_> = url + .path() + .trim_end_matches('/') + .split('/') + .skip(1) + .collect(); + let id = match (url.host_str()?, segments.as_slice()) { + ("youtube.com" | "www.youtube.com" | "m.youtube.com", ["watch"]) => url + .query_pairs() + .find(|(key, _)| key == "v")? + .1 + .into_owned(), + ( + "youtube.com" | "www.youtube.com" | "m.youtube.com", + ["shorts" | "live" | "embed", id], + ) + | ("youtube-nocookie.com" | "www.youtube-nocookie.com", ["embed", id]) + | ("youtu.be", [id]) => (*id).to_owned(), + _ => return None, + }; + // IDs are interpolated into the timedtext query: reject separators and + // encoded query injection before deriving any provider URL. + (!id.is_empty() + && id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || b"_-".contains(&byte))) + .then_some(id) } } @@ -344,6 +348,38 @@ fn safe_truncate_utf8(input: &str, max_bytes: usize) -> &str { mod tests { use super::*; + #[test] + fn accepts_video_variants_and_rejects_query_injection() { + for url in [ + "https://youtube.com/shorts/abc_123-xyz?feature=share", + "https://m.youtube.com/live/abc_123-xyz", + "https://www.youtube.com/embed/abc_123-xyz", + "https://www.youtube-nocookie.com/embed/abc_123-xyz", + ] { + assert_eq!( + YouTubeFetcher::parse_video_id(&Url::parse(url).unwrap()).as_deref(), + Some("abc_123-xyz"), + "{url}" + ); + } + for url in [ + "https://youtube.com/watch/extra?v=abc", + "https://youtu.be/abc/extra", + "https://youtube.com/shorts/", + "https://youtube.com/live/abc/extra", + "https://youtube.com.evil.test/shorts/abc", + "https://youtube.com/watch?v=abc%26lang%3Dfr", + "https://youtube.com/embed/abc%3Fother", + "ftp://youtube.com/watch?v=abc", + ] { + assert_eq!( + YouTubeFetcher::parse_video_id(&Url::parse(url).unwrap()), + None, + "{url}" + ); + } + } + #[test] fn test_parse_youtube_watch() { let url = Url::parse("https://www.youtube.com/watch?v=dQw4w9WgXcQ").unwrap(); diff --git a/crates/fetchkit/src/lib.rs b/crates/fetchkit/src/lib.rs index d6389a1..cab525e 100644 --- a/crates/fetchkit/src/lib.rs +++ b/crates/fetchkit/src/lib.rs @@ -73,6 +73,7 @@ //! - [`TwitterFetcher`] - Twitter/X tweet content with article metadata //! - [`WikipediaFetcher`] - Wikipedia article content via MediaWiki API //! - [`YouTubeFetcher`] - YouTube video metadata via oEmbed +//! - [`SocialMediaFetcher`] - TikTok, Vimeo, and SoundCloud public metadata via oEmbed #[cfg(feature = "bot-auth")] pub mod bot_auth; @@ -110,7 +111,8 @@ pub use error::{FetchError, ToolError}; pub use fetchers::{ ArXivFetcher, DefaultFetcher, DocsSiteFetcher, Fetcher, FetcherRegistry, GitHubCodeFetcher, GitHubIssueFetcher, GitHubRepoFetcher, HackerNewsFetcher, PackageRegistryFetcher, - RSSFeedFetcher, StackOverflowFetcher, TwitterFetcher, WikipediaFetcher, YouTubeFetcher, + RSSFeedFetcher, SocialMediaFetcher, StackOverflowFetcher, TwitterFetcher, WikipediaFetcher, + YouTubeFetcher, }; pub use file_saver::{FileSaveError, FileSaver, LocalFileSaver, SaveResult}; pub use tool::{ diff --git a/crates/fetchkit/tests/fetcher_live.rs b/crates/fetchkit/tests/fetcher_live.rs index dfd4626..fbf89b5 100644 --- a/crates/fetchkit/tests/fetcher_live.rs +++ b/crates/fetchkit/tests/fetcher_live.rs @@ -115,7 +115,7 @@ mod live_github_code { #[tokio::test] async fn fetches_source_file() { let Some(resp) = - fetch_or_skip("https://github.com/rust-lang/rust/blob/master/README.md").await + fetch_or_skip("https://github.com/rust-lang/rust/blob/main/README.md").await else { return; }; @@ -358,16 +358,59 @@ mod live_default { #[tokio::test] async fn fetches_plain_html() { - let Some(resp) = fetch_markdown_or_skip("https://example.com").await else { + let Some(resp) = fetch_markdown_or_skip("https://httpbin.org/html").await else { return; }; assert_eq!(resp.status_code, 200); let content = resp.content.expect("should have content"); assert!( - content.contains("Example Domain"), - "example.com should contain 'Example Domain'" + content.contains("Herman Melville - Moby-Dick"), + "httpbin HTML fixture should contain its Moby-Dick heading" ); assert_eq!(resp.format, Some("markdown".to_string())); } } + +// Matches social_media.rs for changed-fetcher CI selection. +mod live_social_media { + use super::*; + + async fn metadata(url: &str, format: &str) { + let Some(response) = fetch_or_skip(url).await else { + return; + }; + if response.error.is_some() { + eprintln!("UNAVAILABLE: {url} — HTTP {}", response.status_code); + assert!(!(200..300).contains(&response.status_code)); + assert!(response.content.is_none()); + return; + } + assert_eq!(response.status_code, 200); + assert_eq!(response.format.as_deref(), Some(format)); + assert!(response.content.unwrap().contains("Public metadata only")); + } + + #[tokio::test] + async fn tiktok() { + metadata( + "https://www.tiktok.com/@scout2015/video/6718335390845095173", + "tiktok_video", + ) + .await; + } + + #[tokio::test] + async fn vimeo() { + metadata("https://vimeo.com/863362136", "vimeo_video").await; + } + + #[tokio::test] + async fn soundcloud() { + metadata( + "https://soundcloud.com/forss/flickermood", + "soundcloud_track", + ) + .await; + } +} diff --git a/crates/fetchkit/tests/transport.rs b/crates/fetchkit/tests/transport.rs index b9e1266..f45df36 100644 --- a/crates/fetchkit/tests/transport.rs +++ b/crates/fetchkit/tests/transport.rs @@ -579,3 +579,249 @@ fn dns_policy_populates_pinned_addrs_for_resolved_host() { .unwrap() .is_empty()); } + +#[tokio::test] +async fn social_media_registry_returns_metadata_without_loading_embeds() { + let body = serde_json::to_vec(&json!({ + "type": "video", "title": "A public post", "author_name": "Creator", + "author_url": "https://example.test/creator", "description": "

A description & details

", + "thumbnail_url": "https://example.test/image.jpg", + "html": "" + })) + .unwrap(); + for (url, endpoint, canonical, format) in [ + ( + "https://www.tiktok.com/@creator/video/123?tracking=secret", + "www.tiktok.com/oembed", + "https://www.tiktok.com/@creator/video/123", + "tiktok_video", + ), + ( + "https://player.vimeo.com/video/123", + "vimeo.com/api/oembed.json", + "https://vimeo.com/123", + "vimeo_video", + ), + ( + "https://soundcloud.com/creator/track", + "soundcloud.com/oembed", + "https://soundcloud.com/creator/track", + "soundcloud_track", + ), + ( + "https://soundcloud.com/creator/sets/playlist", + "soundcloud.com/oembed", + "https://soundcloud.com/creator/sets/playlist", + "soundcloud_playlist", + ), + ] { + let mock = Arc::new(MockTransport::new().route(endpoint, 200, "application/json", &body)); + let response = fetchkit::FetcherRegistry::with_defaults() + .fetch(FetchRequest::new(url), options_with(mock.clone())) + .await + .unwrap(); + assert_eq!(response.status_code, 200); + assert_eq!(response.format.as_deref(), Some(format)); + let content = response.content.unwrap(); + for expected in [ + "# A public post", + "Creator", + "A **description** & details", + "Public metadata only", + canonical, + ] { + assert!(content.contains(expected), "{content}"); + } + assert!(!content.contains("login", + "{}", + r#"{"type":"link"}"#, + r#"{"type":"video","title":4}"#, + ] { + let mock = Arc::new(MockTransport::new().route( + "/oembed", + 200, + "application/json", + body.as_bytes(), + )); + let result = fetchkit::SocialMediaFetcher::new() + .fetch( + &FetchRequest::new("https://vimeo.com/123"), + &options_with(mock), + ) + .await; + assert!(matches!(result, Err(FetchError::FetcherError(_)))); + } +} + +#[tokio::test] +async fn social_media_enforces_original_and_derived_url_policy_before_transport() { + // THREAT[TM-INPUT-007]/[TM-SSRF-010]: all URL policy applies to API requests too. + for policy in 0..5 { + let mock = Arc::new(MockTransport::new()); + let mut options = options_with(mock.clone()); + match policy { + 0 => options.blocked_hosts = vec!["www.tiktok.com".into()], + 1 => options.allowed_ports = vec![80], + 2 => options.block_prefixes = vec!["https://www.tiktok.com/oembed".into()], + 3 => options.allow_prefixes = vec!["https://tiktok.com/@creator".into()], + 4 => options.block_prefixes = vec!["https://tiktok.com/@creator".into()], + _ => unreachable!(), + } + let result = fetchkit::SocialMediaFetcher::new() + .fetch( + &FetchRequest::new("https://tiktok.com/@creator/video/123"), + &options, + ) + .await; + assert!( + matches!(result, Err(FetchError::BlockedUrl)), + "policy {policy}: {result:?}" + ); + assert!(mock.calls().is_empty()); + } +} + +#[tokio::test] +async fn social_media_handles_minimal_metadata_head_and_size_limits() { + let body = br#"{"type":"video"}"#; + let mock = Arc::new(MockTransport::new().route("/oembed", 200, "application/json", body)); + let request = FetchRequest::new("https://tiktok.com/@creator/video/123"); + let fetcher = fetchkit::SocialMediaFetcher::new(); + let result = fetcher + .fetch(&request, &options_with(mock.clone())) + .await + .unwrap(); + assert!(result.content.unwrap().starts_with("# TikTok")); + let mut options = options_with(mock.clone()); + options.max_body_size = Some(40); + let result = fetcher.fetch(&request, &options).await.unwrap(); + assert_eq!(result.truncated, Some(true)); + assert!(result.content.unwrap().len() <= 40); + options.max_body_size = Some(4); + assert!( + fetcher.fetch(&request, &options).await.is_err(), + "truncated JSON must not masquerade as metadata" + ); + let mut head = request; + head.method = Some(HttpMethod::Head); + let result = fetcher.fetch(&head, &options).await.unwrap(); + assert_eq!(result.status_code, 200); + assert!(result.content.is_none()); + assert_eq!(mock.calls().last().unwrap().1, TransportMethod::Head); + + let body = br#"{"type":"rich","title":"\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9\u00e9"}"#; + let mock = Arc::new(MockTransport::new().route("/oembed", 200, "application/json", body)); + let mut options = options_with(mock); + options.max_body_size = Some(body.len()); + let result = fetcher + .fetch( + &FetchRequest::new("https://soundcloud.com/artist/track"), + &options, + ) + .await + .unwrap(); + assert_eq!(result.truncated, Some(true)); + assert!(result.content.unwrap().len() <= body.len()); +} + +#[tokio::test] +async fn youtube_video_variants_use_canonical_oembed_and_caption_requests() { + for url in [ + "https://youtube.com/shorts/abc_123", + "https://youtube.com/live/abc_123", + "https://www.youtube-nocookie.com/embed/abc_123", + ] { + let mock = Arc::new( + MockTransport::new() + .route( + "/oembed", + 200, + "application/json", + br#"{"title":"Video title","author_name":"Channel"}"#, + ) + .route( + "/api/timedtext", + 200, + "text/xml", + b"Hello captions", + ), + ); + let result = fetchkit::FetcherRegistry::with_defaults() + .fetch(FetchRequest::new(url), options_with(mock.clone())) + .await + .unwrap(); + assert_eq!(result.format.as_deref(), Some("youtube_video")); + let content = result.content.unwrap(); + assert!(content.contains("# Video title")); + assert!(content.contains("Hello captions")); + let calls = mock.calls(); + assert_eq!(calls.len(), 2); + let metadata_url = url::Url::parse(&calls[0].0).unwrap(); + assert_eq!( + metadata_url + .query_pairs() + .find(|(key, _)| key == "url") + .unwrap() + .1, + "https://www.youtube.com/watch?v=abc_123" + ); + } +} + +#[tokio::test] +async fn social_media_rejects_incomplete_body_even_with_valid_json_prefix() { + let body = br#"{"type":"video"} trailing data"#; + let mock = Arc::new(MockTransport::new().route("/oembed", 200, "application/json", body)); + let mut options = options_with(mock); + options.max_body_size = Some(br#"{"type":"video"}"#.len()); + let result = fetchkit::SocialMediaFetcher::new() + .fetch(&FetchRequest::new("https://vimeo.com/123"), &options) + .await; + assert!( + matches!(result, Err(FetchError::FetcherError(message)) if message.contains("Incomplete")) + ); +} diff --git a/docs/social-media.md b/docs/social-media.md new file mode 100644 index 0000000..82ad9f9 --- /dev/null +++ b/docs/social-media.md @@ -0,0 +1,53 @@ +# Social media fetching + +Social URLs are recognized automatically by the Rust library, CLI, MCP server, +and Python bindings. No API keys or extra configuration are needed for these +public metadata fetchers. + +| Platform | Supported URLs | Result format | +| --- | --- | --- | +| TikTok | `https://www.tiktok.com/@user/video/123` | `tiktok_video` | +| YouTube | `/watch?v=ID`, `/shorts/ID`, `/live/ID`, `/embed/ID`, `youtu.be/ID`, `youtube-nocookie.com/embed/ID` | `youtube_video` | +| Vimeo | `https://vimeo.com/123`, `https://player.vimeo.com/video/123` | `vimeo_video` | +| SoundCloud | `https://soundcloud.com/user/track` | `soundcloud_track` | +| SoundCloud playlists | `https://soundcloud.com/user/sets/playlist` | `soundcloud_playlist` | +| X / Twitter (existing) | `https://x.com/user/status/123`, `twitter.com/user/status/123` | `twitter_tweet` | + +```sh +fetchkit fetch 'https://www.tiktok.com/@scout2015/video/6718335390845095173' +fetchkit fetch 'https://www.youtube.com/shorts/jNQXAC9IVRw' +fetchkit fetch 'https://vimeo.com/863362136' +fetchkit fetch 'https://soundcloud.com/forss/flickermood' +``` + +TikTok, Vimeo and SoundCloud return Markdown containing the public title, author, +description and thumbnail URL when supplied by the provider. Their responses are +explicitly labeled **public metadata only**. No video/audio is downloaded or +transcribed. YouTube separately attempts English captions; they are often +unavailable. X/Twitter retains its existing post-content fetcher. + +TikTok's current embed API can omit title/author fields. In that case the result +contains the platform and canonical video URL, without invented content. Embed +HTML and player scripts are ignored. Provider errors (including private/deleted +content, rate limits and redirects) retain their HTTP status and return an error +instead of fabricated metadata. Invalid or oversized JSON returns a fetch error. + +Use canonical public links. TikTok short links (`vm.tiktok.com`, `vt.tiktok.com`), +profiles, private sharing URLs, and other social platforms use the default HTML +fetcher; useful content is not guaranteed. Login walls and access restrictions +are not bypassed. Tracking queries are removed from URLs sent to oEmbed. + +URL policies must allow both the supplied page URL and the provider endpoint: + +- TikTok: `https://www.tiktok.com/oembed` +- YouTube: `https://www.youtube.com/oembed` and, for optional captions, `/api/timedtext` +- Vimeo: `https://vimeo.com/api/oembed.json` +- SoundCloud: `https://soundcloud.com/oembed` + +The new oEmbed fetcher does not follow API redirects or retrieve returned image, +author, or player URLs. Existing DNS restrictions, body limits and URL policy +apply. Rendered output is bounded and reports `size` and `truncated`. + +Provider references: [TikTok](https://developers.tiktok.com/docs/en/embed-videos), +[Vimeo](https://developer.vimeo.com/api/oembed/videos), +[SoundCloud](https://developers.soundcloud.com/docs/oembed). diff --git a/knowledge/foundations/fetchers.md b/knowledge/foundations/fetchers.md index fafe519..de22057 100644 --- a/knowledge/foundations/fetchers.md +++ b/knowledge/foundations/fetchers.md @@ -300,10 +300,31 @@ error names the format and says conversion is not enabled. #### YouTubeFetcher -- Matches: `https://youtube.com/watch?v={id}`, `https://youtu.be/{id}` -- Behavior: Fetches video metadata via oEmbed API +- Matches watch, Shorts, live, embed and `youtu.be` video URLs; privacy-enhanced `youtube-nocookie.com/embed/{id}` is also supported +- Video IDs allow only ASCII alphanumerics, `_` and `-`; extra path segments and query-injection characters are rejected +- Behavior: Fetches video metadata via oEmbed API and best-effort English captions via the legacy timedtext endpoint - Response format field: `"youtube_video"` +#### SocialMediaFetcher + +- Matches canonical TikTok `@user/video/{id}`, Vimeo `/{id}` and player embed URLs, + and SoundCloud `/{user}/{track}` and `/{user}/sets/{playlist}` URLs. +- Uses fixed official oEmbed endpoints without credentials; no endpoint discovery. +- Returns provider-supplied title, author, description and thumbnail URL when available; HTML descriptions are converted to Markdown. + TikTok may return only embed metadata; this produces a platform heading and canonical URL. +- Formats: `tiktok_video`, `vimeo_video`, `soundcloud_track`, `soundcloud_playlist`. +- Always labels output as public metadata only. Does not download media, obtain + transcripts, execute embed HTML, or fetch returned thumbnail/author URLs. +- Query parameters and userinfo are not sent to the oEmbed provider. Short links, + profiles, private sharing paths and unsupported routes use the default fetcher. +- Validates the original and fully constructed API URLs against host/port and + prefix policy; uses the shared bounded transport with DNS pinning and no redirects. +- Preserves non-2xx API status as an error response; invalid/oversized JSON is an + error, while rendered output is UTF-8 safely capped with size/truncation metadata. +- HEAD requests use HEAD on the oEmbed endpoint and return no body. +- Unit and injected-transport tests cover matching, registration, metadata-only + responses, policy denial before transport, provider errors, HEAD and size caps. + #### ArXivFetcher - Matches: `https://arxiv.org/abs/{id}` and `https://arxiv.org/pdf/{id}` diff --git a/knowledge/log.md b/knowledge/log.md index 8ddaa1e..e0d4cd1 100644 --- a/knowledge/log.md +++ b/knowledge/log.md @@ -1,5 +1,9 @@ # Fetchkit Knowledge Update Log +## 2026-10-27 + +* **Social media**: Add public TikTok/Vimeo/SoundCloud oEmbed metadata fetching with fixed endpoints, policy enforcement, bounded output and explicit media limits. Extend YouTube URL matching to Shorts/live/embeds and validate IDs before API query construction. + ## 2026-10-25 * **Publishing authentication**: Bind both Rust crates to GitHub OIDC in `publish.yml`; publishing jobs use short-lived tokens from a pinned official auth action, with no legacy secret fallback. Document recovery from current main with the original immutable release source tag and actual registry consumer verification. diff --git a/knowledge/security/threat-model.md b/knowledge/security/threat-model.md index b4a01c6..12fe398 100644 --- a/knowledge/security/threat-model.md +++ b/knowledge/security/threat-model.md @@ -543,6 +543,14 @@ None — all previously open threats have been mitigated. | Authority-scoped signatures | TM-AUTH | Signature covers `@authority`; per-host binding | | Graceful signing failure | TM-AUTH | Signing errors logged, request proceeds unsigned | +Social media oEmbed requests use fixed TikTok, Vimeo and SoundCloud endpoints, +validate both the original and complete derived URL against policy (TM-INPUT-007), +and use DNS-pinned single-hop transport (TM-SSRF-010). Embed HTML is ignored; +metadata URLs are never followed. Canonical resource URLs omit query parameters +and userinfo. Body/output limits apply (TM-DOS-001). YouTube video identifiers +reject query separators before timedtext URL construction. Coverage lives in +`fetchers/social_media.rs`, `fetchers/youtube.rs`, and `tests/transport.rs`. + ## Release Authentication (TM-RELEASE-001) **Threat**: A leaked long-lived CI publisher secret permits unauthorized crate