Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

### Added

- Public TikTok, Vimeo, and SoundCloud metadata fetching through oEmbed, available automatically in the library, CLI, MCP server, and Python bindings.
- YouTube Shorts, live, embed, and privacy-enhanced embed URL support, with validated video IDs.

## [0.6.0] - 2026-10-23

### Highlights
Expand Down
5 changes: 4 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,13 +30,16 @@ fetchers match first; the default fetcher handles everything else.
- `StackOverflowFetcher` - Stack Overflow and Stack Exchange question URLs
- `PackageRegistryFetcher` - PyPI, crates.io, and npm package pages
- `WikipediaFetcher` - Wikipedia article URLs
- `YouTubeFetcher` - YouTube watch and `youtu.be` URLs
- `YouTubeFetcher` - YouTube watch, Shorts, live, embed, and `youtu.be` URLs
- `SocialMediaFetcher` - TikTok and Vimeo videos; SoundCloud tracks and playlists
- `ArXivFetcher` - arXiv abstract and PDF URLs
- `HackerNewsFetcher` - Hacker News item threads
- `RSSFeedFetcher` - RSS and Atom feed URLs
- `DocsSiteFetcher` - docs sites with `llms.txt`/`llms-full.txt` support
- `DefaultFetcher` - all remaining HTTP/HTTPS URLs with HTML conversion, streaming, timeout handling, and binary detection

See [Social media fetching](docs/social-media.md) for supported URLs and metadata limits.

## Built-in Content Processors

Content processors run after a fetcher retrieves a bounded response body. They
Expand Down
22 changes: 13 additions & 9 deletions crates/fetchkit/src/fetchers/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ mod jupyter_notebook;
mod package_registry;
mod pubmed;
mod rss_feed;
mod social_media;
mod stackoverflow;
mod twitter;
mod wikipedia;
Expand All @@ -42,6 +43,7 @@ pub use jupyter_notebook::JupyterNotebookFetcher;
pub use package_registry::PackageRegistryFetcher;
pub use pubmed::PubMedFetcher;
pub use rss_feed::RSSFeedFetcher;
pub use social_media::SocialMediaFetcher;
pub use stackoverflow::StackOverflowFetcher;
pub use twitter::TwitterFetcher;
pub use wikipedia::WikipediaFetcher;
Expand Down Expand Up @@ -189,6 +191,7 @@ impl FetcherRegistry {
registry.register(Box::new(PackageRegistryFetcher::new()));
registry.register(Box::new(WikipediaFetcher::new()));
registry.register(Box::new(YouTubeFetcher::new()));
registry.register(Box::new(SocialMediaFetcher::new()));
registry.register(Box::new(ArXivFetcher::new()));
registry.register(Box::new(CrossrefFetcher::new()));
registry.register(Box::new(IetfRfcFetcher::new()));
Expand Down Expand Up @@ -400,15 +403,16 @@ mod tests {
assert_eq!(registry.fetchers[10].name(), "package_registry");
assert_eq!(registry.fetchers[11].name(), "wikipedia");
assert_eq!(registry.fetchers[12].name(), "youtube");
assert_eq!(registry.fetchers[13].name(), "arxiv");
assert_eq!(registry.fetchers[14].name(), "crossref");
assert_eq!(registry.fetchers[15].name(), "ietf_rfc");
assert_eq!(registry.fetchers[16].name(), "pubmed");
assert_eq!(registry.fetchers[17].name(), "hackernews");
assert_eq!(registry.fetchers[18].name(), "rss_feed");
assert_eq!(registry.fetchers[19].name(), "docs_site");
assert_eq!(registry.fetchers[20].name(), "default");
assert_eq!(registry.fetchers.len(), 21);
assert_eq!(registry.fetchers[13].name(), "social_media");
assert_eq!(registry.fetchers[14].name(), "arxiv");
assert_eq!(registry.fetchers[15].name(), "crossref");
assert_eq!(registry.fetchers[16].name(), "ietf_rfc");
assert_eq!(registry.fetchers[17].name(), "pubmed");
assert_eq!(registry.fetchers[18].name(), "hackernews");
assert_eq!(registry.fetchers[19].name(), "rss_feed");
assert_eq!(registry.fetchers[20].name(), "docs_site");
assert_eq!(registry.fetchers[21].name(), "default");
assert_eq!(registry.fetchers.len(), 22);
}

#[test]
Expand Down
Loading
Loading