ui change

apps/desktop/src/main.rs

@@ -20,6 +20,7 @@ const HEALTHCHECK_INTERVAL: Duration = Duration::from_millis(250);
 fn main() {
     tauri::Builder::default()
         .manage(BackendState(Mutex::new(None)))
+        .invoke_handler(tauri::generate_handler![open_external_url])
         .setup(|app| {
             // Anything that can fail is handled here so we never let an error
             // propagate out of the setup hook (that would abort() the process
@@ -44,6 +45,31 @@ fn main() {
         });
 }
 
+#[tauri::command]
+fn open_external_url(url: String) -> Result<(), String> {
+    let trimmed = url.trim();
+    if !(trimmed.starts_with("http://") || trimmed.starts_with("https://")) {
+        return Err("only HTTP(S) URLs can be opened externally".to_string());
+    }
+
+    let status = if cfg!(target_os = "windows") {
+        Command::new("rundll32")
+            .args(["url.dll,FileProtocolHandler", trimmed])
+            .status()
+    } else if cfg!(target_os = "macos") {
+        Command::new("open").arg(trimmed).status()
+    } else {
+        Command::new("xdg-open").arg(trimmed).status()
+    }
+    .map_err(|err| err.to_string())?;
+
+    if status.success() {
+        Ok(())
+    } else {
+        Err(format!("failed to open URL externally: {status}"))
+    }
+}
+
 fn start_backend_and_window(app: &tauri::App) -> Result<(), String> {
     let port = reserve_port()?;
     let mut child = spawn_backend(app.handle(), port)?;

backend/content_cleaner/router.py

@@ -1,7 +1,7 @@
 from fastapi import APIRouter
 
-from .schemas import CleanContentRequest, CleanContentResponse
-from .service import clean_html, clean_stored_article
+from .schemas import CleanContentRequest, CleanContentResponse, WebPageResponse
+from .service import clean_html, clean_stored_article, fetch_entry_web_page
 
 router = APIRouter(prefix="/content", tags=["content"])
 
@@ -14,3 +14,8 @@ async def clean_content(request: CleanContentRequest) -> CleanContentResponse:
 @router.get("/entries/{article_id}/clean", response_model=CleanContentResponse)
 async def clean_stored_content(article_id: str) -> CleanContentResponse:
     return clean_stored_article(article_id)
+
+
+@router.get("/entries/{article_id}/web", response_model=WebPageResponse)
+async def fetch_stored_web_page(article_id: str) -> WebPageResponse:
+    return fetch_entry_web_page(article_id)

backend/content_cleaner/schemas.py

@@ -16,3 +16,11 @@ class CleanContentResponse(BaseModel):
     content_hash: str | None = None
     word_count: int
     reading_time_minutes: int
+
+
+class WebPageResponse(BaseModel):
+    article_id: str
+    url: str
+    final_url: str
+    content_type: str | None = None
+    html: str

backend/content_cleaner/service.py

@@ -3,7 +3,9 @@
 import math
 import sqlite3
 from pathlib import Path
-from urllib.parse import urljoin
+from urllib.error import HTTPError, URLError
+from urllib.parse import urljoin, urlsplit
+from urllib.request import Request, urlopen
 
 from bleach import clean as bleach_clean
 from bs4 import BeautifulSoup
@@ -13,7 +15,7 @@
 from app.schemas.content import ArticleContent
 from db import get_article, get_article_content, save_article_content
 
-from .schemas import CleanContentRequest, CleanContentResponse
+from .schemas import CleanContentRequest, CleanContentResponse, WebPageResponse
 
 logger = logging.getLogger(__name__)
 
@@ -102,8 +104,9 @@ def clean_html(request: CleanContentRequest) -> CleanContentResponse:
     """
     raw_html = request.raw_html
 
-    # 1. Parse
+    # 1. Parse and prefer the actual article container when the source is a full web page.
     soup = BeautifulSoup(raw_html, "lxml")
+    soup = _extract_reader_candidate(soup)
 
     # 2. Sanitize — strip scripts, event handlers, dangerous markup
     sanitized = bleach_clean(
@@ -152,7 +155,7 @@ def clean_html(request: CleanContentRequest) -> CleanContentResponse:
 
 
 def clean_stored_article(article_id: str) -> CleanContentResponse:
-    """Read raw HTML from storage, clean it, and persist the result."""
+    """Read article HTML, preferring the canonical URL when feed HTML is only metadata."""
     try:
         content = get_article_content(article_id)
     except sqlite3.OperationalError as err:
@@ -169,18 +172,24 @@ def clean_stored_article(article_id: str) -> CleanContentResponse:
 
     entry = get_article(article_id)
     source_url = entry.url if entry else ""
+    raw_html = content.raw_html
+    if entry is not None and _should_fetch_reader_source(content.raw_html):
+        fetched = _try_fetch_web_page(entry.url)
+        if fetched is not None:
+            raw_html = fetched.html
+            source_url = fetched.final_url
 
     result = clean_html(
         CleanContentRequest(
-            raw_html=content.raw_html,
+            raw_html=raw_html,
             source_url=source_url,
             article_id=article_id,
         )
     )
 
     save_article_content(
         article_id=article_id,
-        raw_html=content.raw_html,
+        raw_html=raw_html,
         cleaned_html=result.cleaned_html,
         cleaned_markdown=result.cleaned_markdown,
         plain_text=result.plain_text,
@@ -195,6 +204,69 @@ def clean_stored_article(article_id: str) -> CleanContentResponse:
     return result
 
 
+def fetch_entry_web_page(article_id: str, timeout_seconds: float = 15.0) -> WebPageResponse:
+    """Fetch the article's canonical URL and return HTML suitable for iframe srcDoc."""
+    try:
+        entry = get_article(article_id)
+    except sqlite3.OperationalError as err:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Article '{article_id}' not found",
+        ) from err
+
+    if entry is None:
+        raise HTTPException(
+            status_code=404,
+            detail=f"Article '{article_id}' not found",
+        )
+
+    fetched = _try_fetch_web_page(entry.url, timeout_seconds=timeout_seconds)
+    if fetched is None:
+        raise HTTPException(status_code=502, detail="Web page request failed")
+
+    html = _inject_base_url(fetched.html, fetched.final_url)
+    return WebPageResponse(
+        article_id=article_id,
+        url=fetched.url,
+        final_url=fetched.final_url,
+        content_type=fetched.content_type,
+        html=html,
+    )
+
+
+def _try_fetch_web_page(url: str, timeout_seconds: float = 15.0) -> WebPageResponse | None:
+    url = _validate_web_url(url)
+    headers = {
+        "User-Agent": "Lumen/0.1 web-preview",
+        "Accept": "text/html,application/xhtml+xml;q=0.9,*/*;q=0.8",
+    }
+    request = Request(url, headers=headers)
+
+    try:
+        with urlopen(request, timeout=timeout_seconds) as response:
+            body = response.read()
+            content_type = response.headers.get("Content-Type")
+            final_url = response.url
+    except HTTPError as exc:
+        logger.info("Web page server returned HTTP %s for %s", exc.code, url)
+        return None
+    except TimeoutError:
+        logger.info("Web page request timed out for %s", url)
+        return None
+    except (URLError, OSError) as exc:
+        logger.info("Web page request failed for %s: %s", url, exc)
+        return None
+
+    html = _decode_response_body(body, content_type)
+    return WebPageResponse(
+        article_id="",
+        url=url,
+        final_url=final_url,
+        content_type=content_type,
+        html=html,
+    )
+
+
 def load_article_content(
     article_id: str,
     db_path: Path | str | None = None,
@@ -224,6 +296,74 @@ def save_cleaned_article_content(
 
 # ── Private helpers ──────────────────────────────────────────────
 
+def _should_fetch_reader_source(raw_html: str) -> bool:
+    text = BeautifulSoup(raw_html or "", "lxml").get_text(separator="\n", strip=True)
+    if not text:
+        return True
+
+    lower = text.lower()
+    hnrss_markers = [
+        "article url:",
+        "comments url:",
+        "points:",
+        "# comments:",
+    ]
+    if sum(1 for marker in hnrss_markers if marker in lower) >= 2:
+        return True
+
+    return False
+
+def _validate_web_url(url: str) -> str:
+    value = url.strip()
+    parts = urlsplit(value)
+    if parts.scheme not in {"http", "https"} or not parts.netloc:
+        raise HTTPException(status_code=409, detail="Article URL is not an absolute HTTP(S) URL")
+    return value
+
+
+def _decode_response_body(body: bytes, content_type: str | None) -> str:
+    encoding = "utf-8"
+    if content_type:
+        for part in content_type.split(";"):
+            key, _, value = part.strip().partition("=")
+            if key.lower() == "charset" and value.strip():
+                encoding = value.strip().strip('"')
+                break
+    try:
+        return body.decode(encoding, errors="replace")
+    except LookupError:
+        return body.decode("utf-8", errors="replace")
+
+
+def _inject_base_url(html: str, final_url: str) -> str:
+    soup = BeautifulSoup(html, "lxml")
+    base = soup.new_tag("base", href=final_url)
+    if soup.head:
+        existing = soup.head.find("base")
+        if existing:
+            existing["href"] = final_url
+        else:
+            soup.head.insert(0, base)
+    elif soup.html:
+        head = soup.new_tag("head")
+        head.append(base)
+        soup.html.insert(0, head)
+    else:
+        return f'<base href="{final_url}">\n{html}'
+    return str(soup)
+
+
+def _extract_reader_candidate(soup: BeautifulSoup) -> BeautifulSoup:
+    for selector in ("article", "main", "[role='main']"):
+        candidate = soup.select_one(selector)
+        if candidate is not None and candidate.get_text(strip=True):
+            return BeautifulSoup(str(candidate), "lxml")
+
+    for tag in soup.select("script, style, noscript, nav, header, footer, aside"):
+        tag.decompose()
+    return soup
+
+
 def _count_words(text: str) -> int:
     """Count words in plain text by splitting on whitespace."""
     return len(text.split()) if text.strip() else 0

backend/db/repositories/article_repo.py

@@ -151,7 +151,10 @@ def list_articles(
     if where_clauses:
         sql += " WHERE " + " AND ".join(where_clauses)
 
-    sql += " ORDER BY articles.published_at DESC, articles.created_at DESC LIMIT ? OFFSET ?"
+    sql += """
+        ORDER BY articles.published_at DESC, articles.created_at DESC, articles.id ASC
+        LIMIT ? OFFSET ?
+    """
     params.extend([limit, offset])
 
     with connection(db_path) as conn:
@@ -351,7 +354,7 @@ def search_articles(
                 LIMIT 1
             )
         WHERE article_fts MATCH ?
-        ORDER BY articles.published_at DESC, articles.created_at DESC
+        ORDER BY articles.published_at DESC, articles.created_at DESC, articles.id ASC
         LIMIT ? OFFSET ?
     """
     with connection(db_path) as conn:
@@ -406,7 +409,7 @@ def search_articles(
                 WHERE articles.title LIKE ?
                     OR articles.summary LIKE ?
                     OR article_search.plain_text LIKE ?
-                ORDER BY articles.published_at DESC, articles.created_at DESC
+                ORDER BY articles.published_at DESC, articles.created_at DESC, articles.id ASC
                 LIMIT ? OFFSET ?
                 """,
                 (like_keyword, like_keyword, like_keyword, limit, offset),