diff --git a/backend/feed_engine/fetcher.py b/backend/feed_engine/fetcher.py index a894138..6f4324f 100644 --- a/backend/feed_engine/fetcher.py +++ b/backend/feed_engine/fetcher.py @@ -112,7 +112,9 @@ from dataclasses import dataclass from urllib.error import HTTPError, URLError from urllib.parse import urlsplit -from urllib.request import Request, urlopen +from urllib.request import HTTPSHandler, Request, build_opener + +import certifi from feed_engine.errors import FeedEngineError @@ -178,10 +180,16 @@ def _fetch_feed_sync( request = Request(url, headers=headers) + if url.startswith("https://"): + ssl_context = ssl.create_default_context(cafile=certifi.where()) + opener = build_opener(HTTPSHandler(context=ssl_context)) + else: + opener = build_opener() + print("[FETCH START]", url) try: - with urlopen(request, timeout=timeout_seconds) as response: + with opener.open(request, timeout=timeout_seconds) as response: print("[FETCH OK]", response.status) return FetchResponse( diff --git a/backend/pyproject.toml b/backend/pyproject.toml index 4125203..63d7e0a 100644 --- a/backend/pyproject.toml +++ b/backend/pyproject.toml @@ -66,6 +66,7 @@ dependencies = [ "bleach>=6.2", "lxml>=5.3", "markdownify>=1.1", + "certifi>=2024", ] [dependency-groups] diff --git a/backend/uv.lock b/backend/uv.lock index 7718381..1b79e30 100644 --- a/backend/uv.lock +++ b/backend/uv.lock @@ -338,6 +338,7 @@ source = { virtual = "." } dependencies = [ { name = "beautifulsoup4" }, { name = "bleach" }, + { name = "certifi" }, { name = "fastapi" }, { name = "httpx" }, { name = "lxml" }, @@ -362,6 +363,7 @@ dev = [ requires-dist = [ { name = "beautifulsoup4", specifier = ">=4.12" }, { name = "bleach", specifier = ">=6.2" }, + { name = "certifi", specifier = ">=2024" }, { name = "fastapi", specifier = ">=0.115" }, { name = "httpx", specifier = ">=0.27" }, { name = "lxml", specifier = ">=5.3" },