Repository navigation
Fix/update playwright chromium #921
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -12,15 +12,21 @@ | |
| import threading | ||
| import time | ||
| import os | ||
| from typing import List, Dict, Optional, Any, Tuple | ||
| from typing import List, Dict, Optional, Any, Tuple, Set | ||
| from concurrent.futures import Future, wait as wait_futures | ||
| import queue | ||
| import uuid | ||
| from urllib.parse import urlparse | ||
|
|
||
| import psutil | ||
| from playwright.async_api import async_playwright, Playwright, Browser, BrowserContext | ||
|
|
||
| from dtable_events.app.config import CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER | ||
| from dtable_events.app.config import ( | ||
| CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER, | ||
| CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS, CONVERT_PDF_ALLOWED_HOSTS, | ||
| CONVERT_PDF_BROWSER_EXECUTABLE_PATH, | ||
| DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT, | ||
| ) | ||
| from dtable_events.convert_page.utils import wait_for_images | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
|
|
@@ -44,12 +50,16 @@ def __init__( | |
| page_timeout: int = 30_000, | ||
| browser_launch_kwargs: Optional[Dict[str, Any]] = None, | ||
| health_check_interval: int = 30, | ||
| block_external_requests: bool = False, | ||
| allowed_hosts: Optional[List[str]] = None, | ||
| ): | ||
| self.num_browsers = max(1, num_browsers) | ||
| self.contexts_per_browser = max(1, contexts_per_browser) | ||
| self.page_timeout = page_timeout | ||
| self.browser_launch_kwargs = browser_launch_kwargs or {} | ||
| self.health_check_interval = health_check_interval | ||
| self.block_external_requests = block_external_requests | ||
| self._allowed_hosts = self._build_allowed_hosts(allowed_hosts or []) | ||
|
|
||
| # thread & loop | ||
| self._thread: Optional[threading.Thread] = None | ||
|
|
@@ -238,19 +248,78 @@ async def _worker_main(self): | |
|
|
||
| async def _launch_browser(self, idx: int) -> Browser: | ||
| """Launch a single Chromium browser instance. Isolated to worker loop.""" | ||
| kwargs = dict(executable_path=os.environ.get('CONVERT_PDF_CHROME_PATH', '/usr/bin/google-chrome'), headless=True) | ||
| kwargs = dict(headless=True) | ||
| kwargs.update(self.browser_launch_kwargs) | ||
| # safe defaults | ||
| if 'args' not in kwargs: | ||
| kwargs['args'] = [ | ||
| '--no-sandbox', | ||
| '--disable-dev-shm-usage', | ||
| '--disable-gpu', | ||
| '--disable-software-rasterizer', | ||
| ] | ||
| if CONVERT_PDF_BROWSER_EXECUTABLE_PATH: | ||
| kwargs['executable_path'] = CONVERT_PDF_BROWSER_EXECUTABLE_PATH | ||
| # Chromium refuses to enable the sandbox when running as root | ||
| # (see https://crbug.com/638180), so only enable it for non-root users. | ||
| if os.geteuid() != 0: | ||
| kwargs['chromium_sandbox'] = True | ||
| browser = await self._playwright.chromium.launch(**kwargs) | ||
| return browser | ||
|
|
||
| def _build_allowed_hosts(self, extra_hosts: List[str]) -> Set[str]: | ||
| """ | ||
| Derive the set of hostnames that are allowed to be requested during rendering. | ||
|
|
||
| Always allows loopback, and derives hosts from the internal service URLs so that | ||
| the rendered page's own resources (html/js/css/images served from dtable-web, | ||
| dtable-server and the file server) are never blocked. Extra hosts come from the | ||
| CONVERT_PDF_ALLOWED_HOSTS configuration. | ||
| """ | ||
| hosts: Set[str] = {'127.0.0.1', 'localhost', '::1'} | ||
| service_urls = [DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT] | ||
| for url in service_urls: | ||
| if not url: | ||
| continue | ||
| try: | ||
| host = urlparse(url).hostname | ||
| except Exception: | ||
| host = None | ||
| if host: | ||
| hosts.add(host) | ||
| for host in extra_hosts: | ||
| if host and isinstance(host, str): | ||
| hosts.add(host.strip()) | ||
| return hosts | ||
|
|
||
| async def _apply_request_filter(self, context: BrowserContext): | ||
| """ | ||
| Attach route handlers to the context that block external requests and WebSocket | ||
| connections when CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS is enabled. Requests whose | ||
| host is in the allowed set are continued / connected, all others are aborted / closed. | ||
| """ | ||
| if not self.block_external_requests: | ||
| return | ||
|
|
||
| async def _route_handler(route): | ||
| try: | ||
| host = urlparse(route.request.url).hostname | ||
| except Exception: | ||
| host = None | ||
| # allow requests without a host (data:/blob: etc.) as well as allowed hosts | ||
| if host is None or host in self._allowed_hosts: | ||
| await route.continue_() | ||
| else: | ||
| logger.debug("block external request: %s", route.request.url) | ||
| await route.abort() | ||
|
|
||
| async def _ws_handler(ws): | ||
| try: | ||
| host = urlparse(ws.url).hostname | ||
| except Exception: | ||
| host = None | ||
| # allow websockets without a host as well as allowed hosts | ||
| if host is None or host in self._allowed_hosts: | ||
| ws.connect_to_server() | ||
| else: | ||
| logger.debug("block external websocket: %s", ws.url) | ||
| await ws.close() | ||
|
|
||
| await context.route("**/*", _route_handler) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Critical] 请求路由不构成网络隔离 影响:过滤默认关闭,且即使配置开启,context.route 也不拦截 WebSocket。真实 Chromium 中,白名单外的本地 HTTP 请求被阻止,但同一目标的 ws:// 连接仍成功;用户可影响的页面仍能访问非白名单网络,未满足任务的网络隔离要求。 建议修复:在独立渲染进程或容器的网络层默认拒绝出站访问,仅经受控通道获取渲染必需资源;应用层同时默认开启过滤,并显式拒绝非白名单 WebSocket。 There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 当前 head 复核:两个 PR 新增的 route_web_socket 已修复此评论中的 WebSocket 绕过子项。使用真实 Chromium 156.0.8078.4 和非白名单本地目标实测,开启过滤后 HTTP、WebSocket 均被阻止,服务端未收到相应请求。 剩余阻塞仍属于本线程的网络隔离根因:CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS 仍默认 False,默认启动时相同 HTTP、WebSocket 目标均可连接;当前改动也未提供渲染进程的网络层出站隔离。建议默认启用过滤,并在独立渲染进程或容器实施默认拒绝出站、仅允许受控资源通道的网络策略;WebSocket 绕过不再作为当前缺陷。 |
||
| await context.route_web_socket("**/*", _ws_handler) | ||
|
|
||
| async def _cleanup_browsers(self): | ||
| # close all contexts per browser | ||
| for i, ctx_list in enumerate(self._contexts): | ||
|
|
@@ -374,6 +443,7 @@ async def _handle_new_batch(self, task: Dict[str, Any], fut: Future): | |
|
|
||
| # create a new context for the batch | ||
| context = await browser.new_context(ignore_https_errors=True) | ||
| await self._apply_request_filter(context) | ||
|
|
||
| # register context | ||
| self._contexts[browser_idx].append(context) | ||
|
|
@@ -432,6 +502,7 @@ async def _handle_task(self, task: Dict[str, Any], fut: Future): | |
| # create a fresh context for the batch and record it | ||
| try: | ||
| new_context = await browser.new_context(viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True) | ||
| await self._apply_request_filter(new_context) | ||
| # replace the old context in contexts list (best-effort) | ||
| try: | ||
| # remove old context reference if present | ||
|
|
@@ -544,6 +615,8 @@ def get_playwright_manager(): | |
| if not playwright_manager: | ||
| playwright_manager = RobustPlaywrightManager( | ||
| num_browsers=CONVERT_PDF_BROWSERS, | ||
| contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER | ||
| contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER, | ||
| block_external_requests=CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS, | ||
| allowed_hosts=CONVERT_PDF_ALLOWED_HOSTS, | ||
| ) | ||
| return playwright_manager | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -32,8 +32,6 @@ | |
| import_excel_csv_add_table_by_dtable_server, update_parsed_file_by_dtable_server, \ | ||
| parse_update_excel_upload_excel_to_json, parse_update_csv_upload_csv_to_json, parse_and_import_excel_csv_to_dtable, \ | ||
| parse_and_import_excel_csv_to_table, parse_and_update_file_to_table, parse_and_append_excel_csv_to_table | ||
| from dtable_events.convert_page.utils import get_chrome_data_dir, get_driver, open_page_view, wait_page_view, \ | ||
| gen_page_design_pdf_view_url, gen_document_pdf_view_url | ||
| from dtable_events.statistics.db import save_email_sending_records, batch_save_email_sending_records | ||
| from dtable_events.data_sync.data_sync_utils import run_sync_emails | ||
| from dtable_events.utils import is_valid_email, uuid_str_to_36_chars, gen_file_upload_url, uuid_str_to_32_chars | ||
|
|
@@ -46,7 +44,6 @@ | |
| SDOC_IMAGES_DIR, get_seadoc_download_link, gen_seadoc_base_dir, export_sdoc_prepare_images_folder,\ | ||
| batch_upload_sdoc_images, get_documents_config, save_documents_config | ||
| from dtable_events.app.log import setup_logger | ||
| from dtable_events.convert_page.utils import get_pdf_print_options, wait_for_images | ||
| from dtable_events.convert_page.manager import get_playwright_manager | ||
|
|
||
| dtable_io_logger = setup_logger('dtable_events_io', propagate=False) | ||
|
|
@@ -900,7 +897,7 @@ def convert_page_design_to_pdf(dtable_uuid, page_id, row_id, username): | |
| url += '?access-token=%s&need_convert=%s' % (access_token, 0) | ||
| filename = '%s_%s_%s.pdf' % (dtable_uuid, page_id, row_id) | ||
| try: | ||
| get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir) | ||
| get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#page-design-render-complete'}], target_dir) | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. [Critical] 空白末页导致整份导出失败 影响:页面设计前页有内容、末页没有 widget 时,前端只在最后页的最后 widget 内插入完成标记,因此该标记永远不存在。新增等待会在 5 秒后超时;真实 Chromium 验证中,原先无 selector 的调用能生成 PDF,此处改动后返回 [] 且整份文件不生成。 建议修复:协调前端在整体渲染完成后插入独立的完成标记,不依赖末页存在 widget,并增加“内容页+空白末页”的导出回归;保留等待,避免改回不可靠的固定延迟。 |
||
| except Exception as e: | ||
| dtable_io_logger.exception('dtable: %s plugin: page-design page: %s row: %s error: %s', dtable_uuid, page_id, row_id, e) | ||
|
|
||
|
|
@@ -927,7 +924,7 @@ def convert_document_to_pdf(dtable_uuid, doc_uuid, row_id, username): | |
| url += '?access-token=%s&need_convert=%s' % (access_token, 0) | ||
| filename = '%s_%s_%s.pdf' % (dtable_uuid, doc_uuid, row_id) | ||
| try: | ||
| get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir) | ||
| get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#document-render-complete'}], target_dir) | ||
| except Exception as e: | ||
| dtable_io_logger.exception('dtable: %s plugin: document doc_uuid: %s row: %s error: %s', dtable_uuid, doc_uuid, row_id, e) | ||
|
|
||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
[Critical] root 分支仍禁用 sandbox
影响:以任务描述的 root 方式运行时,此分支不会设置 chromium_sandbox=True,而 Playwright 默认关闭 sandbox。我用本 PR 的启动函数及 Playwright 1.64.1 实测,浏览器进程仍为 UID 0,命令行自动带有 --no-sandbox;删除手写参数并未消除高权限、无沙箱的风险。
建议修复:强制渲染器以专用非 root 用户运行,并始终启用 chromium_sandbox;检测到 root 时应拒绝启动而不是静默降级,同时更新服务启动配置。