Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions dtable_events/app/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,9 @@ def get_llm_prices(models):
# playwright
CONVERT_PDF_BROWSERS = configs.get('CONVERT_PDF_BROWSERS', default=2)
CONVERT_PDF_SESSIONS_PER_BROWSER = configs.get('CONVERT_PDF_SESSIONS_PER_BROWSER', default=3)
CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS = configs.get('CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS', default=False)
CONVERT_PDF_ALLOWED_HOSTS = configs.get('CONVERT_PDF_ALLOWED_HOSTS', default=[])
CONVERT_PDF_BROWSER_EXECUTABLE_PATH = configs.get('CONVERT_PDF_BROWSER_EXECUTABLE_PATH', default='')

# virus code
VIRUS_SCAN_ENABLED = configs.get('VIRUS_SCAN_ENABLED', default=False)
Expand Down
97 changes: 85 additions & 12 deletions dtable_events/convert_page/manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@
import threading
import time
import os
from typing import List, Dict, Optional, Any, Tuple
from typing import List, Dict, Optional, Any, Tuple, Set
from concurrent.futures import Future, wait as wait_futures
import queue
import uuid
from urllib.parse import urlparse

import psutil
from playwright.async_api import async_playwright, Playwright, Browser, BrowserContext

from dtable_events.app.config import CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER
from dtable_events.app.config import (
CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER,
CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS, CONVERT_PDF_ALLOWED_HOSTS,
CONVERT_PDF_BROWSER_EXECUTABLE_PATH,
DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT,
)
from dtable_events.convert_page.utils import wait_for_images

logger = logging.getLogger(__name__)
Expand All @@ -44,12 +50,16 @@ def __init__(
page_timeout: int = 30_000,
browser_launch_kwargs: Optional[Dict[str, Any]] = None,
health_check_interval: int = 30,
block_external_requests: bool = False,
allowed_hosts: Optional[List[str]] = None,
):
self.num_browsers = max(1, num_browsers)
self.contexts_per_browser = max(1, contexts_per_browser)
self.page_timeout = page_timeout
self.browser_launch_kwargs = browser_launch_kwargs or {}
self.health_check_interval = health_check_interval
self.block_external_requests = block_external_requests
self._allowed_hosts = self._build_allowed_hosts(allowed_hosts or [])

# thread & loop
self._thread: Optional[threading.Thread] = None
Expand Down Expand Up @@ -238,19 +248,78 @@ async def _worker_main(self):

async def _launch_browser(self, idx: int) -> Browser:
"""Launch a single Chromium browser instance. Isolated to worker loop."""
kwargs = dict(executable_path=os.environ.get('CONVERT_PDF_CHROME_PATH', '/usr/bin/google-chrome'), headless=True)
kwargs = dict(headless=True)
kwargs.update(self.browser_launch_kwargs)
# safe defaults
if 'args' not in kwargs:
kwargs['args'] = [
'--no-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
'--disable-software-rasterizer',
]
if CONVERT_PDF_BROWSER_EXECUTABLE_PATH:
kwargs['executable_path'] = CONVERT_PDF_BROWSER_EXECUTABLE_PATH
# Chromium refuses to enable the sandbox when running as root
# (see https://crbug.com/638180), so only enable it for non-root users.
if os.geteuid() != 0:

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] root 分支仍禁用 sandbox

影响:以任务描述的 root 方式运行时,此分支不会设置 chromium_sandbox=True,而 Playwright 默认关闭 sandbox。我用本 PR 的启动函数及 Playwright 1.64.1 实测,浏览器进程仍为 UID 0,命令行自动带有 --no-sandbox;删除手写参数并未消除高权限、无沙箱的风险。

建议修复:强制渲染器以专用非 root 用户运行,并始终启用 chromium_sandbox;检测到 root 时应拒绝启动而不是静默降级,同时更新服务启动配置。

kwargs['chromium_sandbox'] = True
browser = await self._playwright.chromium.launch(**kwargs)
return browser

def _build_allowed_hosts(self, extra_hosts: List[str]) -> Set[str]:
"""
Derive the set of hostnames that are allowed to be requested during rendering.

Always allows loopback, and derives hosts from the internal service URLs so that
the rendered page's own resources (html/js/css/images served from dtable-web,
dtable-server and the file server) are never blocked. Extra hosts come from the
CONVERT_PDF_ALLOWED_HOSTS configuration.
"""
hosts: Set[str] = {'127.0.0.1', 'localhost', '::1'}
service_urls = [DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT]
for url in service_urls:
if not url:
continue
try:
host = urlparse(url).hostname
except Exception:
host = None
if host:
hosts.add(host)
for host in extra_hosts:
if host and isinstance(host, str):
hosts.add(host.strip())
return hosts

async def _apply_request_filter(self, context: BrowserContext):
"""
Attach route handlers to the context that block external requests and WebSocket
connections when CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS is enabled. Requests whose
host is in the allowed set are continued / connected, all others are aborted / closed.
"""
if not self.block_external_requests:
return

async def _route_handler(route):
try:
host = urlparse(route.request.url).hostname
except Exception:
host = None
# allow requests without a host (data:/blob: etc.) as well as allowed hosts
if host is None or host in self._allowed_hosts:
await route.continue_()
else:
logger.debug("block external request: %s", route.request.url)
await route.abort()

async def _ws_handler(ws):
try:
host = urlparse(ws.url).hostname
except Exception:
host = None
# allow websockets without a host as well as allowed hosts
if host is None or host in self._allowed_hosts:
ws.connect_to_server()
else:
logger.debug("block external websocket: %s", ws.url)
await ws.close()

await context.route("**/*", _route_handler)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] 请求路由不构成网络隔离

影响:过滤默认关闭,且即使配置开启,context.route 也不拦截 WebSocket。真实 Chromium 中,白名单外的本地 HTTP 请求被阻止,但同一目标的 ws:// 连接仍成功;用户可影响的页面仍能访问非白名单网络,未满足任务的网络隔离要求。

建议修复:在独立渲染进程或容器的网络层默认拒绝出站访问,仅经受控通道获取渲染必需资源;应用层同时默认开启过滤,并显式拒绝非白名单 WebSocket。

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

当前 head 复核:两个 PR 新增的 route_web_socket 已修复此评论中的 WebSocket 绕过子项。使用真实 Chromium 156.0.8078.4 和非白名单本地目标实测,开启过滤后 HTTP、WebSocket 均被阻止,服务端未收到相应请求。

剩余阻塞仍属于本线程的网络隔离根因:CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS 仍默认 False,默认启动时相同 HTTP、WebSocket 目标均可连接;当前改动也未提供渲染进程的网络层出站隔离。建议默认启用过滤,并在独立渲染进程或容器实施默认拒绝出站、仅允许受控资源通道的网络策略;WebSocket 绕过不再作为当前缺陷。

await context.route_web_socket("**/*", _ws_handler)

async def _cleanup_browsers(self):
# close all contexts per browser
for i, ctx_list in enumerate(self._contexts):
Expand Down Expand Up @@ -374,6 +443,7 @@ async def _handle_new_batch(self, task: Dict[str, Any], fut: Future):

# create a new context for the batch
context = await browser.new_context(ignore_https_errors=True)
await self._apply_request_filter(context)

# register context
self._contexts[browser_idx].append(context)
Expand Down Expand Up @@ -432,6 +502,7 @@ async def _handle_task(self, task: Dict[str, Any], fut: Future):
# create a fresh context for the batch and record it
try:
new_context = await browser.new_context(viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True)
await self._apply_request_filter(new_context)
# replace the old context in contexts list (best-effort)
try:
# remove old context reference if present
Expand Down Expand Up @@ -544,6 +615,8 @@ def get_playwright_manager():
if not playwright_manager:
playwright_manager = RobustPlaywrightManager(
num_browsers=CONVERT_PDF_BROWSERS,
contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER
contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER,
block_external_requests=CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS,
allowed_hosts=CONVERT_PDF_ALLOWED_HOSTS,
)
return playwright_manager
188 changes: 0 additions & 188 deletions dtable_events/convert_page/utils.py
Original file line number Diff line number Diff line change
@@ -1,190 +1,3 @@
import asyncio
import base64
import io
import json
import logging
import time
import os
import requests
import posixpath

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait

from dtable_events.app.config import DTABLE_WEB_SERVICE_URL
from dtable_events.utils import uuid_str_to_36_chars
from dtable_events.dtable_io.utils import gen_inner_file_get_url

from seaserv import seafile_api

logger = logging.getLogger(__name__)

CHROME_DATA_DIR = '/tmp/chrome-user-datas'
DOCUMNET_PLUGIN_FILE_RELATIVE_PATH = 'files/plugins/document'
DOCUMENT_CONFIG_FILE_NAME = 'documents.json'


def get_chrome_data_dir(dir_name='tmp'):
if not os.path.isdir(CHROME_DATA_DIR):
os.makedirs(CHROME_DATA_DIR)
return os.path.join(CHROME_DATA_DIR, dir_name)


def get_driver(user_data_path):
webdriver_options = Options()

webdriver_options.add_argument('--no-sandbox')
webdriver_options.add_argument('--headless')
webdriver_options.add_argument('--disable-gpu')
webdriver_options.add_argument('--disable-dev-shm-usage')
webdriver_options.add_argument(f'--user-data-dir={user_data_path}')

driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=webdriver_options)
return driver


def gen_page_design_pdf_view_url(dtable_uuid, page_id, access_token, row_id=None):
url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/page-design/%s/' % (uuid_str_to_36_chars(dtable_uuid), page_id)
if row_id:
url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/page-design/%s/row/%s/' % (uuid_str_to_36_chars(dtable_uuid), page_id, row_id)

url += '?access-token=%s&need_convert=%s' % (access_token, 0)
return url


def gen_document_pdf_view_url(dtable_uuid, doc_uuid, access_token, row_id):
# row_id maybe None
url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/document/%s/row/%s/' % (uuid_str_to_36_chars(dtable_uuid), doc_uuid, row_id)
url += '?access-token=%s&need_convert=%s' % (access_token, 0)
return url


def open_page_view(driver: webdriver.Chrome, url):
driver.execute_script(f"window.open('{url}')")
return driver.window_handles[-1]


def wait_page_view(driver: webdriver.Chrome, session_id, monitor_dom_id, row_id, output):
def check_images_and_networks(driver, frequency=0.5):
"""
make sure all images complete
make sure no new connections in 0.5s.
TODO: Unreliable and need to be continuously updated.
"""
images_done = driver.execute_script('''
let p = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {};
let entries = p.getEntries();
let images = Array.from(document.images).filter(image => image.src.indexOf('/asset/') !== -1);
if (images.length === 0) return true;
return images.filter(image => image.complete).length == images.length;
''')
if not images_done:
return False

entries_count = None
while True:
now_entries_count = driver.execute_script('''
let p = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {};
return p.getEntries().length;
''')
if entries_count is None:
entries_count = now_entries_count
time.sleep(frequency)
continue
else:
if now_entries_count == entries_count and \
driver.execute_script("return document.readyState === 'complete'"):
return True
break
return False

await_react_render = 60
# sleep_time = 2
# if not row_id:
# await_react_render = 180
# sleep_time = 6

driver.switch_to.window(session_id)

try:
logger.debug('check to wait render')
# make sure react is rendered, timeout await_react_render, rendering is not completed within 3 minutes, and rendering performance needs to be improved
WebDriverWait(driver, await_react_render).until(lambda driver: driver.find_element_by_id(monitor_dom_id) is not None, message='wait react timeout')
logger.debug('check to wait images')
# make sure images from asset are rendered, timeout 120s
WebDriverWait(driver, 120, poll_frequency=1).until(lambda driver: check_images_and_networks(driver), message='wait images and networks timeout')
logger.debug('check to sleep')
# time.sleep(sleep_time) # test how non-sleep effects renderring
except Exception as e:
logger.warning('wait for page error: %s', e)
finally:
calculated_print_options = {
'landscape': False,
'displayHeaderFooter': False,
'printBackground': True,
'preferCSSPageSize': True,
}

resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id
url = driver.command_executor._url + resource
body = json.dumps({'cmd': 'Page.printToPDF', 'params': calculated_print_options})

try:
logger.debug('check to export pdf')
response = driver.command_executor._request('POST', url, body)
logger.debug('check to output')
if not response:
logger.error('execute printToPDF error no response')
v = response.get('value')['data']
if isinstance(output, str):
with open(output, 'wb') as f:
f.write(base64.b64decode(v))
elif isinstance(output, io.BytesIO):
output.write(base64.b64decode(v))
logger.info('check to convert page to pdf success!')
except Exception as e:
logger.exception('execute printToPDF error: {}'.format(e))

# debug page-design view in chrome, console log and network log, don't delete
logger.debug('browser console start')
for log in list(driver.get_log('browser')):
logger.debug(log)
logger.debug('browser console end')
network_logs = driver.execute_script("var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntriesByType('resource') || {}; return network;")
logger.debug('network logs start')
for item in network_logs:
logger.debug('check name: %s start: %s duration: %s end: %s', item.get('name'), item.get('startTime'), item.get('duration'), item.get('responseEnd'))
logger.debug(item)
logger.debug('network logs end')


def gen_document_base_dir(dtable_uuid):
return posixpath.join('/asset', dtable_uuid, DOCUMNET_PLUGIN_FILE_RELATIVE_PATH)


def get_documents_config(repo_id, dtable_uuid, username):
document_plugin_dir = gen_document_base_dir(uuid_str_to_36_chars(dtable_uuid))
config_path = posixpath.join(document_plugin_dir, DOCUMENT_CONFIG_FILE_NAME)
file_id = seafile_api.get_file_id_by_path(repo_id, config_path)
if not file_id:
return []
token = seafile_api.get_fileserver_access_token(repo_id, file_id, 'download', username, use_onetime=True)
url = gen_inner_file_get_url(token, DOCUMENT_CONFIG_FILE_NAME)
resp = requests.get(url)
documents_config = json.loads(resp.content)
return documents_config


def get_pdf_print_options():
return {
'landscape': False,
'display_header_footer': False,
'print_background': True,
'prefer_css_page_size': True,
}


async def wait_for_images(page):
await page.evaluate("""() => {
window.scrollTo(0, document.body.scrollHeight);
Expand All @@ -202,4 +15,3 @@ async def wait_for_images(page):
);
}
""")

7 changes: 2 additions & 5 deletions dtable_events/dtable_io/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,6 @@
import_excel_csv_add_table_by_dtable_server, update_parsed_file_by_dtable_server, \
parse_update_excel_upload_excel_to_json, parse_update_csv_upload_csv_to_json, parse_and_import_excel_csv_to_dtable, \
parse_and_import_excel_csv_to_table, parse_and_update_file_to_table, parse_and_append_excel_csv_to_table
from dtable_events.convert_page.utils import get_chrome_data_dir, get_driver, open_page_view, wait_page_view, \
gen_page_design_pdf_view_url, gen_document_pdf_view_url
from dtable_events.statistics.db import save_email_sending_records, batch_save_email_sending_records
from dtable_events.data_sync.data_sync_utils import run_sync_emails
from dtable_events.utils import is_valid_email, uuid_str_to_36_chars, gen_file_upload_url, uuid_str_to_32_chars
Expand All @@ -46,7 +44,6 @@
SDOC_IMAGES_DIR, get_seadoc_download_link, gen_seadoc_base_dir, export_sdoc_prepare_images_folder,\
batch_upload_sdoc_images, get_documents_config, save_documents_config
from dtable_events.app.log import setup_logger
from dtable_events.convert_page.utils import get_pdf_print_options, wait_for_images
from dtable_events.convert_page.manager import get_playwright_manager

dtable_io_logger = setup_logger('dtable_events_io', propagate=False)
Expand Down Expand Up @@ -900,7 +897,7 @@ def convert_page_design_to_pdf(dtable_uuid, page_id, row_id, username):
url += '?access-token=%s&need_convert=%s' % (access_token, 0)
filename = '%s_%s_%s.pdf' % (dtable_uuid, page_id, row_id)
try:
get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir)
get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#page-design-render-complete'}], target_dir)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

[Critical] 空白末页导致整份导出失败

影响:页面设计前页有内容、末页没有 widget 时,前端只在最后页的最后 widget 内插入完成标记,因此该标记永远不存在。新增等待会在 5 秒后超时;真实 Chromium 验证中,原先无 selector 的调用能生成 PDF,此处改动后返回 [] 且整份文件不生成。

建议修复:协调前端在整体渲染完成后插入独立的完成标记,不依赖末页存在 widget,并增加“内容页+空白末页”的导出回归;保留等待,避免改回不可靠的固定延迟。

except Exception as e:
dtable_io_logger.exception('dtable: %s plugin: page-design page: %s row: %s error: %s', dtable_uuid, page_id, row_id, e)

Expand All @@ -927,7 +924,7 @@ def convert_document_to_pdf(dtable_uuid, doc_uuid, row_id, username):
url += '?access-token=%s&need_convert=%s' % (access_token, 0)
filename = '%s_%s_%s.pdf' % (dtable_uuid, doc_uuid, row_id)
try:
get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir)
get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#document-render-complete'}], target_dir)
except Exception as e:
dtable_io_logger.exception('dtable: %s plugin: document doc_uuid: %s row: %s error: %s', dtable_uuid, doc_uuid, row_id, e)

Expand Down
Loading
Loading