diff --git a/dtable_events/app/config.py b/dtable_events/app/config.py index d0344b16..fbd2e151 100644 --- a/dtable_events/app/config.py +++ b/dtable_events/app/config.py @@ -199,6 +199,9 @@ def get_llm_prices(models): # playwright CONVERT_PDF_BROWSERS = configs.get('CONVERT_PDF_BROWSERS', default=2) CONVERT_PDF_SESSIONS_PER_BROWSER = configs.get('CONVERT_PDF_SESSIONS_PER_BROWSER', default=3) +CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS = configs.get('CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS', default=False) +CONVERT_PDF_ALLOWED_HOSTS = configs.get('CONVERT_PDF_ALLOWED_HOSTS', default=[]) +CONVERT_PDF_BROWSER_EXECUTABLE_PATH = configs.get('CONVERT_PDF_BROWSER_EXECUTABLE_PATH', default='') # virus code VIRUS_SCAN_ENABLED = configs.get('VIRUS_SCAN_ENABLED', default=False) diff --git a/dtable_events/convert_page/manager.py b/dtable_events/convert_page/manager.py index 340bdb08..de475c44 100644 --- a/dtable_events/convert_page/manager.py +++ b/dtable_events/convert_page/manager.py @@ -12,15 +12,21 @@ import threading import time import os -from typing import List, Dict, Optional, Any, Tuple +from typing import List, Dict, Optional, Any, Tuple, Set from concurrent.futures import Future, wait as wait_futures import queue import uuid +from urllib.parse import urlparse import psutil from playwright.async_api import async_playwright, Playwright, Browser, BrowserContext -from dtable_events.app.config import CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER +from dtable_events.app.config import ( + CONVERT_PDF_BROWSERS, CONVERT_PDF_SESSIONS_PER_BROWSER, + CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS, CONVERT_PDF_ALLOWED_HOSTS, + CONVERT_PDF_BROWSER_EXECUTABLE_PATH, + DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT, +) from dtable_events.convert_page.utils import wait_for_images logger = logging.getLogger(__name__) @@ -44,12 +50,16 @@ def __init__( page_timeout: int = 30_000, browser_launch_kwargs: Optional[Dict[str, Any]] = None, health_check_interval: int = 30, + block_external_requests: bool = False, + allowed_hosts: Optional[List[str]] = None, ): self.num_browsers = max(1, num_browsers) self.contexts_per_browser = max(1, contexts_per_browser) self.page_timeout = page_timeout self.browser_launch_kwargs = browser_launch_kwargs or {} self.health_check_interval = health_check_interval + self.block_external_requests = block_external_requests + self._allowed_hosts = self._build_allowed_hosts(allowed_hosts or []) # thread & loop self._thread: Optional[threading.Thread] = None @@ -238,19 +248,78 @@ async def _worker_main(self): async def _launch_browser(self, idx: int) -> Browser: """Launch a single Chromium browser instance. Isolated to worker loop.""" - kwargs = dict(executable_path=os.environ.get('CONVERT_PDF_CHROME_PATH', '/usr/bin/google-chrome'), headless=True) + kwargs = dict(headless=True) kwargs.update(self.browser_launch_kwargs) - # safe defaults - if 'args' not in kwargs: - kwargs['args'] = [ - '--no-sandbox', - '--disable-dev-shm-usage', - '--disable-gpu', - '--disable-software-rasterizer', - ] + if CONVERT_PDF_BROWSER_EXECUTABLE_PATH: + kwargs['executable_path'] = CONVERT_PDF_BROWSER_EXECUTABLE_PATH + # Chromium refuses to enable the sandbox when running as root + # (see https://crbug.com/638180), so only enable it for non-root users. + if os.geteuid() != 0: + kwargs['chromium_sandbox'] = True browser = await self._playwright.chromium.launch(**kwargs) return browser + def _build_allowed_hosts(self, extra_hosts: List[str]) -> Set[str]: + """ + Derive the set of hostnames that are allowed to be requested during rendering. + + Always allows loopback, and derives hosts from the internal service URLs so that + the rendered page's own resources (html/js/css/images served from dtable-web, + dtable-server and the file server) are never blocked. Extra hosts come from the + CONVERT_PDF_ALLOWED_HOSTS configuration. + """ + hosts: Set[str] = {'127.0.0.1', 'localhost', '::1'} + service_urls = [DTABLE_WEB_SERVICE_URL, INNER_DTABLE_WEB_SERVICE_URL, INNER_FILE_SERVER_ROOT] + for url in service_urls: + if not url: + continue + try: + host = urlparse(url).hostname + except Exception: + host = None + if host: + hosts.add(host) + for host in extra_hosts: + if host and isinstance(host, str): + hosts.add(host.strip()) + return hosts + + async def _apply_request_filter(self, context: BrowserContext): + """ + Attach route handlers to the context that block external requests and WebSocket + connections when CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS is enabled. Requests whose + host is in the allowed set are continued / connected, all others are aborted / closed. + """ + if not self.block_external_requests: + return + + async def _route_handler(route): + try: + host = urlparse(route.request.url).hostname + except Exception: + host = None + # allow requests without a host (data:/blob: etc.) as well as allowed hosts + if host is None or host in self._allowed_hosts: + await route.continue_() + else: + logger.debug("block external request: %s", route.request.url) + await route.abort() + + async def _ws_handler(ws): + try: + host = urlparse(ws.url).hostname + except Exception: + host = None + # allow websockets without a host as well as allowed hosts + if host is None or host in self._allowed_hosts: + ws.connect_to_server() + else: + logger.debug("block external websocket: %s", ws.url) + await ws.close() + + await context.route("**/*", _route_handler) + await context.route_web_socket("**/*", _ws_handler) + async def _cleanup_browsers(self): # close all contexts per browser for i, ctx_list in enumerate(self._contexts): @@ -374,6 +443,7 @@ async def _handle_new_batch(self, task: Dict[str, Any], fut: Future): # create a new context for the batch context = await browser.new_context(ignore_https_errors=True) + await self._apply_request_filter(context) # register context self._contexts[browser_idx].append(context) @@ -432,6 +502,7 @@ async def _handle_task(self, task: Dict[str, Any], fut: Future): # create a fresh context for the batch and record it try: new_context = await browser.new_context(viewport={'width': 1920, 'height': 1080}, ignore_https_errors=True) + await self._apply_request_filter(new_context) # replace the old context in contexts list (best-effort) try: # remove old context reference if present @@ -544,6 +615,8 @@ def get_playwright_manager(): if not playwright_manager: playwright_manager = RobustPlaywrightManager( num_browsers=CONVERT_PDF_BROWSERS, - contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER + contexts_per_browser=CONVERT_PDF_SESSIONS_PER_BROWSER, + block_external_requests=CONVERT_PDF_BLOCK_EXTERNAL_REQUESTS, + allowed_hosts=CONVERT_PDF_ALLOWED_HOSTS, ) return playwright_manager diff --git a/dtable_events/convert_page/utils.py b/dtable_events/convert_page/utils.py index ad8b934d..080d6704 100644 --- a/dtable_events/convert_page/utils.py +++ b/dtable_events/convert_page/utils.py @@ -1,190 +1,3 @@ -import asyncio -import base64 -import io -import json -import logging -import time -import os -import requests -import posixpath - -from selenium import webdriver -from selenium.webdriver.chrome.options import Options -from selenium.webdriver.support.ui import WebDriverWait - -from dtable_events.app.config import DTABLE_WEB_SERVICE_URL -from dtable_events.utils import uuid_str_to_36_chars -from dtable_events.dtable_io.utils import gen_inner_file_get_url - -from seaserv import seafile_api - -logger = logging.getLogger(__name__) - -CHROME_DATA_DIR = '/tmp/chrome-user-datas' -DOCUMNET_PLUGIN_FILE_RELATIVE_PATH = 'files/plugins/document' -DOCUMENT_CONFIG_FILE_NAME = 'documents.json' - - -def get_chrome_data_dir(dir_name='tmp'): - if not os.path.isdir(CHROME_DATA_DIR): - os.makedirs(CHROME_DATA_DIR) - return os.path.join(CHROME_DATA_DIR, dir_name) - - -def get_driver(user_data_path): - webdriver_options = Options() - - webdriver_options.add_argument('--no-sandbox') - webdriver_options.add_argument('--headless') - webdriver_options.add_argument('--disable-gpu') - webdriver_options.add_argument('--disable-dev-shm-usage') - webdriver_options.add_argument(f'--user-data-dir={user_data_path}') - - driver = webdriver.Chrome('/usr/local/bin/chromedriver', options=webdriver_options) - return driver - - -def gen_page_design_pdf_view_url(dtable_uuid, page_id, access_token, row_id=None): - url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/page-design/%s/' % (uuid_str_to_36_chars(dtable_uuid), page_id) - if row_id: - url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/page-design/%s/row/%s/' % (uuid_str_to_36_chars(dtable_uuid), page_id, row_id) - - url += '?access-token=%s&need_convert=%s' % (access_token, 0) - return url - - -def gen_document_pdf_view_url(dtable_uuid, doc_uuid, access_token, row_id): - # row_id maybe None - url = DTABLE_WEB_SERVICE_URL.strip('/') + '/dtable/%s/document/%s/row/%s/' % (uuid_str_to_36_chars(dtable_uuid), doc_uuid, row_id) - url += '?access-token=%s&need_convert=%s' % (access_token, 0) - return url - - -def open_page_view(driver: webdriver.Chrome, url): - driver.execute_script(f"window.open('{url}')") - return driver.window_handles[-1] - - -def wait_page_view(driver: webdriver.Chrome, session_id, monitor_dom_id, row_id, output): - def check_images_and_networks(driver, frequency=0.5): - """ - make sure all images complete - make sure no new connections in 0.5s. - TODO: Unreliable and need to be continuously updated. - """ - images_done = driver.execute_script(''' - let p = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; - let entries = p.getEntries(); - let images = Array.from(document.images).filter(image => image.src.indexOf('/asset/') !== -1); - if (images.length === 0) return true; - return images.filter(image => image.complete).length == images.length; - ''') - if not images_done: - return False - - entries_count = None - while True: - now_entries_count = driver.execute_script(''' - let p = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; - return p.getEntries().length; - ''') - if entries_count is None: - entries_count = now_entries_count - time.sleep(frequency) - continue - else: - if now_entries_count == entries_count and \ - driver.execute_script("return document.readyState === 'complete'"): - return True - break - return False - - await_react_render = 60 - # sleep_time = 2 - # if not row_id: - # await_react_render = 180 - # sleep_time = 6 - - driver.switch_to.window(session_id) - - try: - logger.debug('check to wait render') - # make sure react is rendered, timeout await_react_render, rendering is not completed within 3 minutes, and rendering performance needs to be improved - WebDriverWait(driver, await_react_render).until(lambda driver: driver.find_element_by_id(monitor_dom_id) is not None, message='wait react timeout') - logger.debug('check to wait images') - # make sure images from asset are rendered, timeout 120s - WebDriverWait(driver, 120, poll_frequency=1).until(lambda driver: check_images_and_networks(driver), message='wait images and networks timeout') - logger.debug('check to sleep') - # time.sleep(sleep_time) # test how non-sleep effects renderring - except Exception as e: - logger.warning('wait for page error: %s', e) - finally: - calculated_print_options = { - 'landscape': False, - 'displayHeaderFooter': False, - 'printBackground': True, - 'preferCSSPageSize': True, - } - - resource = "/session/%s/chromium/send_command_and_get_result" % driver.session_id - url = driver.command_executor._url + resource - body = json.dumps({'cmd': 'Page.printToPDF', 'params': calculated_print_options}) - - try: - logger.debug('check to export pdf') - response = driver.command_executor._request('POST', url, body) - logger.debug('check to output') - if not response: - logger.error('execute printToPDF error no response') - v = response.get('value')['data'] - if isinstance(output, str): - with open(output, 'wb') as f: - f.write(base64.b64decode(v)) - elif isinstance(output, io.BytesIO): - output.write(base64.b64decode(v)) - logger.info('check to convert page to pdf success!') - except Exception as e: - logger.exception('execute printToPDF error: {}'.format(e)) - - # debug page-design view in chrome, console log and network log, don't delete - logger.debug('browser console start') - for log in list(driver.get_log('browser')): - logger.debug(log) - logger.debug('browser console end') - network_logs = driver.execute_script("var performance = window.performance || window.mozPerformance || window.msPerformance || window.webkitPerformance || {}; var network = performance.getEntriesByType('resource') || {}; return network;") - logger.debug('network logs start') - for item in network_logs: - logger.debug('check name: %s start: %s duration: %s end: %s', item.get('name'), item.get('startTime'), item.get('duration'), item.get('responseEnd')) - logger.debug(item) - logger.debug('network logs end') - - -def gen_document_base_dir(dtable_uuid): - return posixpath.join('/asset', dtable_uuid, DOCUMNET_PLUGIN_FILE_RELATIVE_PATH) - - -def get_documents_config(repo_id, dtable_uuid, username): - document_plugin_dir = gen_document_base_dir(uuid_str_to_36_chars(dtable_uuid)) - config_path = posixpath.join(document_plugin_dir, DOCUMENT_CONFIG_FILE_NAME) - file_id = seafile_api.get_file_id_by_path(repo_id, config_path) - if not file_id: - return [] - token = seafile_api.get_fileserver_access_token(repo_id, file_id, 'download', username, use_onetime=True) - url = gen_inner_file_get_url(token, DOCUMENT_CONFIG_FILE_NAME) - resp = requests.get(url) - documents_config = json.loads(resp.content) - return documents_config - - -def get_pdf_print_options(): - return { - 'landscape': False, - 'display_header_footer': False, - 'print_background': True, - 'prefer_css_page_size': True, - } - - async def wait_for_images(page): await page.evaluate("""() => { window.scrollTo(0, document.body.scrollHeight); @@ -202,4 +15,3 @@ async def wait_for_images(page): ); } """) - diff --git a/dtable_events/dtable_io/__init__.py b/dtable_events/dtable_io/__init__.py index 6f52ee24..c792cad6 100644 --- a/dtable_events/dtable_io/__init__.py +++ b/dtable_events/dtable_io/__init__.py @@ -32,8 +32,6 @@ import_excel_csv_add_table_by_dtable_server, update_parsed_file_by_dtable_server, \ parse_update_excel_upload_excel_to_json, parse_update_csv_upload_csv_to_json, parse_and_import_excel_csv_to_dtable, \ parse_and_import_excel_csv_to_table, parse_and_update_file_to_table, parse_and_append_excel_csv_to_table -from dtable_events.convert_page.utils import get_chrome_data_dir, get_driver, open_page_view, wait_page_view, \ - gen_page_design_pdf_view_url, gen_document_pdf_view_url from dtable_events.statistics.db import save_email_sending_records, batch_save_email_sending_records from dtable_events.data_sync.data_sync_utils import run_sync_emails from dtable_events.utils import is_valid_email, uuid_str_to_36_chars, gen_file_upload_url, uuid_str_to_32_chars @@ -46,7 +44,6 @@ SDOC_IMAGES_DIR, get_seadoc_download_link, gen_seadoc_base_dir, export_sdoc_prepare_images_folder,\ batch_upload_sdoc_images, get_documents_config, save_documents_config from dtable_events.app.log import setup_logger -from dtable_events.convert_page.utils import get_pdf_print_options, wait_for_images from dtable_events.convert_page.manager import get_playwright_manager dtable_io_logger = setup_logger('dtable_events_io', propagate=False) @@ -900,7 +897,7 @@ def convert_page_design_to_pdf(dtable_uuid, page_id, row_id, username): url += '?access-token=%s&need_convert=%s' % (access_token, 0) filename = '%s_%s_%s.pdf' % (dtable_uuid, page_id, row_id) try: - get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir) + get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#page-design-render-complete'}], target_dir) except Exception as e: dtable_io_logger.exception('dtable: %s plugin: page-design page: %s row: %s error: %s', dtable_uuid, page_id, row_id, e) @@ -927,7 +924,7 @@ def convert_document_to_pdf(dtable_uuid, doc_uuid, row_id, username): url += '?access-token=%s&need_convert=%s' % (access_token, 0) filename = '%s_%s_%s.pdf' % (dtable_uuid, doc_uuid, row_id) try: - get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename}], target_dir) + get_playwright_manager().batch_urls_to_pdf_sync([{'url': url, 'filename': filename, 'selector': '#document-render-complete'}], target_dir) except Exception as e: dtable_io_logger.exception('dtable: %s plugin: document doc_uuid: %s row: %s error: %s', dtable_uuid, doc_uuid, row_id, e) diff --git a/dtable_events/utils/dtable_server_api.py b/dtable_events/utils/dtable_server_api.py index 3f0e0031..907c5a24 100644 --- a/dtable_events/utils/dtable_server_api.py +++ b/dtable_events/utils/dtable_server_api.py @@ -343,7 +343,7 @@ def batch_append_rows(self, table_name, rows_data, need_convert_back=None): } if need_convert_back is not None: json_data['need_convert_back'] = need_convert_back - response = requests.post(url, json=json_data, headers=self.headers, timeout=self.timeout) + response = requests.post(url, json=json_data, headers=self.internal_headers, timeout=self.timeout) return parse_response(response) def append_row(self, table_name, row_data, apply_default=None): diff --git a/requirements.txt b/requirements.txt index db7da657..496bcd0b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ redis==4.4.* SQLAlchemy==2.0.18 apscheduler==3.9.* openpyxl==3.0.1 -selenium==3.141.0 waitress==3.0.2 Flask==2.2.* python-dateutil==2.8.* @@ -11,6 +10,6 @@ Markdown==3.6.* mammoth==1.4.* pdfminer.six==20221105 pdf2image~=1.17.0 -playwright==1.48.0 +playwright==1.64.* psutil==6.1.0 pyyaml==6.0.*