Put skaled behind nginx

node_cli/configs/__init__.py

@@ -64,8 +64,20 @@
 STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'static_params.yaml')
 FAIR_STATIC_PARAMS_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'fair_static_params.yaml')
 
-NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2')
-NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf')
+NGINX_TEMPLATE_DIR = os.path.join(CONTAINER_CONFIG_PATH, 'nginx')
+NGINX_TEMPLATE_FILEPATH = os.path.join(NGINX_TEMPLATE_DIR, 'nginx.conf.j2')
+NGINX_BASE_TEMPLATE_FILEPATH = os.path.join(NGINX_TEMPLATE_DIR, 'base.conf.j2')
+NGINX_NJS_SOURCE_PATH = os.path.join(NGINX_TEMPLATE_DIR, 'njs')
+LEGACY_NGINX_TEMPLATE_FILEPATH = os.path.join(CONTAINER_CONFIG_PATH, 'nginx.conf.j2')
+NGINX_DIR = os.path.join(NODE_DATA_PATH, 'nginx')
+NGINX_CONFIG_FILEPATH = os.path.join(NGINX_DIR, 'nginx.conf')
+NGINX_CONF_D_PATH = os.path.join(NGINX_DIR, 'conf.d')
+NGINX_BASE_CONFIG_FILEPATH = os.path.join(NGINX_CONF_D_PATH, 'base.conf')
+NGINX_CHAINS_PATH = os.path.join(NGINX_CONF_D_PATH, 'chains')
+NGINX_NJS_PATH = os.path.join(NGINX_DIR, 'njs')
+# skale-admin takes the same lock for its nginx changes
+NGINX_LOCK_PATH = os.path.join(NGINX_DIR, '.chains.lock')
+LEGACY_NGINX_CONFIG_FILEPATH = os.path.join(NODE_DATA_PATH, 'nginx.conf')
 NGINX_CONTAINER_NAME = 'sk_nginx'
 
 LOG_PATH = os.path.join(NODE_DATA_PATH, 'log')

node_cli/core/host.py

@@ -50,6 +50,7 @@
     SKALE_TMP_DIR,
     UFW_CONFIG_PATH,
     UFW_IPV6_BEFORE_INPUT_CHAIN,
+    LEGACY_NGINX_CONFIG_FILEPATH,
     NGINX_CONFIG_FILEPATH,
 )
 from node_cli.configs.cli_logger import LOG_DATA_PATH
@@ -88,7 +89,8 @@ def prepare_host(env_type: EnvType, allocation: bool = False) -> None:
 
 
 def is_node_inited() -> bool:
-    return os.path.isfile(NGINX_CONFIG_FILEPATH)
+    # the single-file config marks nodes that were not updated to node_data/nginx/ yet
+    return os.path.isfile(NGINX_CONFIG_FILEPATH) or os.path.isfile(LEGACY_NGINX_CONFIG_FILEPATH)
 
 
 def make_dirs():

node_cli/core/nftables.py

@@ -26,6 +26,8 @@
 from pathlib import Path
 from typing import Optional
 
+from filelock import FileLock
+
 from node_cli.configs import (
     DEFAULT_NODE_BASE_PORT,
     ENV,
@@ -36,7 +38,13 @@
     NFTABLES_USER_CONFIG_PATH,
     NODE_CONFIG_PATH,
 )
-from node_cli.utils.helper import cleanup_dir_content, get_ssh_ports, read_json, run_cmd
+from node_cli.utils.helper import (
+    check_ssl_certs,
+    cleanup_dir_content,
+    get_ssh_ports,
+    read_json,
+    run_cmd,
+)
 
 logger = logging.getLogger(__name__)
 
@@ -72,6 +80,9 @@ class SGXPort:
     ZMQ: int = 1031
 
 
+# nginx listens on these only once the node has certificates, see base.conf.j2
+TLS_SERVICE_PORTS = (ServicePort.HTTPS, ServicePort.WATCHDOG_HTTPS)
+
 LEGACY_CHAIN = 'INPUT'
 LEGACY_FAMILY = 'ip'
 LEGACY_TABLE = 'filter'
@@ -609,37 +620,66 @@ def _setup_user_chain(self) -> None:
         self.apply_user_rules()
         self.remove_user_rules_from_main_chain()
 
-    def remove_monitoring_accepts(self) -> None:
-        """Remove every legacy monitoring accept from the managed base chain."""
+    def _remove_tcp_accepts(self, ports: tuple[int, ...]) -> None:
+        """Remove every accept of the ports from the managed base chain, ssh ports stay open."""
         ssh_ports = get_ssh_ports()
-        monitoring_exprs = [
+        exprs = [
             Rule(chain=self.chain, protocol='tcp', first_port=port).to_expr()
-            for port in (ServicePort.EXPORTER, ServicePort.CADVISOR)
+            for port in ports
             if port not in ssh_ports
         ]
         for rule in self.get_rules(self.chain):
             if (
-                self._normalized_expr(rule.get('expr', [])) in monitoring_exprs
+                self._normalized_expr(rule.get('expr', [])) in exprs
                 and rule.get('handle') is not None
             ):
                 self.delete_rule_by_handle(rule['handle'])
 
+    def remove_monitoring_accepts(self) -> None:
+        """Remove every legacy monitoring accept from the managed base chain."""
+        self._remove_tcp_accepts((ServicePort.EXPORTER, ServicePort.CADVISOR))
+
     def _add_service_accepts(self) -> None:
         self._ensure_rule(self.chain, conntrack_accept_expr(), label='connection tracking rule')
         tcp_ports = [
             *get_ssh_ports(),
             ServicePort.DNS,
-            ServicePort.HTTPS,
             ServicePort.HTTP,
             ServicePort.WATCHDOG_HTTP,
-            ServicePort.WATCHDOG_HTTPS,
         ]
         for port in tcp_ports:
             self.add_rule(Rule(chain=self.chain, protocol='tcp', first_port=port))
+        self.sync_tls_accepts()
         self.remove_misordered_udp_drop()
         self.add_rule(Rule(chain=self.chain, protocol='udp', first_port=ServicePort.DNS))
         self._ensure_rule(self.chain, loopback_accept_expr(), label='loopback rule')
 
+    def sync_tls_accepts(self) -> None:
+        """Open the TLS service ports with certificates, close them without."""
+        path = Path(NFTABLES_CHAIN_FOLDER_PATH) / 'tls-ports.conf'
+        with FileLock(path.with_suffix('.lock'), timeout=120):
+            ports = ', '.join(map(str, TLS_SERVICE_PORTS)) if check_ssl_certs() else ''
+            elements = f' elements = {{ {ports} }};' if ports else ''
+            declaration = f'set skale_tls_ports {{ type inet_service;{elements} }}'
+            commands = (
+                f'add set {self.family} {self.table} skale_tls_ports {{ type inet_service; }}\n'
+                f'flush set {self.family} {self.table} skale_tls_ports\n'
+            )
+            if ports:
+                commands += (
+                    f'add element {self.family} {self.table} skale_tls_ports {{ {ports} }}\n'
+                )
+            rc, _, error = self.nft.cmd(commands)
+            if rc != 0:
+                raise NFTablesError(f'Failed to sync TLS ports: {error}')
+            temporary = path.with_suffix('.tmp')
+            temporary.write_text(declaration + '\n')
+            temporary.replace(path)
+            expr = [dport_match('tcp', 0, 0), {'counter': None}, {'accept': None}]
+            expr[0]['match']['right'] = '@skale_tls_ports'
+            self._ensure_rule(self.chain, expr, label='TLS service ports')
+            self._remove_tcp_accepts(TLS_SERVICE_PORTS)
+
     def _add_icmp_accepts(self) -> None:
         self.remove_source_quench_rule()
         for icmp_type in ICMP_ACCEPT_TYPES:
@@ -840,6 +880,13 @@ def configure_nftables(keep_accept_policy: bool = False) -> None:
     remove_legacy_saved_rules()
 
 
+def sync_tls_ports() -> None:
+    """Make a configured firewall follow the certificates and save it for the next boot."""
+    nft_mgr = NFTablesManager()
+    nft_mgr.sync_tls_accepts()
+    save_nftables_base_rules(nft_mgr.get_base_ruleset())
+
+
 def cleanup_nftables() -> None:
     """Reset the firewall after node cleanup and persist the minimal state."""
     logger.info('Cleaning up firewall rules')
@@ -870,8 +917,7 @@ def save_nftables_base_rules(ruleset: str) -> None:
         f'\t\tinclude "{NFTABLES_USER_CONFIG_PATH}"',
         '\t}',
     ]
-    ruleset_lines[1:1] = user_chain_lines
-    ruleset_lines.insert(-2, chain_include_line)
+    ruleset_lines[1:1] = [chain_include_line, *user_chain_lines]
     with open(NFTABLES_SKALE_BASE_CONFIG_PATH, 'w') as f:
         f.write('\n'.join(ruleset_lines))
     logger.info('Rules saved successfully to %s', NFTABLES_SKALE_BASE_CONFIG_PATH)

node_cli/core/nginx.py

@@ -17,44 +17,156 @@
 #   You should have received a copy of the GNU Affero General Public License
 #   along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
+import glob
+import hashlib
 import logging
 import os.path
+import shutil
+import time
+from collections.abc import Iterator
+from contextlib import contextmanager
+from pathlib import Path
+from typing import Callable, Optional
+
+from filelock import FileLock, Timeout
+from jinja2 import Environment
 
 from node_cli.cli.info import TYPE
-from node_cli.configs import NODE_CERTS_PATH, NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH
+from node_cli.configs import (
+    LEGACY_NGINX_CONFIG_FILEPATH,
+    LEGACY_NGINX_TEMPLATE_FILEPATH,
+    NGINX_BASE_CONFIG_FILEPATH,
+    NGINX_BASE_TEMPLATE_FILEPATH,
+    NGINX_CHAINS_PATH,
+    NGINX_CONFIG_FILEPATH,
+    NGINX_LOCK_PATH,
+    NGINX_NJS_PATH,
+    NGINX_NJS_SOURCE_PATH,
+    NGINX_TEMPLATE_DIR,
+    NGINX_TEMPLATE_FILEPATH,
+    NODE_CERTS_PATH,
+)
+from node_cli.core.nftables import ServicePort
+from node_cli.migrations.nginx_layout import migrate_nginx_layout
 from node_cli.utils.node_type import NodeType
-from node_cli.utils.docker_utils import restart_nginx_container, docker_client
-from node_cli.utils.helper import process_template
+from node_cli.utils.docker_utils import (
+    NginxConfigError,
+    docker_client,
+    nginx_answer,
+    reload_nginx_container,
+)
+from node_cli.utils.helper import check_ssl_certs, process_template, read_file, safe_mkdir
 
 logger = logging.getLogger(__name__)
 
 
 SSL_KEY_NAME = 'ssl_key'
 SSL_CRT_NAME = 'ssl_cert'
 
+# loopback-only location in base.conf that answers with the file's fingerprint
+PROBE_PATH = '/.skale-proxy'
+BASE_HEADER = '# node base config, rendered by node-cli and skale-admin: fingerprint '
+# nginx binds all listeners of a new config or none, so one base.conf server speaks for all
+BASE_PROBE_URL = f'http://127.0.0.1:{ServicePort.WATCHDOG_HTTP}{PROBE_PATH}'
+# nginx retries a busy listener port five times, 500 ms apart, before it keeps the old config
+APPLY_TIMEOUT_SECONDS = 10
+POLL_INTERVAL_SECONDS = 0.5
+LOCK_TIMEOUT_SECONDS = 120
+
 
-def generate_nginx_config() -> None:
+def generate_nginx_config() -> Optional[str]:
+    """Renders the nginx config, returns base.conf or None for streams without it"""
     ssl_on = check_ssl_certs()
     skale_node = is_skale_node_nginx()
     template_data = {
         'ssl': ssl_on,
         'skale_node': skale_node,
     }
-    logger.info(f'Processing nginx template. ssl: {ssl_on}, skale_node: {skale_node}')
+    logger.info(f'Processing nginx templates. ssl: {ssl_on}, skale_node: {skale_node}')
+    if not os.path.isdir(NGINX_TEMPLATE_DIR):
+        # a config stream from before node_data/nginx/ mounts one server file into conf.d
+        process_template(
+            LEGACY_NGINX_TEMPLATE_FILEPATH, LEGACY_NGINX_CONFIG_FILEPATH, template_data
+        )
+        return None
+    # chains/ is filled by skale-admin, njs/ gets the static handlers from the config repo
+    safe_mkdir(NGINX_CHAINS_PATH)
+    safe_mkdir(NGINX_NJS_PATH)
     process_template(NGINX_TEMPLATE_FILEPATH, NGINX_CONFIG_FILEPATH, template_data)
+    base = render_base_config(ssl_on, skale_node)
+    write_atomic(NGINX_BASE_CONFIG_FILEPATH, base)
+    for script in glob.glob(os.path.join(NGINX_NJS_SOURCE_PATH, '*.js')):
+        shutil.copy(script, NGINX_NJS_PATH)
+    migrate_nginx_layout()
+    return base
+
+
+def render_base_config(ssl_on: bool, skale_node: bool) -> str:
+    """base.conf fingerprinted with its certificate, rendered identically by skale-admin"""
+    template = Environment().from_string(read_file(NGINX_BASE_TEMPLATE_FILEPATH))
+    data = {'ssl': ssl_on, 'skale_node': skale_node}
+    draft = template.render(data, fingerprint='')
+    cert_path = Path(NODE_CERTS_PATH, SSL_CRT_NAME)
+    cert = cert_path.read_bytes() if ssl_on and cert_path.is_file() else b''
+    fingerprint = hashlib.sha256(draft.encode() + cert).hexdigest()[:16]
+    return template.render(data, fingerprint=fingerprint)
+
+
+def write_atomic(path: str, text: str) -> None:
+    """nginx never reads a half-written file: conf.d/*.conf does not match the temporary name"""
+    tmp_path = f'{path}.tmp'
+    Path(tmp_path).write_text(text)
+    os.replace(tmp_path, path)
 
 
-def check_ssl_certs():
-    crt_path = os.path.join(NODE_CERTS_PATH, SSL_CRT_NAME)
-    key_path = os.path.join(NODE_CERTS_PATH, SSL_KEY_NAME)
-    return os.path.exists(crt_path) and os.path.exists(key_path)
+def base_fingerprint(text: str) -> Optional[str]:
+    first_line = text.split('\n', 1)[0]
+    return first_line.removeprefix(BASE_HEADER) if first_line.startswith(BASE_HEADER) else None
+
+
+def wait_for(predicate: Callable[[], bool], timeout: float) -> bool:
+    deadline = time.monotonic() + timeout
+    while not predicate():
+        if time.monotonic() >= deadline:
+            return False
+        time.sleep(POLL_INTERVAL_SECONDS)
+    return True
 
 
 def is_skale_node_nginx() -> bool:
     return TYPE == NodeType.SKALE
 
 
+@contextmanager
+def nginx_lock() -> Iterator[None]:
+    """Serialises with skale-admin, which reloads nginx for chain files and certificates"""
+    try:
+        with FileLock(NGINX_LOCK_PATH, timeout=LOCK_TIMEOUT_SECONDS):
+            yield
+    except Timeout as err:
+        raise NginxConfigError('skale-admin holds the nginx lock') from err
+
+
 def reload_nginx() -> None:
+    """Re-render and reload; `nginx -s reload` exits 0 even when nginx keeps its old config"""
     dutils = docker_client()
-    generate_nginx_config()
-    restart_nginx_container(dutils=dutils)
+    base_path = Path(NGINX_BASE_CONFIG_FILEPATH)
+    with nginx_lock():
+        previous = base_path.read_text() if base_path.is_file() else None
+        base = generate_nginx_config()
+        try:
+            reload_nginx_container(dutils=dutils)
+            served = base is None or wait_for(
+                lambda: base_served(base, dutils), APPLY_TIMEOUT_SECONDS
+            )
+            if not served:
+                raise NginxConfigError('nginx kept its old configuration, see docker logs sk_nginx')
+        except Exception:
+            if previous is not None:
+                # a file nginx never ran could stop it at its next start
+                write_atomic(NGINX_BASE_CONFIG_FILEPATH, previous)
+            raise
+
+
+def base_served(base: str, dutils) -> bool:
+    return nginx_answer(BASE_PROBE_URL, dutils=dutils) == f'base {base_fingerprint(base)}'

node_cli/core/ssl/upload.py

@@ -22,12 +22,17 @@
 
 from node_cli.core.ssl.check import check_cert_openssl
 from node_cli.core.ssl.utils import is_ssl_folder_empty, copy_cert_key_pair
+from node_cli.utils.docker_utils import NginxConfigError
 from node_cli.utils.helper import ok_result, err_result
+from node_cli.core.nftables import sync_tls_ports
 from node_cli.core.nginx import reload_nginx
 
 
 logger = logging.getLogger(__name__)
 
+# the certificates are already in place, so a plain retry is refused as a second upload
+RETRY_HINT = 'Run the upload again with --force'
+
 
 def upload_cert(cert_path, key_path, force, no_client=False):
     try:
@@ -38,5 +43,19 @@ def upload_cert(cert_path, key_path, force, no_client=False):
     if not is_ssl_folder_empty() and not force:
         return err_result(CERTS_UPLOADED_ERR_MSG)
     copy_cert_key_pair(cert_path, key_path)
-    reload_nginx()
+    try:
+        reload_nginx()
+    except NginxConfigError as err:
+        logger.exception('nginx did not apply the new certificates')
+        return err_result(
+            f'Certificates are saved, but nginx is not serving them. {err}. {RETRY_HINT}'
+        )
+    try:
+        sync_tls_ports()
+    except Exception as err:
+        logger.exception('Firewall does not follow the new certificates')
+        return err_result(
+            f'Certificates are saved, but the firewall may still block the TLS ports. {err}. '
+            f'{RETRY_HINT}'
+        )
     return ok_result()