diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e4e72a5..df1506e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,6 +47,9 @@ jobs: - name: Verify what reaches root or a superuser (certificates, downloads, venvs, neutralisation) run: python tools/verify_privilege.py + - name: Verify the PostgreSQL install steps (port, scram, listen_addresses, pg_hba) + run: python tools/verify_postgres_setup.py + - name: Verify nginx and fail2ban accept the generated configuration run: | sudo apt-get update -qq diff --git a/CHANGELOG.md b/CHANGELOG.md index f5979e8..72d4478 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ All notable changes to this project are documented here. The format is based on with the host `python3` only when it can build the version, else with a CPython installed by uv (pinned, SHA-256-checked) into `/opt/odoo-python`. setuptools is pinned per version (`<58` up to 13, `<81` up to 16) and Odoo 12's `pyldap` is replaced. Before, Odoo 14 did not install on Ubuntu 24.04 (its requirements stop at - Python 3.10) and Odoo 15–16 did not start there (setuptools 81 removed the `pkg_resources` they import). + Python 3.10) and Odoo 15–16 did not start there (setuptools 82 removed the `pkg_resources` they import). - **OCB.** The install asks for the core — official Odoo or OCA's OCB; a replica runs the source's core. - New instances get `without_demo = all` (Odoo ≤ 18) and `data_dir = /var/lib/odoo/`, outside the home. The PostgreSQL floor of the Odoo version is checked when PostgreSQL is installed. @@ -43,6 +43,25 @@ All notable changes to this project are documented here. The format is based on ### Fixed +- **Installs that failed on supported hosts.** + - **Python.** Odoo 15 on Debian 12 (Python 3.11) and Odoo 14 on a host Python 3.10 hit the `gevent==21.8.0` pin, + which has no wheel there and no longer builds; they now get uv's interpreter. + - **PostgreSQL 13** (Debian 11) stored the role's password as md5, which the scram `pg_hba` rule refuses. + - **nginx.** The first HTTPS install on Ubuntu 22.04 and Debian 11 wrote `proxy_cookie_flags` for an nginx that + was not installed yet (1.18 rejects it). + - **An existing database role** kept its old password while `odoo.conf` got the new one; the install now asks + which password to use. + - **arm64** got the amd64 wkhtmltopdf package. +- **Corrected facts.** Odoo 16's Python maximum is 3.12, as `odoo/__init__.py` states, not 3.13: Debian 13's + 3.13 now gets uv's 3.12. Odoo 15–18 state their range outright (official, not derived). setuptools removed + `pkg_resources` in 82. +- **Update configuration** keeps `odoo.conf`'s other sections and multi-line values, and writes each key once. + Before, a key written in two cases made the file unreadable for Odoo. +- **PostgreSQL steps.** + - Local commands use the instance's port. + - Remote access survives a distribution's `conf.d` and restarts PostgreSQL only when needed. + - uv is always the pinned one. + - **A neutralised copy could still reach the outside.** - Several rules named columns some versions lack, so they were skipped silently and left those copies in production: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 27aebc2..67a634d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -54,10 +54,11 @@ python3 tools/verify_neutralisation.py --odoo 18=/path/venv/bin/python:/path/odo python3 tools/verify_ops_configs.py # nginx -t and fail2ban on the generated config python3 tools/verify_secrets.py # secrets in no process argument, preview or error python3 tools/verify_privilege.py # certificates, downloads, venv replication, neutralisation as owner +python3 tools/verify_postgres_setup.py # PostgreSQL install steps: port, scram, listen_addresses, pg_hba ``` -`tools/verify_data_safety.py`, `tools/verify_secrets.py`, `tools/verify_privilege.py` and -`tools/verify_ops_configs.py` also run in CI. +`tools/verify_data_safety.py`, `tools/verify_secrets.py`, `tools/verify_privilege.py`, +`tools/verify_postgres_setup.py` and `tools/verify_ops_configs.py` also run in CI. `tools/verify_data_safety.py` executes the generated scheduled-backup script, the retention prune and the delete step against stub binaries, and the template copy, forced drop and purge discovery against a PostgreSQL cluster of its own (it needs `initdb` and refuses to run as root). Run it after @@ -69,6 +70,11 @@ against stubs, and the neutralisation as the copy's owner against a PostgreSQL c trigger in the copy tries to make its owner a superuser. Run it after changing those planners or `_post_db_mode_commands`. +`tools/verify_postgres_setup.py` runs the PostgreSQL steps of an install against a cluster of its own on a port +other than 5432, storing md5 by default as PostgreSQL 13 does and with a `conf.d` file, and checks the role's +scram password, `listen_addresses` set with one restart, and the `pg_hba` rule added once. Run it after changing +those planners. + `tools/verify_install_runtime.py` is not part of CI: it downloads the pinned uv (and refuses a wrong checksum), installs the interpreter the matrix picks, builds a real venv for each Odoo checkout you pass and starts `odoo-bin --version`; for Odoo <= 16 it also shows the unpinned setuptools breaks it. It needs network diff --git a/docs/installation.md b/docs/installation.md index 076f9bd..c795884 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -130,7 +130,7 @@ the whole configuration; otherwise the previous vhost and links are put back, so enabled to stop nginx (and every instance) at its next restart. The vhosts accept uploads up to 2 GB and compress with gzip. The HTTPS vhost follows Odoo's deployment guide: -TLS 1.2/1.3 with its cipher list, HSTS, and the `session_id` cookie marked `secure` (nginx ≥ 1.19.8). +TLS 1.2/1.3 with its cipher list, HSTS, and the `session_id` cookie marked `secure` (nginx ≥ 1.19.3). For HTTPS you pick a certificate strategy: diff --git a/docs/platforms.md b/docs/platforms.md index e913817..7ab394e 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -45,16 +45,17 @@ assuming a single stack: | OS family | Debian/Ubuntu (**apt**) | Package steps target apt. A non-apt OS is detected and warned, not driven blindly. | | Ubuntu | 22.04 (jammy), 24.04 (noble) | Validated primary targets. Newer releases work; the wkhtmltopdf table maps noble → the jammy build. | | Debian | 11 (bullseye), 12 (bookworm) | wkhtmltopdf assets pinned for both. | -| nginx | 1.18 → 1.25+ | HTTP/2 directive form chosen by detected version; `proxy_cookie_flags` written from 1.19.8. `tools/verify_ops_configs.py` runs `nginx -t` on the generated vhosts with the nginx the host distribution ships (1.24 on Ubuntu 24.04). | +| nginx | 1.18 → 1.25+ | HTTP/2 directive form chosen by detected version; `proxy_cookie_flags` written from 1.19.3. The version is the installed nginx's, or the one apt would install when the plan installs it; when neither is known the vhost is written for an older nginx. `tools/verify_ops_configs.py` runs `nginx -t` on the generated vhosts with the nginx the host distribution ships (1.24 on Ubuntu 24.04). | | PostgreSQL | 13+ | Local drops use `dropdb --force` (13+); Odoo's own floor per version is below. Every supported Ubuntu/Debian ships 13 or newer. | | Odoo Community | 12 → 19 | Per-version interpreter, setuptools, config keys — see the table below. Official Odoo or OCB. | | wkhtmltopdf | 0.12.6.1-3 (patched) | jammy/noble, bookworm, bullseye assets, each SHA-256-pinned; other codenames use the distro package or skip. | -| CPU architecture | x86_64 (amd64), aarch64 | The pinned uv exists for both; an Odoo version that needs uv's Python cannot be installed on another architecture. The patched wkhtmltopdf assets are amd64 only (use the distro package elsewhere). | +| CPU architecture | x86_64 (amd64), aarch64 | The pinned uv exists for both; an Odoo version that needs uv's Python cannot be installed on another architecture. The patched wkhtmltopdf has amd64 and arm64 builds for Ubuntu 22.04/24.04 and Debian 11/12; elsewhere the distribution package is offered when it has one. | ## Odoo versions -Python ranges are those Odoo declares; a maximum marked *derived* is the newest interpreter bucket the branch's -`requirements.txt` declares (only Odoo 19 states one outright). The fallback interpreter is the one uv installs +Python ranges are those Odoo declares: Odoo 15–19 state their range outright (`MIN_PY_VERSION` / +`MAX_PY_VERSION` in `odoo/__init__.py`, `odoo/release.py` from 19). Odoo 14's maximum is *derived*: the newest +interpreter bucket its `requirements.txt` declares. The fallback interpreter is the one uv installs when the host's cannot build the version. The facts are declared once, in `instance_manager/support.py`; `tools/verify_install_runtime.py` builds a venv for a given Odoo checkout with them and starts it. @@ -62,14 +63,17 @@ when the host's cannot build the version. The facts are declared once, in `insta |------|--------|---------------|------------|------------------|-------| | 12 | 3.5 – no maximum stated | 3.8 | `<58` | not stated | host used only up to 3.8; `pyldap` → `python-ldap==3.1.0` | | 13 | 3.6 – no maximum stated | 3.8 | `<58` | not stated | host used only up to 3.8 | -| 14 | 3.7 – 3.10 (derived) | 3.8 | `<81` | 12 | | -| 15 | 3.7 – 3.12 (derived) | 3.12 | `<81` | 12 | | -| 16 | 3.7 – 3.13 (derived) | 3.12 | `<81` | 12 | never a host 3.10 (gevent pin) | -| 17 | 3.10 – 3.14 (derived) | 3.12 | current | 12 | never a host 3.10 (gevent pin) | -| 18 | 3.10 – 3.14 (derived) | 3.12 | current | 12 | never a host 3.10 (gevent pin) | +| 14 | 3.7 – 3.10 (derived) | 3.8 | `<81` | 12 | never a host 3.10 (gevent pin) | +| 15 | 3.7 – 3.12 (official) | 3.12 | `<81` | 12 | never a host 3.10 or 3.11 (gevent pin) | +| 16 | 3.7 – 3.12 (official) | 3.12 | `<81` | 12 | never a host 3.10 (gevent pin) | +| 17 | 3.10 – 3.14 (official) | 3.12 | current | 12 | never a host 3.10 (gevent pin) | +| 18 | 3.10 – 3.14 (official) | 3.12 | current | 12 | never a host 3.10 (gevent pin) | | 19 | 3.10 – 3.14 (official) | 3.12 | current | 13 | demo data off by default in Odoo itself | -On Ubuntu 24.04 (Python 3.12) the host interpreter builds Odoo 15–19; Odoo 12–14 get uv's 3.8. +On Ubuntu 24.04 (Python 3.12) the host interpreter builds Odoo 15–19; Odoo 12–14 get uv's 3.8. The gevent pin is +the `gevent==21.8.0` row of each branch's `requirements.txt`: that release has no wheel for those interpreters and +its source no longer builds, so uv's interpreter is used instead. `setuptools<81` keeps `pkg_resources`, which +Odoo ≤ 16 imports and setuptools 82 removed. ## Out of scope diff --git a/instance_manager/i18n.py b/instance_manager/i18n.py index 93dd33a..4fcda5d 100644 --- a/instance_manager/i18n.py +++ b/instance_manager/i18n.py @@ -783,7 +783,6 @@ def tf(template: str, *args: object, **kwargs: object) -> str: "Odoo admin_passwd (master password)": "admin_passwd de Odoo (contraseña maestra)", "Expose the Odoo database manager over HTTP (list_db)? Recommended: No": "¿Exponer el gestor de bases de datos de Odoo por HTTP (list_db)? Recomendado: No", "The database manager will be reachable over HTTP (create/duplicate/drop/restore), guarded only by the master password. Keep a strong master password and a dbfilter.": "El gestor de bases de datos quedará accesible por HTTP (crear/duplicar/eliminar/restaurar), protegido solo por la contraseña maestra. Mantén una contraseña maestra fuerte y un dbfilter.", - "Database manager disabled (list_db = False). Create the first database via CLI (odoo-bin -d -i base --stop-after-init) or by temporarily re-enabling it.": "Gestor de bases de datos desactivado (list_db = False). Crea la primera base de datos por CLI (odoo-bin -d -i base --stop-after-init) o reactivándolo temporalmente.", "dbfilter (binds host/database)": "dbfilter (vincula host/base de datos)", "Restrict the instance to specific database(s) with a dbfilter (recommended)?": "¿Restringir la instancia a una o varias bases de datos con un dbfilter (recomendado)?", "No dbfilter set: the instance will not filter databases (all are reachable). Recommended only when the manager is disabled or a single database is served.": "Sin dbfilter: la instancia no filtrará bases de datos (todas quedan accesibles). Recomendado solo si el gestor está desactivado o se sirve una única base de datos.", @@ -981,4 +980,14 @@ def tf(template: str, *args: object, **kwargs: object) -> str: 'Restore the dump as role {}': 'Restaurar el dump como rol {}', "'cli' servers send through odoo.conf's smtp_server": "los servidores 'cli' envían por el smtp_server de odoo.conf", 'The copy failed: its unfinished database {} is dropped.': 'La copia ha fallado: se elimina su base de datos a medias {}.', + "Listen on every address (listen_addresses = '*')": "Escuchar en todas las direcciones (listen_addresses = '*')", + 'Reload PostgreSQL (pg_hba)': 'Recargar PostgreSQL (pg_hba)', + 'Install uv {} (checksum-verified) unless it is there': 'Instalar uv {} (verificado por checksum) salvo que ya esté', + 'Set the role to the password entered (update other instances that use it)': 'Poner al rol la contraseña introducida (actualiza las otras instancias que lo usen)', + "Keep the role's password (type it now)": 'Mantener la contraseña del rol (escríbela ahora)', + 'The PostgreSQL role {} already exists. Which password should this instance use?': 'El rol de PostgreSQL {} ya existe. ¿Qué contraseña debe usar esta instancia?', + 'Current password of role {}': 'Contraseña actual del rol {}', + 'Database manager disabled (list_db = False). Create the first database as the instance user: sudo -u /venv/bin/python /odoo/odoo-bin -c -d -i base --stop-after-init, or re-enable the manager for a while.': 'Gestor de bases de datos desactivado (list_db = False). Crea la primera base de datos como el usuario de la instancia: sudo -u /venv/bin/python /odoo/odoo-bin -c -d -i base --stop-after-init, o reactiva el gestor un momento.', + 'verify-full checks the server against a CA: place it at {}/.postgresql/root.crt, readable by the instance user, before the install runs.': 'verify-full comprueba el servidor contra una CA: colócala en {}/.postgresql/root.crt, legible por el usuario de la instancia, antes de ejecutar la instalación.', + 'The gevent port must differ from the HTTP port.': 'El puerto gevent debe ser distinto del puerto HTTP.', } diff --git a/instance_manager/planners.py b/instance_manager/planners.py index 104418f..f2e6d7d 100644 --- a/instance_manager/planners.py +++ b/instance_manager/planners.py @@ -26,6 +26,12 @@ def _is_local_db_host(db_host: str) -> bool: } +# The pg_hba rule the tool writes asks for scram-sha-256, and a password stored as +# md5 cannot pass it. PostgreSQL 13 (Debian 11) still stores md5 by default; 14 and +# later store scram. Set for the session that sets the password. +SCRAM = "SET password_encryption = 'scram-sha-256'; " + + def _dollar_tag(*bodies: str) -> str: """A dollar-quote tag that none of ``bodies`` contains: a password holding ``$$`` would otherwise end the quoted block early.""" @@ -35,18 +41,24 @@ def _dollar_tag(*bodies: str) -> str: return f"$oim{index}$" -def _db_role_create_if_missing_sql(config: InstanceConfig) -> str: +def _db_role_create_if_missing_sql(config: InstanceConfig, reset_password: bool = False) -> str: + """Create the role with the password, or — when it exists — keep it, setting the + password only when ``reset_password`` (the operator chose to).""" db_user_literal = _sql_literal(config.db_user) db_password_literal = _sql_literal(config.db_password) tag = _dollar_tag(db_password_literal) return ( - f"DO {tag} " + SCRAM + f"DO {tag} " "BEGIN " f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname='{db_user_literal}') THEN " f"CREATE ROLE {config.db_user} WITH LOGIN CREATEDB PASSWORD '{db_password_literal}'; " "ELSE " - f"ALTER ROLE {config.db_user} WITH LOGIN CREATEDB; " - f"RAISE NOTICE 'Role {db_user_literal} already exists; reusing it without changing the password.'; " + + (f"ALTER ROLE {config.db_user} WITH LOGIN CREATEDB PASSWORD '{db_password_literal}'; " + f"RAISE NOTICE 'Role {db_user_literal} already exists; its password is set to the new one.'; " + if reset_password else + f"ALTER ROLE {config.db_user} WITH LOGIN CREATEDB; " + f"RAISE NOTICE 'Role {db_user_literal} already exists; reusing it without changing the password.'; ") + + "END IF; " "END " f"{tag};" @@ -54,15 +66,22 @@ def _db_role_create_if_missing_sql(config: InstanceConfig) -> str: def _db_connectivity_check(config: InstanceConfig) -> Command: - """Log in as the instance's role, the way Odoo will (TCP for a local host).""" + """Log in as the instance's role, the way Odoo will: TCP for a local host, and + the SSL mode odoo.conf sets — checked against the CA where libpq looks for the + instance user's (``~/.postgresql/root.crt``).""" check_host = config.db_host if _is_local_db_host(check_host): check_host = "127.0.0.1" + env = pg_env(config.db_password) + if config.db_sslmode: + env["PGSSLMODE"] = config.db_sslmode + if config.db_sslmode.startswith("verify"): + env["PGSSLROOTCERT"] = f"{config.odoo_home}/.postgresql/root.crt" return Command( 'Validate DB user login', f"psql -X -h {shlex.quote(check_host)} -p {int(config.db_port)} -U {shlex.quote(config.db_user)} " "-d postgres -tAc 'SELECT 1;' >/dev/null", - env=pg_env(config.db_password), + env=env, ) @@ -131,11 +150,17 @@ def _conf_key(line: str) -> str | None: return line.split("=", 1)[0].strip() -def render_merged_odoo_conf(config: InstanceConfig, existing: dict[str, str]) -> str: +def _conf_line(key: str, value: str) -> str: + """``key = value``, a value of several lines continued as Odoo's parser reads it.""" + return f"{key} = " + value.replace("\n", "\n ") + + +def render_merged_odoo_conf(config: InstanceConfig, existing: dict[str, str], other_sections: str = "") -> str: """The instance's ``odoo.conf`` regenerated without losing what it held: the - keys in ``CONF_KEYS_KEPT_ON_UPDATE`` keep their current value, and every key the + keys in ``CONF_KEYS_KEPT_ON_UPDATE`` keep their current value, every key the tool does not write (``smtp_*``, ``server_wide_modules``, ``db_name``, …) is - carried over below. Pure — ``existing`` is the parsed current file.""" + carried over below, and the file's other sections (``[queue_job]``, …) follow as + they were. Pure — ``existing`` is the parsed ``[options]`` of the current file.""" lines = _odoo_conf_content(config).rstrip("\n").split("\n") written: set[str] = set() for index, line in enumerate(lines): @@ -144,16 +169,18 @@ def render_merged_odoo_conf(config: InstanceConfig, existing: dict[str, str]) -> continue written.add(key) if key in CONF_KEYS_KEPT_ON_UPDATE and existing.get(key, "") != "": - lines[index] = f"{key} = {existing[key]}" + lines[index] = _conf_line(key, existing[key]) for key in CONF_KEYS_KEPT_ON_UPDATE: if key not in written and existing.get(key, "") != "": - lines.append(f"{key} = {existing[key]}") + lines.append(_conf_line(key, existing[key])) written.add(key) if written & set(_BUS_PORT_KEYS): written |= set(_BUS_PORT_KEYS) - carried = [f"{key} = {value}" for key, value in existing.items() if key not in written] + carried = [_conf_line(key, value) for key, value in existing.items() if key not in written] if carried: lines += ["", "; kept from the previous configuration", *carried] + if other_sections: + lines += ["", other_sections] return "\n".join(lines) + "\n" @@ -163,6 +190,7 @@ def plan_update_instance_config( *, new_db_password: bool, restart: bool, + other_sections: str = "", ) -> list[Command]: """Rewrite an existing instance's ``odoo.conf`` (merged) and unit, and apply them. @@ -171,7 +199,7 @@ def plan_update_instance_config( the next start could not connect. The service is restarted when it runs, since ``systemctl start`` on a running unit loads nothing.""" commands = write_text_file_command( - config.odoo_conf_file, render_merged_odoo_conf(config, existing), "640", + config.odoo_conf_file, render_merged_odoo_conf(config, existing, other_sections), "640", secrets=(config.odoo_admin_passwd, config.db_password, *(value for key, value in existing.items() if "pass" in key or "secret" in key)), ) @@ -182,10 +210,10 @@ def plan_update_instance_config( commands += write_text_file_command(service_path, _systemd_content(config), "644") commands.append(Command('Reload systemd', "systemctl daemon-reload")) if new_db_password and not config.is_remote_db_host: - sql = f"ALTER ROLE {config.db_user} WITH PASSWORD '{_sql_literal(config.db_password)}';" + sql = f"{SCRAM}ALTER ROLE \"{config.db_user}\" WITH PASSWORD '{_sql_literal(config.db_password)}';" commands.append( _psql_stdin_command( - "sudo -u postgres psql -X -q -v ON_ERROR_STOP=1", sql, (config.db_password,), + f"{_local_psql(config)} -q -v ON_ERROR_STOP=1", sql, (config.db_password,), 'Set the new password on the local PostgreSQL role', ) ) @@ -291,7 +319,9 @@ def _proxy_headers(extra: str = "") -> str: def _secure_location_headers(nginx_version: tuple[int, int, int] | None) -> str: # add_header in a location replaces the server's, so each location carries it. lines = ' add_header Strict-Transport-Security "max-age=31536000; includeSubDomains";\n' - if nginx_version is None or nginx_version >= (1, 19, 8): + # proxy_cookie_flags appeared in nginx 1.19.3; an unknown version is taken as + # older, since nginx -t rejects a directive it does not know. + if nginx_version is not None and nginx_version >= (1, 19, 3): lines += " proxy_cookie_flags session_id samesite=lax secure;\n" return lines @@ -881,12 +911,18 @@ def plan_odoo_base_setup( return commands +def _local_psql(config: InstanceConfig) -> str: + """psql as postgres on the local server, at the instance's port (a host may run + a second cluster).""" + return f"sudo -u postgres psql -X -p {int(config.db_port)}" + + def _pg_hba_append_command(config: InstanceConfig) -> str: """Append the app server's rule to pg_hba.conf unless the same line is there. The values are validated (role, IP) and the line is shell-quoted as one word.""" rule = f"host all {config.db_user} {host_cidr(config.app_server_ip)} scram-sha-256" return ( - 'PG_HBA=$(sudo -u postgres psql -X -t -P format=unaligned -c "SHOW hba_file;") && ' + f'PG_HBA=$({_local_psql(config)} -t -P format=unaligned -c "SHOW hba_file;") && ' f'{{ grep -qxF -- {shlex.quote(rule)} "$PG_HBA" || printf \'%s\\n\' {shlex.quote(rule)} >> "$PG_HBA"; }}' ) @@ -919,7 +955,7 @@ def _venv_commands(config: InstanceConfig) -> list[Command]: create = ( f"env UV_PYTHON_INSTALL_DIR={shlex.quote(support.UV_PYTHON_DIR)} UV_PYTHON_DOWNLOADS=never " "UV_PYTHON_PREFERENCE=only-managed " - f"uv venv --seed --allow-existing --no-project --python {shlex.quote(config.python)} " + f"{support.UV_BIN} venv --seed --allow-existing --no-project --python {shlex.quote(config.python)} " f"{shlex.quote(venv)}" ) label = tf('Create the venv with Python {} (uv)', config.python) @@ -948,8 +984,9 @@ def _venv_commands(config: InstanceConfig) -> list[Command]: def plan_ensure_uv(arch: str) -> list[Command]: - """Install the pinned uv into /usr/local/bin when it is missing, from GitHub's - release asset, checked against its published SHA-256 before anything runs.""" + """Install the pinned uv into /usr/local/bin unless that exact version is there, + from GitHub's release asset, checked against its published SHA-256 before + anything runs. Another uv on root's PATH does not count.""" asset = support.UV_ASSETS.get(arch) if asset is None: return [] @@ -957,7 +994,8 @@ def plan_ensure_uv(arch: str) -> list[Command]: url = f"https://github.com/astral-sh/uv/releases/download/{support.UV_VERSION}/{filename}" stem = filename.removesuffix(".tar.gz") script = ( - "command -v uv >/dev/null 2>&1 && exit 0; " + f'[ "$({support.UV_BIN} --version 2>/dev/null | cut -d" " -f2)" = {shlex.quote(support.UV_VERSION)} ] && exit 0; ' + 'tmp=$(mktemp -d) && trap \'rm -rf "$tmp"\' EXIT && ' f'curl -fsSL -o "$tmp/{filename}" {shlex.quote(url)} && ' f'echo {shlex.quote(sha256 + " ")}"$tmp/{filename}" | sha256sum -c - && ' @@ -969,7 +1007,7 @@ def plan_ensure_uv(arch: str) -> list[Command]: 'Ensure curl is available', "command -v curl >/dev/null 2>&1 || (apt-get update && DEBIAN_FRONTEND=noninteractive apt-get -y -o Dpkg::Options::=--force-confold -o DPkg::Lock::Timeout=600 install curl)", ), - Command(tf('Install uv {} (checksum-verified) if missing', support.UV_VERSION), script), + Command(tf('Install uv {} (checksum-verified) unless it is there', support.UV_VERSION), script), ] @@ -981,7 +1019,8 @@ def plan_uv_python(python: str) -> list[Command]: Command( tf('Install Python {} with uv into {}', python, support.UV_PYTHON_DIR), f"install -d -m 755 {directory} && " - f"UV_PYTHON_INSTALL_DIR={directory} uv python install {shlex.quote(python)} && " + # --no-bin: no `python3.x` link in root's PATH; venvs name the interpreter. + f"UV_PYTHON_INSTALL_DIR={directory} {support.UV_BIN} python install --no-bin {shlex.quote(python)} && " f"chmod -R a+rX,go-w {directory}", ) ] @@ -996,7 +1035,7 @@ def _postgres_floor_commands(config: InstanceConfig) -> list[Command]: return [ Command( tf('Check PostgreSQL is {} or newer (Odoo {} requirement)', floor, version.major), - "v=$(sudo -u postgres psql -X -tAc 'SHOW server_version_num') && " + f"v=$({_local_psql(config)} -tAc 'SHOW server_version_num') && " f'[ "$v" -ge {floor * 10000} ] || ' f"{{ echo \"PostgreSQL $v is older than {floor}, the floor of Odoo {version.major}.\" >&2; exit 1; }}", ) @@ -1004,12 +1043,12 @@ def _postgres_floor_commands(config: InstanceConfig) -> list[Command]: def plan_db_setup( - config: InstanceConfig, ensure_remote_access: bool = True + config: InstanceConfig, ensure_remote_access: bool = True, reset_password: bool = False ) -> list[Command]: config.normalize_defaults() config.ensure_strong_secrets() config.validate_identifiers() - role_sql = _db_role_create_if_missing_sql(config) + role_sql = _db_role_create_if_missing_sql(config, reset_password) commands: list[Command] = [ Command( @@ -1018,7 +1057,7 @@ def plan_db_setup( Command('Enable and start PostgreSQL', "systemctl enable --now postgresql"), *_postgres_floor_commands(config), _psql_stdin_command( - "sudo -u postgres psql -X -v ON_ERROR_STOP=1", role_sql, (config.db_password,), + f"{_local_psql(config)} -v ON_ERROR_STOP=1", role_sql, (config.db_password,), 'Ensure PostgreSQL role (create if missing)', ), _db_connectivity_check(config), @@ -1027,32 +1066,38 @@ def plan_db_setup( if ensure_remote_access: commands.extend( [ + # ALTER SYSTEM wins over postgresql.conf and conf.d; listen_addresses + # needs a restart, which drops every instance's connections, so only + # when the value changes. The result is read back. Command( - "Allow listen_addresses='*'", - 'PG_CONF=$(sudo -u postgres psql -t -P format=unaligned -c "SHOW config_file;") && sed -ri "s/^#?\\s*listen_addresses\\s*=.*/listen_addresses = \'*\'/" "$PG_CONF"', + "Listen on every address (listen_addresses = '*')", + f"psql=({_local_psql(config)} -tA -v ON_ERROR_STOP=1); " + 'if [ "$("${psql[@]}" -c "SHOW listen_addresses")" != "*" ]; then ' + '"${psql[@]}" -c "ALTER SYSTEM SET listen_addresses = \'*\'" && systemctl restart postgresql; fi && ' + '[ "$("${psql[@]}" -c "SHOW listen_addresses")" = "*" ]', ), Command( 'Add pg_hba rule for the app-server IP', _pg_hba_append_command(config), ), - Command('Restart PostgreSQL', "systemctl restart postgresql"), + Command('Reload PostgreSQL (pg_hba)', "systemctl reload postgresql"), ] ) return commands -def plan_ensure_db_role(config: InstanceConfig) -> list[Command]: +def plan_ensure_db_role(config: InstanceConfig, reset_password: bool = False) -> list[Command]: config.normalize_defaults() config.ensure_strong_secrets() config.validate_identifiers() commands: list[Command] = [] if _is_local_db_host(config.db_host): - role_sql = _db_role_create_if_missing_sql(config) + role_sql = _db_role_create_if_missing_sql(config, reset_password) commands.append( _psql_stdin_command( - "sudo -u postgres psql -X -v ON_ERROR_STOP=1", role_sql, (config.db_password,), + f"{_local_psql(config)} -v ON_ERROR_STOP=1", role_sql, (config.db_password,), 'Ensure local PostgreSQL role (create if missing)', ) ) @@ -1481,43 +1526,45 @@ def plan_ufw_delete_rule(number: int, expected_line: str) -> list[Command]: _WKHTMLTOPDF_BASE_URL = ( "https://github.com/wkhtmltopdf/packaging/releases/download/0.12.6.1-3" ) -# Detected OS codename -> (asset filename, sha256). amd64, Qt-patched 0.12.6.1-3. -# The jammy build is verified against Odoo's own pinned checksum (its SHA-1 -# 967390a759707337b46d1c02452e2bb6b2dc6d59 matches the Odoo 18 Dockerfile) and -# also runs on noble (24.04). Codenames without a compatible asset resolve to -# None so the caller recommends the distro package or skip — never a guessed URL. -_WKHTMLTOPDF_ASSETS: dict[str, tuple[str, str]] = { - "jammy": ( - "wkhtmltox_0.12.6.1-3.jammy_amd64.deb", - "4f723b2691ad8638a9df960e0421d346d7315083e3583a334f33362280ddba15", - ), - "noble": ( - "wkhtmltox_0.12.6.1-3.jammy_amd64.deb", - "4f723b2691ad8638a9df960e0421d346d7315083e3583a334f33362280ddba15", - ), - "bookworm": ( - "wkhtmltox_0.12.6.1-3.bookworm_amd64.deb", - "98ba0d157b50d36f23bd0dedf4c0aa28c7b0c50fcdcdc54aa5b6bbba81a3941d", - ), - "bullseye": ( - "wkhtmltox_0.12.6.1-3.bullseye_amd64.deb", - "9c687f0c58cf50e01f2a6375d2e34372f8feeec56a84690ea113d298fccadd98", - ), +# (OS codename, machine) -> (asset filename, sha256), Qt-patched 0.12.6.1-3. The +# jammy amd64 build is verified against Odoo's own pinned checksum (its SHA-1 +# 967390a759707337b46d1c02452e2bb6b2dc6d59 matches the Odoo 18 Dockerfile) and also +# runs on noble (24.04); the other checksums were computed from the release's +# assets (GitHub publishes no digest for this release). Pairs without a compatible +# asset — Debian 13, Ubuntu 26.04, other machines — resolve to None, so the caller +# offers the distribution package or skip, never a guessed URL. +_WKHTMLTOPDF_ASSETS: dict[tuple[str, str], tuple[str, str]] = { + ("jammy", "x86_64"): ("wkhtmltox_0.12.6.1-3.jammy_amd64.deb", + "4f723b2691ad8638a9df960e0421d346d7315083e3583a334f33362280ddba15"), + ("jammy", "aarch64"): ("wkhtmltox_0.12.6.1-3.jammy_arm64.deb", + "2095f20256661ebf0983b9311168596c9d012666e21a94bc24f304db6ac69ec5"), + ("bookworm", "x86_64"): ("wkhtmltox_0.12.6.1-3.bookworm_amd64.deb", + "98ba0d157b50d36f23bd0dedf4c0aa28c7b0c50fcdcdc54aa5b6bbba81a3941d"), + ("bookworm", "aarch64"): ("wkhtmltox_0.12.6.1-3.bookworm_arm64.deb", + "b6606157b27c13e044d0abbe670301f88de4e1782afca4f9c06a5817f3e03a9c"), + ("bullseye", "x86_64"): ("wkhtmltox_0.12.6.1-3.bullseye_amd64.deb", + "9c687f0c58cf50e01f2a6375d2e34372f8feeec56a84690ea113d298fccadd98"), + ("bullseye", "aarch64"): ("wkhtmltox_0.12.6.1-3.bullseye_arm64.deb", + "e73435a82cf21ba0387bfee32a193f221fa43e48dda6ed38b12e4c1b70c69728"), } +# Ubuntu 24.04 runs the 22.04 build. +_WKHTMLTOPDF_SAME_AS = {"noble": "jammy"} -def resolve_wkhtmltopdf_asset(codename: str) -> tuple[str, str, str] | None: - """``(url, filename, sha256)`` for the patched wkhtmltopdf matching ``codename``, - or ``None`` when no compatible verified asset is pinned (the caller then - recommends the distro package or skip; it never guesses a URL).""" - entry = _WKHTMLTOPDF_ASSETS.get((codename or "").strip().lower()) +def resolve_wkhtmltopdf_asset(codename: str, arch: str = "x86_64") -> tuple[str, str, str] | None: + """``(url, filename, sha256)`` for the patched wkhtmltopdf matching ``codename`` + and the machine ``arch`` (``x86_64``, ``aarch64``), or ``None`` when no compatible + verified asset is pinned (the caller then offers the distro package or skip; it + never guesses a URL).""" + name = (codename or "").strip().lower() + entry = _WKHTMLTOPDF_ASSETS.get((_WKHTMLTOPDF_SAME_AS.get(name, name), arch)) if not entry: return None filename, sha256 = entry return f"{_WKHTMLTOPDF_BASE_URL}/{filename}", filename, sha256 -def plan_install_wkhtmltopdf(mode: str, codename: str = "") -> list[Command]: +def plan_install_wkhtmltopdf(mode: str, codename: str = "", arch: str = "x86_64") -> list[Command]: """Plan a wkhtmltopdf install. ``mode == "patched"``: download the codename's pinned Qt-patched ``.deb``, @@ -1534,7 +1581,7 @@ def plan_install_wkhtmltopdf(mode: str, codename: str = "") -> list[Command]: ] if mode != "patched": return [] - asset = resolve_wkhtmltopdf_asset(codename) + asset = resolve_wkhtmltopdf_asset(codename, arch) if asset is None: return [] url, filename, sha256 = asset diff --git a/instance_manager/support.py b/instance_manager/support.py index 83af124..03e8ddd 100644 --- a/instance_manager/support.py +++ b/instance_manager/support.py @@ -1,13 +1,13 @@ """Per-version facts for installing Odoo: which Python, which setuptools, which PostgreSQL floor, and which core repository. -Pure data, no I/O. Every bound names the evidence behind it, because only Odoo 19 -states a Python maximum outright: the other maxima are derived from the newest -interpreter bucket each branch's own ``requirements.txt`` declares, and must not be -presented as something Odoo requires. The facts are the ones the sibling project -odoo_dwg keeps in its support matrix (``docs/reference/support-matrix.md``, -re-derived by its ``tools/verify_support_matrix.py``), where every Odoo version -12-19 was built and started on Ubuntu 24.04. +Pure data, no I/O. Every bound names the evidence behind it: Odoo 15-19 state a +Python range outright (``MIN_PY_VERSION``/``MAX_PY_VERSION`` in ``odoo/__init__.py``, +``odoo/release.py`` from 19), and 14's maximum is derived from the newest +interpreter bucket of its ``requirements.txt``, so it must not be presented as +something Odoo requires. The interpreter uv installs is one this tool and its +sibling odoo_dwg have built and started each version on (Ubuntu 24.04); +``tools/verify_install_runtime.py`` re-checks it. """ from __future__ import annotations @@ -41,6 +41,9 @@ "0e9a3499b0587d449c9ff684c0160da607826e4af1cee220bc87f378702d3e08", ), } +# The uv the tool installs and calls by this path: root's PATH may hold another +# (~/.local/bin/uv), and an instance user's secure_path holds none. +UV_BIN = "/usr/local/bin/uv" # Where uv keeps the interpreters it installs: shared, root-owned, readable by every # instance user. Outside /opt/odoo so it is never listed as an instance. UV_PYTHON_DIR = "/opt/odoo-python" @@ -60,6 +63,10 @@ def _docs(version: str) -> str: return f"https://www.odoo.com/documentation/{version}/{page}" +def _init_max(version: str) -> str: + return f"odoo/odoo@{version} odoo/__init__.py MAX_PY_VERSION" + + def _bucket(version: str, distro: str) -> str: return f"odoo/odoo@{version} requirements.txt — newest interpreter bucket targets {distro}" @@ -92,30 +99,37 @@ def branch(self) -> str: None, "the 13.0 install page states no floor"), 14: VersionSupport(14, "3.7", _docs("14.0"), "3.10", DERIVED, _bucket("14.0", "Ubuntu 22.04 Jammy"), "3.8", 12, _docs("14.0")), - 15: VersionSupport(15, "3.7", _docs("15.0"), "3.12", DERIVED, _bucket("15.0", "Ubuntu 24.04 Noble"), + 15: VersionSupport(15, "3.7", _docs("15.0"), "3.12", OFFICIAL, _init_max("15.0"), "3.12", 12, _docs("15.0")), - 16: VersionSupport(16, "3.7", _docs("16.0"), "3.13", DERIVED, _bucket("16.0", "Debian 13 Trixie"), + 16: VersionSupport(16, "3.7", _docs("16.0"), "3.12", OFFICIAL, _init_max("16.0"), "3.12", 12, _docs("16.0")), - 17: VersionSupport(17, "3.10", _docs("17.0"), "3.14", DERIVED, - _bucket("17.0", "Ubuntu 26.04 Resolute"), "3.12", 12, _docs("17.0")), - 18: VersionSupport(18, "3.10", _docs("18.0"), "3.14", DERIVED, - _bucket("18.0", "Ubuntu 26.04 Resolute"), "3.12", 12, _docs("18.0")), + 17: VersionSupport(17, "3.10", _docs("17.0"), "3.14", OFFICIAL, _init_max("17.0"), + "3.12", 12, _docs("17.0")), + 18: VersionSupport(18, "3.10", _docs("18.0"), "3.14", OFFICIAL, _init_max("18.0"), + "3.12", 12, _docs("18.0")), 19: VersionSupport(19, "3.10", "odoo/odoo@19.0 odoo/release.py MIN_PY_VERSION", "3.14", OFFICIAL, "odoo/odoo@19.0 odoo/release.py MAX_PY_VERSION", "3.12", 13, _docs("19.0")), } # Odoo <= 16 imports pkg_resources at startup (odoo/modules/module.py), which -# setuptools removed in 81; Odoo <= 13 requires vatnumber==1.2, whose setup.py -# still uses use_2to3, removed in setuptools 58. +# setuptools 82 removed (81 still ships it, deprecated); the pin stays below 81, the +# series every one of them was built and started with. Odoo <= 13 requires +# vatnumber==1.2, whose setup.py still uses use_2to3, removed in setuptools 58. PKG_RESOURCES_LAST_MAJOR = 16 USE_2TO3_LAST_MAJOR = 13 -# Odoo 16-19 pin gevent==21.8.0 / greenlet==1.1.2 for python_version == '3.10' -# (their Jammy row). That gevent predates CPython 3.10 — no cp310 wheel — and its -# sdist no longer compiles under a current Cython, so a 3.10 host cannot install -# them with pip (odoo_dwg hit this and repairs it with uv overrides for its -# migration steps). An instance avoids the row instead: uv's 3.12. -GEVENT_PY310_FIRST_MAJOR = 16 +# The interpreters each branch pins gevent==21.8.0 / greenlet==1.1.2 for (their +# Jammy row): 14 for python_version > '3.9', 15 for > '3.9' and < '3.12', 16 for +# > '3.9' and <= '3.10', 17-19 for == '3.10'. That gevent predates CPython 3.10 — no +# cp310 or cp311 wheel — and its sdist no longer compiles under a current Cython, +# so such a host cannot install it with pip (odoo_dwg hit this and repairs it with +# uv overrides for its migration steps). An instance avoids the row: uv's +# recommended interpreter instead. +GEVENT_2108_PYTHONS: dict[int, tuple[str, ...]] = { + 14: ("3.10", "3.11", "3.12", "3.13", "3.14"), + 15: ("3.10", "3.11"), + 16: ("3.10",), 17: ("3.10",), 18: ("3.10",), 19: ("3.10",), +} # Requirements an instance installs in place of one its branch pins: # ``{major: (dropped project, replacement)}``. Odoo 12 pins pyldap==2.4.28, a @@ -157,13 +171,14 @@ def host_python_usable(support: VersionSupport, host_python: str | None) -> bool """Whether the host's python3 can build this version's requirements. Inside the range, except: with no stated maximum (12, 13) only up to the - recommendation — on 3.12 their pinned gevent does not even build — and never - 3.10 for 16-19 (see ``GEVENT_PY310_FIRST_MAJOR``).""" + recommendation — on 3.12 their pinned gevent does not even build — and never an + interpreter the branch pins gevent 21.8.0 for (see ``GEVENT_2108_PYTHONS``).""" if not host_python or not python_in_range(support, host_python): return False if support.python_max is None and python_tuple(host_python) > python_tuple(support.recommended_python): return False - return not (support.major >= GEVENT_PY310_FIRST_MAJOR and python_tuple(host_python)[:2] == (3, 10)) + minor = ".".join(str(part) for part in python_tuple(host_python)[:2]) + return minor not in GEVENT_2108_PYTHONS.get(support.major, ()) def choose_python(support: VersionSupport, host_python: str | None) -> tuple[str, str]: diff --git a/instance_manager/system.py b/instance_manager/system.py index e6de873..bef0362 100644 --- a/instance_manager/system.py +++ b/instance_manager/system.py @@ -1,5 +1,6 @@ from __future__ import annotations +import configparser import os import platform import re @@ -230,27 +231,49 @@ def list_instances(base_path: str = "/opt/odoo") -> list[str]: return list_dirs(base_path) +def _conf_text(conf_path: str) -> str: + try: + with open(conf_path, encoding="utf-8", errors="replace") as file_handle: + return file_handle.read() + except OSError: + return "" + + def read_odoo_conf(conf_path: str) -> dict[str, str]: - values: dict[str, str] = {} - if not os.path.exists(conf_path): + """The ``[options]`` of an odoo.conf as Odoo reads it (``RawConfigParser``: keys + lower-cased, a continued value kept whole with its newlines). A file with no + section header is read line by line, as before.""" + text = _conf_text(conf_path) + if not text: + return {} + parser = configparser.RawConfigParser(strict=False, interpolation=None) + try: + parser.read_string(text) + except configparser.MissingSectionHeaderError: + values: dict[str, str] = {} + for raw_line in text.splitlines(): + line = raw_line.strip() + if line and not line.startswith(("#", ";", "[")) and "=" in line: + key, value = line.split("=", 1) + values[key.strip().lower()] = value.strip() return values + except configparser.Error: + return {} + return dict(parser["options"]) if parser.has_section("options") else {} - with open(conf_path, encoding="utf-8") as file_handle: - for raw_line in file_handle: - line = raw_line.strip() - if ( - not line - or line.startswith("#") - or line.startswith(";") - or line.startswith("[") - ): - continue - if "=" not in line: - continue - key, value = line.split("=", 1) - values[key.strip()] = value.strip() - return values +def read_conf_other_sections(conf_path: str) -> str: + """The text of every section of an odoo.conf other than ``[options]`` (a + module's own settings, such as ``[queue_job]``), as written.""" + kept: list[str] = [] + section = "" + for line in _conf_text(conf_path).splitlines(): + stripped = line.strip() + if stripped.startswith("[") and stripped.endswith("]"): + section = stripped[1:-1].strip().lower() + if section and section != "options": + kept.append(line) + return "\n".join(kept).strip("\n") def detect_host_python() -> str | None: @@ -333,30 +356,28 @@ def detect_os_release() -> dict[str, str]: return values +def apt_candidate(package: str) -> str: + """The version apt would install for ``package`` ("" when it has none).""" + result = run(f"apt-cache policy {shlex.quote(package)} 2>/dev/null", check=False) + match = re.search(r"Candidate:\s*(\S+)", result.stdout) + return "" if not match or match.group(1) == "(none)" else match.group(1) + + def detect_nginx_version() -> tuple[int, int, int] | None: - """Parse ``nginx -v`` (``nginx version: nginx/1.24.0``) into a version tuple.""" - if not command_ok("command -v nginx >/dev/null 2>&1"): - return None - # nginx prints its version banner to stderr. - result = run("nginx -v 2>&1", check=False) - match = re.search(r"nginx/(\d+)\.(\d+)\.(\d+)", result.stdout + result.stderr) + """The nginx a vhost is written for: the installed one (``nginx -v`` prints + ``nginx version: nginx/1.24.0``), else the one apt would install — a plan that + installs nginx is built before it is there. None when neither is known.""" + if command_ok("command -v nginx >/dev/null 2>&1"): + # nginx prints its version banner to stderr. + result = run("nginx -v 2>&1", check=False) + match = re.search(r"nginx/(\d+)\.(\d+)\.(\d+)", result.stdout + result.stderr) + else: + match = re.match(r"(?:\d+:)?(\d+)\.(\d+)\.(\d+)", apt_candidate("nginx")) if not match: return None return (int(match.group(1)), int(match.group(2)), int(match.group(3))) -def detect_postgres_version() -> int | None: - """Detected local PostgreSQL server major version via ``SHOW server_version``.""" - result = run( - 'sudo -u postgres psql -tAc "SHOW server_version" 2>/dev/null', - check=False, - ) - match = re.search(r"(\d+)", result.stdout.strip()) - if result.returncode == 0 and match: - return int(match.group(1)) - return None - - def list_databases( db_host: str, db_port: int, diff --git a/instance_manager/workflows/install.py b/instance_manager/workflows/install.py index 9e6c85d..00372bb 100644 --- a/instance_manager/workflows/install.py +++ b/instance_manager/workflows/install.py @@ -34,11 +34,13 @@ plan_uv_python, resolve_wkhtmltopdf_asset, ) -from ..prompts import ask_bool, ask_int, ask_text, choose, select_file_path +from ..prompts import ask_bool, ask_int, ask_secret, ask_text, choose, select_file_path from ..system import ( Command, apply_commands, + apt_candidate, db_role_absent, + db_role_exists, detect_arch, detect_cpu_count, detect_host_python, @@ -149,7 +151,7 @@ def _prompt_production_hardening(config: InstanceConfig) -> None: print( level_text( "INFO", - 'Database manager disabled (list_db = False). Create the first database via CLI (odoo-bin -d -i base --stop-after-init) or by temporarily re-enabling it.', + 'Database manager disabled (list_db = False). Create the first database as the instance user: sudo -u /venv/bin/python /odoo/odoo-bin -c -d -i base --stop-after-init, or re-enable the manager for a while.', ) ) @@ -202,6 +204,11 @@ def _prompt_production_hardening(config: InstanceConfig) -> None: 'prefer (allows cleartext fallback)': "prefer", 'disable': "disable", }.get(selection, "require") + if config.db_sslmode == "verify-full": + print(level_text("WARN", tf( + "verify-full checks the server against a CA: place it at {}/.postgresql/root.crt, readable by the instance user, before the install runs.", + config.odoo_home, + ))) if config.db_sslmode in {"prefer", "disable"}: print( level_text( @@ -218,7 +225,10 @@ def _maybe_plan_wkhtmltopdf() -> list[Command]: OS codename to pick the verified patched build.""" os_release = detect_os_release() codename = os_release.get("VERSION_CODENAME", "") - has_patched = resolve_wkhtmltopdf_asset(codename) is not None + arch = detect_arch() + has_patched = resolve_wkhtmltopdf_asset(codename, arch) is not None + # Debian 13 and Ubuntu 26.04 ship no wkhtmltopdf package. + has_distro = bool(apt_candidate("wkhtmltopdf")) print( level_text( @@ -229,12 +239,13 @@ def _maybe_plan_wkhtmltopdf() -> list[Command]: options: list[str] = [] if has_patched: options.append('Patched 0.12.6 (recommended, checksum-verified)') - options.append('Distribution package (un-patched, reduced fidelity)') + if has_distro: + options.append('Distribution package (un-patched, reduced fidelity)') options.append('Skip (PDF reports will fail until installed)') selection = choose('wkhtmltopdf installation', options, default_index=0) if selection.startswith('Patched'): - return plan_install_wkhtmltopdf("patched", codename) + return plan_install_wkhtmltopdf("patched", codename, arch) if selection.startswith('Distribution'): return plan_install_wkhtmltopdf("distro") if not has_patched and selection == "": @@ -307,6 +318,9 @@ def _collect_instance_config(with_odoo: bool = True) -> InstanceConfig: config.http_port = ask_int('Internal Odoo HTTP port', suggested_http) config.gevent_port = ask_int('Internal Odoo gevent port', suggested_gevent) + while config.gevent_port == config.http_port: + print(level_text("ERROR", 'The gevent port must differ from the HTTP port.')) + config.gevent_port = ask_int('Internal Odoo gevent port', suggested_gevent) config.db_host = ask_text('DB host', config.db_host, required=True, validate=db_host_error) config.db_port = ask_int('DB port', config.db_port) config.db_user = ask_text('DB user', config.instance, required=True) @@ -398,7 +412,7 @@ def _build_partial_install_cleanup( commands.append( Command( '[Cleanup] Remove the instance PostgreSQL role (if it exists)', - "sudo -u postgres psql -X -v ON_ERROR_STOP=1 -c " + f"sudo -u postgres psql -X -p {int(config.db_port)} -v ON_ERROR_STOP=1 -c " + _quote('DROP ROLE IF EXISTS "' + config.db_user + '";') + " || true", ) ) @@ -623,15 +637,42 @@ def _suggest_instance_ports(base_http: int, base_gevent: int) -> tuple[int, int] return base_http, base_gevent +def _existing_role_password(config: InstanceConfig) -> bool | None: + """When the local role exists already, the password odoo.conf gets must be its + password: set it to the one entered (``True``), or keep it and take it from the + operator (``False``). ``None``: cancelled.""" + if config.is_remote_db_host or not db_role_exists(config.db_user): + return False + choice = choose( + tf('The PostgreSQL role {} already exists. Which password should this instance use?', config.db_user), + [ + 'Set the role to the password entered (update other instances that use it)', + "Keep the role's password (type it now)", + 'Cancel', + ], + default_index=None, + ) + if choice in {"", 'Cancel'}: + return None + if choice.startswith('Set'): + return True + config.db_password = ask_secret(tf('Current password of role {}', config.db_user)) + return False + + def install_odoo_only() -> None: config = _collect_instance_config() + reset_password = _existing_role_password(config) + if reset_password is None: + print(level_text("INFO", 'Operation cancelled.')) + return service_autostart = ask_bool( 'Enable the Odoo service to start on boot?', True, ) commands: list[Command] = [] commands.extend(_plan_runtime(config)) - commands.extend(plan_ensure_db_role(config)) + commands.extend(plan_ensure_db_role(config, reset_password)) commands.extend(plan_odoo_base_setup(config, service_autostart=service_autostart)) commands.extend(_maybe_plan_wkhtmltopdf()) @@ -652,17 +693,25 @@ def install_odoo_only() -> None: def install_db_only() -> None: config = _collect_instance_config(with_odoo=False) + reset_password = _existing_role_password(config) + if reset_password is None: + print(level_text("INFO", 'Operation cancelled.')) + return ensure_remote_access = ask_bool( 'Configure listen_addresses and pg_hba for remote access?', True ) # Dropped on failure only if this run creates it: an existing role is reused. role_is_new = db_role_absent(config.db_user) - commands = plan_db_setup(config, ensure_remote_access=ensure_remote_access) + commands = plan_db_setup(config, ensure_remote_access=ensure_remote_access, reset_password=reset_password) _execute_install_with_cleanup(commands, config, cleanup_db_role=role_is_new, with_odoo=False) def install_odoo_and_db() -> None: config = _collect_instance_config() + reset_password = _existing_role_password(config) + if reset_password is None: + print(level_text("INFO", 'Operation cancelled.')) + return service_autostart = ask_bool( 'Enable the Odoo service to start on boot?', True, @@ -672,7 +721,7 @@ def install_odoo_and_db() -> None: commands.extend(_plan_runtime(config)) # Odoo and PostgreSQL on one host talk over loopback: nothing is opened to the # network (listen_addresses stays local, no pg_hba rule is added). - commands.extend(plan_db_setup(config, ensure_remote_access=False)) + commands.extend(plan_db_setup(config, ensure_remote_access=False, reset_password=reset_password)) commands.extend(plan_odoo_base_setup(config, service_autostart=service_autostart)) commands.extend(_maybe_plan_wkhtmltopdf()) diff --git a/instance_manager/workflows/manage.py b/instance_manager/workflows/manage.py index 9a5ab67..d5b8777 100644 --- a/instance_manager/workflows/manage.py +++ b/instance_manager/workflows/manage.py @@ -32,6 +32,7 @@ detect_nginx_version, path_exists, pg_env, + read_conf_other_sections, read_odoo_conf, service_active, service_exists, @@ -492,6 +493,7 @@ def update_existing_configs(instance: str) -> None: existing, new_db_password=new_db_password, restart=service_active(config.odoo_service), + other_sections=read_conf_other_sections(config.odoo_conf_file), ) ) if new_db_password and config.is_remote_db_host: diff --git a/openspec/changes/archive/2026-10-08-provisioning-facts/proposal.md b/openspec/changes/archive/2026-10-08-provisioning-facts/proposal.md new file mode 100644 index 0000000..931785b --- /dev/null +++ b/openspec/changes/archive/2026-10-08-provisioning-facts/proposal.md @@ -0,0 +1,64 @@ +# Provisioning facts, re-derived from their sources + +## Why + +The fact audit checked every hard-coded version, flag and package against its primary source: Odoo's +checkouts, PyPI, nginx.org, the wkhtmltopdf release, and runs of the real tools. These were false or risky: + +- **gevent 21.8.0.** It has no wheel for Python 3.10 or 3.11, and its source no longer builds. Odoo 14 pins it + for every interpreter above 3.9, and Odoo 15 for 3.10 and 3.11; only 16–19 (3.10) were avoided. So Odoo 15 + did not install on Debian 12 (Python 3.11). +- **Python maxima.** Odoo 15–18 state `MIN_PY_VERSION`/`MAX_PY_VERSION` in `odoo/__init__.py`. Odoo 16's maximum + is 3.12, not the 3.13 derived earlier, so 16 was built on Debian 13's 3.13. +- **setuptools.** It removed `pkg_resources` in 82, not 81. The `<81` pin is still right, but its reason was + wrong. +- **PostgreSQL 13** (Debian 11) stores passwords as md5 by default. The scram `pg_hba` rule the tool writes then + refuses the app server. +- **nginx.** + - The version was detected before nginx was installed, so it was unknown and treated as new. On 22.04 and + Debian 11 the first HTTPS install then wrote `proxy_cookie_flags` for nginx 1.18, and `nginx -t` failed. + - The directive appeared in 1.19.3, not 1.19.8. +- **odoo.conf merge.** + - It lifted other sections into `[options]` and cut multi-line values to their first line. + - It wrote a key twice when its case differed, which Odoo's parser rejects. +- **Local PostgreSQL steps.** + - Local `psql` ignored the port. + - Remote access ran `sed` on `postgresql.conf`, which `conf.d` overrides. + - It restarted PostgreSQL even when nothing changed. +- **uv.** `command -v uv` accepted any uv on root's PATH, and the instance user's `secure_path` holds none. +- **wkhtmltopdf.** + - The pinned packages were amd64 only, although upstream builds arm64. + - The distribution package was offered where there is none (Debian 13, Ubuntu 26.04). +- **An existing role.** It was reused with its old password while `odoo.conf` got the new one, so the install + always failed. + +## What changes + +- **Python matrix.** Official maxima for 15–18 (16 is 3.12). The gevent 21.8.0 rows are listed per branch, and + uv's interpreter is used for those hosts. +- **PostgreSQL passwords.** `password_encryption = 'scram-sha-256'` is set whenever a password is set. An + existing role's password is either set or kept, as the operator chooses. +- **Local PostgreSQL steps.** Local `psql -p `. Remote access uses `ALTER SYSTEM`, restarts only on a + change, reads the value back, and reloads for `pg_hba`. +- **Database login check.** It uses `odoo.conf`'s SSL mode and the instance user's CA. +- **nginx.** The version is the installed one or apt's candidate; unknown counts as old. `proxy_cookie_flags` + from 1.19.3. +- **odoo.conf.** It is read as Odoo reads it, other sections are kept, and multi-line values are written back. +- **uv.** It is called at `/usr/local/bin/uv` with its version checked, and installs interpreters with `--no-bin`. +- **wkhtmltopdf.** Assets keyed by codename and machine, the arm64 checksums computed, and the distro package + offered only when apt has one. +- **Smaller fixes.** + - The gevent port must differ from the HTTP port. + - The CLI hint creates a database as the instance user. + - The unused PostgreSQL version probe is removed. + +## Impact + +- Specs: `instance-provisioning`, `web-proxy-tls`, `instance-configuration`. +- Code: `support.py`, `planners.py`, `system.py`, `workflows/{install,manage}.py`. +- Tests: `tests/test_provisioning_facts.py`. +- Verification: + - `tools/verify_postgres_setup.py` (CI) runs the PostgreSQL steps against a cluster on another port that + stores md5 by default and has a `conf.d`; + - `tools/verify_secrets.py` checks scram storage; + - `tools/verify_install_runtime.py` checks the uv path and its skip. diff --git a/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-configuration/spec.md b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-configuration/spec.md new file mode 100644 index 0000000..1eee487 --- /dev/null +++ b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-configuration/spec.md @@ -0,0 +1,41 @@ +## MODIFIED Requirements + +### Requirement: Configuration update merges and applies + +Updating an existing instance's configuration SHALL first back up the current config, systemd unit, and Nginx +vhosts into a single private timestamped directory, then rewrite `odoo.conf` **merged** with the current one and +rewrite the unit — without reinstalling anything — and optionally regenerate the Nginx vhost. The keys an +operator tunes by hand (`addons_path`, `data_dir`, `logfile`, `http_interface`, `without_demo`) SHALL keep their +current value, and keys the tool does not write SHALL be carried over. The current file SHALL be read as Odoo +reads it (`[options]` with keys lower-cased and a continued value kept whole), and its other sections (a +module's own, such as `[queue_job]`) SHALL follow unchanged. The Odoo version SHALL be read from the +instance's checkout and the domain from its vhost. A new DB password SHALL be set on the local role as well; a +running service SHALL be restarted so the change is live. + +#### Scenario: Existing files are backed up before regeneration + +- **WHEN** the operator updates an instance configuration +- **THEN** the plan copies the current config, unit, and Nginx vhosts into a single + `/var/backups//config_preupdate//` directory (mode `700`) before writing new versions + +#### Scenario: The current configuration is merged, not replaced + +- **WHEN** the current `odoo.conf` holds `data_dir`, extra `addons_path` entries (over several lines), `smtp_*` + or other keys, keys written in another case, or other sections +- **THEN** the rewritten file keeps them, with each key once, and Odoo reads it + +#### Scenario: Nothing is reinstalled + +- **WHEN** the configuration update is applied +- **THEN** the plan runs no package install, clone or `pip install` + +#### Scenario: The change is live when the plan ends + +- **WHEN** the service is running +- **THEN** the plan restarts it after writing the configuration; and when a new DB password was entered for a + local database, the plan sets it on the role before the restart + +#### Scenario: Autostart state is preserved across update + +- **WHEN** the configuration is regenerated +- **THEN** the service's enabled-at-boot state is left as it was diff --git a/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-provisioning/spec.md b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-provisioning/spec.md new file mode 100644 index 0000000..663bf25 --- /dev/null +++ b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/instance-provisioning/spec.md @@ -0,0 +1,153 @@ +## MODIFIED Requirements + +### Requirement: Database role provisioning + +The tool SHALL ensure the instance's PostgreSQL role exists with LOGIN and CREATEDB, creating it only if missing, +storing every password it sets as `scram-sha-256` (the `pg_hba` rule it writes requires it, and PostgreSQL 13 +stores md5 by default), and SHALL validate the role can log in with the SSL mode `odoo.conf` sets. When a local +role exists already, the operator SHALL choose: set it to the password entered, or keep its password and type +it, so `odoo.conf` holds the role's real password. Local `psql` SHALL name the instance's port. + +#### Scenario: Missing local role is created + +- **WHEN** the DB host is local and the role does not exist +- **THEN** the plan creates the role with LOGIN CREATEDB using the configured password, stored as scram + +#### Scenario: An existing role keeps or gets the password, as chosen + +- **WHEN** the role already exists +- **THEN** the plan re-asserts LOGIN CREATEDB and either sets the entered password (when the operator chose to) + or leaves the password alone, the operator having typed it for `odoo.conf` + +#### Scenario: Remote DB host skips role creation + +- **WHEN** the DB host is remote +- **THEN** the plan skips role creation (no admin credentials assumed) and only validates the configured user's + login, with the SSL mode and, for `verify-*`, the CA at the instance user's `~/.postgresql/root.crt` + +### Requirement: Remote database access + +When installing PostgreSQL with remote access enabled, the plan SHALL set `listen_addresses = '*'` with +`ALTER SYSTEM` (which a distribution's `conf.d` file does not override), restart PostgreSQL only when the value +changes, read the value back, append a host-scoped `pg_hba` rule for the app server IP using `scram-sha-256`, +and reload PostgreSQL for it. + +#### Scenario: Remote access is opened for the app server + +- **WHEN** remote access is enabled during a DB install +- **THEN** the plan sets `listen_addresses = '*'` (restarting once if it changed), ensures a `pg_hba` line for + ` /32 scram-sha-256` (idempotently), and reloads PostgreSQL + +#### Scenario: A second run restarts nothing + +- **WHEN** the same plan runs again +- **THEN** PostgreSQL is not restarted and the `pg_hba` rule is not added twice + +### Requirement: Python interpreter per Odoo version + +The tool SHALL know, for each supported Odoo version (12-19), the Python range it accepts — with the evidence +behind each bound (official: Odoo 15-19 state `MIN_PY_VERSION`/`MAX_PY_VERSION`; or derived from the branch's +`requirements.txt`) — and SHALL build the venv with the host `python3` only when that interpreter can build the +version: inside the range; for a version with no stated maximum, not newer than the interpreter the matrix +recommends; and never an interpreter the branch pins `gevent==21.8.0` for (14: above 3.9; 15: 3.10 and 3.11; +16-19: 3.10), which has no wheel for it and whose source no longer builds. uv SHALL be called by the absolute +path the tool installs it at, its version checked, and its interpreters installed without links in root's +PATH. Otherwise it SHALL install the matrix's interpreter with uv, +pinned to one release and verified against its published SHA-256, into a shared root-owned directory the +instance users can read but not write. The operator SHALL see the range and the choice before the plan. + +#### Scenario: An out-of-range host gets uv's interpreter + +- **WHEN** Odoo 14 is installed on a host whose `python3` is 3.12 +- **THEN** the plan installs the pinned uv (checksum-verified), installs CPython 3.8 with it, and builds the + venv with that interpreter + +#### Scenario: An in-range host interpreter is used + +- **WHEN** Odoo 18 is installed on a host whose `python3` is 3.12 +- **THEN** the venv is built with the host `python3` and nothing is downloaded + +#### Scenario: setuptools follows the version + +- **WHEN** the venv is prepared +- **THEN** it installs `setuptools<58` for Odoo <= 13 (`vatnumber` uses `use_2to3`), `setuptools<81` for Odoo + <= 16 (they import `pkg_resources` at startup, which setuptools 82 removed), and the current setuptools + otherwise + +#### Scenario: A gevent 21.8.0 row is avoided + +- **WHEN** Odoo 15 is installed on a host whose `python3` is 3.11 (Debian 12), or Odoo 16 on 3.13 (Debian 13) +- **THEN** the venv is built with uv's 3.12 + +### Requirement: Optional wkhtmltopdf provisioning + +During provisioning the tool SHALL offer to install `wkhtmltopdf`, explaining that Odoo's PDF reports +(invoices, quotations, and similar) require it. The recommended option SHALL install the Qt-patched +0.12.6 build selected for the detected OS codename and machine (`x86_64`, `aarch64`) from a pinned table +(asset URL + SHA-256), verified by checksum before installation; the tool SHALL also offer the distribution +package as a clearly labelled reduced-fidelity fallback when the distribution has one, and a skip option. When the operator skips, the tool SHALL warn +that PDF report generation will fail until wkhtmltopdf is installed. + +#### Scenario: Operator is offered wkhtmltopdf with the reports rationale + +- **WHEN** the operator provisions an Odoo instance +- **THEN** the tool offers to install wkhtmltopdf and states that PDF reports require it + +#### Scenario: Recommended install uses the checksum-verified patched build for the codename + +- **WHEN** the operator accepts the recommended wkhtmltopdf option +- **THEN** the plan selects the patched 0.12.6 asset mapped to the detected OS codename (or the closest + ABI-compatible build when the codename has no native asset), downloads it into a private temporary + directory, and installs it only if its SHA-256 matches the pinned checksum, in the same step + +#### Scenario: Distribution package is offered as a reduced-fidelity fallback + +- **WHEN** the operator chooses the distribution package instead +- **THEN** the plan installs the distro `wkhtmltopdf`, labelled as un-patched/reduced-fidelity + +#### Scenario: Unmapped codename avoids guessing a download + +- **WHEN** the detected codename has no mapped patched asset and no ABI-compatible fallback +- **THEN** the tool does not fabricate a download URL and instead recommends the distribution package + or skipping + +#### Scenario: Skipping warns that PDF reports will fail + +- **WHEN** the operator skips wkhtmltopdf installation +- **THEN** the tool warns that PDF report generation will fail until wkhtmltopdf is installed + +### Requirement: Environment detection and version-adaptive configuration + +The tool SHALL detect the version-sensitive components of the target host — OS family and codename +(from `/etc/os-release`), the nginx version, and the PostgreSQL version — and combine them with the +operator-supplied Odoo major to render version-correct configuration rather than assuming a fixed +stack. Detection SHALL run in the execution layer (the pure planners receive the resolved values), and +when a component is unknown or unsupported the tool SHALL fall back to the safest known form and report +what it assumed. + +#### Scenario: Odoo config key matches the Odoo major + +- **WHEN** the instance config is written for a known Odoo major +- **THEN** Odoo ≥ 16 receives `gevent_port` and Odoo ≤ 15 receives `longpolling_port` for the same + configured port value + +#### Scenario: Detection feeds rendering from the execution layer + +- **WHEN** the host's OS codename, nginx version, or PostgreSQL version is needed for rendering +- **THEN** it is probed in the execution layer and passed into the planners, which remain free of I/O + +#### Scenario: Unsupported OS family is reported, not assumed + +- **WHEN** the detected OS is not part of the Debian/Ubuntu (apt) family the package steps target +- **THEN** the tool warns that package installation steps may not apply instead of running them blindly + +#### Scenario: A PostgreSQL older than the version's floor is refused + +- **WHEN** the local server is older than the Odoo version's floor (12, or 13 for Odoo 19 — all of which store + `scram-sha-256`) +- **THEN** the plan stops before creating the role + +#### Scenario: nginx not installed yet + +- **WHEN** the plan installs nginx, so none is there while the vhost is rendered +- **THEN** the vhost is written for the version apt would install, or for an older nginx when that is unknown diff --git a/openspec/changes/archive/2026-10-08-provisioning-facts/specs/web-proxy-tls/spec.md b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/web-proxy-tls/spec.md new file mode 100644 index 0000000..1c835c4 --- /dev/null +++ b/openspec/changes/archive/2026-10-08-provisioning-facts/specs/web-proxy-tls/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: Proxy vhost contents + +Generated vhosts SHALL proxy to the instance's internal HTTP and gevent ports, include a dedicated +live-chat/bus upstream with connection-upgrade handling, set forwarded headers for proxy mode, accept uploads +up to 2048 MB and compress with gzip. The HTTPS vhost SHALL follow Odoo's deployment guide: its TLS settings +(TLS 1.2 and 1.3, its cipher list, `ssl_prefer_server_ciphers off`, `ssl_session_timeout 30m`), HSTS, and the +`session_id` cookie marked `secure` with `proxy_cookie_flags` on nginx ≥ 1.19.3 (the version installed, or the +one apt would install; when neither is known the directive is left out, as for an older nginx). The vhost SHALL adapt to the +detected environment: the HTTP/2 form matches the detected nginx version (`listen … ssl http2` on nginx < +1.25.1, `listen … ssl` + `http2 on;` on ≥ 1.25.1), and the live-chat location matches the Odoo major +(`/websocket` on Odoo ≥ 16, `/longpolling/poll` on Odoo ≤ 15). + +#### Scenario: Websocket and forwarded headers are configured + +- **WHEN** a vhost is generated +- **THEN** it defines `odoo_` and `odoochat_` upstreams, a live-chat location with + `Upgrade`/`Connection` headers, and `X-Forwarded-*`/`X-Real-IP` headers on all locations + +#### Scenario: HTTP/2 directive matches the detected nginx version + +- **WHEN** an HTTPS vhost is generated on a host whose nginx version is detected +- **THEN** nginx ≥ 1.25.1 uses `listen … ssl;` with a separate `http2 on;`, and nginx < 1.25.1 uses the + `listen … ssl http2;` form, so `nginx -t` passes on either version + +#### Scenario: Live-chat location matches the Odoo major + +- **WHEN** a vhost is generated for an instance of a known Odoo major +- **THEN** Odoo ≥ 16 gets a `/websocket` location and Odoo ≤ 15 gets a `/longpolling/poll` location, + both proxying to the instance gevent/longpolling upstream + +#### Scenario: The HTTPS vhost follows Odoo's deployment guide + +- **WHEN** an HTTPS vhost is generated for nginx ≥ 1.19.3 +- **THEN** each location sends HSTS and marks the `session_id` cookie `secure`, and the server uses the guide's + TLS settings diff --git a/openspec/changes/archive/2026-10-08-provisioning-facts/tasks.md b/openspec/changes/archive/2026-10-08-provisioning-facts/tasks.md new file mode 100644 index 0000000..fdf103d --- /dev/null +++ b/openspec/changes/archive/2026-10-08-provisioning-facts/tasks.md @@ -0,0 +1,10 @@ +# Tasks + +- [x] 1. Spec deltas (instance-provisioning, web-proxy-tls, instance-configuration); openspec 1.4.1 validates. +- [x] 2. support.py: official maxima 15-18, gevent 21.8.0 rows per branch, setuptools reason, UV_BIN. +- [x] 3. Role SQL with scram; existing role password set or kept; local psql port; ALTER SYSTEM; reload. +- [x] 4. nginx candidate version, unknown as old, proxy_cookie_flags from 1.19.3. +- [x] 5. odoo.conf read as Odoo reads it; other sections and multi-line values kept. +- [x] 6. uv absolute path, version check, --no-bin; wkhtmltopdf per machine, distro only when available. +- [x] 7. SSL mode in the login check; CA hint; CLI hint; distinct ports; unused probe removed. +- [x] 8. Tests; verify_postgres_setup (CI), verify_secrets, verify_install_runtime on Odoo 14 and 16. diff --git a/openspec/specs/instance-configuration/spec.md b/openspec/specs/instance-configuration/spec.md index daba4cd..5034792 100644 --- a/openspec/specs/instance-configuration/spec.md +++ b/openspec/specs/instance-configuration/spec.md @@ -151,7 +151,9 @@ Updating an existing instance's configuration SHALL first back up the current co vhosts into a single private timestamped directory, then rewrite `odoo.conf` **merged** with the current one and rewrite the unit — without reinstalling anything — and optionally regenerate the Nginx vhost. The keys an operator tunes by hand (`addons_path`, `data_dir`, `logfile`, `http_interface`, `without_demo`) SHALL keep their -current value, and keys the tool does not write SHALL be carried over. The Odoo version SHALL be read from the +current value, and keys the tool does not write SHALL be carried over. The current file SHALL be read as Odoo +reads it (`[options]` with keys lower-cased and a continued value kept whole), and its other sections (a +module's own, such as `[queue_job]`) SHALL follow unchanged. The Odoo version SHALL be read from the instance's checkout and the domain from its vhost. A new DB password SHALL be set on the local role as well; a running service SHALL be restarted so the change is live. @@ -163,8 +165,9 @@ running service SHALL be restarted so the change is live. #### Scenario: The current configuration is merged, not replaced -- **WHEN** the current `odoo.conf` holds `data_dir`, extra `addons_path` entries, `smtp_*` or other keys -- **THEN** the rewritten file keeps them +- **WHEN** the current `odoo.conf` holds `data_dir`, extra `addons_path` entries (over several lines), `smtp_*` + or other keys, keys written in another case, or other sections +- **THEN** the rewritten file keeps them, with each key once, and Odoo reads it #### Scenario: Nothing is reinstalled diff --git a/openspec/specs/instance-provisioning/spec.md b/openspec/specs/instance-provisioning/spec.md index eb192c0..e14d085 100644 --- a/openspec/specs/instance-provisioning/spec.md +++ b/openspec/specs/instance-provisioning/spec.md @@ -91,35 +91,46 @@ config and systemd unit, and register the service. ### Requirement: Database role provisioning -The tool SHALL ensure the instance's PostgreSQL role exists with LOGIN and -CREATEDB, creating it only if missing and never overwriting the password of an -existing role, and SHALL validate the role can log in. +The tool SHALL ensure the instance's PostgreSQL role exists with LOGIN and CREATEDB, creating it only if missing, +storing every password it sets as `scram-sha-256` (the `pg_hba` rule it writes requires it, and PostgreSQL 13 +stores md5 by default), and SHALL validate the role can log in with the SSL mode `odoo.conf` sets. When a local +role exists already, the operator SHALL choose: set it to the password entered, or keep its password and type +it, so `odoo.conf` holds the role's real password. Local `psql` SHALL name the instance's port. #### Scenario: Missing local role is created - **WHEN** the DB host is local and the role does not exist -- **THEN** the plan creates the role with LOGIN CREATEDB using the configured password +- **THEN** the plan creates the role with LOGIN CREATEDB using the configured password, stored as scram -#### Scenario: Existing role is preserved +#### Scenario: An existing role keeps or gets the password, as chosen - **WHEN** the role already exists -- **THEN** the plan re-asserts LOGIN CREATEDB but does not change the existing password, emitting a notice that the role is reused +- **THEN** the plan re-asserts LOGIN CREATEDB and either sets the entered password (when the operator chose to) + or leaves the password alone, the operator having typed it for `odoo.conf` #### Scenario: Remote DB host skips role creation - **WHEN** the DB host is remote -- **THEN** the plan skips role creation (no admin credentials assumed) and only validates the configured user's login +- **THEN** the plan skips role creation (no admin credentials assumed) and only validates the configured user's + login, with the SSL mode and, for `verify-*`, the CA at the instance user's `~/.postgresql/root.crt` ### Requirement: Remote database access -When installing PostgreSQL with remote access enabled, the plan SHALL set -`listen_addresses='*'` and append a host-scoped `pg_hba` rule for the app -server IP using `scram-sha-256`, then restart PostgreSQL. +When installing PostgreSQL with remote access enabled, the plan SHALL set `listen_addresses = '*'` with +`ALTER SYSTEM` (which a distribution's `conf.d` file does not override), restart PostgreSQL only when the value +changes, read the value back, append a host-scoped `pg_hba` rule for the app server IP using `scram-sha-256`, +and reload PostgreSQL for it. #### Scenario: Remote access is opened for the app server - **WHEN** remote access is enabled during a DB install -- **THEN** the plan edits the resolved `postgresql.conf` for `listen_addresses='*'`, ensures a `pg_hba` line for ` /32 scram-sha-256` (idempotently), and restarts PostgreSQL +- **THEN** the plan sets `listen_addresses = '*'` (restarting once if it changed), ensures a `pg_hba` line for + ` /32 scram-sha-256` (idempotently), and reloads PostgreSQL + +#### Scenario: A second run restarts nothing + +- **WHEN** the same plan runs again +- **THEN** PostgreSQL is not restarted and the `pg_hba` rule is not added twice ### Requirement: Safe configuration defaults @@ -259,9 +270,9 @@ tool SHALL NOT force a `db_sslmode`. During provisioning the tool SHALL offer to install `wkhtmltopdf`, explaining that Odoo's PDF reports (invoices, quotations, and similar) require it. The recommended option SHALL install the Qt-patched -0.12.6 build selected for the detected OS codename from a pinned table (asset URL + SHA-256), verified -by checksum before installation; the tool SHALL also offer the distribution package as a clearly -labelled reduced-fidelity fallback, and a skip option. When the operator skips, the tool SHALL warn +0.12.6 build selected for the detected OS codename and machine (`x86_64`, `aarch64`) from a pinned table +(asset URL + SHA-256), verified by checksum before installation; the tool SHALL also offer the distribution +package as a clearly labelled reduced-fidelity fallback when the distribution has one, and a skip option. When the operator skips, the tool SHALL warn that PDF report generation will fail until wkhtmltopdf is installed. #### Scenario: Operator is offered wkhtmltopdf with the reports rationale @@ -317,18 +328,27 @@ what it assumed. - **WHEN** the detected OS is not part of the Debian/Ubuntu (apt) family the package steps target - **THEN** the tool warns that package installation steps may not apply instead of running them blindly -#### Scenario: PostgreSQL without scram support is flagged +#### Scenario: A PostgreSQL older than the version's floor is refused + +- **WHEN** the local server is older than the Odoo version's floor (12, or 13 for Odoo 19 — all of which store + `scram-sha-256`) +- **THEN** the plan stops before creating the role -- **WHEN** the detected PostgreSQL version does not support `scram-sha-256` (older than 10) -- **THEN** the tool reports it rather than silently writing an unsupported `pg_hba` auth method +#### Scenario: nginx not installed yet + +- **WHEN** the plan installs nginx, so none is there while the vhost is rendered +- **THEN** the vhost is written for the version apt would install, or for an older nginx when that is unknown ### Requirement: Python interpreter per Odoo version The tool SHALL know, for each supported Odoo version (12-19), the Python range it accepts — with the evidence -behind each bound (official, or derived from the branch's `requirements.txt`) — and SHALL build the venv with -the host `python3` only when that interpreter can build the version: inside the range; for a version with no -stated maximum, not newer than the interpreter the matrix recommends; and never Python 3.10 for Odoo 16-19, -whose 3.10 row pins a gevent pip cannot build. Otherwise it SHALL install the matrix's interpreter with uv, +behind each bound (official: Odoo 15-19 state `MIN_PY_VERSION`/`MAX_PY_VERSION`; or derived from the branch's +`requirements.txt`) — and SHALL build the venv with the host `python3` only when that interpreter can build the +version: inside the range; for a version with no stated maximum, not newer than the interpreter the matrix +recommends; and never an interpreter the branch pins `gevent==21.8.0` for (14: above 3.9; 15: 3.10 and 3.11; +16-19: 3.10), which has no wheel for it and whose source no longer builds. uv SHALL be called by the absolute +path the tool installs it at, its version checked, and its interpreters installed without links in root's +PATH. Otherwise it SHALL install the matrix's interpreter with uv, pinned to one release and verified against its published SHA-256, into a shared root-owned directory the instance users can read but not write. The operator SHALL see the range and the choice before the plan. @@ -347,7 +367,13 @@ instance users can read but not write. The operator SHALL see the range and the - **WHEN** the venv is prepared - **THEN** it installs `setuptools<58` for Odoo <= 13 (`vatnumber` uses `use_2to3`), `setuptools<81` for Odoo - <= 16 (they import `pkg_resources` at startup), and the current setuptools otherwise + <= 16 (they import `pkg_resources` at startup, which setuptools 82 removed), and the current setuptools + otherwise + +#### Scenario: A gevent 21.8.0 row is avoided + +- **WHEN** Odoo 15 is installed on a host whose `python3` is 3.11 (Debian 12), or Odoo 16 on 3.13 (Debian 13) +- **THEN** the venv is built with uv's 3.12 ### Requirement: Odoo core choice diff --git a/openspec/specs/web-proxy-tls/spec.md b/openspec/specs/web-proxy-tls/spec.md index 7514001..c21de1f 100644 --- a/openspec/specs/web-proxy-tls/spec.md +++ b/openspec/specs/web-proxy-tls/spec.md @@ -46,7 +46,8 @@ Generated vhosts SHALL proxy to the instance's internal HTTP and gevent ports, i live-chat/bus upstream with connection-upgrade handling, set forwarded headers for proxy mode, accept uploads up to 2048 MB and compress with gzip. The HTTPS vhost SHALL follow Odoo's deployment guide: its TLS settings (TLS 1.2 and 1.3, its cipher list, `ssl_prefer_server_ciphers off`, `ssl_session_timeout 30m`), HSTS, and the -`session_id` cookie marked `secure` with `proxy_cookie_flags` on nginx ≥ 1.19.8. The vhost SHALL adapt to the +`session_id` cookie marked `secure` with `proxy_cookie_flags` on nginx ≥ 1.19.3 (the version installed, or the +one apt would install; when neither is known the directive is left out, as for an older nginx). The vhost SHALL adapt to the detected environment: the HTTP/2 form matches the detected nginx version (`listen … ssl http2` on nginx < 1.25.1, `listen … ssl` + `http2 on;` on ≥ 1.25.1), and the live-chat location matches the Odoo major (`/websocket` on Odoo ≥ 16, `/longpolling/poll` on Odoo ≤ 15). @@ -71,7 +72,7 @@ detected environment: the HTTP/2 form matches the detected nginx version (`liste #### Scenario: The HTTPS vhost follows Odoo's deployment guide -- **WHEN** an HTTPS vhost is generated for nginx ≥ 1.19.8 +- **WHEN** an HTTPS vhost is generated for nginx ≥ 1.19.3 - **THEN** each location sends HSTS and marks the `session_id` cookie `secure`, and the server uses the guide's TLS settings diff --git a/tests/test_injection_privilege.py b/tests/test_injection_privilege.py index 4dd9b51..c9815c7 100644 --- a/tests/test_injection_privilege.py +++ b/tests/test_injection_privilege.py @@ -154,7 +154,7 @@ def test_a_password_with_dollar_quotes_keeps_the_block_whole(self) -> None: config.normalize_defaults() config.db_password = "a$$b$oim0$c" sql = planners._db_role_create_if_missing_sql(config) - self.assertTrue(sql.startswith("DO $oim1$ ")) + self.assertTrue(sql.startswith("SET password_encryption = 'scram-sha-256'; DO $oim1$ ")) self.assertTrue(sql.endswith("$oim1$;")) diff --git a/tests/test_install_per_version.py b/tests/test_install_per_version.py index 9f8acff..2dfb2a8 100644 --- a/tests/test_install_per_version.py +++ b/tests/test_install_per_version.py @@ -42,10 +42,24 @@ def test_unbounded_versions_stop_at_the_recommendation(self) -> None: self.assertEqual(support.choose_python(support.ODOO_SUPPORT[12], "3.12")[1], support.UV) self.assertEqual(support.choose_python(support.ODOO_SUPPORT[13], "3.8"), ("3.8", support.HOST)) - def test_python_310_is_avoided_from_16(self) -> None: - # 16-19 pin gevent==21.8.0 for 3.10, which pip cannot build. - self.assertEqual(support.choose_python(support.ODOO_SUPPORT[16], "3.10"), ("3.12", support.UV)) - self.assertEqual(support.choose_python(support.ODOO_SUPPORT[15], "3.10"), ("3.10", support.HOST)) + def test_the_gevent_2108_rows_are_avoided(self) -> None: + # Each branch's requirements.txt pins gevent==21.8.0, which pip cannot build, + # for: 14 > 3.9, 15 3.10-3.11, 16-19 3.10. + for major, python, expected in ((14, "3.10", ("3.8", support.UV)), (15, "3.10", ("3.12", support.UV)), + (15, "3.11", ("3.12", support.UV)), (16, "3.10", ("3.12", support.UV)), + (19, "3.10", ("3.12", support.UV)), (15, "3.12", ("3.12", support.HOST)), + (16, "3.11", ("3.11", support.HOST))): + with self.subTest(major=major, python=python): + self.assertEqual(support.choose_python(support.ODOO_SUPPORT[major], python), expected) + + def test_odoo_states_the_maximum_from_15(self) -> None: + # MAX_PY_VERSION in odoo/__init__.py (15.0-18.0) and odoo/release.py (19.0). + for major, maximum in ((15, "3.12"), (16, "3.12"), (17, "3.14"), (18, "3.14"), (19, "3.14")): + with self.subTest(major=major): + self.assertEqual(support.ODOO_SUPPORT[major].python_max, maximum) + self.assertEqual(support.ODOO_SUPPORT[major].python_max_tier, support.OFFICIAL) + # Debian 13's 3.13 is outside 16's range: uv's 3.12. + self.assertEqual(support.choose_python(support.ODOO_SUPPORT[16], "3.13"), ("3.12", support.UV)) def test_no_host_python_gets_uv(self) -> None: self.assertEqual(support.choose_python(support.ODOO_SUPPORT[18], None)[1], support.UV) @@ -187,7 +201,8 @@ def test_update_reinstalls_nothing_and_restarts(self) -> None: for absent in ("apt-get", "git clone", "pip install"): self.assertNotIn(absent, text) alter = next(c for c in commands if "OIM_SQL" in c.command) - self.assertEqual(alter.env["OIM_SQL"], "ALTER ROLE shop WITH PASSWORD 'pw';") + self.assertEqual(alter.env["OIM_SQL"], + "SET password_encryption = 'scram-sha-256'; ALTER ROLE \"shop\" WITH PASSWORD 'pw';") # The password is in no command text and the preview masks it. self.assertNotIn("'pw'", text) self.assertNotIn("'pw'", "\n".join(c.shown for c in commands)) diff --git a/tests/test_ops_configs.py b/tests/test_ops_configs.py index 1c02dff..93f5705 100644 --- a/tests/test_ops_configs.py +++ b/tests/test_ops_configs.py @@ -24,8 +24,11 @@ def test_https_follows_odoos_guide(self) -> None: self.assertIn("ssl_session_timeout 30m;", text) self.assertIn("gzip on;", text) - def test_cookie_flags_need_nginx_1_19_8(self) -> None: - self.assertNotIn("proxy_cookie_flags", planners._nginx_https_content(_config(), (1, 18, 0))) + def test_cookie_flags_need_nginx_1_19_3(self) -> None: + self.assertNotIn("proxy_cookie_flags", planners._nginx_https_content(_config(), (1, 19, 2))) + self.assertIn("proxy_cookie_flags", planners._nginx_https_content(_config(), (1, 19, 3))) + # Unknown: written as for an older nginx, which nginx -t accepts. + self.assertNotIn("proxy_cookie_flags", planners._nginx_https_content(_config(), None)) def test_http_allows_uploads_and_sends_no_hsts(self) -> None: text = planners._nginx_http_content(_config()) diff --git a/tests/test_provisioning_facts.py b/tests/test_provisioning_facts.py new file mode 100644 index 0000000..76895be --- /dev/null +++ b/tests/test_provisioning_facts.py @@ -0,0 +1,120 @@ +"""Provisioning facts: odoo.conf read and merged as Odoo reads it, scram passwords, +nginx directives by version.""" + +from __future__ import annotations + +import configparser +import tempfile +import unittest +from pathlib import Path + +from instance_manager import planners, system +from instance_manager.models import InstanceConfig + +OLD_CONF = """[options] +admin_passwd = old +addons_path = /opt/odoo/shop/odoo/addons, + /opt/odoo/shop/addons-oca +Data_Dir = /var/lib/odoo/shop +data_dir = /var/lib/odoo/shop +smtp_server = mail.example.com + +[queue_job] +channels = root:2 +""" + + +class OdooConfTests(unittest.TestCase): + def _merged(self) -> str: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "shop.conf" + path.write_text(OLD_CONF, encoding="utf-8") + existing = system.read_odoo_conf(str(path)) + other = system.read_conf_other_sections(str(path)) + config = InstanceConfig(instance="shop", data_dir="/var/lib/odoo/shop") + config.normalize_defaults() + config.ensure_strong_secrets() + return planners.render_merged_odoo_conf(config, existing, other) + + def test_read_as_odoo_reads_it(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "shop.conf" + path.write_text(OLD_CONF, encoding="utf-8") + values = system.read_odoo_conf(str(path)) + self.assertEqual(values["addons_path"], "/opt/odoo/shop/odoo/addons,\n/opt/odoo/shop/addons-oca") + self.assertNotIn("channels", values) + self.assertNotIn("Data_Dir", values) + + def test_the_merge_parses_whole_and_keeps_other_sections(self) -> None: + parser = configparser.RawConfigParser() # strict, as a duplicate would fail + parser.read_string(self._merged()) + self.assertEqual(parser.sections(), ["options", "queue_job"]) + self.assertEqual(parser["options"]["addons_path"], "/opt/odoo/shop/odoo/addons,\n/opt/odoo/shop/addons-oca") + self.assertEqual(parser["options"]["smtp_server"], "mail.example.com") + self.assertEqual(parser["queue_job"]["channels"], "root:2") + + def test_a_file_without_header_and_bad_bytes_is_still_read(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + path = Path(tmp) / "x.conf" + path.write_bytes(b"db_user = shop\nlogfile = /var/log/\xff.log\n") + values = system.read_odoo_conf(str(path)) + self.assertEqual(values["db_user"], "shop") + + +class PasswordStorageTests(unittest.TestCase): + def test_passwords_are_set_as_scram(self) -> None: + config = InstanceConfig(instance="shop") + config.normalize_defaults() + config.ensure_strong_secrets() + self.assertTrue(planners._db_role_create_if_missing_sql(config).startswith( + "SET password_encryption = 'scram-sha-256'; ")) + + +if __name__ == "__main__": + unittest.main() + + +class WkhtmltopdfAssetTests(unittest.TestCase): + def test_assets_follow_the_machine(self) -> None: + from instance_manager.planners import resolve_wkhtmltopdf_asset + self.assertIn("jammy_amd64", resolve_wkhtmltopdf_asset("noble", "x86_64")[1]) + self.assertIn("jammy_arm64", resolve_wkhtmltopdf_asset("noble", "aarch64")[1]) + self.assertIn("bookworm_arm64", resolve_wkhtmltopdf_asset("bookworm", "aarch64")[1]) + # Debian 13 and Ubuntu 26.04 have no compatible build; nor has another machine. + self.assertIsNone(resolve_wkhtmltopdf_asset("trixie", "x86_64")) + self.assertIsNone(resolve_wkhtmltopdf_asset("jammy", "riscv64")) + + +class AptCandidateTests(unittest.TestCase): + def test_versions_from_apt_cache_policy(self) -> None: + from unittest import mock + out = "nginx:\n Installed: (none)\n Candidate: 1.18.0-6ubuntu14.4\n" + with mock.patch.object(system, "run", return_value=mock.Mock(stdout=out, returncode=0)), \ + mock.patch.object(system, "command_ok", return_value=False): + self.assertEqual(system.apt_candidate("nginx"), "1.18.0-6ubuntu14.4") + self.assertEqual(system.detect_nginx_version(), (1, 18, 0)) + none = "wkhtmltopdf:\n Installed: (none)\n Candidate: (none)\n" + with mock.patch.object(system, "run", return_value=mock.Mock(stdout=none, returncode=0)): + self.assertEqual(system.apt_candidate("wkhtmltopdf"), "") + + +class ExistingRoleTests(unittest.TestCase): + def test_an_existing_role_keeps_or_gets_the_password_as_chosen(self) -> None: + config = InstanceConfig(instance="shop") + config.normalize_defaults() + config.db_password = "new-pw" + keep = planners._db_role_create_if_missing_sql(config) + reset = planners._db_role_create_if_missing_sql(config, reset_password=True) + self.assertIn("ALTER ROLE shop WITH LOGIN CREATEDB; ", keep) + self.assertIn("ALTER ROLE shop WITH LOGIN CREATEDB PASSWORD 'new-pw'; ", reset) + + def test_local_psql_names_the_port(self) -> None: + config = InstanceConfig(instance="shop", db_port=5433) + config.normalize_defaults() + config.ensure_strong_secrets() + joined = "\n".join(c.command for c in planners.plan_db_setup(config, ensure_remote_access=True)) + self.assertNotIn("sudo -u postgres psql -X -v", joined) + self.assertIn("sudo -u postgres psql -X -p 5433", joined) + self.assertIn("ALTER SYSTEM SET listen_addresses", joined) + self.assertIn("systemctl reload postgresql", joined) + self.assertNotIn("sed -ri", joined) diff --git a/tools/verify_install_runtime.py b/tools/verify_install_runtime.py index e0a1672..7bc1c05 100644 --- a/tools/verify_install_runtime.py +++ b/tools/verify_install_runtime.py @@ -63,18 +63,24 @@ def _uv(root: Path, env: dict[str, str]) -> Path | None: if not commands: print(f"skip no pinned uv for {arch}") return None - script = commands[-1].command.replace("command -v uv >/dev/null 2>&1 && exit 0; ", "") + script = commands[-1].command dest = root / "bin" dest.mkdir() - good = script.replace("/usr/local/bin/", f"{dest}/") + good = script.replace("/usr/local/bin/", f"{dest}/").replace(support.UV_BIN, str(dest / "uv")) result = _run(good, env) version = subprocess.run([str(dest / "uv"), "--version"], capture_output=True, text=True) check("uv installs from the pinned, checksum-verified asset", result.returncode == 0 and support.UV_VERSION in version.stdout, result.stderr) + uv = dest / "uv" + stamp = uv.stat().st_mtime_ns + result = _run(good.replace("curl ", "false && curl "), env) + check("with that version in place, the step does nothing", + result.returncode == 0 and uv.stat().st_mtime_ns == stamp, result.stderr) bad_dest = root / "bad" bad_dest.mkdir() sha = support.UV_ASSETS[arch][1] - bad = script.replace(sha, "0" * 64).replace("/usr/local/bin/", f"{bad_dest}/") + bad = script.replace(sha, "0" * 64).replace("/usr/local/bin/", f"{bad_dest}/").replace( + support.UV_BIN, str(bad_dest / "uv")) result = _run(bad, env) check("a wrong checksum refuses the install", result.returncode != 0 and not any(bad_dest.iterdir())) return dest @@ -114,7 +120,7 @@ def _venv(root: Path, major: int, clone: Path, env: dict[str, str]) -> None: subprocess.run([str(pip), "install", "-q", "--upgrade", "setuptools"], capture_output=True, env=env) found = subprocess.run([str(venv_python), "-c", "import importlib.metadata as m; print(m.version('setuptools'))"], capture_output=True, text=True, env=env).stdout.strip() - if int(found.split(".")[0] or 0) < 81: + if int(found.split(".")[0] or 0) < 82: # pip resolves the newest setuptools the interpreter supports: on # 3.8 that is 75.x, which still ships pkg_resources. print(f"info Odoo {major}: unpinned setuptools on Python {python} is {found}, " @@ -147,7 +153,8 @@ def main() -> int: if uv_bin is None: return 1 env["PATH"] = f"{stubs}:{uv_bin}:{env['PATH']}" - with mock.patch.object(support, "UV_PYTHON_DIR", str(uv_python)): + with mock.patch.object(support, "UV_PYTHON_DIR", str(uv_python)), \ + mock.patch.object(support, "UV_BIN", str(uv_bin / "uv")): for major, clone in sorted(clones.items()): _venv(root, major, clone, env) print(f"\n{len(FAILURES)} failure(s)" if FAILURES else "\nall checks passed") diff --git a/tools/verify_postgres_setup.py b/tools/verify_postgres_setup.py new file mode 100644 index 0000000..e9682bc --- /dev/null +++ b/tools/verify_postgres_setup.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Run the PostgreSQL steps of an install against a cluster of our own. + +The cluster listens on a port that is not 5432 and ``PGPORT`` is unset, so every +step must name the instance's port. It stores passwords as md5 by default, as +PostgreSQL 13 does. Then, with ``systemctl`` stubbed onto ``pg_ctl``: + +- the version floor check passes; +- the role is created with a scram password (the pg_hba rule asks for scram); +- ``listen_addresses`` is set to ``*`` with ``ALTER SYSTEM`` — which also wins over a + ``conf.d`` file — and PostgreSQL restarted once; a second run restarts nothing; +- the pg_hba rule is added once, and PostgreSQL reloaded, not restarted. + +``initdb`` refuses root, so this is a developer-machine (and CI) tool. + + python3 tools/verify_postgres_setup.py +""" + +from __future__ import annotations + +import os +import subprocess +import sys +import tempfile +from pathlib import Path + +HERE = Path(__file__).resolve().parent +sys.path.insert(0, str(HERE.parent)) +sys.path.insert(0, str(HERE)) + +from verify_data_safety import Cluster, _postgres_bindir, _write_exe # noqa: E402 + +from instance_manager import planners # noqa: E402 +from instance_manager.models import InstanceConfig # noqa: E402 + +FAILURES: list[str] = [] + + +def check(name: str, ok: bool, detail: str = "") -> None: + print(f"{'ok ' if ok else 'FAIL'} {name}" + (f" — {detail[-400:]}" if detail and not ok else ""), flush=True) + if not ok: + FAILURES.append(name) + + +def main() -> int: + bindir = _postgres_bindir() + if bindir is None or os.geteuid() == 0: + check("initdb and a non-root user are available", not os.environ.get("CI")) + print("skip needs initdb and a non-root user") + return 1 if FAILURES else 0 + with tempfile.TemporaryDirectory(prefix="oim-verify-pg-") as tmp: + root = Path(tmp) + stubs = root / "stubs" + stubs.mkdir() + cluster = Cluster(root, bindir) + calls = root / "calls" + _write_exe(stubs / "sudo", 'while [ "${1#-}" != "$1" ]; do if [ "$1" = "-u" ]; then shift; fi; shift; done\n' + 'exec "$@"\n') + pg_ctl = f"{bindir}/pg_ctl -D {cluster.data} -w -l {root}/pg.log" + # A restart without the start's listen_addresses='' override, so the + # setting the step makes takes effect. + _write_exe(stubs / "systemctl", f'echo "systemctl $*" >> {calls}\n' + f'case "$1" in restart) {pg_ctl} -o "-p {cluster.port} -k {root}" restart ' + '>/dev/null;; reload) ' + f"{bindir}/pg_ctl -D {cluster.data} reload >/dev/null;; esac\n") + try: + cluster.start() + cluster.value("ALTER SYSTEM SET password_encryption = 'md5'") + cluster.value("SELECT pg_reload_conf()") + # A distribution's conf.d setting, which a sed on postgresql.conf would miss. + (cluster.data / "conf.d").mkdir() + (cluster.data / "conf.d" / "listen.conf").write_text("listen_addresses = 'localhost'\n", encoding="utf-8") + with open(cluster.data / "postgresql.conf", "a", encoding="utf-8") as conf: + conf.write("include_dir = 'conf.d'\n") + env = cluster.env(stubs) + env.pop("PGPORT") + env["PGHOST"] = str(root) + + config = InstanceConfig(instance="shop", db_port=cluster.port, app_server_ip="127.0.0.2") + config.db_password = "S3cret-pg-setup" + config.odoo_admin_passwd = "x" + steps = [c for c in planners.plan_db_setup(config, ensure_remote_access=True) + if not c.description.startswith(("Install PostgreSQL", "Enable and start", "Validate DB user"))] + for command in steps: + result = subprocess.run(["bash", "-c", command.command], capture_output=True, text=True, + env={**env, **command.env}) + check(f"step runs: {command.description}", result.returncode == 0, result.stderr + result.stdout) + + check("the role's password is stored as scram", + cluster.value("SELECT rolpassword LIKE 'SCRAM-SHA-256$%' FROM pg_authid WHERE rolname = 'shop'") == "t") + check("PostgreSQL listens on every address", cluster.value("SHOW listen_addresses") == "*") + log = calls.read_text(encoding="utf-8").split("\n") if calls.exists() else [] + check("restarted once, for listen_addresses", log.count("systemctl restart postgresql") == 1, str(log)) + check("reloaded for pg_hba", "systemctl reload postgresql" in log, str(log)) + + calls.unlink() + for command in steps: + subprocess.run(["bash", "-c", command.command], capture_output=True, text=True, + env={**env, **command.env}) + log = calls.read_text(encoding="utf-8").split("\n") if calls.exists() else [] + check("a second run restarts nothing", "systemctl restart postgresql" not in log, str(log)) + # An existing role: kept as it is, or given the new password on request. + def stored() -> str: + return cluster.value("SELECT rolpassword FROM pg_authid WHERE rolname = 'shop'") + + before = stored() + config.db_password = "S3cret-pg-setup-new" + for reset, label in ((False, "kept without reset"), (True, "replaced on reset")): + step = next(c for c in planners.plan_db_setup(config, ensure_remote_access=False, reset_password=reset) + if c.description.startswith("Ensure PostgreSQL role")) + result = subprocess.run(["bash", "-c", step.command], capture_output=True, text=True, + env={**env, **step.env}) + changed = stored() != before + check(f"an existing role's password is {label}", result.returncode == 0 and changed == reset, + result.stderr) + check("and stored as scram", stored().startswith("SCRAM-SHA-256$")) + hba = (cluster.data / "pg_hba.conf").read_text(encoding="utf-8") + check("the pg_hba rule is there once", + hba.count("host all shop 127.0.0.2/32 scram-sha-256") == 1, hba[-300:]) + finally: + cluster.stop() + print(f"\n{len(FAILURES)} failure(s)" if FAILURES else "\nall checks passed") + return 1 if FAILURES else 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/verify_privilege.py b/tools/verify_privilege.py index 9d01471..f2bfc25 100644 --- a/tools/verify_privilege.py +++ b/tools/verify_privilege.py @@ -139,7 +139,7 @@ def _wkhtmltopdf(root: Path) -> None: calls.unlink(missing_ok=True) planners_asset = ("https://example.invalid/w.deb", "w.deb", digest) original = planners.resolve_wkhtmltopdf_asset - planners.resolve_wkhtmltopdf_asset = lambda codename, asset=planners_asset: asset + planners.resolve_wkhtmltopdf_asset = lambda codename, arch="x86_64", asset=planners_asset: asset try: step = planners.plan_install_wkhtmltopdf("patched", "noble")[-1] finally: diff --git a/tools/verify_secrets.py b/tools/verify_secrets.py index c6fbe97..c072e78 100644 --- a/tools/verify_secrets.py +++ b/tools/verify_secrets.py @@ -107,7 +107,11 @@ def _role(root: Path) -> None: (root / "pg").mkdir() try: cluster.start() - config = InstanceConfig(instance="shop") + # PostgreSQL 13's default: the role SQL must still store a scram password. + cluster.value("ALTER SYSTEM SET password_encryption = 'md5'") + cluster.value("SELECT pg_reload_conf()") + # The step names the instance's port; PGPORT alone would not reach it. + config = InstanceConfig(instance="shop", db_port=cluster.port) config.db_password, config.odoo_admin_passwd = SECRET, "x" step = next(c for c in planners.plan_ensure_db_role(config) if "OIM_SQL" in c.command) check("the role SQL is not in the command text", SECRET not in step.command and SECRET not in step.shown) @@ -116,6 +120,9 @@ def _role(root: Path) -> None: check("the role is created through stdin", result.returncode == 0, result.stderr) check("with a password set", cluster.value("SELECT rolpassword IS NOT NULL FROM pg_authid WHERE rolname = 'shop'") == "t") + check("stored as scram even where the server defaults to md5 (PostgreSQL 13)", + cluster.value("SELECT rolpassword LIKE 'SCRAM-SHA-256$%' FROM pg_authid WHERE rolname = 'shop'") == "t", + cluster.value("SELECT left(rolpassword, 14) FROM pg_authid WHERE rolname = 'shop'")) finally: cluster.stop()