From 02ecdcef0301e5a9a63890de1cf00dab683b9844 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Sun, 11 Oct 2026 11:34:23 -0300 Subject: [PATCH 01/13] feat(charm): consume the single-kernel library for the remaining VM modules --- actions.yaml | 7 + src/charm.py | 2706 +--------------- src/cluster_topology_observer.py | 1 + .../postgresql_vm_test_charm/actions.yaml | 108 + tests/unit/test_charm.py | 2782 +---------------- tests/unit/test_cluster.py | 323 -- tests/unit/test_cluster_topology_observer.py | 71 - tests/unit/test_rotate_logs.py | 129 - tests/unit/test_tls.py | 22 +- 9 files changed, 176 insertions(+), 5973 deletions(-) create mode 100644 tests/charms/postgresql_vm_test_charm/actions.yaml delete mode 100644 tests/unit/test_cluster.py delete mode 100644 tests/unit/test_cluster_topology_observer.py delete mode 100644 tests/unit/test_rotate_logs.py diff --git a/actions.yaml b/actions.yaml index 6dfbb12269d..a627f84d8fd 100644 --- a/actions.yaml +++ b/actions.yaml @@ -97,3 +97,10 @@ restore: restore-to-time: type: string description: Point-in-time-recovery target in PSQL format. +status-detail: + description: Gets statuses of the charm + params: + recompute: + type: boolean + default: false + description: a boolean indicating whether a unit should recompute all statuses. diff --git a/src/charm.py b/src/charm.py index b0232f791e4..16cf3484078 100755 --- a/src/charm.py +++ b/src/charm.py @@ -1,211 +1,30 @@ -#!/usr/bin/env -S LD_LIBRARY_PATH=lib python3 -# Copyright 2021 Canonical Ltd. +# Copyright 2023 Canonical Ltd. # See LICENSE file for licensing details. -"""Charmed Machine Operator for the PostgreSQL database.""" +"""Charmed Machine Operator for PostgreSQL.""" -import json import logging -import os -import shutil -import subprocess -import sys -from contextlib import suppress -from datetime import UTC, datetime from functools import cached_property -from pathlib import Path -from typing import TYPE_CHECKING, Any, Literal, cast, get_args -from urllib.parse import urlparse - -# First platform-specific import, will fail on wrong architecture -try: - import psycopg2 - import psycopg2.errors -except ModuleNotFoundError: - from ops.main import main - from single_kernel_postgresql.utils.arch import ( - WrongArchitectureWarningCharm, - is_wrong_architecture, - ) - - # If the charm was deployed inside a host with different architecture - # (possibly due to user specifying an incompatible revision) - # then deploy an empty blocked charm with a warning. - if is_wrong_architecture() and __name__ == "__main__": - main(WrongArchitectureWarningCharm) - raise import charm_refresh -import ops.log -from charmlibs import snap -from charms.data_platform_libs.v0.data_interfaces import DataPeerData, DataPeerUnitData -from charms.data_platform_libs.v1.data_models import TypedCharmBase +import ops from charms.grafana_agent.v0.cos_agent import COSAgentProvider, ProtocolNotFoundError -from charms.rolling_ops.v0.rollingops import RollingOpsManager, RunWithLock -from ops import ( - ActionEvent, - ActiveStatus, - BlockedStatus, - CharmEvents, - HookEvent, - InstallEvent, - JujuVersion, - LeaderElectedEvent, - MaintenanceStatus, - ModelError, - Relation, - RelationDepartedEvent, - RelationEvent, - SecretChangedEvent, - SecretNotFoundError, - SecretRemoveEvent, - StartEvent, - Unit, - WaitingStatus, - main, -) +from charms.rolling_ops.v0.rollingops import RollingOpsManager from ops_tracing import Tracing, set_destination -from single_kernel_postgresql.compat.postgresql import PostgreSQLBaseError -from single_kernel_postgresql.config.enums import Substrates -from single_kernel_postgresql.config.exceptions import ( - NotReadyError, - RemoveRaftMemberFailedError, - SwitchoverFailedError, - SwitchoverNotSyncError, -) +from single_kernel_postgresql.charms.vm_charm import PostgreSQLVMCharm from single_kernel_postgresql.config.literals import ( - APP_SCOPE, - BACKUP_USER, - DATABASE, - DATABASE_DEFAULT_NAME, - DATABASE_PORT, - LOGICAL_REPLICATION_VALIDATION_ERROR_STATUS, METRICS_PORT, - MONITORING_PASSWORD_KEY, - MONITORING_USER, - PATRONI_PASSWORD_KEY, PEER_RELATION, PGBACKREST_METRICS_PORT, - PLUGIN_OVERRIDES, - RAFT_PASSWORD_KEY, - REPLICATION_CONSUMER_RELATION, - REPLICATION_OFFER_RELATION, - REPLICATION_PASSWORD_KEY, - REPLICATION_USER, - REWIND_PASSWORD_KEY, - REWIND_USER, - S3_RELATION_NAME, - SECRET_DELETED_LABEL, - SECRET_INTERNAL_LABEL, - SECRET_KEY_OVERRIDES, - SNAP_USER, - SPI_MODULE, - SYSTEM_USERS, TRACING_PROTOCOL, TRACING_RELATION_NAME, - UNIT_SCOPE, - USER, - USER_PASSWORD_KEY, ) from single_kernel_postgresql.core.config import CharmConfig -from single_kernel_postgresql.core.state import CharmState -from single_kernel_postgresql.events.async_replication import PostgreSQLAsyncReplication -from single_kernel_postgresql.events.backup import BackupEventsHandler -from single_kernel_postgresql.events.database import DatabaseEventsHandler -from single_kernel_postgresql.events.ldap import LDAP -from single_kernel_postgresql.events.logical_replication import PostgreSQLLogicalReplication -from single_kernel_postgresql.events.observer import ( - ClusterTopologyChangeCharmEvents, -) -from single_kernel_postgresql.events.tls import TLS -from single_kernel_postgresql.events.tls_transfer import TLSTransfer -from single_kernel_postgresql.events.watcher import WatcherEventsHandler -from single_kernel_postgresql.lib.charms.data_platform_libs.v0.data_interfaces import ( - DatabaseProvides, -) -from single_kernel_postgresql.lib.charms.data_platform_libs.v0.s3 import S3Requirer -from single_kernel_postgresql.managers.async_replication import AsyncReplicationManager -from single_kernel_postgresql.managers.backup import BackupManager -from single_kernel_postgresql.managers.cluster import ClusterManager -from single_kernel_postgresql.managers.config import ConfigManager -from single_kernel_postgresql.managers.database import DatabaseManager -from single_kernel_postgresql.managers.logical_replication import LogicalReplicationManager -from single_kernel_postgresql.managers.observer import ObserverManager -from single_kernel_postgresql.managers.patroni import PatroniManager -from single_kernel_postgresql.managers.refresh import RefreshManager -from single_kernel_postgresql.managers.raft import RaftManager -from single_kernel_postgresql.managers.restore import RestoreManager -from single_kernel_postgresql.managers.tls import TLSManager -from single_kernel_postgresql.utils import label2name, new_password -from single_kernel_postgresql.utils.backup import ( - CANNOT_RESTORE_PITR, - S3_BLOCK_MESSAGES, - parse_backup_id, -) -from single_kernel_postgresql.utils.postgresql import ( - ACCESS_GROUP_IDENTITY, - ACCESS_GROUPS, - REQUIRED_PLUGINS, - ROLE_BACKUP, - ROLE_STATS, - PostgreSQL, - PostgreSQLCreatePredefinedRolesError, - PostgreSQLCreateUserError, - PostgreSQLEnableDisableExtensionError, - PostgreSQLGetCurrentTimelineError, - PostgreSQLGrantDatabasePrivilegesToUserError, - PostgreSQLListUsersError, - PostgreSQLUndefinedHostError, - PostgreSQLUpdateUserPasswordError, -) -from single_kernel_postgresql.utils.s3 import S3Client -from single_kernel_postgresql.workload.vm import VMWorkload -from tenacity import RetryError, Retrying, stop_after_attempt, stop_after_delay, wait_fixed - -if TYPE_CHECKING: - from single_kernel_postgresql.charms.abstract_charm import AbstractPostgreSQLCharm - -from cluster import Patroni -from cluster_topology_observer import ( - ClusterTopologyChangeCharmEvents, - ClusterTopologyObserver, - start_raft_observer, -) -from constants import ( - MONITORING_SNAP_SERVICE, - PGBACKREST_MONITORING_SNAP_SERVICE, - POSTGRESQL_DATA_DIR, - RAFT_PARTNER_PREFIX, - RAFT_PORT, - TEMP_DATA_DIR, - UPDATE_CERTS_BIN_PATH, -) -from rotate_logs import RotateLogs logger = logging.getLogger(__name__) -logging.getLogger("httpx").setLevel(logging.WARNING) -logging.getLogger("httpcore").setLevel(logging.WARNING) -logging.getLogger("asyncio").setLevel(logging.WARNING) -logging.getLogger("boto3").setLevel(logging.WARNING) -logging.getLogger("botocore").setLevel(logging.WARNING) - -PRIMARY_NOT_REACHABLE_MESSAGE = "waiting for primary to be reachable from this unit" -EXTENSIONS_DEPENDENCY_MESSAGE = "Unsatisfied plugin dependencies. Please check the logs" -EXTENSION_OBJECT_MESSAGE = "Cannot disable plugins: Existing objects depend on it. See logs" - -SCOPES = Literal["app", "unit"] -PASSWORD_USERS = [*SYSTEM_USERS, "patroni"] -class CannotConnectError(Exception): - """Cannot run smoke check on connected Database.""" - - -class StorageUnavailableError(Exception): - """Cannot find storage mountpoint.""" - - -def charm_tracing_config(endpoint_requirer: COSAgentProvider) -> None: +def charm_tracing_config(endpoint_requirer) -> None: """Utility function to set tracing destination.""" if not endpoint_requirer.is_ready(): return @@ -228,242 +47,48 @@ def charm_tracing_config(endpoint_requirer: COSAgentProvider) -> None: set_destination(endpoint, None) -class PostgresqlOperatorCharm(TypedCharmBase[CharmConfig]): +class PostgresqlOperatorCharm(PostgreSQLVMCharm): """Charmed Operator for the PostgreSQL database.""" config_type = CharmConfig - on: "CharmEvents" = ClusterTopologyChangeCharmEvents() - _postgresql: PostgreSQL | None = None - # Override data_models.py TypedCharmBase config + # Prefer the dash (-) option name with the underscore (_) fallback; the lib + # CharmState.config reads the same convention through its own cached property. @cached_property def config(self) -> CharmConfig: """Return a config instance validated and parsed using the provided pydantic class.""" config = { - # Prefer value of option name with dash (-) and fallback to name with underscore (_) config_option: self.model.config.get( config_option.replace("_", "-"), self.model.config.get(config_option) ) for config_option in self.config_type.keys() # noqa: SIM118 } - config: dict[str, Any] = { - config_option: value for config_option, value in config.items() if value is not None - } + config = {k: v for k, v in config.items() if v is not None} return self.config_type(**config) def __init__(self, *args): - super().__init__(*args) - # Show logger name (module name) in logs + super().__init__( + *args, + restart_lock_acquire=lambda: self.on[ + str(self.restart_manager.name) + ].acquire_lock.emit(), + ) + + # Show logger name (module name) in logs. root_logger = logging.getLogger() for handler in root_logger.handlers: if isinstance(handler, ops.log.JujuLogHandler): handler.setFormatter(logging.Formatter("{name}:{message}", style="{")) - self.peer_relation_app = DataPeerData( - self.model, - relation_name=PEER_RELATION, - secret_field_name=SECRET_INTERNAL_LABEL, - deleted_label=SECRET_DELETED_LABEL, - ) - self.peer_relation_unit = DataPeerUnitData( - self.model, - relation_name=PEER_RELATION, - secret_field_name=SECRET_INTERNAL_LABEL, - deleted_label=SECRET_DELETED_LABEL, - ) - - # TODO switch to the abstract class base - # State - # ops permits only one S3Requirer object per charm for the s3-parameters relation; - # build it here (before CharmState) and hand the same instance to the backups manager. - self.s3_requirer = S3Requirer(self, S3_RELATION_NAME) - self.state = CharmState( - charm=self, - substrate=self.substrate, - s3_requirer=self.s3_requirer, - ) - - # Managers - self.patroni_manager = PatroniManager(state=self.state, workload=self.workload) - self.cluster_manager = ClusterManager(state=self.state, workload=self.workload) - self.observer_manager = ObserverManager(state=self.state, workload=self.workload) - # Built here so the raft manager can hold it; the async subsystem below - # reuses the same instance. - self.watcher_handler = WatcherEventsHandler(self, self.workload, self.state) - self.raft_manager = RaftManager( - state=self.state, - workload=self.workload, - patroni_manager=self.patroni_manager, - watcher_handler=self.watcher_handler, - update_config=self.update_config, - set_unit_status=self.set_unit_status, - peer_relation_changed=self.on[PEER_RELATION].relation_changed, - remove_from_members_ips=self._remove_from_members_ips, - ) - - self._rotate_logs = RotateLogs(self) - self.framework.observe(self.on.raft_reconnect, self._on_raft_reconnect) - self.framework.observe(self.on.databases_change, self._on_databases_change) - self.framework.observe(self.on.install, self._on_install) - self.framework.observe(self.on.leader_elected, self._on_leader_elected) - self.framework.observe(self.on.config_changed, self._on_config_changed) - self.framework.observe(self.on.get_primary_action, self._on_get_primary) - self.framework.observe( - self.on[PEER_RELATION].relation_changed, self._on_peer_relation_changed - ) - self.framework.observe(self.on.secret_changed, self._on_peer_relation_changed) - # add specific handler for updated system-user secrets - self.framework.observe(self.on.secret_changed, self._on_secret_changed) - self.framework.observe( - self.on[PEER_RELATION].relation_departed, self._on_peer_relation_departed - ) - self.framework.observe(self.on.start, self._on_start) - self.framework.observe(self.on.promote_to_primary_action, self._on_promote_to_primary) - self.framework.observe(self.on.update_status, self._on_update_status) - self.framework.observe(self.on.secret_remove, self._on_secret_remove) - for storage_name in self.meta.storages: - self.framework.observe( - self.on[storage_name].storage_detaching, self._on_storage_detaching - ) - self.framework.observe(self.on.remove, self._on_remove) - self.cluster_name = self.app.name - self._member_name = self.unit.name.replace("/", "-") - self._certs_path = "/usr/local/share/ca-certificates" - self._storage_path = self.meta.storages["data"].location - - self.s3_client = S3Client(self.workload) - self.backup = BackupManager( - state=self.state, - workload=self.workload, - s3_client=self.s3_client, - patroni_manager=self.patroni_manager, - update_config=self.update_config, - resource_provider=self.workload, - is_standby_cluster=lambda: self.is_standby_cluster, - set_unit_status=self.set_unit_status, - refresh_primary_status=self._set_primary_status_message, - ) - self.restore_manager = RestoreManager( - state=self.state, - workload=self.workload, - patroni_manager=self.patroni_manager, - update_config=self.update_config, - backup_manager=self.backup, - is_standby_cluster=lambda: self.is_standby_cluster, - ) - self.ldap = LDAP(self, self.state) - self.backup_events = BackupEventsHandler( - self, # ty: ignore[invalid-argument-type] - self.state, - self.backup, - self.restore_manager, - ) - # TLS events handler owns the two cert requirers; build it before the TLS - # manager so the manager can constructor-inject them for its live-fetch getters. - self.tls = TLS(self, self.state) - self.tls_manager = TLSManager( - state=self.state, - workload=self.workload, - client_certificate=self.tls.client_certificate, - peer_certificate=self.tls.peer_certificate, - ) - self.database_manager = DatabaseManager( - state=self.state, - workload=self.workload, - database_provides=DatabaseProvides(self, relation_name=DATABASE), - set_unit_status=self.set_unit_status, - ) - self.database = DatabaseEventsHandler( - self, self.state, self.database_manager, self.patroni_manager, self.tls_manager - ) - self.logical_replication_manager = LogicalReplicationManager( - state=self.state, - workload=self.workload, - # Per-call bridges: the client and the primary lookup are freshly - # constructed per access (Patroni primary lookup + app secret). - postgresql=lambda: self.postgresql, - primary_endpoint=lambda: self.primary_endpoint, - update_config=self.update_config, - set_unit_status=self.set_unit_status, - ) - self.logical_replication = PostgreSQLLogicalReplication( - self, self.state, self.logical_replication_manager - ) - self.config_manager = ConfigManager( - state=self.state, - workload=self.workload, - tls_manager=self.tls_manager, - patroni_manager=self.patroni_manager, - database_manager=self.database_manager, - ldap_handler=self.ldap, - resource_provider=self.get_resource_provider, - request_restart=self.request_restart, - restart_services=self.restart_services, - set_unit_status=self.set_unit_status, - logical_replication_slots=self.logical_replication.replication_slots, - ) - # Reload PostgreSQL after the lib TLS handler has actually pushed the cert files. - # tls_files_pushed fires only on a completed push (the handler routes both - # certificate_available and relation_broken through that push), so a deferred push - # never triggers a reload against files that were never written. - self.framework.observe(self.tls.tls_files_pushed, self._reload_tls_after_push) - self.tls_transfer = TLSTransfer(self, PEER_RELATION) - self.async_replication_manager = AsyncReplicationManager( - state=self.state, - workload=self.workload, - patroni_manager=self.patroni_manager, - update_config=self.update_config, - set_unit_status=self.set_unit_status, - set_primary_status_message=self.set_primary_status_message, - set_app_status=lambda: self.async_replication.set_app_status(), - # K8s-only bridges: on VM the pgdata re-initialisation goes through - # _reinitialise_pgdata (workload seam) and there is no leader annotation. - create_pgdata=lambda: None, - fix_leader_annotation=lambda: True, - re_emit_relation_changed=lambda: ( - self.async_replication._re_emit_async_relation_changed_event() - ), - watcher=self.watcher_handler, - ) - self.async_replication = PostgreSQLAsyncReplication( - self, - self.state, - self.async_replication_manager, - self.patroni_manager, - self.workload, - watcher=self.watcher_handler, - ) - + # The rolling-ops manager is a charm lib; it stays charm-side and its + # callback lands in the lib events handler. self.restart_manager = RollingOpsManager( - charm=self, relation="restart", callback=self._restart - ) - self.refresh_manager = RefreshManager( - state=self.state, - workload=self.workload, - # The charm implements the abstract charm's dispatch surface - # duck-typed; the lib managers type it as AbstractPostgreSQLCharm. - charm=cast("AbstractPostgreSQLCharm", self), - set_default_status=self._set_primary_status_message, + charm=self, + relation="restart", + callback=self.postgresql_events_handler._restart, ) - # Do not use collect status events elsewhere—otherwise ops will prioritize statuses - # incorrectly - # https://canonical-charm-refresh.readthedocs-hosted.com/latest/add-to-charm/status/#implementation - self.framework.observe(self.on.collect_unit_status, self._reconcile_refresh_status) - - # Support for disabling the operator. - disable_file = Path(f"{os.environ.get('CHARM_DIR')}/disable") - if disable_file.exists(): - logger.warning( - f"\n\tDisable file `{disable_file.resolve()}` found, the charm will skip all events." - "\n\tTo resume normal operations, please remove the file." - ) - self.unit.status = BlockedStatus("Disabled") - sys.exit(0) - - self.refresh_manager.on_init() - self.observer_manager.start_observer() - self._rotate_logs.start_log_rotation() + # COS requirers (charm libs stay charm-side). self._grafana_agent = COSAgentProvider( self, metrics_endpoints=[ @@ -482,2287 +107,8 @@ def __init__(self, *args): self.tracing = Tracing(self, tracing_relation_name=TRACING_RELATION_NAME) charm_tracing_config(self._grafana_agent) - @property - def workload(self) -> VMWorkload: - """Access current workload instance. - - Returns the workload object. - - Returns: - BaseWorkload: The VMWorkload instance for this charm - """ - return VMWorkload(charm_dir=self.charm_dir) - - @property - def substrate(self) -> Substrates: - """Access current substrate type. - - Returns: - Substrates: always Substrates.VM for this charm - """ - return Substrates.VM - - def _reload_tls_after_push(self, event) -> None: - """Reload PostgreSQL after the lib TLS handler has pushed the cert files. - - Observes the handler's ``tls_files_pushed`` event, which fires only after a - successful push (cert renewal or relation_broken), so the files are on disk and - the internal CA is present by the time this runs -- no local readiness guard is - needed. A transient config-apply failure (Patroni API unreachable, member not - started) defers and retries rather than leaving stale TLS state or failing the hook. - """ - try: - if not self.update_config(): - event.defer() - except Exception: - logger.exception("TLS reload (update_config) failed; deferring") - event.defer() - - def _regenerate_internal_cert(self, *, reload: bool = True) -> None: - """Generate the internal peer cert, push it to the workload, and (optionally) reload. - - reload=False is used at cluster bootstrap: the leader renders patroni.yml on - leader-elected and each replica renders it in _on_peer_relation_changed just - before starting Patroni, so a reload here would be redundant. The internal peer - cert does not toggle ssl in the config -- only the operator/client cert does, via - is_tls_enabled -- so skipping the reload cannot leave a stale ssl setting. - """ - self.tls_manager.generate_internal_peer_cert() - self.tls_manager.push_tls_files() - if reload: - self.update_config() - - def set_unit_status( - self, status: ops.StatusBase, /, *, refresh: charm_refresh.Machines | None = None - ): - """Set unit status without overriding higher priority refresh status.""" - # RefreshManager invokes the charm-injected set_default_status callable during - # its construction (reconcile_refresh_status), before this attribute is - # assigned; set the ops status directly in that window, mirroring the direct - # sets the library itself performs while reconciling. - refresh_manager = getattr(self, "refresh_manager", None) - if refresh_manager is None: - self.unit.status = status - return - refresh_manager.set_unit_status(status, refresh=refresh) - - def _reconcile_refresh_status(self, _=None) -> None: - """Reconcile the unit status with the refresh status on collect-unit-status.""" - self.refresh_manager.reconcile_refresh_status(_) - - @property - def refresh(self) -> charm_refresh.Machines | None: - """The charm_refresh object owned by the refresh manager. - - Returns None while the refresh manager is still constructing itself, since - status reconciliation runs inside its __init__ before the attribute exists. - """ - if (refresh_manager := getattr(self, "refresh_manager", None)) is None: - return None - return refresh_manager.refresh - - def set_default_unit_status(self) -> None: - """Set the unit status that applies when no refresh status is active.""" - self._set_primary_status_message() - - def _recompute_async_app_status(self) -> None: - """Set the application status from the async-replication state. - - Owned by the async-replication module until that phase migrates; the refresh - manager calls this during status reconciliation. - """ - self.async_replication.set_app_status() - - def get_async_primary_cluster_endpoint(self) -> str | None: - """Endpoint of the primary cluster of the async replication partner, if any.""" - return self.async_replication.get_primary_cluster_endpoint() - - def has_async_replication_relation(self) -> bool: - """Whether this unit is related to an async replication partner.""" - return self.async_replication._relation is not None - - def update_relation_endpoints(self) -> None: - """Refresh the client and async relation endpoints after a switchover.""" - self._update_relation_endpoints() - - def update_pebble_layers(self) -> None: - """Reconcile the workload's Pebble layers (K8s only).""" - - def ensure_pgdata_dirs_and_symlinks(self) -> None: - """Create the storage directories and symlinks for the PostgreSQL data paths (K8s only).""" - - def post_refresh_side_effects(self) -> None: - """Run the post-snap-refresh side effects owned by not-yet-migrated modules. - - The exporter and pgBackRest exporter setup, the pgBackRest service - start/stop, and the watcher unit address update are owned by modules that - have not yet migrated to the single kernel library. - """ - self._setup_exporter() - self.backup.start_stop_pgbackrest_service() - self._setup_pgbackrest_exporter() - self.watcher_handler.update_unit_address() - - def set_app_status(self, status: ops.StatusBase) -> None: - """Set the application status without overriding a higher-priority refresh status. - - Bridge for the single-kernel async-replication handler, which writes the app - status through the charm instead of touching ``self.app.status`` directly. - """ - if self.refresh is not None and self.refresh.app_status_higher_priority: - self.app.status = self.refresh.app_status_higher_priority - return - self.app.status = status - - def set_primary_status_message(self) -> None: - """Recompute the unit's primary/standby status message. - - Bridge for the single-kernel async-replication handler (DPE-10203). - """ - self._set_primary_status_message() - - def _restore_unit_status(self, status: ops.StatusBase) -> None: - """Restore a previously cached unit status. - - The status getter can return statuses that the setter rejects (e.g. an - "error" status left over from a previously failed hook). Skip restoring - any status juju does not allow us to set, to avoid an InvalidStatusError - that would deadlock the unit. - """ - if isinstance(status, ActiveStatus | BlockedStatus | MaintenanceStatus | WaitingStatus): - self.set_unit_status(status) - - def _on_databases_change(self, _): - """Handle databases change event.""" - self.update_config() - logger.debug("databases changed") - timestamp = datetime.now() - self.unit_peer_data.update({"timestamp": str(timestamp)}) - logger.debug(f"authorisation rules changed at {timestamp}") - - def patroni_scrape_config(self) -> list[dict]: - """Generates scrape config for the Patroni metrics endpoint.""" - return [ - { - "metrics_path": "/metrics", - "static_configs": [{"targets": [f"{self.state.unit_ip}:8008"]}], - "tls_config": {"insecure_skip_verify": True}, - "scheme": "https", - } - ] - - @property - def app_peer_data(self) -> dict: - """Application peer relation data object.""" - return self.all_peer_data.get(self.app, {}) - - @property - def unit_peer_data(self) -> dict: - """Unit peer relation data object.""" - return self.all_peer_data.get(self.unit, {}) - - @property - def all_peer_data(self) -> dict: - """Return all peer data if available.""" - if self._peers is None: - return {} - - # RelationData has dict like API - return self._peers.data # type: ignore - - @cached_property - def cpu_count(self) -> int: - """Property with numbers of cpus.""" - if cpus := os.cpu_count(): - return cpus - return 0 - - def peer_relation_data(self, scope: SCOPES) -> DataPeerData: - """Returns the peer relation data per scope.""" - if scope == APP_SCOPE: - return self.peer_relation_app - else: - return self.peer_relation_unit - - def _translate_field_to_secret_key(self, key: str) -> str: - """Change 'key' to secrets-compatible key field.""" - key = SECRET_KEY_OVERRIDES.get(key, key) - new_key = key.replace("_", "-") - return new_key.strip("-") - - def get_secret(self, scope: SCOPES, key: str) -> str | None: - """Get secret from the secret storage.""" - if scope not in get_args(SCOPES): - raise RuntimeError("Unknown secret scope.") - - if not (peers := self.model.get_relation(PEER_RELATION)): - return None - secret_key = self._translate_field_to_secret_key(key) - return self.peer_relation_data(scope).get_secret(peers.id, secret_key) - - def set_secret(self, scope: SCOPES, key: str, value: str | None) -> str | None: - """Set secret from the secret storage.""" - if scope not in get_args(SCOPES): - raise RuntimeError("Unknown secret scope.") - - if not value: - return self.remove_secret(scope, key) - - if not (peers := self.model.get_relation(PEER_RELATION)): - return None - secret_key = self._translate_field_to_secret_key(key) - self.peer_relation_data(scope).set_secret(peers.id, secret_key, value) - - def remove_secret(self, scope: SCOPES, key: str) -> None: - """Removing a secret.""" - if scope not in get_args(SCOPES): - raise RuntimeError("Unknown secret scope.") - - if not (peers := self.model.get_relation(PEER_RELATION)): - return None - secret_key = self._translate_field_to_secret_key(key) - self.peer_relation_data(scope).delete_relation_data(peers.id, [secret_key]) - - def get_secret_from_id(self, secret_id: str) -> dict[str, str]: - """Resolve the given id of a Juju secret and return the content as a dict. - - This method can be used to retrieve any secret, not just those used via the peer relation. - If the secret is not owned by the charm, it has to be granted access to it. - - Args: - secret_id (str): The id of the secret. - - Returns: - dict: The content of the secret. - """ - try: - secret_content = self.model.get_secret(id=secret_id).get_content(refresh=True) - except (SecretNotFoundError, ModelError): - raise - - return secret_content - - @property - def is_cluster_initialised(self) -> bool: - """Returns whether the cluster is already initialised.""" - return "cluster_initialised" in self.app_peer_data - - @property - def is_cluster_restoring_backup(self) -> bool: - """Returns whether the cluster is restoring a backup.""" - return "restoring-backup" in self.app_peer_data - - @property - def is_cluster_restoring_to_time(self) -> bool: - """Returns whether the cluster is restoring a backup to a specific time.""" - return "restore-to-time" in self.app_peer_data - - @property - def is_unit_departing(self) -> bool: - """Returns whether the unit is departing.""" - return "departing" in self.unit_peer_data - - @property - def is_unit_stopped(self) -> bool: - """Returns whether the unit is stopped.""" - return "stopped" in self.unit_peer_data - - def _ensure_storage_layout(self) -> None: - """Ensure the temp tablespace dir exists and versioned parents are _daemon_-owned. - - Data migration between storage roots and versioned 16/main - subdirectories is handled by the snap hooks (pre-refresh for - reverse, post-refresh for forward). TEMP_DATA_DIR may live on - a tmpfs mount that is wiped on reboot, so we recreate it - unconditionally. CREATE TABLESPACE requires the directory to - be writable by the PostgreSQL _daemon_ user, so we chown it. - - The 16/ parent dirs must also be _daemon_-owned: the snap daemon - runs as _daemon_ and needs write permission on the parent to - remove/rename the versioned subdirectory. Without this, the temp - parent breaks DROP/CREATE TABLESPACE during rollback, and the data - parent breaks ``patronictl reinit`` (Patroni can only empty 16/main, - not remove it, logging permission errors). The snap's - migrate-data.sh creates these parents as root, so we fix them here. - """ - temp_dir = Path(TEMP_DATA_DIR) - temp_dir.mkdir(parents=True, exist_ok=True) - shutil.chown(temp_dir, user=SNAP_USER, group=SNAP_USER) - if temp_dir.parent.exists(): - shutil.chown(temp_dir.parent, user=SNAP_USER, group=SNAP_USER) - - data_parent = Path(POSTGRESQL_DATA_DIR).parent - if data_parent.exists(): - shutil.chown(data_parent, user=SNAP_USER, group=SNAP_USER) - - @cached_property - def postgresql(self) -> PostgreSQL: - """Returns an instance of the object used to interact with the database.""" - return PostgreSQL( - substrate=Substrates.VM, - primary_host=self.primary_endpoint, - # Connecting to local Postgresql socket - current_host="/tmp/snap-private-tmp/snap.charmed-postgresql/tmp/", # noqa: S108 - user=USER, - password=str(self.get_secret(APP_SCOPE, f"{USER}-password")), - database=DATABASE_DEFAULT_NAME, - system_users=SYSTEM_USERS, - ) - - @cached_property - def primary_endpoint(self) -> str | None: - """Returns the endpoint of the primary instance or None when no primary available.""" - if not self._peers: - logger.debug("primary endpoint early exit: Peer relation not joined yet.") - return None - try: - primary = ( - self.patroni_manager.get_primary() or self.patroni_manager.get_standby_leader() - ) - primary_endpoint = self.patroni_manager.get_member_ip(primary) if primary else None - # Force a retry if there is no primary or the member that was - # returned is not in the list of the current cluster members - # (like when the cluster was not updated yet after a failed switchover). - if not primary_endpoint: - logger.warning(f"Missing primary IP for {primary}") - primary_endpoint = None - elif primary_endpoint not in self._units_ips: - if len(self._peers.units) == 0: - logger.info( - f"The unit didn't join {PEER_RELATION} relation? Using {primary_endpoint}" - ) - elif len(self._units_ips) == 1 and len(self._peers.units) > 1: - logger.warning(f"Possibly incomplete peer data, keep using {primary_endpoint}") - else: - logger.debug("Early exit primary_endpoint: Primary IP not in cached peer list") - primary_endpoint = None - except RetryError: - return None - else: - return primary_endpoint - - def _on_secret_remove(self, event: SecretRemoveEvent) -> None: - if self.model.juju_version < JujuVersion("3.6.11"): - logger.warning( - "Skipping secret revision removal due to https://github.com/juju/juju/issues/20782" - ) - return - - # A secret removal (entire removal, not just a revision removal) causes - # https://github.com/juju/juju/issues/20794. This check is to avoid the - # errors that would happen if we tried to remove the revision in that case - # (in the revision removal, the label is present). - if event.secret.label is None: - logger.debug("Secret with no label cannot be removed") - return - logger.debug(f"Removing secret with label {event.secret.label} revision {event.revision}") - event.remove_revision() - - def _on_get_primary(self, event: ActionEvent) -> None: - """Get primary instance.""" - try: - primary = self.patroni_manager.get_primary(unit_name_pattern=True) - event.set_results({"primary": primary}) - except RetryError as e: - logger.error(f"failed to get primary with error {e}") - - def updated_synchronous_node_count(self) -> bool: - """Tries to update synchronous_node_count configuration and reports the result.""" - try: - self.patroni_manager.update_synchronous_node_count() - return True - except RetryError: - logger.debug("Unable to set synchronous_node_count") - return False - - def _on_peer_relation_departed_early_exit(self, event: RelationDepartedEvent) -> bool: - if not event.departing_unit: - logger.debug("Early exit on_peer_relation_departed: No departing unit") - return True - if event.departing_unit == self.unit: - logger.debug("Early exit on_peer_relation_departed: Skipping departing unit") - return True - - if self.has_raft_keys(): - logger.debug("Early exit on_peer_relation_departed: Raft recovery in progress") - return True - return False - - def _on_peer_relation_departed(self, event: RelationDepartedEvent) -> None: - """The leader removes the departing units from the list of cluster members.""" - # Don't handle this event in the same unit that is departing. - if self._on_peer_relation_departed_early_exit(event): - return - - # Remove the departing member from the raft cluster. - try: - # checked for none in the early exit method - departing_member = event.departing_unit.name.replace("/", "-") # type: ignore - if member_ip := self.patroni_manager.get_member_ip(departing_member): - self.raft_manager.remove_raft_member(f"{member_ip}:{RAFT_PORT}") - except RemoveRaftMemberFailedError: - logger.debug( - "Deferring on_peer_relation_departed: Failed to remove member from raft cluster" - ) - event.defer() - return - except RetryError: - unit = event.departing_unit.name if event.departing_unit else None - logger.warning(f"Early exit on_peer_relation_departed: Cannot get {unit} member IP") - return - - # Allow leader to update the cluster members. - if not self.unit.is_leader(): - return - - if not self.is_cluster_initialised or not self.updated_synchronous_node_count(): - logger.debug("Deferring on_peer_relation_departed: cluster not initialized") - event.defer() - return - - # Remove cluster members one at a time. - for member_ip in self._get_ips_to_remove(): - # Check that all members are ready before removing unit from the cluster. - if not self.patroni_manager.are_all_members_ready(): - logger.info("Deferring reconfigure: another member doing sync right now") - event.defer() - return - - # Update the list of the current members. - self._remove_from_members_ips(member_ip) - self.update_config() - - if self.primary_endpoint: - self._update_relation_endpoints() - else: - self.set_unit_status(WaitingStatus(PRIMARY_NOT_REACHABLE_MESSAGE)) - return - - # Update the sync-standby endpoint in the async replication data. - self.async_replication.update_async_replication_data() - - def _stuck_raft_cluster_check(self) -> None: - """Check for stuck raft cluster and reinitialise if safe.""" - raft_stuck = False - all_units_stuck = True - candidate = self.app_peer_data.get("raft_selected_candidate") - for key, data in self.all_peer_data.items(): - if key == self.app: - continue - if "raft_stuck" in data: - raft_stuck = True - else: - all_units_stuck = False - if not candidate and "raft_candidate" in data: - candidate = key - - if not raft_stuck: - return - - if not all_units_stuck: - logger.warning("Stuck raft not yet detected on all units") - return - - if not candidate: - logger.warning("Stuck raft has no candidate") - return - if "raft_selected_candidate" not in self.app_peer_data: - logger.info(f"{candidate.name} selected for new raft leader") - self.app_peer_data["raft_selected_candidate"] = candidate.name - - def _stuck_raft_cluster_rejoin(self) -> None: - """Reconnect cluster to new raft.""" - primary = None - for key, data in self.all_peer_data.items(): - if key == self.app: - continue - if "raft_primary" in data: - primary = key - break - - if primary and "raft_reset_primary" not in self.app_peer_data: - logger.info("Updating the primary endpoint") - self.app_peer_data.pop("members_ips", None) - if self._peers: - for unit in self._peers.units: - if ip := self._get_unit_ip(unit): - self._add_to_members_ips(ip) - if self.state.unit_ip: - self._add_to_members_ips(self.state.unit_ip) - self.app_peer_data["raft_reset_primary"] = "True" - self._update_relation_endpoints() - if ( - "raft_rejoin" not in self.app_peer_data - and "raft_followers_stopped" in self.app_peer_data - and "raft_reset_primary" in self.app_peer_data - ): - logger.info("Notify units they can rejoin") - self.app_peer_data["raft_rejoin"] = "True" - - def _stuck_raft_cluster_stopped_check(self) -> None: - """Check that the cluster is stopped.""" - if not self._peers or "raft_followers_stopped" in self.app_peer_data: - return - - for key, data in self._peers.data.items(): - if key == self.app: - continue - if "raft_stopped" not in data: - return - - logger.info("Cluster is shut down") - self.app_peer_data["raft_followers_stopped"] = "True" - - def _stuck_raft_cluster_cleanup(self) -> None: - if self._peers: - for key, data in self._peers.data.items(): - if key == self.app: - continue - for flag in data: - if flag.startswith("raft_"): - return - - logger.info("Cleaning up raft app data") - self.app_peer_data.pop("raft_rejoin", None) - self.app_peer_data.pop("raft_reset_primary", None) - self.app_peer_data.pop("raft_selected_candidate", None) - self.app_peer_data.pop("raft_followers_stopped", None) - - def _raft_reinitialisation(self) -> None: - """Handle raft cluster loss of quorum.""" - # Skip to cleanup if rejoining - if "raft_rejoin" not in self.app_peer_data: - if self.unit.is_leader(): - self._stuck_raft_cluster_check() - - if ( - candidate := self.app_peer_data.get("raft_selected_candidate") - ) and "raft_stopped" not in self.unit_peer_data: - self.unit_peer_data.pop("raft_stuck", None) - self.unit_peer_data.pop("raft_candidate", None) - self.raft_manager.remove_raft_data() - logger.info(f"Stopping {self.unit.name}") - self.unit_peer_data["raft_stopped"] = "True" - self.watcher_handler.disable_watcher() - if self.watcher_handler.is_active: - logger.info("waiting for RAFT watcher to disconnect.") - return - - if self.unit.is_leader(): - self._stuck_raft_cluster_stopped_check() - - if ( - candidate == self.unit.name - and "raft_primary" not in self.unit_peer_data - and "raft_followers_stopped" in self.app_peer_data - ): - self.set_unit_status(MaintenanceStatus("Reinitialising raft")) - logger.info(f"Reinitialising {self.unit.name} as primary") - self.raft_manager.reinitialise_raft_data() - self.unit_peer_data["raft_primary"] = "True" - - if self.unit.is_leader(): - self._stuck_raft_cluster_rejoin() - - if "raft_rejoin" in self.app_peer_data: - logger.info("Cleaning up raft unit data") - self.unit_peer_data.pop("raft_primary", None) - self.unit_peer_data.pop("raft_stopped", None) - self.update_config() - self.patroni_manager.start_patroni() - self._set_primary_status_message() - - if self.unit.is_leader(): - self._stuck_raft_cluster_cleanup() - - def has_raft_keys(self): - """Checks for the presence of raft recovery keys in peer data.""" - for key in self.app_peer_data: - if key.startswith("raft_"): - return True - - return any(key.startswith("raft_") for key in self.unit_peer_data) - - def _peer_relation_changed_checks(self, event: HookEvent) -> bool: - """Split of to reduce complexity.""" - # Prevents the cluster to be reconfigured before it's bootstrapped in the leader. - if not self.is_cluster_initialised: - logger.debug("Early exit on_peer_relation_changed: cluster not initialized") - return False - - # Check whether raft is stuck. - if self.has_raft_keys(): - self._raft_reinitialisation() - logger.debug("Early exit on_peer_relation_changed: stuck raft recovery") - return False - - # If the unit is the leader, it can reconfigure the cluster. - if self.unit.is_leader() and not self._reconfigure_cluster(event): - event.defer() - return False - - # Don't update this member before it's part of the members list. - if self.state.unit_ip not in self.members_ips: - logger.debug("Early exit on_peer_relation_changed: Unit not in the members list") - return False - return True - - def _on_peer_relation_changed(self, event: HookEvent): - """Reconfigure cluster members when something changes.""" - if not self._peer_relation_changed_checks(event): - return - - # Update the list of the cluster members in the replicas to make them know each other. - try: - # Update the members of the cluster in the Patroni configuration on this unit. - self.update_config() - except RetryError: - self.set_unit_status(MaintenanceStatus("cluster member update failed, retrying")) - return - except ValueError as e: - self.set_unit_status(BlockedStatus("Configuration Error. Please check the logs")) - logger.error("Invalid configuration: %s", str(e)) - return - - # Should not override a blocked status - if isinstance(self.unit.status, BlockedStatus): - logger.debug("on_peer_relation_changed early exit: Unit in blocked status") - return - - if ( - self.is_cluster_restoring_backup or self.is_cluster_restoring_to_time - ) and not self._was_restore_successful(): - logger.debug("on_peer_relation_changed early exit: Backup restore check failed") - return - - # Start can be called here multiple times as it's idempotent. - # At this moment, it starts Patroni at the first time the data is received - # in the relation. - self.patroni_manager.start_patroni() - - # Assert the member is up and running before marking the unit as active. - if not self.patroni_manager.member_started: - logger.debug("Deferring on_peer_relation_changed: awaiting for member to start") - self.set_unit_status(WaitingStatus("awaiting for member to start")) - event.defer() - return - - # In Raft mode with a watcher, ensure this member is properly registered in the DCS. - # A new member may be running but not registered if it was added to Raft after starting. - if ( - self.watcher_handler.is_watcher_connected - and not self.patroni_manager.is_member_registered_in_cluster() - ): - logger.info("Member running but not registered in Raft cluster - restarting Patroni") - self.patroni_manager.restart_patroni() - event.defer() - return - - self._start_stop_pgbackrest_service(event) - - if not self._handle_s3_initialization(event): - return - - # Update watcher relation with fresh peer IPs when peer data changes - # This ensures pg-endpoints stay current when unit IPs change - if self.unit.is_leader(): - self.watcher_handler.update_endpoints() - self.async_replication.update_async_replication_data() - - self._update_new_unit_status() - - # Split off into separate function, because of complexity _on_peer_relation_changed - def _handle_s3_initialization(self, event: HookEvent) -> bool: - """Handle S3 initialization during peer relation changes. - - Returns: - True if processing should continue, False if we should return early. - """ - # This is intended to be executed only when leader is reinitializing S3 connection - # due to the leader change. - if ( - "s3-initialization-start" in self.app_peer_data - and "s3-initialization-done" not in self.unit_peer_data - and self.is_primary - and not self.backup.initialise_s3_repository() - ): - return False - - # Clean-up unit initialization data after successful sync to the leader. - if "s3-initialization-done" in self.app_peer_data and not self.unit.is_leader(): - self.unit_peer_data.update({ - "stanza": "", - "s3-initialization-block-message": "", - "s3-initialization-done": "", - "s3-initialization-start": "", - }) - - return True - - def _on_secret_changed(self, event: SecretChangedEvent) -> None: - """Handle the secret_changed event.""" - if not self.unit.is_leader(): - return - - if (admin_secret_id := self.config.system_users) and admin_secret_id == event.secret.id: - try: - self._update_admin_password(admin_secret_id) - except PostgreSQLUpdateUserPasswordError: - event.defer() - - # Split off into separate function, because of complexity _on_peer_relation_changed - def _start_stop_pgbackrest_service(self, event: HookEvent) -> None: - # Start or stop the pgBackRest TLS server service when TLS certificate change. - if not self.backup.start_stop_pgbackrest_service(): - logger.debug( - "Deferring on_peer_relation_changed: awaiting for TLS server service to start on primary" - ) - event.defer() - return - - self.backup.coordinate_stanza_fields() - - if "exporter-started" not in self.unit_peer_data: - self._setup_exporter() - if "pgbackrest-exporter-started" not in self.unit_peer_data: - self._setup_pgbackrest_exporter() - - def _update_new_unit_status(self) -> None: - """Update the status of a new unit that recently joined the cluster.""" - # Only update the connection endpoints if there is a primary. - # A cluster can have all members as replicas for some time after - # a failed switchover, so wait until the primary is elected. - if self.primary_endpoint: - self._update_relation_endpoints() - self.async_replication.handle_read_only_mode() - # Update watcher relation with current cluster endpoints - self.watcher_handler.update_endpoints() - else: - self.set_unit_status(WaitingStatus(PRIMARY_NOT_REACHABLE_MESSAGE)) - - def _reconfigure_cluster(self, event: HookEvent | RelationEvent) -> bool: - """Reconfigure the cluster by adding and removing members IPs to it. - - Returns: - Whether it was possible to reconfigure the cluster. - """ - # Remove departing units when the leader changes. - if self.is_cluster_initialised and not self.raft_manager.cleanup_raft_cluster(): - logger.debug("Deferring on_peer_relation_changed: failed to remove raft member") - return False - try: - self._add_members(event) - except Exception: - logger.debug("Deferring on_peer_relation_changed: Unable to add members") - return False - return True - - def _update_member_ip(self) -> bool: - """Update the member IP in the unit databag. - - Returns: - Whether the IP was updated. - """ - self.update_endpoint_addresses() - # Stop Patroni (and update the member IP) if it was previously isolated - # from the cluster network. Patroni will start back when its IP address is - # updated in all the units through the peer relation changed event (in that - # hook, the configuration is updated and the service is started - or only - # reloaded in the other units). - stored_ip = self.unit_peer_data.get("ip") - current_ip = self.state.unit_ip - if stored_ip is None: - self.unit_peer_data.update({"ip": current_ip}) - return False - elif current_ip != stored_ip: - logger.info(f"ip changed from {stored_ip} to {current_ip}") - self.unit_peer_data.update({"ip-to-remove": stored_ip, "ip": current_ip}) - self.patroni_manager.stop_patroni() - self._update_certificate() - # Update watcher relation - unit address for all units, endpoints only for leader - self.watcher_handler.update_unit_address() - if self.unit.is_leader(): - self.watcher_handler.update_endpoints() - return True - else: - self.unit_peer_data.update({"ip-to-remove": ""}) - return False - - def _add_members(self, event): - """Add new cluster members. - - This method is responsible for adding new members to the cluster - when new units are added to the application. This event is deferred if - one of the current units is copying data from the primary, to avoid - multiple units copying data at the same time, which can cause slow - transfer rates in these processes and overload the primary instance. - """ - try: - # Compare set of Patroni cluster members and Juju hosts - # to avoid the unnecessary reconfiguration. - if ( - self.patroni_manager.cluster_members == self._hosts - and self._units_ips <= self.members_ips - ): - logger.debug("Early exit add_members: Patroni members equal Juju hosts") - return - - logger.info("Reconfiguring cluster") - self.set_unit_status(MaintenanceStatus("reconfiguring cluster")) - for member in self._hosts - self.patroni_manager.cluster_members: - logger.debug("Adding %s to cluster", member) - self.add_cluster_member(member) - - if missing_ips := self._units_ips - self.members_ips: - for ip in missing_ips: - logger.info("Adding new IP %s to the members list", ip) - self._add_to_members_ips(ip) - self.update_config() - - self.patroni_manager.update_synchronous_node_count() - except NotReadyError: - logger.info("Deferring reconfigure: another member doing sync right now") - event.defer() - except RetryError: - logger.info("Deferring reconfigure: couldn't retrieve current cluster members") - event.defer() - - def add_cluster_member(self, member: str) -> None: - """Add member to the cluster if all members are already up and running. - - Raises: - NotReadyError if either the new member or the current members are not ready. - """ - unit = self.model.get_unit(label2name(member)) - if member_ip := self._get_unit_ip(unit): - if not self.patroni_manager.are_all_members_ready(): - logger.info("not all members are ready") - raise NotReadyError("not all members are ready") - - # Add the member to the list that should be updated in each other member. - self._add_to_members_ips(member_ip) - - # Update Patroni configuration file. - try: - self.update_config() - except RetryError: - self.set_unit_status(MaintenanceStatus("cluster member update failed, retrying")) - else: - self.set_unit_status(WaitingStatus("waiting for peer IP")) - - def _get_unit_ip(self, unit: Unit, relation_name: str = PEER_RELATION) -> str | None: - """Get the IP address of a specific unit. - - Args: - unit: The unit to get the IP address for. - relation_name: The name of the relation to use for getting the IP address. - """ - try: - if self._peers: - return self._peers.data[unit].get(f"{relation_name}-address") - except KeyError: - return None - - @property - def _hosts(self) -> set[str]: - """List of the current Juju hosts. - - Returns: - a set containing the current Juju hosts - with the names using - instead of / - to match Patroni members names - """ - hosts = [self.unit.name.replace("/", "-")] - if self._peers: - for unit in self._peers.units: - hosts.append(unit.name.replace("/", "-")) - return set(hosts) - - @property - def _planned_units(self) -> int: - """Number of planned units, resilient to a transient goal-state failure. - - ops implements ``Application.planned_units()`` via ``goal-state``, which fails - ("saas application ... not found") while a cross-model SAAS force-removed during a - dead-DC teardown still lingers in goal-state. Fall back to the count of currently known - units so the hook reconciles instead of crashing the ``_patroni`` property and every - hook that touches it (DPE-10203). - """ - try: - return self.app.planned_units() - except ModelError: - return len(self._hosts) - - @property - def is_connectivity_enabled(self) -> bool: - """Return whether this unit can be connected externally.""" - return self.unit_peer_data.get("connectivity", "on") == "on" - - @property - def is_ldap_charm_related(self) -> bool: - """Return whether this unit has an LDAP charm related.""" - return self.app_peer_data.get("ldap_enabled", "False") == "True" - - @property - def is_ldap_enabled(self) -> bool: - """Return whether this unit has LDAP enabled.""" - return self.is_ldap_charm_related and self.is_cluster_initialised - - @property - def is_primary(self) -> bool: - """Return whether this unit is the primary instance.""" - return self.unit.name == self.patroni_manager.get_primary(unit_name_pattern=True) - - @property - def is_standby_leader(self) -> bool: - """Return whether this unit is the standby leader instance.""" - return self.unit.name == self.patroni_manager.get_standby_leader(unit_name_pattern=True) - - @property - def is_standby_cluster(self) -> bool: - """Return whether this unit belongs to a standby (read-only) cluster.""" - if ( - self.model.get_relation(REPLICATION_CONSUMER_RELATION) is None - and self.model.get_relation(REPLICATION_OFFER_RELATION) is None - ): - return False - return not self.async_replication.is_primary_cluster() - - @property - def is_tls_enabled(self) -> bool: - """Return whether TLS is enabled.""" - return all(self.tls_manager.get_client_tls_files()) - - @property - def _peer_members_ips(self) -> set[str]: - """Fetch current list of peer members IPs. - - Returns: - A list of peer members addresses (strings). - """ - # Get all members IPs and remove the current unit IP from the list. - addresses = self.members_ips - current_unit_ip = self.state.unit_ip - if current_unit_ip in addresses: - addresses.remove(current_unit_ip) - return addresses - - @property - def _units_ips(self) -> set[str]: - """Fetch current list of peers IPs. - - Returns: - A list of peers addresses (strings). - """ - # Get all members IPs and remove the current unit IP from the list. - addresses = set() - - if self.state.unit_ip: - addresses.add(self.state.unit_ip) - if self._peers: - for unit in self._peers.units: - if ip := self._get_unit_ip(unit): - addresses.add(ip) - return addresses - - @property - def members_ips(self) -> set[str]: - """Returns the list of IPs addresses of the current members of the cluster.""" - if not self._peers: - return set() - return set(json.loads(self._peers.data[self.app].get("members_ips", "[]"))) - - def _add_to_members_ips(self, ip: str) -> None: - """Add one IP to the members list.""" - self._update_members_ips(ip_to_add=ip) - - def _remove_from_members_ips(self, ip: str) -> None: - """Remove IPs from the members list.""" - self._update_members_ips(ip_to_remove=ip) - - def _update_members_ips( - self, ip_to_add: str | None = None, ip_to_remove: str | None = None - ) -> None: - """Update cluster member IPs on application data. - - Member IPs on application data are used to determine when a unit of PostgreSQL - should be added or removed from the PostgreSQL cluster. - - NOTE: this function does not update the IPs on the PostgreSQL cluster - in the Patroni configuration. - """ - # Allow leader to reset which members are part of the cluster. - if not self.unit.is_leader(): - return - - ips = json.loads(self.app_peer_data.get("members_ips", "[]")) - if ip_to_add and ip_to_add not in ips: - ips.append(ip_to_add) - elif ip_to_remove: - with suppress(ValueError): - ips.remove(ip_to_remove) - self.app_peer_data["members_ips"] = json.dumps(ips) - - def update_endpoint_addresses(self) -> None: - """Update ip addresses for relation endpoints on unit peer databag.""" - logger.debug("Updating relation endpoints addresses") - updates = {} - for key, val in ( - (f"{PEER_RELATION}-address", self.state.unit_ip), - (f"{DATABASE}-address", self.state.database_ip), - (f"{REPLICATION_OFFER_RELATION}-address", self.state.replication_offer_ip), - (f"{REPLICATION_CONSUMER_RELATION}-address", self.state.replication_consumer_ip), - ): - if val: - updates[key] = val - self.unit_peer_data.update(updates) - self.watcher_handler.update_endpoints() - - def _on_raft_reconnect(self, _) -> None: - raft_status = self.raft_manager.get_raft_status() - logger.debug(f"Local raft status: {raft_status}") - if ( - not raft_status - or not self.state.unit_ip - or not self.is_cluster_initialised - or self.state.unit_ip not in self.members_ips - or self.has_raft_keys() - or (not self.members_ips and not self.watcher_handler.watcher_raft_address) - ): - return - - if all( - raft_status[partner] == 2 - for partner in raft_status - if partner.startswith(RAFT_PARTNER_PREFIX) - ): - logger.debug("All raft members are active.") - return - - logger.info("Potentially stuck Raft connection detected. Re-adding Raft member.") - local_addr = f"{self.state.unit_ip}:{RAFT_PORT}" - remote_addr = ( - watcher_addr - if (watcher_addr := self.watcher_handler.watcher_raft_address) - and self.watcher_handler.is_active - else f"{next(member for member in self.members_ips if member != self.state.unit_ip)}:{RAFT_PORT}" - ) - try: - self.raft_manager.remove_raft_member( - local_addr, remote_address=remote_addr, set_raft_flags=False - ) - except Exception: - logger.exception("Unable to remove Raft member") - return - try: - self.raft_manager.add_raft_member(local_addr, remote_address=remote_addr) - except Exception: - logger.exception("Unable to add Raft member") - return - - def _on_install(self, event: InstallEvent) -> None: - """Install prerequisites for the application.""" - logger.debug("Install start time: %s", datetime.now()) - self._check_detached_storage() - - self.set_unit_status(MaintenanceStatus("installing PostgreSQL")) - - # Install the charmed PostgreSQL snap. - self.workload.install_snap_package(revision=None) - - cache = snap.SnapCache() - postgres_snap = cache[charm_refresh.snap_name()] - try: - postgres_snap.alias("patronictl") - except snap.SnapError: - logger.warning("Unable to create patronictl alias") - try: - postgres_snap.alias("psql") - except snap.SnapError: - logger.warning("Unable to create psql alias") - - self.set_unit_status(WaitingStatus("waiting to start PostgreSQL")) - - def _on_leader_elected(self, event: LeaderElectedEvent) -> None: # noqa: C901 - """Handle the leader-elected event.""" - # consider configured system user passwords - system_user_passwords = {} - if admin_secret_id := self.config.system_users: - try: - system_user_passwords = self.get_secret_from_id(secret_id=admin_secret_id) - except (ModelError, SecretNotFoundError) as e: - # only display the error but don't return to make sure all users have passwords - logger.error(f"Error setting internal passwords: {e}") - self.set_unit_status(BlockedStatus("Password setting for system users failed.")) - event.defer() - - # The leader sets the needed passwords if they weren't set before. - for key in ( - USER_PASSWORD_KEY, - REPLICATION_PASSWORD_KEY, - REWIND_PASSWORD_KEY, - MONITORING_PASSWORD_KEY, - RAFT_PASSWORD_KEY, - PATRONI_PASSWORD_KEY, - ): - if self.get_secret(APP_SCOPE, key) is None: - if key in system_user_passwords: - # use provided passwords for system-users if available - self.set_secret(APP_SCOPE, key, system_user_passwords[key]) - logger.info(f"Using configured password for {key}") - else: - # generate a password for this user if not provided - self.set_secret(APP_SCOPE, key, new_password()) - logger.info(f"Generated new password for {key}") - - if self.has_raft_keys(): - self._raft_reinitialisation() - return - - # Update the list of the current PostgreSQL hosts when a new leader is elected. - # Add this unit to the list of cluster members - # (the cluster should start with only this member). - if self.state.unit_ip and self.state.unit_ip not in self.members_ips: - self._add_to_members_ips(self.state.unit_ip) - - # Remove departing units when the leader changes. - for ip in self._get_ips_to_remove(): - logger.info("Removing %s from the cluster", ip) - self._remove_from_members_ips(ip) - - if not self._reconfigure_cluster(event): - logger.debug("On leader elected failed to reconfigure cluster.") - - if not self.get_secret(APP_SCOPE, "internal-ca"): - self.tls_manager.generate_internal_peer_ca() - self.update_config() - - # Don't update connection endpoints in the first time this event run for - # this application because there are no primary and replicas yet. - if not self.is_cluster_initialised: - logger.debug("Early exit on_leader_elected: Cluster not initialized") - return - - # Only update the connection endpoints if there is a primary. - # A cluster can have all members as replicas for some time after - # a failed switchover, so wait until the primary is elected. - if self.primary_endpoint: - self._update_relation_endpoints() - self.async_replication.update_async_replication_data() - else: - self.set_unit_status(WaitingStatus(PRIMARY_NOT_REACHABLE_MESSAGE)) - - def _on_config_changed(self, event) -> None: # noqa: C901 - """Handle configuration changes, like enabling plugins.""" - if not self._peers: - # update endpoint addresses - logger.debug("Defer on_config_changed: no peer relation") - event.defer() - return - self.update_endpoint_addresses() - - if not self.is_cluster_initialised: - logger.debug("Defer on_config_changed: cluster not initialised yet") - event.defer() - return - - if self.refresh is None: - logger.warning("Warning _on_config_changed: Refresh could be in progress") - elif self.refresh.in_progress: - logger.debug("Defer on_config_changed: Refresh in progress") - event.defer() - return - - if self._update_member_ip(): - # Update the sync-standby endpoint in the async replication data. - self.async_replication.update_async_replication_data() - return - - try: - self._validate_config_options() - # update config on every run - self.update_config() - except psycopg2.OperationalError: - logger.debug("Defer on_config_changed: Cannot connect to database") - event.defer() - return - except ValueError as e: - self.set_unit_status(BlockedStatus("Configuration Error. Please check the logs")) - logger.error("Invalid configuration: %s", str(e)) - return - - if self.is_blocked and "Configuration Error" in self.unit.status.message: - self.set_unit_status(ActiveStatus()) - - # Update the sync-standby endpoint in the async replication data. - self.async_replication.update_async_replication_data() - - if not self.logical_replication.apply_changed_config(event): - return - - if not self.unit.is_leader(): - return - - # Enable and/or disable the extensions. - self.enable_disable_extensions() - - if admin_secret_id := self.config.system_users: - try: - self._update_admin_password(admin_secret_id) - except PostgreSQLUpdateUserPasswordError: - event.defer() - - def enable_disable_extensions(self, database: str | None = None) -> None: - """Enable/disable PostgreSQL extensions set through config options. - - Args: - database: optional database where to enable/disable the extension. - """ - if self.patroni_manager.get_primary() is None: - logger.debug("Early exit enable_disable_extensions: standby cluster") - return - original_status = self.unit.status - extensions = {} - # collect extensions - for plugin in self.config.plugin_keys(): - enable = self.config[plugin] - - # Enable or disable the plugin/extension. - extension = "_".join(plugin.split("_")[1:-1]) - if extension == "spi": - for ext in SPI_MODULE: - extensions[ext] = enable - continue - extension = PLUGIN_OVERRIDES.get(extension, extension) - if self._check_extension_dependencies(extension, enable): - self.set_unit_status(BlockedStatus(EXTENSIONS_DEPENDENCY_MESSAGE)) - return - extensions[extension] = enable - if self.is_blocked and self.unit.status.message == EXTENSIONS_DEPENDENCY_MESSAGE: - self.set_unit_status(ActiveStatus()) - original_status = self.unit.status - self.set_unit_status(WaitingStatus("Updating extensions")) - try: - self.postgresql.enable_disable_extensions(extensions, database) - except psycopg2.errors.DependentObjectsStillExist as e: # type: ignore - logger.error( - "Failed to disable plugin: %s\nWas the plugin enabled manually? If so, update charm config with `juju config postgresql plugin--enable=True`", - str(e), - ) - self.set_unit_status(BlockedStatus(EXTENSION_OBJECT_MESSAGE)) - return - except (PostgreSQLEnableDisableExtensionError, PostgreSQLUndefinedHostError) as e: - logger.exception("failed to change plugins: %s", str(e)) - if original_status.message == EXTENSION_OBJECT_MESSAGE: - self.set_unit_status(ActiveStatus()) - return - self._restore_unit_status(original_status) - - def _check_extension_dependencies(self, extension: str, enable: bool) -> bool: - skip = False - if enable and extension in REQUIRED_PLUGINS: - for ext in REQUIRED_PLUGINS[extension]: - if not self.config[f"plugin_{ext}_enable"]: - skip = True - logger.exception( - "cannot enable %s, extension required %s to be enabled before", - extension, - ext, - ) - return skip - - def _get_ips_to_remove(self) -> set[str]: - """List the IPs that were part of the cluster but departed.""" - old = self.members_ips - current = self._units_ips - return old - current - - def _can_start(self, event: StartEvent) -> bool: - """Returns whether the workload can be started on this unit.""" - self._check_detached_storage() - - # Safeguard against starting while refreshing. - if self.refresh is None: - logger.warning("Warning on_start: Refresh could be in progress") - elif self.refresh.in_progress: - # TODO: we should probably start workload if scale up while refresh in progress - logger.debug("Defer on_start: Refresh in progress") - event.defer() - return False - - if self.has_raft_keys(): - logger.debug("Early exit on_start: Raft recovery in process") - return False - - return True - - def _on_start(self, event: StartEvent) -> None: - """Handle the start event.""" - if not self._can_start(event): - return - - try: - postgres_password = self._get_password() - except ModelError: - logger.debug("_on_start: secrets not yet available") - postgres_password = None - # If the leader was not elected (and the needed passwords were not generated yet), - # the cluster cannot be bootstrapped yet. - if not postgres_password or not self._replication_password: - logger.info("leader not elected and/or passwords not yet generated") - self.set_unit_status(WaitingStatus("awaiting passwords generation")) - event.defer() - return - - if not self.get_secret(APP_SCOPE, "internal-ca"): - logger.info("leader not elected and/or internal CA not yet generated") - event.defer() - return - if not self.get_secret(UNIT_SCOPE, "internal-cert"): - self._regenerate_internal_cert(reload=False) - - self._update_member_ip() - - self._ensure_storage_layout() - - # Open port - try: - self.unit.open_port("tcp", 5432) - except ModelError: - logger.exception("failed to open port") - - start_raft_observer() - # Only the leader can bootstrap the cluster. - # On replicas, only prepare for starting the instance later. - if not self.unit.is_leader(): - self._start_replica(event) - self._restart_services_after_reboot() - return - - # Bootstrap the cluster in the leader unit. - self._start_primary(event) - self._restart_services_after_reboot() - - def _restart_services_after_reboot(self): - """Restart the Patroni and pgBackRest after a reboot.""" - if self.state.unit_ip in self.members_ips: - self.patroni_manager.start_patroni() - self.backup.start_stop_pgbackrest_service() - - def _restart_metrics_service(self, postgres_snap: snap.Snap) -> None: - """Restart the monitoring service if the password was rotated.""" - try: - snap_password = postgres_snap.get("exporter.password") - except snap.SnapError: - logger.warning("Early exit: Trying to reset metrics service with no configuration set") - return None - - if snap_password != self.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY): - self._setup_exporter(postgres_snap) - - def _restart_ldap_sync_service(self, postgres_snap: snap.Snap) -> None: - """Restart the LDAP sync service in case any configuration changed.""" - if not self.patroni_manager.member_started: - logger.debug("Restart LDAP sync early exit: Patroni has not started yet") - return - - sync_service = postgres_snap.services["ldap-sync"] - - if not self.is_primary and sync_service["active"]: - logger.debug("Stopping LDAP sync service. It must only run in the primary") - postgres_snap.stop(services=["ldap-sync"]) - - if self.is_primary and not self.is_ldap_enabled: - logger.debug("Stopping LDAP sync service") - postgres_snap.stop(services=["ldap-sync"]) - return - - if self.is_primary and self.is_ldap_enabled: - self._setup_ldap_sync(postgres_snap) - - def _setup_exporter(self, postgres_snap: snap.Snap | None = None) -> None: - """Set up postgresql_exporter options.""" - if postgres_snap is None: - cache = snap.SnapCache() - postgres_snap = cache[charm_refresh.snap_name()] - - postgres_snap.set({ - "exporter.user": MONITORING_USER, - "exporter.password": self.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY), - }) - - if postgres_snap.services[MONITORING_SNAP_SERVICE]["active"] is False: - postgres_snap.start(services=[MONITORING_SNAP_SERVICE], enable=True) - else: - postgres_snap.restart(services=[MONITORING_SNAP_SERVICE]) - - self.unit_peer_data.update({"exporter-started": "True"}) - - def _setup_pgbackrest_exporter(self, postgres_snap: snap.Snap | None = None) -> None: - """Set up pgbackrest_exporter.""" - if postgres_snap is None: - cache = snap.SnapCache() - postgres_snap = cache[charm_refresh.snap_name()] - - if postgres_snap.services[PGBACKREST_MONITORING_SNAP_SERVICE]["active"] is False: - postgres_snap.start(services=[PGBACKREST_MONITORING_SNAP_SERVICE], enable=True) - else: - postgres_snap.restart(services=[PGBACKREST_MONITORING_SNAP_SERVICE]) - - self.unit_peer_data.update({"pgbackrest-exporter-started": "True"}) - - def _setup_ldap_sync(self, postgres_snap: snap.Snap | None = None) -> None: - """Set up postgresql_ldap_sync options.""" - if postgres_snap is None: - cache = snap.SnapCache() - postgres_snap = cache[charm_refresh.snap_name()] - - ldap_params = self.ldap.get_ldap_parameters() - ldap_url = urlparse(ldap_params["ldapurl"]) - ldap_host = ldap_url.hostname - ldap_port = ldap_url.port - - ldap_base_dn = ldap_params["ldapbasedn"] - ldap_bind_username = ldap_params["ldapbinddn"] - ldap_bind_password = ldap_params["ldapbindpasswd"] - ldap_group_mappings = self.postgresql.build_postgresql_group_map(self.config.ldap_map) - - postgres_snap.set({ - "ldap-sync.ldap_host": ldap_host, - "ldap-sync.ldap_port": ldap_port, - "ldap-sync.ldap_base_dn": ldap_base_dn, - "ldap-sync.ldap_bind_username": ldap_bind_username, - "ldap-sync.ldap_bind_password": ldap_bind_password, - "ldap-sync.ldap_group_identity": json.dumps(ACCESS_GROUP_IDENTITY), - "ldap-sync.ldap_group_mappings": json.dumps(ldap_group_mappings), - "ldap-sync.postgres_host": "127.0.0.1", - "ldap-sync.postgres_port": DATABASE_PORT, - "ldap-sync.postgres_database": DATABASE_DEFAULT_NAME, - "ldap-sync.postgres_username": USER, - "ldap-sync.postgres_password": self._get_password(), - }) - - logger.debug("Starting LDAP sync service") - postgres_snap.restart(services=["ldap-sync"]) - - def _setup_users(self) -> None: - # A standby cluster's PostgreSQL is a read-only hot standby; these objects are - # provisioned on the primary cluster and replicated here, so the write DDL below - # would fail with ReadOnlySqlTransaction. Skip it (DPE-10284). - if self.is_standby_cluster: - logger.debug("Early exit _setup_users: standby cluster is read-only") - return - - self.postgresql.create_predefined_instance_roles() - - # Create the default postgres database user that is needed for some - # applications (not charms) like Landscape Server. - - # This event can be run on a replica if the machines are restarted. - # For that case, check whether the postgres user already exits. - users = self.postgresql.list_users() - # Create the backup user. - if BACKUP_USER not in users: - self.postgresql.create_user( - BACKUP_USER, new_password(), extra_user_roles=[ROLE_BACKUP] - ) - self.postgresql.grant_database_privileges_to_user(BACKUP_USER, "postgres", ["connect"]) - if MONITORING_USER not in users: - # Create the monitoring user. - self.postgresql.create_user( - MONITORING_USER, - self.get_secret(APP_SCOPE, MONITORING_PASSWORD_KEY), - extra_user_roles=[ROLE_STATS], - ) - - self.postgresql.set_up_database(temp_location=TEMP_DATA_DIR) - - access_groups = self.postgresql.list_access_groups() - if access_groups != set(ACCESS_GROUPS): - self.postgresql.create_access_groups() - self.postgresql.grant_internal_access_group_memberships() - - self.database_manager.oversee_users(self.postgresql) - - def _start_primary(self, event: StartEvent) -> None: - """Bootstrap the cluster.""" - # Set some information needed by Patroni to bootstrap the cluster. - if not self.patroni_manager.bootstrap_cluster(): - self.set_unit_status(BlockedStatus("failed to start Patroni")) - return - - # Assert the member is up and running before marking it as initialised. - if not self.patroni_manager.member_started: - logger.debug("Deferring on_start: awaiting for member to start") - self.set_unit_status(WaitingStatus("awaiting for member to start")) - event.defer() - return - - if not self._can_connect_to_postgresql: - logger.debug("Deferring on_start: awaiting for database to start") - self.unit.status = WaitingStatus("awaiting for database to start") - event.defer() - return - - if not self.primary_endpoint: - logger.debug("Deferrring on_start: awaitng start of the primary") - self.unit.status = WaitingStatus("awaiting start of the primary") - event.defer() - return - - try: - self._setup_users() - except PostgreSQLCreatePredefinedRolesError as e: - logger.exception(e) - self.unit.status = BlockedStatus("Failed to create pre-defined roles") - return - except PostgreSQLGrantDatabasePrivilegesToUserError as e: - logger.exception(e) - self.unit.status = BlockedStatus("Failed to grant database privileges to user") - return - except PostgreSQLCreateUserError as e: - logger.exception(e) - self.set_unit_status(BlockedStatus("Failed to create postgres user")) - return - except PostgreSQLListUsersError: - logger.warning("Deferriing on_start: Unable to list users") - event.defer() - return - - # Set the flag to enable the replicas to start the Patroni service. - self.app_peer_data["cluster_initialised"] = "True" - # Flag to know if triggers need to be removed after refresh - self.app_peer_data["refresh_remove_trigger"] = "True" - - # Clear unit data if this unit became a replica after a failover/switchover. - self._update_relation_endpoints() - - # Enable/disable PostgreSQL extensions if they were set before the cluster - # was fully initialised. - self.enable_disable_extensions() - - logger.debug("Active workload time: %s", datetime.now()) - self._set_primary_status_message() - - def _start_replica(self, event) -> None: - """Configure the replica if the cluster was already initialised.""" - if not self.is_cluster_initialised: - logger.debug("Deferring on_start: awaiting for cluster to start") - self.set_unit_status(WaitingStatus("awaiting for cluster to start")) - event.defer() - return - - # Member already started, so we can set an ActiveStatus. - # This can happen after a reboot. - if self.patroni_manager.member_started: - self.set_unit_status(ActiveStatus()) - return - - # Configure Patroni in the replica but don't start it yet. - self.patroni_manager.configure_patroni_on_unit() - - def _update_admin_password(self, admin_secret_id: str) -> None: - """Check if the password of a system user was changed and update it in the database.""" - if not self.patroni_manager.are_all_members_ready(): - # Ensure all members are ready before reloading Patroni configuration to avoid errors - # e.g. API not responding in one instance because PostgreSQL / Patroni are not ready - raise PostgreSQLUpdateUserPasswordError( - "Failed changing the password: Not all members healthy or finished initial sync." - ) - - replication_offer_relation = self.model.get_relation(REPLICATION_OFFER_RELATION) - other_cluster_primary_ip = "" - if ( - replication_offer_relation is not None - and not self.async_replication.is_primary_cluster() - ): - other_cluster_endpoints = self.async_replication.get_all_primary_cluster_endpoints() - other_cluster_primary = self.patroni_manager.get_primary( - alternative_endpoints=other_cluster_endpoints - ) - other_cluster_primary_ip = next( - replication_offer_relation.data[unit].get("ip") - or replication_offer_relation.data[unit].get("private-address") - for unit in replication_offer_relation.units - if unit.name.replace("/", "-") == other_cluster_primary - ) - elif self.model.get_relation(REPLICATION_CONSUMER_RELATION) is not None: - logger.error( - "Failed changing the password: This can be ran only in the cluster from the offer side." - ) - self.set_unit_status(BlockedStatus("Password update for system users failed.")) - return - - try: - updateable_users = list(SYSTEM_USERS) - # get the secret content and check each user configured there - # only SYSTEM_USERS with changed passwords are processed, all others ignored - updated_passwords = self.get_secret_from_id(secret_id=admin_secret_id) - for user, password in list(updated_passwords.items()): - if user not in updateable_users: - logger.error( - f"Can only update system users: {', '.join(updateable_users)} not {user}" - ) - updated_passwords.pop(user) - continue - if password == self.get_secret(APP_SCOPE, f"{user}-password"): - updated_passwords.pop(user) - except (ModelError, SecretNotFoundError) as e: - logger.error(f"Error updating internal passwords: {e}") - self.set_unit_status(BlockedStatus("Password update for system users failed.")) - return - - try: - # perform the actual password update for the remaining users - for user, password in updated_passwords.items(): - logger.info(f"Updating password for user {user}") - self.postgresql.update_user_password( - user, - password, - database_host=other_cluster_primary_ip if other_cluster_primary_ip else None, - ) - # Update the password in the secret store after updating it in the database - self.set_secret(APP_SCOPE, f"{user}-password", password) - except PostgreSQLUpdateUserPasswordError as e: - logger.exception(e) - self.set_unit_status(BlockedStatus("Password update for system users failed.")) - return - - # Update and reload Patroni configuration in this unit to use the new password. - # Other units Patroni configuration will be reloaded in the peer relation changed event. - self.update_config() - - def _on_promote_to_primary(self, event: ActionEvent) -> None: - if event.params.get("scope") == "cluster": - return self.async_replication.promote_to_primary(event) - elif event.params.get("scope") == "unit": - return self.promote_primary_unit(event) - else: - event.fail("Scope should be either cluster or unit") - - def promote_primary_unit(self, event: ActionEvent) -> None: - """Handles promote to primary for unit scope.""" - if event.params.get("force"): - if self.has_raft_keys(): - self.unit_peer_data.update({"raft_candidate": "True"}) - if self.unit.is_leader(): - self._raft_reinitialisation() - return - event.fail("Raft is not stuck") - else: - if self.has_raft_keys(): - event.fail("Raft is stuck. Set force to reinitialise with new primary") - return - try: - self.patroni_manager.switchover(self._member_name) - self.unit_peer_data.update({"timestamp": str(datetime.now())}) - if self.unit.is_leader(): - self.database_manager.update_endpoints(**self._client_endpoint_inputs()) - self.async_replication.update_async_replication_data() - except SwitchoverNotSyncError: - event.fail("Unit is not sync standby") - except SwitchoverFailedError: - event.fail("Switchover failed or timed out, check the logs for details") - - def _on_update_status(self, _) -> None: - """Update the unit status message and users list in the database.""" - if not self._can_run_on_update_status(): - return - - if ( - self.is_cluster_restoring_backup or self.is_cluster_restoring_to_time - ) and not self._was_restore_successful(): - return - - if self._handle_processes_failures(): - return - - self.database_manager.oversee_users(self.postgresql) - if self.primary_endpoint: - self._update_relation_endpoints() - - if not self.patroni_manager.member_started and self.patroni_manager.is_member_isolated: - self.patroni_manager.restart_patroni() - self.observer_manager.start_observer() - return - - # Update the sync-standby endpoint in the async replication data. - self.async_replication.update_async_replication_data() - - # Clear a promoted-cluster-counter orphaned by a dead-DC teardown whose relation-broken - # never fired (Juju CMR limitation); otherwise a newly-formed async relation re-counts it - # and create-replication wrongly reports "There is already a replication set up.". - self.async_replication.clear_stale_promotion() - - self.backup.coordinate_stanza_fields() - - self.logical_replication.retry_validations() - - self._set_primary_status_message() - - # Restart topology observer if it is gone - self.observer_manager.start_observer() - - # Keep this unit data current for watcher AZ/IP checks. - self.watcher_handler.update_unit_address() - - if self.unit.is_leader() and "refresh_remove_trigger" not in self.app_peer_data: - self.postgresql.drop_hba_triggers() - self.app_peer_data["refresh_remove_trigger"] = "True" - - def _was_restore_successful(self) -> bool: - if self.is_cluster_restoring_to_time and all(self.restore_manager.is_pitr_failed()): - logger.error( - "Restore failed: database service failed to reach point-in-time-recovery target. " - "You can launch another restore with different parameters" - ) - self.restore_manager.log_pitr_last_transaction_time() - self.set_unit_status(BlockedStatus(CANNOT_RESTORE_PITR)) - return False - - if "failed" in self.patroni_manager.get_member_status(self._member_name): - logger.error("Restore failed: database service failed to start") - self.set_unit_status(BlockedStatus("Failed to restore backup")) - return False - - if not self.patroni_manager.member_started: - logger.debug("Restore check early exit: Patroni has not started yet") - return False - - try: - self._setup_users() - except Exception as e: - logger.exception(e) - return False - - restoring_backup = self.app_peer_data.get("restoring-backup") - restore_timeline = self.app_peer_data.get("restore-timeline") - restore_to_time = self.app_peer_data.get("restore-to-time") - try: - current_timeline = self.postgresql.get_current_timeline() - except PostgreSQLGetCurrentTimelineError: - logger.debug("Restore check early exit: can't get current wal timeline") - return False - - self.enable_disable_extensions() - - # Remove the restoring backup flag and the restore stanza name. - self.app_peer_data.update({ - "restoring-backup": "", - "restore-stanza": "", - "restore-to-time": "", - "restore-timeline": "", - }) - self.update_config() - self.restore_manager.restore_patroni_restart_condition() - - logger.info( - "Restored" - f"{f' to {restore_to_time}' if restore_to_time else ''}" - f"{f' from timeline {restore_timeline}' if restore_timeline and not restoring_backup else ''}" - f"{f' from backup {parse_backup_id(restoring_backup)[0]}' if restoring_backup else ''}" - f". Currently tracking the newly created timeline {current_timeline}." - ) - - can_use_s3_repository, validation_message = self.backup.can_use_s3_repository() - if not can_use_s3_repository: - self.app_peer_data.update({ - "stanza": "", - "s3-initialization-start": "", - "s3-initialization-done": "", - "s3-initialization-block-message": validation_message, - }) - - return True - - def _can_run_on_update_status(self) -> bool: - if not self.is_cluster_initialised: - return False - - if self.has_raft_keys(): - logger.debug("Early exit on_update_status: Raft recovery in progress") - return False - - if self.refresh is None: - logger.debug("Early exit on_update_status: Refresh could be in progress") - return False - if self.refresh.in_progress: - logger.debug("Early exit on_update_status: Refresh in progress") - return False - - if ( - self.is_blocked - and self.unit.status.message not in S3_BLOCK_MESSAGES - and self.unit.status.message != LOGICAL_REPLICATION_VALIDATION_ERROR_STATUS - and self.unit.status.message - != self.app_peer_data.get("logical-replication-validation-status-message") - and self.unit.status.message - != self.app_peer_data.get("logical-replication-last-block-message") - ): - # If charm was failing to disable plugin, try again (user may have removed the objects) - if self.unit.status.message == EXTENSION_OBJECT_MESSAGE: - self.enable_disable_extensions() - logger.debug("on_update_status early exit: Unit is in Blocked status") - return False - - return True - - def _handle_processes_failures(self) -> bool: - """Handle Patroni and PostgreSQL OS processes failures. - - Returns: - a bool indicating whether the charm performed any action. - """ - # Restart the PostgreSQL process if it was frozen (in that case, the Patroni - # process is running by the PostgreSQL process not). - if self.state.unit_ip in self.members_ips and self.patroni_manager.member_inactive: - if not os.path.exists(POSTGRESQL_DATA_DIR): - # The data directory is created during bootstrap. If it does not exist yet, - # the member has not been initialised (e.g. update-status firing before the - # start event completes), so there is no frozen process to recover. - logger.debug( - "Early exit handle_processes_failures: data directory does not exist yet" - ) - return False - data_directory_contents = os.listdir(POSTGRESQL_DATA_DIR) - if len(data_directory_contents) == 1 and data_directory_contents[0] == "pg_wal": - os.rename( - os.path.join(POSTGRESQL_DATA_DIR, "pg_wal"), - os.path.join(POSTGRESQL_DATA_DIR, f"pg_wal-{datetime.now(UTC).isoformat()}"), - ) - logger.info("PostgreSQL data directory was not empty. Moved pg_wal") - return True - try: - logger.info("restarted PostgreSQL because it was not running") - self.patroni_manager.restart_patroni() - self.observer_manager.start_observer() - return True - except RetryError: - logger.error("failed to restart PostgreSQL after checking that it was not running") - return False - - return False - - def _set_primary_status_message(self) -> None: - """Display 'Primary' in the unit status message if the current unit is the primary.""" - try: - if self.unit.is_leader() and "s3-initialization-block-message" in self.app_peer_data: - self.set_unit_status( - BlockedStatus(self.app_peer_data["s3-initialization-block-message"]) - ) - return - # if self.unit.is_leader() and ( - # self.app_peer_data.get("logical-replication-validation") == "error" - # or self.logical_replication.has_remote_publisher_errors() - # ): - # self.unit.status = BlockedStatus(LOGICAL_REPLICATION_VALIDATION_ERROR_STATUS) - # return - if self.unit.is_leader() and ( - self.app_peer_data.get("logical-replication-validation") == "error" - or self.logical_replication.has_remote_publisher_errors() - ): - self.unit.status = BlockedStatus( - self.app_peer_data.get("logical-replication-validation-status-message") - or self.logical_replication_manager.remote_publisher_error_message() - or LOGICAL_REPLICATION_VALIDATION_ERROR_STATUS - ) - return - if ( - self.patroni_manager.get_primary(unit_name_pattern=True) == self.unit.name - or self.is_standby_leader - ): - danger_state = "" - if not self.raft_manager.has_raft_quorum(): - danger_state = " (read-only)" - elif len(self.patroni_manager.get_running_cluster_members()) < self._planned_units: - danger_state = " (degraded)" - unit_status = "Standby" if self.is_standby_leader else "Primary" - self.set_unit_status(ActiveStatus(f"{unit_status}{danger_state}")) - elif self.patroni_manager.member_started: - self.set_unit_status(ActiveStatus()) - except (RetryError, ConnectionError) as e: - logger.error(f"failed to get primary with error {e}") - - def _update_certificate(self) -> None: - """Updates the TLS certificate if the unit IP changes.""" - # Request the certificate only if there is already one. If there isn't, - # the certificate will be generated in the relation joined event when - # relating to the TLS Certificates Operator. - if all(self.tls_manager.get_client_tls_files()) or all( - self.tls_manager.get_peer_tls_files() - ): - self.tls.refresh_tls_certificates_event.emit() - if self.get_secret(UNIT_SCOPE, "internal-cert"): - self._regenerate_internal_cert() - - @property - def is_blocked(self) -> bool: - """Returns whether the unit is in a blocked state.""" - return isinstance(self.unit.status, BlockedStatus) - - def _get_password(self) -> str | None: - """Get operator user password. - - Returns: - The password from the peer relation or None if the - password has not yet been set by the leader. - """ - return self.get_secret(APP_SCOPE, USER_PASSWORD_KEY) - - @property - def _replication_password(self) -> str | None: - """Get replication user password. - - Returns: - The password from the peer relation or None if the - password has not yet been set by the leader. - """ - return self.get_secret(APP_SCOPE, REPLICATION_PASSWORD_KEY) - - def _on_storage_detaching(self, _) -> None: - """Stop the workload so Juju can unmount the storage on app teardown.""" - # On scale-down the surviving cluster still needs this unit's Patroni to - # remove it from raft; only stop when the whole app is going away. - if self.app.planned_units() > 0: - return - self.observer_manager.stop_observer() - self._rotate_logs.stop_log_rotation() - try: - # Disable too, so a mid-teardown restart of the unit can't re-enable the - # services and re-grab the storage mounts before Juju finishes unmounting. - snap.SnapCache()[charm_refresh.snap_name()].stop(disable=True) - except snap.SnapError: - logger.exception("Failed to stop charmed-postgresql snap services") - - def _on_remove(self, _) -> None: - """Remove the charmed-postgresql snap on app teardown, before the machine goes away.""" - # Juju only unmounts the storages after this hook, and snapd refuses to remove the - # snap while anything is mounted under its data directories, so unmount them here. - # The services were already stopped on storage-detaching, and unmounting leaves - # the storage contents untouched. - for name, storage in self.meta.storages.items(): - if not storage.location: - continue - # `mountpoint` also detects bind mounts (unlike `os.path.ismount`), which is - # how Juju attaches rootfs storage. - if subprocess.run(["/usr/bin/mountpoint", "-q", storage.location]).returncode != 0: # noqa: S603 - continue - try: - subprocess.run( # noqa: S603 - ["/usr/bin/umount", storage.location], - check=True, - capture_output=True, - text=True, - ) - except subprocess.CalledProcessError as e: - logger.error( - "Skipping charmed-postgresql snap removal: failed to unmount %s storage: %s", - name, - e.stderr, - ) - return - try: - snap.SnapCache()[charm_refresh.snap_name()].ensure(snap.SnapState.Absent) - except (snap.SnapError, snap.SnapNotFoundError): - # Don't fail the hook: an error here would block the removal of the unit. - logger.exception("Failed to remove charmed-postgresql snap") - - def _is_storage_attached(self) -> bool: - """Returns if storage is attached.""" - try: - # Storage path is constant - subprocess.check_call(["/usr/bin/mountpoint", "-q", self._storage_path]) # noqa: S603 #type: ignore - return True - except subprocess.CalledProcessError: - return False - - @property - def _peers(self) -> Relation | None: - """Fetch the peer relation. - - Returns: - A:class:`ops.model.Relation` object representing - the peer relation. - """ - return self.model.get_relation(PEER_RELATION) - - def push_ca_file_into_workload(self, secret_name: str) -> bool: - """Move CA certificates file into the PostgreSQL storage path.""" - certs = self.get_secret(UNIT_SCOPE, secret_name) - if certs is not None: - certs_file = Path(self._certs_path, f"{secret_name}.crt") - certs_file.write_text(certs) - subprocess.check_call([UPDATE_CERTS_BIN_PATH]) # noqa: S603 - - try: - return self.update_config() - except Exception: - logger.exception("CA file failed to push. Error in config update") - return False - - def clean_ca_file_from_workload(self, secret_name: str) -> bool: - """Cleans up CA certificates from the PostgreSQL storage path.""" - certs_file = Path(self._certs_path, f"{secret_name}.crt") - certs_file.unlink() - - subprocess.check_call([UPDATE_CERTS_BIN_PATH]) # noqa: S603 - - try: - return self.update_config() - except Exception: - logger.exception("CA file failed to clean. Error in config update") - return False - - def _check_detached_storage(self) -> None: - """Wait for storage to become available. - - Workaround for lxd containers not getting storage attached on startups. - - Args: - event: the event that triggered this handler - """ - cached_status = self.unit.status - for attempt in Retrying(stop=stop_after_attempt(10), wait=wait_fixed(1), reraise=True): - with attempt: - if not self._is_storage_attached(): - logger.error("Data directory not attached.") - self.unit.status = WaitingStatus("Data directory not attached") - raise StorageUnavailableError() - self._restore_unit_status(cached_status) - - def _restart(self, event: RunWithLock) -> None: - """Restart PostgreSQL.""" - if not self.patroni_manager.are_all_members_ready(): - logger.debug("Early exit _restart: not all members ready yet") - event.defer() - return - - try: - self.patroni_manager.restart_postgresql() - self.unit_peer_data["postgresql_restarted"] = "True" - except RetryError: - error_message = "failed to restart PostgreSQL" - logger.exception(error_message) - self.set_unit_status(BlockedStatus(error_message)) - return - - try: - for attempt in Retrying(wait=wait_fixed(3), stop=stop_after_delay(300)): - with attempt: - if not self._can_connect_to_postgresql: - raise CannotConnectError - except Exception: - logger.exception("Unable to reconnect to postgresql") - - # Start or stop the pgBackRest TLS server service when TLS certificate change. - self.backup.start_stop_pgbackrest_service() - - @property - def _can_connect_to_postgresql(self) -> bool: - if not self.postgresql.password or not self.postgresql.current_host: - return False - try: - for attempt in Retrying(stop=stop_after_delay(10), wait=wait_fixed(3)): - with attempt: - if not self.postgresql.get_postgresql_timezones(): - logger.debug("Cannot connect to database (CannotConnectError)") - raise CannotConnectError - except RetryError: - logger.debug("Cannot connect to database (RetryError)") - return False - return True - - def get_resource_provider(self) -> VMWorkload: - """Return this unit's (cpu_cores, memory_bytes) introspector. Bridge for ConfigManager.""" - return self.workload - - def request_restart(self) -> None: - """Clear the restarted flag and acquire the rolling-restart lock. Bridge for ConfigManager.""" - self.unit_peer_data.pop("postgresql_restarted", None) - self.on[str(self.restart_manager.name)].acquire_lock.emit() - - def restart_services(self) -> None: - """Restart the monitoring and LDAP sync snap services if needed. Bridge for ConfigManager.""" - cache = snap.SnapCache() - postgres_snap = cache[charm_refresh.snap_name()] - self._restart_metrics_service(postgres_snap) - self._restart_ldap_sync_service(postgres_snap) - - def update_config( - self, - is_creating_backup: bool = False, - no_peers: bool = False, - *, - refresh: charm_refresh.Machines | None = None, - ) -> bool: - """Updates Patroni config file based on the existence of the TLS files.""" - if refresh is None: - refresh = self.refresh - primary_cluster_endpoint = self.async_replication.get_primary_cluster_endpoint() - result = self.config_manager.update_config( - self.postgresql, - is_creating_backup=is_creating_backup, - relations_user_databases_map=self.relations_user_databases_map, - async_primary_cluster_endpoint=primary_cluster_endpoint, - async_partner_addresses=self.async_replication.get_partner_addresses(), - async_standby_endpoints=self.async_replication.get_standby_endpoints(), - watcher_raft_address=self.watcher_handler.watcher_raft_address - if self.watcher_handler.is_active - else None, - no_peers=no_peers, - refresh=refresh, - ) - # The lib's apply_api_config only SETS the DCS standby_cluster (when another - # cluster is primary) and never CLEARS it. A force-promote bumps the - # promoted-cluster-counter but — while the dead-DC relation still lingers — does - # not call promote_standby_cluster(), so without this the reconciler never clears - # the stale standby and the cluster stays a read-only standby leader (DPE-10203). - if ( - result - and not no_peers - and self.patroni_manager.member_started - and primary_cluster_endpoint is None - ): - self.patroni_manager.bulk_update_parameters_controller_by_patroni( - {}, {"standby_cluster": None} - ) - return result - - def _validate_config_options(self) -> None: - """Validates specific config options that need access to the database or to the TLS status.""" - if ( - self.config.instance_default_text_search_config - not in self.postgresql.get_postgresql_text_search_configs() - ): - raise ValueError( - "instance_default_text_search_config config option has an invalid value" - ) - - if not self.postgresql.validate_group_map(self.config.ldap_map): - raise ValueError("ldap_map config option has an invalid value") - - if self.config.request_date_style and not self.postgresql.validate_date_style( - self.config.request_date_style - ): - raise ValueError("request_date_style config option has an invalid value") - - if self.config.request_time_zone not in self.postgresql.get_postgresql_timezones(): - raise ValueError("request_time_zone config option has an invalid value") - - if ( - self.config.storage_default_table_access_method - not in self.postgresql.get_postgresql_default_table_access_methods() - ): - raise ValueError( - "storage_default_table_access_method config option has an invalid value" - ) - - def _client_endpoint_inputs(self) -> dict: - """Gather the update_endpoints inputs the manager takes caller-side. - - The Patroni cluster-status query is a live REST call: gather it only on the - leader (update_endpoints returns early otherwise), so non-leader units fire - no extra REST calls. - """ - return { - "online_members": ( - self.patroni_manager.online_cluster_members() if self.unit.is_leader() else None - ), - "client_tls_files": self.tls_manager.get_client_tls_files(), - } - - def _update_relation_endpoints(self) -> None: - """Updates endpoints and read-only endpoint in all relations.""" - self.database_manager.update_endpoints(**self._client_endpoint_inputs()) - - @property - def client_relations(self) -> list[Relation]: - """Return the list of established client relations.""" - return self.model.relations.get("database", []) - - @property - def relations_user_databases_map(self) -> dict: - """Returns a user->databases map for all relations.""" - # Copy relations users directly instead of waiting for them to be created - user_database_map = self._collect_user_relations() - - if not self.is_cluster_initialised or not self.patroni_manager.member_started: - user_database_map.update({ - USER: "all", - REPLICATION_USER: "all", - REWIND_USER: "all", - }) - return user_database_map - hosts = ( - [True, False] - if self.is_connectivity_enabled and self.primary_endpoint - else [self.is_connectivity_enabled] - ) - for current_host in hosts: - try: - for user in self.postgresql.list_users(current_host=current_host): - if user in ( - "backup", - "monitoring", - "operator", - "postgres", - "replication", - "rewind", - "charmed_databases_owner", - ): - continue - if databases := ",".join( - sorted( - self.postgresql.list_accessible_databases_for_user( - user, current_host=current_host - ) - ) - ): - user_database_map[user] = databases - else: - logger.debug(f"User {user} has no databases to connect to") - # Add "landscape" superuser by default to the list when the "db-admin" relation is present. - if any( - True for relation in self.client_relations if relation.name == "db-admin" - ): - user_database_map["landscape"] = "all" - if self.postgresql.list_access_groups(current_host=current_host) != set( - ACCESS_GROUPS - ): - user_database_map.update({ - USER: "all", - REPLICATION_USER: "all", - REWIND_USER: "all", - }) - return user_database_map - except PostgreSQLBaseError as e: - logger.debug(f"Failing to get users with {e}") - continue - logger.debug("relations_user_databases_map: Unable to get users") - user_database_map.update({ - USER: "all", - REPLICATION_USER: "all", - REWIND_USER: "all", - }) - return user_database_map - - def _collect_user_relations(self) -> dict[str, str]: - return self.database_manager.collect_user_relations() - - def get_plugins(self) -> list[str]: - """Return a list of installed plugins.""" - plugins = [ - "_".join(plugin.split("_")[1:-1]) - for plugin in self.config.plugin_keys() - if self.config[plugin] - ] - plugins = [PLUGIN_OVERRIDES.get(plugin, plugin) for plugin in plugins] - if "spi" in plugins: - plugins.remove("spi") - for ext in SPI_MODULE: - plugins.append(ext) - return plugins - if __name__ == "__main__": + from ops.main import main + main(PostgresqlOperatorCharm) diff --git a/src/cluster_topology_observer.py b/src/cluster_topology_observer.py index ac089010c20..da2cd4678d5 100644 --- a/src/cluster_topology_observer.py +++ b/src/cluster_topology_observer.py @@ -15,6 +15,7 @@ from jinja2 import Template from ops import ActiveStatus, CharmEvents, EventBase, EventSource, Object from single_kernel_postgresql.config.enums import Substrates + from single_kernel_postgresql.utils import render_file if TYPE_CHECKING: diff --git a/tests/charms/postgresql_vm_test_charm/actions.yaml b/tests/charms/postgresql_vm_test_charm/actions.yaml new file mode 100644 index 00000000000..33e2f8be4df --- /dev/null +++ b/tests/charms/postgresql_vm_test_charm/actions.yaml @@ -0,0 +1,108 @@ +# Copyright 2021 Canonical Ltd. +# See LICENSE file for licensing details. + +create-backup: + description: Creates a backup to s3 storage. + params: + type: + type: string + description: The backup type, the default value is 'full'. + Full backup is a full copy of all data. + Differential backup is a copy only of changed data since the last full backup. + Incremental backup is a copy only of changed data since the last backup (any type). + Possible values - full, differential, incremental. +create-replication: + description: Set up asynchronous replication between two clusters. + params: + name: + type: string + description: The name of the replication (defaults to 'default'). + default: default +get-primary: + description: Get the unit which is the primary/leader in the replication. +list-backups: + description: Lists backups in s3 storage. +pre-refresh-check: + description: Check if charm is ready to refresh + additionalProperties: false +force-refresh-start: + description: | + Potential of data loss and downtime + + Force refresh of first unit + + Must run with at least one of the parameters `=false` + params: + check-compatibility: + type: boolean + default: true + description: | + Potential of data loss and downtime + + If `false`, force refresh if new version of PostgreSQL and/or charm is not compatible with previous version + run-pre-refresh-checks: + type: boolean + default: true + description: | + Potential of data loss and downtime + + If `false`, force refresh if app is unhealthy or not ready to refresh (and unit status shows "Pre-refresh check failed") + check-workload-container: + type: boolean + default: true + description: | + Potential of data loss and downtime during and after refresh + + If `false`, allow refresh to PostgreSQL container version that has not been validated to work with the charm revision + additionalProperties: false +resume-refresh: + description: | + Refresh next unit(s) (after you have manually verified that refreshed units are healthy) + + If the `pause-after-unit-refresh` config is set to `all`, this action will refresh the next unit. + + If `pause-after-unit-refresh` is set to `first`, this action will refresh all remaining units. + Exception: if automatic health checks fail after a unit has refreshed, the refresh will pause. + + If `pause-after-unit-refresh` is set to `none`, this action will have no effect unless it is called with `check-health-of-refreshed-units` as `false`. + params: + check-health-of-refreshed-units: + type: boolean + default: true + description: | + Potential of data loss and downtime + + If `false`, force refresh (of next unit) if 1 or more refreshed units are unhealthy + + Warning: if first unit to refresh is unhealthy, consider running `force-refresh-start` action on that unit instead of using this parameter. + If first unit to refresh is unhealthy because compatibility checks, pre-refresh checks, or workload container checks are failing, this parameter is more destructive than the `force-refresh-start` action. + additionalProperties: false +promote-to-primary: + description: Promotes the cluster of choice to a primary cluster. Must be ran against the leader unit when promoting a cluster + or against the unit to be promoted within the cluster. + params: + scope: + type: string + description: Whether to promote a unit or a cluster. Must be set to either unit or cluster. + force: + type: boolean + description: Force the promotion of a cluster when there is already a primary cluster. +restore: + description: Restore a database backup using pgBackRest. + S3 credentials are retrieved from a relation with the S3 integrator charm. + params: + backup-id: + type: string + description: A backup-id to identify the backup to restore (format = %Y-%m-%dT%H:%M:%SZ) + restore-to-time: + type: string + description: Point-in-time-recovery target in PSQL format. +status-detail: + description: Gets statuses of the charm + params: + recompute: + type: boolean + default: false + description: a boolean indicating whether a unit should recompute all statuses. - continue upgrade if 1+ upgraded units have non-active status +======= + description: a boolean indicating whether a unit should recompute all statuses. diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 424d0b8562a..868734e9a53 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -1,66 +1,43 @@ # Copyright 2021 Canonical Ltd. # See LICENSE file for licensing details. -import contextlib -import itertools -import json -import logging -import os -import subprocess -from datetime import UTC, datetime -from unittest.mock import MagicMock, Mock, PropertyMock, call, patch, sentinel - # from ops.framework import EventBase -import charm_refresh -import psycopg2 +import pathlib +from unittest.mock import patch + import pytest -from charmlibs import snap -from ops import ( - ActiveStatus, - BlockedStatus, - ErrorStatus, - JujuVersion, - MaintenanceStatus, - ModelError, - RelationDataTypeError, - RelationEvent, - Unit, - UnknownStatus, - WaitingStatus, -) from ops.testing import Harness -from psycopg2 import OperationalError -from single_kernel_postgresql.config.exceptions import ( - NotReadyError, - RemoveRaftMemberFailedError, - SwitchoverFailedError, - SwitchoverNotSyncError, -) -from single_kernel_postgresql.config.literals import PEER_RELATION, SECRET_INTERNAL_LABEL -from single_kernel_postgresql.utils.backup import CANNOT_RESTORE_PITR -from single_kernel_postgresql.utils.postgresql import ( - PostgreSQLCreateUserError, - PostgreSQLEnableDisableExtensionError, -) -from tenacity import RetryError, wait_fixed +from single_kernel_postgresql.config.literals import PEER_RELATION -from charm import ( - EXTENSIONS_DEPENDENCY_MESSAGE, - PostgresqlOperatorCharm, - StorageUnavailableError, -) -from constants import ( - POSTGRESQL_DATA_DIR, - UPDATE_CERTS_BIN_PATH, -) +from charm import PostgresqlOperatorCharm CREATE_CLUSTER_CONF_PATH = "/etc/postgresql-common/createcluster.d/pgcharm.conf" # used for assert functions +@pytest.fixture(autouse=True) +def _no_render(tmp_path): + """Point the Patroni config render at nothing for the unit suite. + + The real render needs the snap's live paths. + """ + with ( + patch("single_kernel_postgresql.managers.config.render_file"), + patch( + "single_kernel_postgresql.workload.vm.VMWorkload.is_patroni_running", + return_value=False, + ), + ): + yield + + @pytest.fixture(autouse=True) def harness(): - harness = Harness(PostgresqlOperatorCharm) + harness = Harness( + PostgresqlOperatorCharm, + meta=pathlib.Path("metadata.yaml").read_text(), + actions=pathlib.Path("actions.yaml").read_text(), + ) harness.begin() harness.add_relation(PEER_RELATION, harness.charm.app.name) harness.add_relation("restart", harness.charm.app.name) @@ -94,2710 +71,3 @@ def test_config_fallback(harness): harness.update_config({"connection-authentication-timeout": 120}) del harness.charm.config assert charm.config.connection_authentication_timeout == 120 - - -def test_on_install(harness): - with ( - patch("charm.snap.SnapCache") as _snap_cache, - patch( - "single_kernel_postgresql.workload.vm.VMWorkload.install_snap_package" - ) as _install_snap_package, - patch("charm.PostgresqlOperatorCharm._check_detached_storage"), - patch( - "charm.PostgresqlOperatorCharm._is_storage_attached", - side_effect=[False, True, True], - ) as _is_storage_attached, - ): - pg_snap = _snap_cache.return_value[charm_refresh.snap_name()] - - # Test without adding Patroni resource. - harness.charm.on.install.emit() - # Assert that the needed calls were made. - _install_snap_package.assert_called_once_with(revision=None) - assert pg_snap.alias.call_count == 2 - pg_snap.alias.assert_any_call("psql") - pg_snap.alias.assert_any_call("patronictl") - - # Assert the status set by the event handler. - assert isinstance(harness.model.unit.status, WaitingStatus) - - -def test_on_storage_detaching(harness): - with ( - patch("charm.snap.SnapCache") as _snap_cache, - patch("charm.ObserverManager.stop_observer") as _stop_observer, - patch("charm.RotateLogs.stop_log_rotation") as _stop_log_rotation, - patch.object(harness.charm.app, "planned_units") as _planned_units, - ): - _selected_snap = _snap_cache.return_value.__getitem__.return_value - - # Scale-down (units remain): the surviving cluster still needs this unit's - # Patroni reachable to remove it from raft, so the workload must NOT be - # stopped here. - _planned_units.return_value = 2 - storage_id = harness.add_storage("data", attach=True)[0] - harness.detach_storage(storage_id) - _stop_observer.assert_not_called() - _stop_log_rotation.assert_not_called() - _selected_snap.stop.assert_not_called() - - # Full teardown (no units remain): release the storage by stopping the - # background processes and disabling (so a mid-teardown restart can't - # re-grab the mount) all the snap services. - _planned_units.return_value = 0 - for storage_name in ("archive", "data", "logs", "temp"): - _stop_observer.reset_mock() - _stop_log_rotation.reset_mock() - _selected_snap.reset_mock() - - storage_id = harness.add_storage(storage_name, attach=True)[0] - harness.detach_storage(storage_id) - - _stop_observer.assert_called_once_with() - _stop_log_rotation.assert_called_once_with() - _selected_snap.stop.assert_called_once_with(disable=True) - - -def test_on_remove(harness): - with ( - patch("charm.snap.SnapCache") as _snap_cache, - patch("charm.subprocess.run") as _run, - patch.object(harness.charm.app, "planned_units") as _planned_units, - ): - _selected_snap = _snap_cache.return_value.__getitem__.return_value - # Report every storage as mounted. - _run.return_value.returncode = 0 - - # Full teardown (no units remain): unmount the storages and remove the snap. - _planned_units.return_value = 0 - harness.charm.on.remove.emit() - for storage in harness.charm.meta.storages.values(): - _run.assert_any_call( - ["/usr/bin/umount", storage.location], - check=True, - capture_output=True, - text=True, - ) - _selected_snap.ensure.assert_called_once_with(snap.SnapState.Absent) - - -def test_patroni_scrape_config(harness): - result = harness.charm.patroni_scrape_config() - - assert result == [ - { - "metrics_path": "/metrics", - "scheme": "https", - "static_configs": [{"targets": ["192.0.2.0:8008"]}], - "tls_config": {"insecure_skip_verify": True}, - }, - ] - - -def test_primary_endpoint(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._units_ips", - new_callable=PropertyMock, - return_value={"1.1.1.1", "1.1.1.2"}, - ), - patch( - "charm.PostgresqlOperatorCharm._peers", new_callable=PropertyMock, return_value=True - ), - patch("charm.PatroniManager.get_member_ip", return_value="1.1.1.1") as _get_member_ip, - patch("charm.PatroniManager.get_primary", return_value=sentinel.primary) as _get_primary, - ): - assert harness.charm.primary_endpoint == "1.1.1.1" - - _get_member_ip.assert_called_once_with(sentinel.primary) - _get_primary.assert_called_once_with() - - -def test_primary_endpoint_no_peers(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._peers", new_callable=PropertyMock, return_value=None - ), - patch( - "charm.PostgresqlOperatorCharm._units_ips", - new_callable=PropertyMock, - return_value={"1.1.1.1", "1.1.1.2"}, - ), - patch("charm.PatroniManager.get_member_ip", return_value="1.1.1.1") as _get_member_ip, - patch("charm.PatroniManager.get_primary", return_value=sentinel.primary) as _get_primary, - # patch("charm.PostgresqlOperatorCharm._patroni", new_callable=PropertyMock) as _patroni, - ): - assert harness.charm.primary_endpoint is None - - assert not _get_member_ip.called - assert not _get_primary.called - - -def test_on_leader_elected(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints", new_callable=PropertyMock - ) as _update_relation_endpoints, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - ) as _primary_endpoint, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PostgresqlOperatorCharm._reconfigure_cluster"), - patch("charm.TLSManager.generate_internal_peer_cert"), - ): - # Assert that there is no password in the peer relation. - assert harness.charm._peers.data[harness.charm.app].get("operator-password", None) is None - - # Check that a new password was generated on leader election. - _primary_endpoint.return_value = "1.1.1.1" - harness.set_leader() - password = harness.charm.get_secret("app", "operator-password") - _update_config.assert_called_once() - _update_relation_endpoints.assert_not_called() - assert password is not None - - # Mark the cluster as initialised. - harness.charm._peers.data[harness.charm.app].update({"cluster_initialised": "True"}) - - # Trigger a new leader election and check that the password is still the same - # and also that update_endpoints was called after the cluster was initialised. - harness.set_leader(False) - harness.set_leader() - assert harness.charm.get_secret("app", "operator-password") == password - _update_relation_endpoints.assert_called_once() - assert not (isinstance(harness.model.unit.status, BlockedStatus)) - - # Check for a WaitingStatus when the primary is not reachable yet. - _primary_endpoint.return_value = None - harness.set_leader(False) - harness.set_leader() - _update_relation_endpoints.assert_called_once() # Assert it was not called again. - assert isinstance(harness.model.unit.status, WaitingStatus) - - -def test_is_cluster_initialised(harness): - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test when the cluster was not initialised yet. - assert not (harness.charm.is_cluster_initialised) - - # Test when the cluster was already initialised. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.app.name, {"cluster_initialised": "True"} - ) - assert harness.charm.is_cluster_initialised - - -def test_on_config_changed(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._update_member_ip", return_value=False - ) as _update_member_ip, - patch( - "charm.PostgresqlOperatorCharm._validate_config_options" - ) as _validate_config_options, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch( - "charm.PostgresqlOperatorCharm.enable_disable_extensions" - ) as _enable_disable_extensions, - patch( - "charm.PostgresqlOperatorCharm.is_cluster_initialised", new_callable=PropertyMock - ) as _is_cluster_initialised, - patch("charm.PostgresqlOperatorCharm.update_endpoint_addresses"), - patch( - "single_kernel_postgresql.events.logical_replication." - "PostgreSQLLogicalReplication.apply_changed_config", - return_value=True, - ), - ): - # Test when the cluster was not initialised yet. - _is_cluster_initialised.return_value = False - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_not_called() - - # Test when the unit is not the leader. - _is_cluster_initialised.return_value = True - harness.charm.on.config_changed.emit() - _validate_config_options.assert_called_once() - _enable_disable_extensions.assert_not_called() - - # Test unable to connect to db - _update_config.reset_mock() - _validate_config_options.side_effect = OperationalError - harness.charm.on.config_changed.emit() - assert not _update_config.called - _validate_config_options.side_effect = None - - # Test after the cluster was initialised. - with harness.hooks_disabled(): - harness.set_leader() - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_called_once() - - # Test when the unit is in a blocked state due to extensions request, - # but there are no established legacy relations. - _enable_disable_extensions.reset_mock() - harness.charm.unit.status = BlockedStatus( - "extensions requested through relation, enable them through config options" - ) - harness.charm.on.config_changed.emit() - _enable_disable_extensions.assert_called_once() - - # Test when there is an error related to the config options. - _update_member_ip.reset_mock() - _enable_disable_extensions.reset_mock() - harness.charm.unit.status = BlockedStatus("Configuration Error") - harness.charm.on.config_changed.emit() - assert isinstance(harness.model.unit.status, ActiveStatus) - _update_member_ip.assert_called_once() - _enable_disable_extensions.assert_called_once() - - # Test when the unit has updated its member IP. - _update_member_ip.reset_mock() - _enable_disable_extensions.reset_mock() - _update_member_ip.return_value = True - harness.charm.on.config_changed.emit() - _update_member_ip.assert_called_once() - _enable_disable_extensions.assert_not_called() - - -def test_check_extension_dependencies(harness): - with ( - patch("charm.PatroniManager.get_primary") as _get_primary, - patch("subprocess.check_output", return_value=b"C"), - patch.object(PostgresqlOperatorCharm, "postgresql", Mock()), - patch("charm.PostgresqlOperatorCharm.update_endpoint_addresses"), - ): - _get_primary.return_value = harness.charm.unit - - # Test when plugins dependencies exception is not caused - config = { - "plugin_address_standardizer_enable": False, - "plugin_postgis_enable": False, - "plugin_address_standardizer_data_us_enable": False, - "plugin_jsonb_plperl_enable": False, - "plugin_plperl_enable": False, - "plugin_postgis_raster_enable": False, - "plugin_postgis_tiger_geocoder_enable": False, - "plugin_fuzzystrmatch_enable": False, - "plugin_postgis_topology_enable": False, - } - harness.update_config(config) - harness.charm.enable_disable_extensions() - assert not (isinstance(harness.model.unit.status, BlockedStatus)) - - # Test when plugins dependencies exception caused - config["plugin_address_standardizer_enable"] = True - harness.update_config(config) - del harness.charm.config - harness.charm.enable_disable_extensions() - assert isinstance(harness.model.unit.status, BlockedStatus) - assert harness.model.unit.status.message == EXTENSIONS_DEPENDENCY_MESSAGE - - -def test_pg_cron_config(harness): - assert harness.charm.config.plugin_pg_cron_enable is False - with ( - harness.hooks_disabled(), - patch.object(harness.charm, "patroni_manager"), - patch.object(harness.charm, "postgresql") as postgresql, - ): - for enabled in (True, False): - harness.update_config({"plugin-pg-cron-enable": enabled}) - del harness.charm.config - harness.charm.enable_disable_extensions() - assert postgresql.enable_disable_extensions.call_args.args[0]["pg_cron"] is enabled - assert ("pg_cron" in harness.charm.get_plugins()) is enabled - - -def test_enable_disable_extensions(harness, caplog): - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", new_callable=PropertyMock - ) as _get_primary, - patch("single_kernel_postgresql.core.state.CharmState.unit_ip"), - patch.object(harness.charm, "patroni_manager"), - patch("subprocess.check_output", return_value=b"C"), - patch.object(harness.charm, "postgresql") as postgresql_mock, - ): - _get_primary.return_value = harness.charm.unit - - # Test when all extensions install/uninstall succeed. - postgresql_mock.enable_disable_extension.side_effect = None - with caplog.at_level(logging.ERROR): - assert len(caplog.records) == 0 - harness.charm.enable_disable_extensions() - assert postgresql_mock.enable_disable_extensions.call_count == 1 - - # Test when one extension install/uninstall fails. - postgresql_mock.reset_mock() - postgresql_mock.enable_disable_extensions.side_effect = ( - PostgreSQLEnableDisableExtensionError - ) - with caplog.at_level(logging.ERROR): - harness.charm.enable_disable_extensions() - assert postgresql_mock.enable_disable_extensions.call_count == 1 - assert "failed to change plugins: " in caplog.text - - # Test when one config option should be skipped (because it's not related - # to a plugin/extension). - postgresql_mock.reset_mock() - postgresql_mock.enable_disable_extensions.side_effect = None - with caplog.at_level(logging.ERROR): - # Block if extension-dependent object error is raised - postgresql_mock.reset_mock() - postgresql_mock.enable_disable_extensions.side_effect = [ - psycopg2.errors.DependentObjectsStillExist, - None, - ] - harness.charm.enable_disable_extensions() - assert isinstance(harness.charm.unit.status, BlockedStatus) - # Should resolve afterwards - harness.charm.enable_disable_extensions() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - -@pytest.mark.parametrize("unsettable_status", [ErrorStatus(), UnknownStatus()]) -def test_enable_disable_extensions_does_not_restore_unsettable_status(harness, unsettable_status): - # enable_disable_extensions caches the current unit status and restores it - # afterwards. The getter can return statuses the backend rejects on set - # (e.g. "error" or "unknown") left over from a previously failed hook; - # restoring such a cached status must not be attempted, otherwise the backend - # raises InvalidStatusError/ModelError and deadlocks the unit. - with ( - patch("charm.PatroniManager.get_primary") as _get_primary, - patch("single_kernel_postgresql.core.state.CharmState.unit_ip"), - patch("subprocess.check_output", return_value=b"C"), - patch.object(harness.charm, "postgresql", Mock()) as postgresql_mock, - ): - _get_primary.return_value = harness.charm.unit - postgresql_mock.enable_disable_extensions.side_effect = None - - # Cache an unsettable status that would otherwise be restored verbatim. - harness.charm.unit._status = unsettable_status - - # Must not raise when the cached status is restored at the end. - harness.charm.enable_disable_extensions() - - # The unsettable cached status was skipped, not restored. - assert not isinstance(harness.charm.unit.status, ErrorStatus | UnknownStatus) - - -def test_on_start_no_password(harness): - """Test start is deferred when passwords are not yet generated.""" - with ( - patch("charm.PostgresqlOperatorCharm._check_detached_storage"), - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - ): - _get_password.return_value = None - harness.charm.on.start.emit() - assert isinstance(harness.model.unit.status, WaitingStatus) - - # ModelError when fetching the password has the same outcome. - _get_password.side_effect = ModelError - harness.charm.on.start.emit() - assert isinstance(harness.model.unit.status, WaitingStatus) - - -def test_on_start_bootstrap_failure(harness): - """Test start is blocked when cluster bootstrap fails.""" - with ( - patch( - "charm.PostgresqlOperatorCharm._restart_services_after_reboot" - ) as _restart_services_after_reboot, - patch("charm.PatroniManager.get_primary", return_value=sentinel.primary), - patch( - "single_kernel_postgresql.core.peer_relation.PostgreSQLApplication.replication_password", - new_callable=PropertyMock, - ) as _replication_password, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch("charm.PostgresqlOperatorCharm._ensure_storage_layout"), - patch("charm.PostgresqlOperatorCharm._check_detached_storage"), - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch("charm.TLSManager.generate_internal_peer_cert"), - patch("charm.PatroniManager.bootstrap_cluster") as _bootstrap_cluster, - patch("charm.PostgresqlOperatorCharm.update_config"), - patch("charm.start_raft_observer"), - ): - _get_password.return_value = "fake-operator-password" - _replication_password.return_value = "fake-replication-password" - _bootstrap_cluster.return_value = False - - # TODO: test replicas start (DPE-494). - with harness.hooks_disabled(): - harness.set_leader() - harness.charm.on.start.emit() - _bootstrap_cluster.assert_called_once() - _restart_services_after_reboot.assert_called_once() - assert isinstance(harness.model.unit.status, BlockedStatus) - - -def test_on_start_create_user_error(harness): - """Test start is blocked when creating the default postgres user fails.""" - with ( - patch( - "charm.PostgresqlOperatorCharm._restart_services_after_reboot" - ) as _restart_services_after_reboot, - patch( - "charm.PostgresqlOperatorCharm._replication_password", new_callable=PropertyMock - ) as _replication_password, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch("charm.PostgresqlOperatorCharm._ensure_storage_layout"), - patch("charm.PostgresqlOperatorCharm._check_detached_storage"), - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch("charm.TLSManager.generate_internal_peer_cert"), - patch("charm.PatroniManager.bootstrap_cluster") as _bootstrap_cluster, - patch( - "charm.PatroniManager.member_started", - new_callable=PropertyMock, - ) as _member_started, - patch( - "charm.PostgresqlOperatorCharm._can_connect_to_postgresql", - new_callable=PropertyMock, - return_value=True, - ), - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value=True, - ), - patch.object(harness.charm, "postgresql") as _postgresql, - patch("charm.PostgresqlOperatorCharm.update_config"), - patch("charm.start_raft_observer"), - ): - _get_password.return_value = "fake-operator-password" - _replication_password.return_value = "fake-replication-password" - _bootstrap_cluster.return_value = True - _member_started.return_value = True - _postgresql.list_users.return_value = [] - _postgresql.create_user.side_effect = PostgreSQLCreateUserError - - with harness.hooks_disabled(): - harness.set_leader() - harness.charm.on.start.emit() - _postgresql.create_user.assert_called_once() - _restart_services_after_reboot.assert_called_once() - assert isinstance(harness.model.unit.status, BlockedStatus) - - -def test_on_start_success(harness): - """Test successful cluster bootstrapping on the primary unit.""" - with ( - patch( - "charm.PostgresqlOperatorCharm._restart_services_after_reboot" - ) as _restart_services_after_reboot, - patch( - "charm.PostgresqlOperatorCharm._set_primary_status_message" - ) as _set_primary_status_message, - patch( - "charm.PostgresqlOperatorCharm.enable_disable_extensions" - ) as _enable_disable_extensions, - patch("charm.PostgresqlOperatorCharm._update_relation_endpoints"), - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.oversee_users" - ) as _oversee_users, - patch( - "charm.PostgresqlOperatorCharm._replication_password", - new_callable=PropertyMock, - ) as _replication_password, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch("charm.PostgresqlOperatorCharm._ensure_storage_layout"), - patch("charm.PostgresqlOperatorCharm._check_detached_storage"), - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch("charm.TLSManager.generate_internal_peer_cert"), - patch("charm.PatroniManager.bootstrap_cluster") as _bootstrap_cluster, - patch( - "charm.PatroniManager.member_started", - new_callable=PropertyMock, - ) as _member_started, - patch( - "charm.PostgresqlOperatorCharm._can_connect_to_postgresql", - new_callable=PropertyMock, - return_value=True, - ), - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value=True, - ), - patch.object(harness.charm, "postgresql") as _postgresql, - patch("charm.PostgresqlOperatorCharm.update_config"), - patch("charm.start_raft_observer"), - ): - _get_password.return_value = "fake-operator-password" - _replication_password.return_value = "fake-replication-password" - _bootstrap_cluster.return_value = True - _member_started.return_value = True - _postgresql.list_users.return_value = [] - - with harness.hooks_disabled(): - harness.set_leader() - harness.charm.on.start.emit() - assert _postgresql.create_user.call_count == 2 # backup user + monitoring user - _oversee_users.assert_called_once() - _enable_disable_extensions.assert_called_once() - _set_primary_status_message.assert_called_once() - _restart_services_after_reboot.assert_called_once() - - -def test_setup_users_skips_writes_on_standby_cluster(harness): - """A standby cluster is read-only, so _setup_users must not issue write DDL (DPE-10284).""" - with ( - patch( - "charm.PostgresqlOperatorCharm.is_standby_cluster", - new_callable=PropertyMock, - return_value=True, - ), - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.oversee_users" - ) as _oversee_users, - patch("charm.PostgresqlOperatorCharm.postgresql") as _postgresql, - ): - # Even though this would raise on a read-only standby, the guard must - # short-circuit before create_predefined_instance_roles is ever called. - _postgresql.create_predefined_instance_roles.side_effect = ( - psycopg2.errors.ReadOnlySqlTransaction - ) - - harness.charm._setup_users() - - _postgresql.create_predefined_instance_roles.assert_not_called() - _postgresql.create_user.assert_not_called() - _postgresql.set_up_database.assert_not_called() - _oversee_users.assert_not_called() - - -def test_setup_users_runs_on_primary_cluster(harness): - """On a primary cluster _setup_users provisions the predefined roles and users.""" - with ( - patch( - "charm.PostgresqlOperatorCharm.is_standby_cluster", - new_callable=PropertyMock, - return_value=False, - ), - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.oversee_users" - ) as _oversee_users, - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch.object(harness.charm, "postgresql") as _postgresql, - ): - _postgresql.list_users.return_value = [] - _postgresql.list_access_groups.return_value = set() - - harness.charm._setup_users() - - _postgresql.create_predefined_instance_roles.assert_called_once() - assert _postgresql.create_user.call_count == 2 # backup user + monitoring user - _oversee_users.assert_called_once() - - -def test_is_standby_cluster(harness): - """is_standby_cluster is relation-based and independent of the Patroni API.""" - with ( - patch("charm.PostgreSQLAsyncReplication.is_primary_cluster") as _is_primary_cluster, - patch.object(harness.charm.model, "get_relation") as _get_relation, - ): - # No replication relation at all -> not a standby cluster. - _get_relation.return_value = None - assert harness.charm.is_standby_cluster is False - - # Replication relation present, and this app is the primary cluster. - _get_relation.return_value = Mock() - _is_primary_cluster.return_value = True - assert harness.charm.is_standby_cluster is False - - # Replication relation present, and this app is NOT the primary cluster -> standby. - _is_primary_cluster.return_value = False - assert harness.charm.is_standby_cluster is True - - -def test_on_start_replica(harness): - with ( - patch("charm.snap.SnapCache") as _snap_cache, - patch( - "single_kernel_postgresql.workload.base.BaseWorkload.get_postgresql_version" - ) as _get_postgresql_version, - patch( - "charm.PostgresqlOperatorCharm._restart_services_after_reboot" - ) as _restart_services_after_reboot, - patch("charm.PatroniManager.configure_patroni_on_unit") as _configure_patroni_on_unit, - patch( - "charm.PatroniManager.member_started", - new_callable=PropertyMock, - ) as _member_started, - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints", new_callable=PropertyMock - ) as _update_relation_endpoints, - patch("ops.framework.EventBase.defer") as _defer, - patch("charm.PostgresqlOperatorCharm._replication_password") as _replication_password, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch("charm.PostgresqlOperatorCharm._ensure_storage_layout"), - patch( - "charm.PostgresqlOperatorCharm._is_storage_attached", - return_value=True, - ) as _is_storage_attached, - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch("charm.TLSManager.generate_internal_peer_cert"), - patch("charm.start_raft_observer"), - ): - _get_postgresql_version.return_value = "16.6" - - # Set the current unit to be a replica (non leader unit). - harness.set_leader(False) - - # Mock the passwords. - _get_password.return_value = "fake-operator-password" - _replication_password.return_value = "fake-replication-password" - - # Test an uninitialized cluster. - harness.charm._peers.data[harness.charm.app].update({"cluster_initialised": ""}) - harness.charm.on.start.emit() - _defer.assert_called_once() - _restart_services_after_reboot.assert_called_once() - - # Set an initial waiting status again (like after a machine restart). - harness.model.unit.status = WaitingStatus("fake message") - - # Mark the cluster as initialised and with the workload up and running. - _restart_services_after_reboot.reset_mock() - harness.charm._peers.data[harness.charm.app].update({"cluster_initialised": "True"}) - _member_started.return_value = True - harness.charm.on.start.emit() - _configure_patroni_on_unit.assert_not_called() - _restart_services_after_reboot.assert_called_once() - assert isinstance(harness.model.unit.status, ActiveStatus) - - # Set an initial waiting status (like after the install hook was triggered). - harness.model.unit.status = WaitingStatus("fake message") - - # Check that the unit status doesn't change when the workload is not running. - # In that situation only Patroni is configured in the unit (but not started). - _restart_services_after_reboot.reset_mock() - _member_started.return_value = False - harness.charm.on.start.emit() - _configure_patroni_on_unit.assert_called_once() - _restart_services_after_reboot.assert_called_once() - assert isinstance(harness.model.unit.status, WaitingStatus) - - -def test_on_start_no_patroni_member(harness): - with ( - patch("subprocess.check_output", return_value=b"C"), - patch("charm.snap.SnapCache") as _snap_cache, - patch( - "single_kernel_postgresql.workload.base.BaseWorkload.get_postgresql_version" - ) as _get_postgresql_version, - patch.object(harness.charm, "postgresql") as _postgresql, - patch.object(harness.charm, "patroni_manager") as _patroni_manager, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch("charm.PostgresqlOperatorCharm._ensure_storage_layout"), - patch( - "charm.PostgresqlOperatorCharm._is_storage_attached", return_value=True - ) as _is_storage_attached, - patch("charm.PostgresqlOperatorCharm.get_secret"), - patch("charm.TLSManager.generate_internal_peer_cert"), - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.get_username_mapping", - return_value={}, - ), - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.get_databases_prefix_mapping", - return_value={}, - ), - patch("charm.start_raft_observer"), - ): - # Mock the passwords. - _get_postgresql_version.return_value = "16.6" - _patroni_manager.member_started = False - _get_password.return_value = "fake-operator-password" - bootstrap_cluster = _patroni_manager.bootstrap_cluster - bootstrap_cluster.return_value = True - - with harness.hooks_disabled(): - harness.set_leader() - harness.charm.on.start.emit() - bootstrap_cluster.assert_called_once() - _postgresql.create_user.assert_not_called() - assert isinstance(harness.model.unit.status, WaitingStatus) - assert harness.model.unit.status.message == "awaiting for member to start" - - -@pytest.mark.parametrize( - "scope,raft_flag", - [ - ("unit", "raft_stuck"), - ("unit", "raft_candidate"), - ("unit", "raft_stopped"), - ("unit", "raft_primary"), - ("app", "raft_selected_candidate"), - ("app", "raft_followers_stopped"), - ("app", "raft_reset_primary"), - ("app", "raft_rejoin"), - ], -) -def test_on_start_after_blocked_state(harness, scope, raft_flag): - with ( - patch("charm.PatroniManager.bootstrap_cluster") as _bootstrap_cluster, - patch("charm.PatroniManager.start_patroni") as _start_patroni, - patch("charm.PostgresqlOperatorCharm._replication_password") as _replication_password, - patch("charm.PostgresqlOperatorCharm._get_password") as _get_password, - patch( - "charm.PostgresqlOperatorCharm._is_storage_attached", return_value=True - ) as _is_storage_attached, - ): - # Set an initial blocked status (like after losing the raft quorum before a reboot). - initial_status = BlockedStatus("Raft majority loss, run: promote-to-primary") - harness.model.unit.status = initial_status - rel_id = harness.model.get_relation(PEER_RELATION).id - entity = harness.charm.unit.name if scope == "unit" else harness.charm.app.name - with harness.hooks_disabled(): - harness.update_relation_data(rel_id, entity, {raft_flag: "True"}) - - # Patroni must not be started while the raft recovery is pending. - harness.charm.on.start.emit() - _get_password.assert_not_called() - _replication_password.assert_not_called() - _bootstrap_cluster.assert_not_called() - _start_patroni.assert_not_called() - # Assert the status didn't change. - assert harness.model.unit.status == initial_status - - -def test_ensure_storage_layout(harness, tmp_path): - """_ensure_storage_layout creates TEMP_DATA_DIR and chowns the versioned parents. - - Data migration between storage roots and versioned paths is handled - by the snap hooks. The charm ensures TEMP_DATA_DIR exists (it may live - on a tmpfs mount that is wiped on reboot) and makes the 16/ parent of - both the temp and the data dirs _daemon_-owned, so Patroni can remove - the versioned subdirectory (e.g. during patronictl reinit). - """ - temp_root = tmp_path / "temp" / "16" / "main" - data_root = tmp_path / "data" / "16" / "main" - # The snap's migrate-data.sh creates the data parent (as root); the - # charm only fixes its ownership, it does not create the data dir. - data_root.parent.mkdir(parents=True) - with ( - patch("charm.TEMP_DATA_DIR", str(temp_root)), - patch("charm.POSTGRESQL_DATA_DIR", str(data_root)), - patch("charm.shutil") as mock_shutil, - ): - harness.charm._ensure_storage_layout() - assert temp_root.is_dir() - # The charm does not create the data dir leaf — only the snap does. - assert not data_root.exists() - # Both versioned 16/ parents (and the temp leaf) are chowned to _daemon_. - chowned = {call.args[0] for call in mock_shutil.chown.call_args_list} - assert temp_root in chowned - assert temp_root.parent in chowned - assert data_root.parent in chowned - assert not (tmp_path / "archive").exists() - assert not (tmp_path / "logs").exists() - - -def test_migrate_temp_tablespace_location_skips_when_no_endpoint(harness): - """If primary_endpoint is not yet set, the migration is skipped.""" - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value=None, - ), - ): - result = harness.charm.refresh_manager.migrate_temp_tablespace_location() - - assert result is True - - -def test_migrate_temp_tablespace_location_migrates_from_old_path(harness, tmp_path): - """When temp tablespace is at the old storage root, it is migrated to the versioned dir.""" - temp_data_dir = tmp_path / "16" / "main" - temp_storage_path = str(temp_data_dir.parent) - temp_data_dir.mkdir(parents=True) - - connection = MagicMock() - cursor = MagicMock() - cursor.fetchone.return_value = (temp_storage_path,) # still at old path - connection.cursor.return_value = cursor - postgresql = MagicMock() - postgresql._connect_to_database.return_value = connection - workload = MagicMock() - workload.paths.temp = temp_data_dir - - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value="10.0.0.1", - ), - patch.object( - harness.charm.refresh_manager, "_resolve_primary_host", return_value="10.0.0.1" - ), - patch( - "charm.PostgresqlOperatorCharm.postgresql", - new_callable=PropertyMock, - return_value=postgresql, - ), - patch( - "charm.PostgresqlOperatorCharm.workload", - new_callable=PropertyMock, - return_value=workload, - ), - ): - assert harness.charm.refresh_manager.migrate_temp_tablespace_location() - - cursor.execute.assert_has_calls([ - call("SELECT pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname='temp';"), - call("DROP TABLESPACE temp;"), - call(f"CREATE TABLESPACE temp LOCATION '{temp_data_dir}';"), - call("GRANT CREATE ON TABLESPACE temp TO public;"), - ]) - - -def test_migrate_temp_tablespace_location_skips_when_already_at_versioned_path(harness, tmp_path): - """When temp tablespace is already at the versioned path, no migration is performed.""" - temp_data_dir = tmp_path / "16" / "main" - temp_data_dir.mkdir(parents=True) - - connection = MagicMock() - cursor = MagicMock() - cursor.fetchone.return_value = (str(temp_data_dir),) # already at versioned path - connection.cursor.return_value = cursor - postgresql = MagicMock() - postgresql._connect_to_database.return_value = connection - workload = MagicMock() - workload.paths.temp = temp_data_dir - - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value="10.0.0.1", - ), - patch.object( - harness.charm.refresh_manager, "_resolve_primary_host", return_value="10.0.0.1" - ), - patch( - "charm.PostgresqlOperatorCharm.postgresql", - new_callable=PropertyMock, - return_value=postgresql, - ), - patch( - "charm.PostgresqlOperatorCharm.workload", - new_callable=PropertyMock, - return_value=workload, - ), - ): - assert harness.charm.refresh_manager.migrate_temp_tablespace_location() - - # Only the SELECT should have been executed — no DROP/CREATE - cursor.execute.assert_called_once_with( - "SELECT pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname='temp';" - ) - - -def test_migrate_temp_tablespace_location_skips_when_tablespace_missing(harness, tmp_path): - """When the tablespace doesn't exist in pg_catalog, no migration is needed.""" - connection = MagicMock() - cursor = MagicMock() - cursor.fetchone.return_value = None # tablespace was never created or already dropped - connection.cursor.return_value = cursor - postgresql = MagicMock() - postgresql._connect_to_database.return_value = connection - workload = MagicMock() - workload.paths.temp = tmp_path / "16" / "main" - - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value="10.0.0.1", - ), - patch.object( - harness.charm.refresh_manager, "_resolve_primary_host", return_value="10.0.0.1" - ), - patch( - "charm.PostgresqlOperatorCharm.postgresql", - new_callable=PropertyMock, - return_value=postgresql, - ), - patch( - "charm.PostgresqlOperatorCharm.workload", - new_callable=PropertyMock, - return_value=workload, - ), - ): - assert harness.charm.refresh_manager.migrate_temp_tablespace_location() - - # Only the SELECT should have been executed - cursor.execute.assert_called_once_with( - "SELECT pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname='temp';" - ) - - -def test_migrate_temp_tablespace_location_skips_when_unexpected_location(harness, tmp_path): - """When the tablespace is at an unexpected location, migration is skipped with a warning.""" - connection = MagicMock() - cursor = MagicMock() - cursor.fetchone.return_value = ("/some/unexpected/path",) - connection.cursor.return_value = cursor - postgresql = MagicMock() - postgresql._connect_to_database.return_value = connection - workload = MagicMock() - workload.paths.temp = tmp_path / "16" / "main" - - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value="10.0.0.1", - ), - patch.object( - harness.charm.refresh_manager, "_resolve_primary_host", return_value="10.0.0.1" - ), - patch( - "charm.PostgresqlOperatorCharm.postgresql", - new_callable=PropertyMock, - return_value=postgresql, - ), - patch( - "charm.PostgresqlOperatorCharm.workload", - new_callable=PropertyMock, - return_value=workload, - ), - patch("single_kernel_postgresql.managers.refresh.logger") as logger, - ): - assert harness.charm.refresh_manager.migrate_temp_tablespace_location() - - cursor.execute.assert_called_once_with( - "SELECT pg_tablespace_location(oid) FROM pg_tablespace WHERE spcname='temp';" - ) - logger.warning.assert_called_once() - - -def test_migrate_temp_tablespace_location_returns_false_on_db_error(harness): - """When a psycopg2 error occurs, the method returns False.""" - postgresql = MagicMock() - postgresql._connect_to_database.side_effect = psycopg2.Error("connection failed") - - with ( - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value="10.0.0.1", - ), - patch.object( - harness.charm.refresh_manager, "_resolve_primary_host", return_value="10.0.0.1" - ), - patch( - "charm.PostgresqlOperatorCharm.postgresql", - new_callable=PropertyMock, - return_value=postgresql, - ), - ): - assert not harness.charm.refresh_manager.migrate_temp_tablespace_location() - - -def test_ensure_storage_layout_recreates_temp_dir_on_reboot(harness, tmp_path): - """TEMP_DATA_DIR is recreated after a tmpfs wipe on reboot.""" - temp_root = tmp_path / "temp" / "16" / "main" - with ( - patch("charm.TEMP_DATA_DIR", str(temp_root)), - patch("charm.shutil"), - ): - harness.charm._ensure_storage_layout() - assert temp_root.is_dir() - - -def test_on_update_status(harness): - with ( - patch("charm.ObserverManager.start_observer") as _start_observer, - patch( - "charm.PostgresqlOperatorCharm._set_primary_status_message" - ) as _set_primary_status_message, - patch("charm.PatroniManager.restart_patroni") as _restart_patroni, - patch("charm.PostgreSQLAsyncReplication.clear_stale_promotion") as _clear_stale_promotion, - patch("charm.PatroniManager.is_member_isolated") as _is_member_isolated, - patch("charm.PatroniManager.member_started", new_callable=PropertyMock) as _member_started, - patch( - "charm.PostgresqlOperatorCharm.is_standby_leader", new_callable=PropertyMock - ) as _is_standby_leader, - patch( - "charm.PostgresqlOperatorCharm.is_primary", new_callable=PropertyMock - ) as _is_primary, - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock(return_value=True), - ) as _primary_endpoint, - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.oversee_users" - ) as _oversee_users, - patch("charm.PatroniManager.last_postgresql_logs") as _last_postgresql_logs, - patch("charm.PatroniManager.patroni_logs") as _patroni_logs, - patch("charm.PatroniManager.get_member_status") as _get_member_status, - patch( - "charm.BackupManager.can_use_s3_repository", return_value=(True, None) - ) as _can_use_s3_repository, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.RestoreManager.log_pitr_last_transaction_time"), - patch("charm.PostgreSQL.drop_hba_triggers") as _drop_hba_triggers, - ): - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test before the cluster is initialised. - harness.charm.on.update_status.emit() - _set_primary_status_message.assert_not_called() - - # Test after the cluster was initialised, but with the unit in a blocked state. - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data( - rel_id, harness.charm.app.name, {"cluster_initialised": "True"} - ) - harness.charm.unit.status = BlockedStatus("fake blocked status") - harness.charm.on.update_status.emit() - _set_primary_status_message.assert_not_called() - - # Test the point-in-time-recovery fail. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - { - "cluster_initialised": "True", - "restoring-backup": "valid", - "restore-to-time": "valid", - }, - ) - harness.charm.unit.status = ActiveStatus() - _patroni_logs.return_value = "2022-02-24 02:00:00 UTC patroni.exceptions.PatroniFatalException: Failed to bootstrap cluster" - harness.charm.on.update_status.emit() - _set_primary_status_message.assert_not_called() - assert harness.charm.unit.status.message == CANNOT_RESTORE_PITR - - # Test with the unit in a status different that blocked. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"cluster_initialised": "True", "restoring-backup": "", "restore-to-time": ""}, - ) - harness.charm.unit.status = ActiveStatus() - harness.charm.on.update_status.emit() - # A stale promoted-cluster-counter from a dead-DC teardown is reconciled here too. - _clear_stale_promotion.assert_called_once_with() - _set_primary_status_message.assert_called_once() - # A stale promoted-cluster-counter from a dead-DC teardown is reconciled here too. - _clear_stale_promotion.assert_called_once_with() - - # Test call to restart when the member is isolated from the cluster. - _set_primary_status_message.reset_mock() - _start_observer.reset_mock() - _member_started.return_value = False - _is_member_isolated.return_value = True - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.unit.name, {"postgresql_restarted": ""} - ) - harness.charm.on.update_status.emit() - _restart_patroni.assert_called_once() - _start_observer.assert_called_once_with() - _drop_hba_triggers.assert_called_once_with() - - -def test_on_update_status_after_restore_operation(harness): - with ( - patch("charm.ObserverManager.start_observer"), - patch( - "charm.PostgresqlOperatorCharm._set_primary_status_message" - ) as _set_primary_status_message, - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", - new_callable=PropertyMock, - return_value=True, - ) as _primary_endpoint, - patch( - "single_kernel_postgresql.managers.database.DatabaseManager.oversee_users" - ) as _oversee_users, - patch( - "charm.PostgresqlOperatorCharm._handle_processes_failures" - ) as _handle_processes_failures, - patch("charm.BackupManager.can_use_s3_repository") as _can_use_s3_repository, - patch( - "single_kernel_postgresql.utils.postgresql.PostgreSQL.get_current_timeline" - ) as _get_current_timeline, - patch("charm.PostgresqlOperatorCharm._setup_users") as _setup_users, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PatroniManager.member_started", new_callable=PropertyMock) as _member_started, - patch("charm.PatroniManager.get_member_status") as _get_member_status, - patch( - "charm.PostgresqlOperatorCharm.enable_disable_extensions" - ) as _enable_disable_extensions, - patch("charm.PostgreSQL.drop_hba_triggers") as _drop_hba_triggers, - ): - _get_current_timeline.return_value = "2" - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test when the restore operation fails. - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data( - rel_id, - harness.charm.app.name, - { - "cluster_initialised": "True", - "restoring-backup": "20230101-090000F", - "refresh_remove_trigger": "True", - }, - ) - _get_member_status.return_value = "failed" - harness.charm.on.update_status.emit() - _update_config.assert_not_called() - _handle_processes_failures.assert_not_called() - _oversee_users.assert_not_called() - _update_relation_endpoints.assert_not_called() - _set_primary_status_message.assert_not_called() - _enable_disable_extensions.assert_not_called() - assert isinstance(harness.charm.unit.status, BlockedStatus) - - # Test when the restore operation hasn't finished yet. - harness.charm.unit.status = ActiveStatus() - _get_member_status.return_value = "running" - _member_started.return_value = False - harness.charm.on.update_status.emit() - _update_config.assert_not_called() - _handle_processes_failures.assert_not_called() - _oversee_users.assert_not_called() - _update_relation_endpoints.assert_not_called() - _set_primary_status_message.assert_not_called() - _enable_disable_extensions.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Assert that the backup id is still in the application relation databag. - assert harness.get_relation_data(rel_id, harness.charm.app) == { - "cluster_initialised": "True", - "restoring-backup": "20230101-090000F", - "refresh_remove_trigger": "True", - } - - # Test when the restore operation finished successfully. - _member_started.return_value = True - _can_use_s3_repository.return_value = (True, None) - _handle_processes_failures.return_value = False - harness.charm.on.update_status.emit() - _update_config.assert_called_once() - _handle_processes_failures.assert_called_once() - _oversee_users.assert_called_once() - _update_relation_endpoints.assert_called_once() - _set_primary_status_message.assert_called_once() - _enable_disable_extensions.assert_called_once_with() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Assert that the backup id is not in the application relation databag anymore. - assert harness.get_relation_data(rel_id, harness.charm.app) == { - "cluster_initialised": "True", - "refresh_remove_trigger": "True", - } - - # Test when it's not possible to use the configured S3 repository. - _update_config.reset_mock() - _set_primary_status_message.reset_mock() - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"restoring-backup": "20230101-090000F"}, - ) - _can_use_s3_repository.return_value = (False, "fake validation message") - harness.charm.on.update_status.emit() - _update_config.assert_called_once() - _set_primary_status_message.assert_called_once() - # Assert that the backup id is not in the application relation databag anymore. - assert harness.get_relation_data(rel_id, harness.charm.app) == { - "cluster_initialised": "True", - "s3-initialization-block-message": "fake validation message", - "refresh_remove_trigger": "True", - } - assert not _drop_hba_triggers.called - - -def test_is_storage_attached(harness): - with patch( - "subprocess.check_call", - side_effect=[None, subprocess.CalledProcessError(1, "fake command")], - ) as _check_call: - # Test with attached storage. - is_storage_attached = harness.charm._is_storage_attached() - _check_call.assert_called_once_with([ - "/usr/bin/mountpoint", - "-q", - harness.charm._storage_path, - ]) - assert is_storage_attached - - # Test with detached storage. - is_storage_attached = harness.charm._is_storage_attached() - assert not (is_storage_attached) - - -def test_check_detached_storage(harness): - with ( - patch("charm.PostgresqlOperatorCharm._is_storage_attached") as _is_storage_attached, - patch("charm.wait_fixed", return_value=wait_fixed(0)), - ): - _is_storage_attached.return_value = False - with pytest.raises(StorageUnavailableError): - harness.charm._check_detached_storage() - assert isinstance(harness.charm.unit.status, WaitingStatus) - - -@pytest.mark.parametrize("unsettable_status", [ErrorStatus(), UnknownStatus()]) -def test_check_detached_storage_does_not_restore_unsettable_status(harness, unsettable_status): - # The unit.status getter can return statuses that the juju backend rejects - # on set (e.g. "error" or "unknown"). Restoring such a cached status must not - # be attempted, otherwise the backend raises InvalidStatusError/ModelError. - harness.charm.unit._status = unsettable_status - with patch("charm.PostgresqlOperatorCharm._is_storage_attached", return_value=True): - # Storage is attached, so the cached status would be restored verbatim; - # this must not raise. - harness.charm._check_detached_storage() - - -def test_restart(harness): - with ( - patch("charm.PatroniManager.restart_postgresql") as _restart_postgresql, - patch("charm.PatroniManager.are_all_members_ready") as _are_all_members_ready, - patch("charm.PostgresqlOperatorCharm._can_connect_to_postgresql", return_value=True), - ): - _are_all_members_ready.side_effect = [False, True, True] - - # Test when not all members are ready. - mock_event = MagicMock() - harness.charm._restart(mock_event) - mock_event.defer.assert_called_once() - _restart_postgresql.assert_not_called() - - # Test a successful restart. - mock_event.defer.reset_mock() - harness.charm._restart(mock_event) - assert not (isinstance(harness.charm.unit.status, BlockedStatus)) - mock_event.defer.assert_not_called() - - # Test a failed restart. - _restart_postgresql.side_effect = RetryError(last_attempt=1) - harness.charm._restart(mock_event) - assert isinstance(harness.charm.unit.status, BlockedStatus) - mock_event.defer.assert_not_called() - - -def test_request_restart(harness): - """Bridge for ConfigManager: clears the restarted flag and acquires the restart lock.""" - with patch( - "charms.rolling_ops.v0.rollingops.RollingOpsManager._on_acquire_lock" - ) as _on_acquire_lock: - rel_id = harness.model.get_relation(PEER_RELATION).id - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.unit.name, {"postgresql_restarted": "True"} - ) - - harness.charm.request_restart() - - assert "postgresql_restarted" not in harness.charm.unit_peer_data - _on_acquire_lock.assert_called_once() - - -def test_restart_services(harness): - """Bridge for ConfigManager: restarts the metrics and LDAP sync snap services.""" - with ( - patch("charm.snap.SnapCache") as _snap_cache, - patch( - "charm.PostgresqlOperatorCharm._restart_metrics_service" - ) as _restart_metrics_service, - patch( - "charm.PostgresqlOperatorCharm._restart_ldap_sync_service" - ) as _restart_ldap_sync_service, - ): - postgres_snap = _snap_cache.return_value.__getitem__.return_value - - harness.charm.restart_services() - - _restart_metrics_service.assert_called_once_with(postgres_snap) - _restart_ldap_sync_service.assert_called_once_with(postgres_snap) - - -def test_update_config_delegates_to_config_manager(harness): - """update_config forwards to the lib's ConfigManager with the charm-owned refresh.""" - with patch.object(harness.charm, "config_manager") as _config_manager: - _config_manager.update_config.return_value = True - - result = harness.charm.update_config(no_peers=True) - - assert result is True - _config_manager.update_config.assert_called_once() - _, kwargs = _config_manager.update_config.call_args - assert kwargs["no_peers"] is True - assert kwargs["refresh"] is harness.charm.refresh - - -def test_update_config_clears_stale_standby_when_primary(harness): - """Test update_config clears a stale DCS standby_cluster when this cluster is the primary. - - A force-promote bumps the promoted-cluster-counter but, while the dead-DC relation still - lingers, does not call promote_standby_cluster() — so the reconciler must clear the stale - standby_cluster itself on the next update-config, or the cluster stays a read-only standby - leader (DPE-10203). - """ - with ( - patch.object(harness.charm, "patroni_manager") as _patroni_manager, - patch.object(harness.charm, "config_manager") as _config_manager, - patch.object(harness.charm.async_replication, "get_primary_cluster_endpoint") as _endpoint, - ): - # This cluster is the primary -> no primary endpoint. - _endpoint.return_value = None - _config_manager.update_config.return_value = True - _patroni_manager.member_started = True - - assert harness.charm.update_config() is True - - base_patch = _patroni_manager.bulk_update_parameters_controller_by_patroni.call_args[0][1] - - # standby_cluster must be explicitly cleared (patched to None) so the DCS converges; - # merely omitting it would leave the stale standby from before the promotion in place. - assert base_patch["standby_cluster"] is None - - -def test_validate_config_options(harness): - with ( - patch("charm.PostgresqlOperatorCharm.postgresql", new_callable=PropertyMock) as _charm_lib, - ): - _charm_lib.return_value.get_postgresql_text_search_configs.return_value = [] - _charm_lib.return_value.validate_date_style.return_value = False - _charm_lib.return_value.validate_group_map.return_value = False - _charm_lib.return_value.get_postgresql_timezones.return_value = [] - - # Test instance_default_text_search_config exception - with harness.hooks_disabled(): - harness.update_config({"instance_default_text_search_config": "pg_catalog.test"}) - - with pytest.raises(ValueError) as e: - harness.charm._validate_config_options() - assert ( - str(e.value) - == "instance_default_text_search_config config option has an invalid value" - ) - - _charm_lib.return_value.get_postgresql_text_search_configs.assert_called_once_with() - _charm_lib.return_value.get_postgresql_text_search_configs.return_value = [ - "pg_catalog.test" - ] - - # Test ldap_map exception - with harness.hooks_disabled(): - harness.update_config({"ldap_map": "ldap_group="}) - del harness.charm.config - - with pytest.raises(ValueError) as e: - harness.charm._validate_config_options() - assert str(e.value) == "ldap_map config option has an invalid value" - - _charm_lib.return_value.validate_group_map.assert_called_once_with("ldap_group=") - _charm_lib.return_value.validate_group_map.return_value = True - - # Test request_date_style exception - with harness.hooks_disabled(): - harness.update_config({"request_date_style": "ISO, TEST"}) - del harness.charm.config - - with pytest.raises(ValueError) as e: - harness.charm._validate_config_options() - assert str(e.value) == "request_date_style config option has an invalid value" - - _charm_lib.return_value.validate_date_style.assert_called_once_with("ISO, TEST") - _charm_lib.return_value.validate_date_style.return_value = True - - # Test request_time_zone exception - with harness.hooks_disabled(): - harness.update_config({"request_time_zone": "TEST_ZONE"}) - del harness.charm.config - - with pytest.raises(ValueError) as e: - harness.charm._validate_config_options() - assert str(e.value) == "request_time_zone config option has an invalid value" - - _charm_lib.return_value.get_postgresql_timezones.assert_called_once_with() - _charm_lib.return_value.get_postgresql_timezones.return_value = ["TEST_ZONE"] - - # Test locales exception - with harness.hooks_disabled(): - harness.update_config({"response_lc_monetary": "test_TEST"}) - del harness.charm.config - - with pytest.raises(ValueError) as e: - harness.charm._validate_config_options() - message = "1 validation error for CharmConfig\nresponse_lc_monetary\n Input should be " - assert str(e.value).startswith(message) - - -def test_config_validation_invalid_worker_values(harness): - """Test that pydantic validates worker process config values.""" - # Test invalid string value (not "auto" or a number) - with harness.hooks_disabled(): - harness.update_config({"cpu-max-worker-processes": "invalid"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - - with pytest.raises(ValueError) as e: - _ = harness.charm.config - - # Pydantic should reject this - assert "validation error" in str(e.value).lower() - - # Test negative number - with harness.hooks_disabled(): - harness.update_config({"cpu-max-worker-processes": "-5"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - - with pytest.raises(ValueError) as e: - _ = harness.charm.config - - # Pydantic should reject this - assert "validation error" in str(e.value).lower() - - # Test value less than 2 - should be accepted at config level but fail during calculation - with harness.hooks_disabled(): - harness.update_config({"cpu-max-worker-processes": "2", "cpu-max-parallel-workers": "7"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - - # The config should accept it (as it gets validated later in the calculation method) - assert harness.charm.config.cpu_max_parallel_workers == 7 - - -def test_config_validation_valid_worker_values(harness): - """Test that pydantic accepts valid worker process config values.""" - with harness.hooks_disabled(): - # Test "auto" - harness.update_config({"cpu-max-worker-processes": "auto"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - assert harness.charm.config.cpu_max_worker_processes == "auto" - - # Test positive integer - harness.update_config({"cpu-max-worker-processes": "16"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - assert harness.charm.config.cpu_max_worker_processes == 16 - - # Test large positive integer - harness.update_config({"cpu-max-parallel-workers": "1000"}) - with contextlib.suppress(AttributeError): - del harness.charm.config - assert harness.charm.config.cpu_max_parallel_workers == 1000 - - -def test_on_peer_relation_changed(harness): - with ( - patch("charm.snap.SnapCache"), - patch("charm.PostgresqlOperatorCharm._update_new_unit_status") as _update_new_unit_status, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", new_callable=PropertyMock - ) as _primary_endpoint, - patch("charm.BackupManager.coordinate_stanza_fields") as _coordinate_stanza_fields, - patch( - "charm.BackupManager.start_stop_pgbackrest_service" - ) as _start_stop_pgbackrest_service, - patch("charm.PostgresqlOperatorCharm.is_standby_leader") as _is_standby_leader, - patch("charm.PostgresqlOperatorCharm.is_primary") as _is_primary, - patch("charm.PatroniManager.member_started", new_callable=PropertyMock) as _member_started, - patch("charm.PatroniManager.start_patroni") as _start_patroni, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PostgresqlOperatorCharm._update_member_ip") as _update_member_ip, - patch("charm.PostgresqlOperatorCharm._reconfigure_cluster") as _reconfigure_cluster, - patch("ops.framework.EventBase.defer") as _defer, - ): - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test an uninitialized cluster. - mock_event = Mock() - mock_event.unit = None - - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.app.name, {"cluster_initialised": ""} - ) - harness.charm._on_peer_relation_changed(mock_event) - _reconfigure_cluster.assert_not_called() - - # Test an initialized cluster and this is the leader unit - # (but it fails to reconfigure the cluster). - mock_event.defer.reset_mock() - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"cluster_initialised": "True", "members_ips": '["192.0.2.0"]'}, - ) - harness.set_leader() - _reconfigure_cluster.return_value = False - harness.charm._on_peer_relation_changed(mock_event) - _reconfigure_cluster.assert_called_once_with(mock_event) - mock_event.defer.assert_called_once() - - # Test when the leader can reconfigure the cluster. - mock_event.defer.reset_mock() - _reconfigure_cluster.reset_mock() - _reconfigure_cluster.return_value = True - _member_started.return_value = True - _primary_endpoint.return_value = "192.0.2.0" - harness.model.unit.status = WaitingStatus("awaiting for cluster to start") - harness.charm._on_peer_relation_changed(mock_event) - mock_event.defer.assert_not_called() - _reconfigure_cluster.assert_called_once_with(mock_event) - _update_config.assert_called_once() - _start_patroni.assert_called_once() - _update_new_unit_status.assert_called_once() - - # Test when the unit fails to update the Patroni configuration. - _update_config.reset_mock() - _start_patroni.reset_mock() - _update_new_unit_status.reset_mock() - _update_config.side_effect = RetryError(last_attempt=1) - harness.charm._on_peer_relation_changed(mock_event) - _update_config.assert_called_once() - _start_patroni.assert_not_called() - _update_new_unit_status.assert_not_called() - assert isinstance(harness.model.unit.status, MaintenanceStatus) - - # Test event is early exiting when in blocked status. - _update_config.side_effect = None - _member_started.return_value = False - harness.model.unit.status = BlockedStatus() - harness.charm._on_peer_relation_changed(mock_event) - _start_patroni.assert_not_called() - - # Test when Patroni hasn't started yet in the unit. - harness.model.unit.status = ActiveStatus() - _update_config.side_effect = None - _member_started.return_value = False - harness.charm._on_peer_relation_changed(mock_event) - _start_patroni.assert_called_once() - _update_new_unit_status.assert_not_called() - assert isinstance(harness.model.unit.status, WaitingStatus) - - # Test when Patroni has already started but this is a replica with a - # huge or unknown lag. - relation = harness.model.get_relation(PEER_RELATION, rel_id) - _member_started.return_value = True - for values in [True, False]: - _defer.reset_mock() - _start_stop_pgbackrest_service.reset_mock() - _is_primary.return_value = values - _is_standby_leader.return_value = values - harness.charm.unit.status = ActiveStatus() - harness.charm.on.database_peers_relation_changed.emit(relation) - if _is_primary.return_value == values: - _defer.assert_not_called() - _start_stop_pgbackrest_service.assert_called_once() - assert isinstance(harness.charm.unit.status, ActiveStatus) - else: - _defer.assert_called_once() - _start_stop_pgbackrest_service.assert_not_called() - assert isinstance(harness.charm.unit.status, MaintenanceStatus) - - # Test when it was not possible to start the pgBackRest service yet. - relation = harness.model.get_relation(PEER_RELATION, rel_id) - _member_started.return_value = True - _defer.reset_mock() - _coordinate_stanza_fields.reset_mock() - _start_stop_pgbackrest_service.return_value = False - harness.charm.on.database_peers_relation_changed.emit(relation) - _defer.assert_called_once() - _coordinate_stanza_fields.assert_not_called() - - # Test the last calls been made when it was possible to start the - # pgBackRest service. - _defer.reset_mock() - _start_stop_pgbackrest_service.return_value = True - harness.charm.on.database_peers_relation_changed.emit(relation) - _defer.assert_not_called() - _coordinate_stanza_fields.assert_called_once() - - -def test_reconfigure_cluster(harness): - with ( - patch("charm.PostgresqlOperatorCharm._add_members") as _add_members, - patch( - "charm.PostgresqlOperatorCharm.is_cluster_initialised", - new_callable=PropertyMock, - return_value=False, - ) as _is_cluster_initialised, - patch( - "charm.RaftManager.cleanup_raft_cluster", - return_value=False, - ) as _cleanup_raft_cluster, - ): - # Cluster not initialised - mock_event = MagicMock(spec=RelationEvent) - mock_event.unit = harness.charm.unit - mock_event.relation = Mock() - - assert harness.charm._reconfigure_cluster(mock_event) - - assert not _cleanup_raft_cluster.called - _add_members.assert_called_once_with(mock_event) - _add_members.reset_mock() - - # Removing members failed - _is_cluster_initialised.return_value = True - - assert not harness.charm._reconfigure_cluster(mock_event) - - _cleanup_raft_cluster.assert_called_once_with() - assert not _add_members.called - _cleanup_raft_cluster.reset_mock() - _add_members.reset_mock() - - # Happy scenario - _cleanup_raft_cluster.return_value = True - - assert harness.charm._reconfigure_cluster(mock_event) - - _cleanup_raft_cluster.assert_called_once_with() - _add_members.assert_called_once_with(mock_event) - - -def test_update_certificate(harness): - with ( - patch("charm.TLSManager.get_client_tls_files") as _get_client_tls_files, - patch("charm.TLS.refresh_tls_certificates_event") as _refresh_tls_certificates_event, - ): - # If there is no current TLS files, _request_certificate should be called - # only when the certificates relation is established. - _get_client_tls_files.return_value = (None, None, None) - harness.charm._update_certificate() - _refresh_tls_certificates_event.emit.assert_not_called() - - # Test with already present TLS files (when they will be replaced by new ones). - _get_client_tls_files.return_value = (sentinel.key, sentinel.ca, sentinel.cert) - - harness.charm._update_certificate() - _refresh_tls_certificates_event.emit.assert_called_once_with() - - -def test_update_member_ip(harness): - with ( - patch("charm.PostgresqlOperatorCharm._update_certificate") as _update_certificate, - patch("charm.PatroniManager.stop_patroni") as _stop_patroni, - patch("charm.PostgresqlOperatorCharm.update_endpoint_addresses"), - patch("charm.PostgresqlOperatorCharm.update_config"), - patch.object(harness.charm.watcher_handler, "update_unit_address"), - patch.object(harness.charm.watcher_handler, "update_endpoints"), - ): - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test when the IP address of the unit hasn't changed. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.unit.name, - { - "ip": "192.0.2.0", - }, - ) - assert not (harness.charm._update_member_ip()) - relation_data = harness.get_relation_data(rel_id, harness.charm.unit.name) - assert relation_data.get("ip-to-remove") is None - _stop_patroni.assert_not_called() - _update_certificate.assert_not_called() - - # Test when the IP address of the unit has changed. - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.unit.name, - { - "ip": "2.2.2.2", - }, - ) - assert harness.charm._update_member_ip() - relation_data = harness.get_relation_data(rel_id, harness.charm.unit.name) - assert relation_data.get("ip") == "192.0.2.0" - assert relation_data.get("ip-to-remove") == "2.2.2.2" - _stop_patroni.assert_called_once() - _update_certificate.assert_called_once() - - -def test_push_ca_file_into_workload(harness): - with ( - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("pathlib.Path.write_text") as _write_text, - patch("subprocess.check_call") as _check_call, - ): - harness.charm.set_secret("unit", "ca-app", "test-ca") - - assert harness.charm.push_ca_file_into_workload("ca-app") - _write_text.assert_called_once() - _check_call.assert_called_once_with([UPDATE_CERTS_BIN_PATH]) - _update_config.assert_called_once() - - -def test_clean_ca_file_from_workload(harness): - with ( - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("pathlib.Path.write_text") as _write_text, - patch("pathlib.Path.unlink") as _unlink, - patch("subprocess.check_call") as _check_call, - ): - harness.charm.set_secret("unit", "ca-app", "test-ca") - - assert harness.charm.push_ca_file_into_workload("ca-app") - _write_text.assert_called_once() - _check_call.assert_called_once_with([UPDATE_CERTS_BIN_PATH]) - _update_config.assert_called_once() - - _check_call.reset_mock() - _update_config.reset_mock() - - assert harness.charm.clean_ca_file_from_workload("ca-app") - _unlink.assert_called_once() - _check_call.assert_called_once_with([UPDATE_CERTS_BIN_PATH]) - - -def test_client_relations(harness): - # Test when the charm has no relations. - assert len(harness.charm.client_relations) == 0 - - # Test when the charm has some relations. - harness.add_relation("database", "application") - database_relation = harness.model.get_relation("database") - assert harness.charm.client_relations == [database_relation] - - -def test_add_cluster_member(harness): - with ( - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PostgresqlOperatorCharm._get_unit_ip", return_value="1.1.1.1"), - patch("charm.PostgresqlOperatorCharm._add_to_members_ips") as _add_to_members_ips, - patch("charm.PatroniManager.are_all_members_ready") as _are_all_members_ready, - ): - harness.charm.add_cluster_member("postgresql-0") - - _add_to_members_ips.assert_called_once_with("1.1.1.1") - _update_config.assert_called_once_with() - _update_config.reset_mock() - - # Charm goes into maintenance when update_config fails - _update_config.side_effect = RetryError(last_attempt=None) - harness.charm.add_cluster_member("postgresql-0") - _update_config.assert_called_once_with() - assert isinstance(harness.charm.unit.status, MaintenanceStatus | WaitingStatus) - _update_config.reset_mock() - - # Not ready error if not all members are ready - _are_all_members_ready.return_value = False - with pytest.raises(NotReadyError): - harness.charm.add_cluster_member("postgresql-0") - - -def test_stuck_raft_cluster_check(harness): - # doesn't raise flags if there are no raft flags - assert not harness.charm._stuck_raft_cluster_check() - - # Raft is stuck - rel_id = harness.model.get_relation(PEER_RELATION).id - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data(rel_id, harness.charm.unit.name, {"raft_stuck": "True"}) - - harness.charm._stuck_raft_cluster_check() - assert "raft_selected_candidate" not in harness.charm.app_peer_data - - # Raft candidate - with harness.hooks_disabled(): - harness.update_relation_data(rel_id, harness.charm.unit.name, {"raft_candidate": "True"}) - harness.charm._stuck_raft_cluster_check() - assert harness.charm.app_peer_data["raft_selected_candidate"] == harness.charm.unit.name - - # Don't override existing candidate - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.app.name, {"raft_selected_candidate": "something_else"} - ) - harness.charm._stuck_raft_cluster_check() - assert harness.charm.app_peer_data["raft_selected_candidate"] != harness.charm.unit.name - - -def test_stuck_raft_cluster_cleanup(harness): - rel_id = harness.model.get_relation(PEER_RELATION).id - - # Cleans up app data - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - { - "raft_rejoin": "True", - "raft_reset_primary": "True", - "raft_selected_candidate": "unit_name", - }, - ) - harness.charm._stuck_raft_cluster_cleanup() - - assert "raft_rejoin" not in harness.charm.app_peer_data - assert "raft_reset_primary" not in harness.charm.app_peer_data - assert "raft_selected_candidate" not in harness.charm.app_peer_data - - # Don't clean up if there's unit data flags - with harness.hooks_disabled(): - harness.update_relation_data(rel_id, harness.charm.unit.name, {"raft_primary": "True"}) - harness.update_relation_data( - rel_id, - harness.charm.app.name, - { - "raft_rejoin": "True", - "raft_reset_primary": "True", - "raft_selected_candidate": "unit_name", - }, - ) - harness.charm._stuck_raft_cluster_cleanup() - - assert "raft_rejoin" in harness.charm.app_peer_data - assert "raft_reset_primary" in harness.charm.app_peer_data - assert "raft_selected_candidate" in harness.charm.app_peer_data - - -def test_stuck_raft_cluster_rejoin(harness): - rel_id = harness.model.get_relation(PEER_RELATION).id - - with ( - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch("charm.PostgresqlOperatorCharm._add_to_members_ips") as _add_to_members_ips, - ): - # No data - harness.charm._stuck_raft_cluster_rejoin() - - assert "raft_reset_primary" not in harness.charm.app_peer_data - assert "raft_rejoin" not in harness.charm.app_peer_data - - # Raises primary flag - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.unit.name, - { - "raft_primary": "test_primary", - f"{PEER_RELATION}-address": "192.0.2.0", - }, - ) - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"raft_followers_stopped": "test_candidate"}, - ) - - harness.charm._stuck_raft_cluster_rejoin() - - assert "raft_reset_primary" in harness.charm.app_peer_data - assert "raft_rejoin" in harness.charm.app_peer_data - assert "members_ips" not in harness.charm.app_peer_data - _add_to_members_ips.assert_called_once_with("192.0.2.0") - _update_relation_endpoints.assert_called_once_with() - - -def test_raft_reinitialisation(harness): - rel_id = harness.model.get_relation(PEER_RELATION).id - - with ( - patch( - "charm.PostgresqlOperatorCharm._stuck_raft_cluster_check" - ) as _stuck_raft_cluster_check, - patch( - "charm.PostgresqlOperatorCharm._stuck_raft_cluster_rejoin" - ) as _stuck_raft_cluster_rejoin, - patch( - "charm.PostgresqlOperatorCharm._stuck_raft_cluster_cleanup" - ) as _stuck_raft_cluster_cleanup, - patch("charm.RaftManager.remove_raft_data") as _remove_raft_data, - patch("charm.RaftManager.reinitialise_raft_data") as _reinitialise_raft_data, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PostgresqlOperatorCharm._set_primary_status_message"), - patch("charm.PatroniManager.start_patroni"), - ): - # No data - harness.charm._raft_reinitialisation() - - # Different candidate - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data( - rel_id, harness.charm.unit.name, {"raft_stuck": "True", "raft_candidate": "True"} - ) - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"raft_selected_candidate": "test_candidate"}, - ) - - harness.charm._raft_reinitialisation() - _stuck_raft_cluster_rejoin.assert_called_once_with() - _stuck_raft_cluster_check.assert_called_once_with() - assert not _stuck_raft_cluster_cleanup.called - _remove_raft_data.assert_called_once_with() - assert not _reinitialise_raft_data.called - - # Current candidate - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data( - rel_id, - harness.charm.unit.name, - {"raft_stuck": "", "raft_candidate": "True", "raft_stopped": "True"}, - ) - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"raft_selected_candidate": "postgresql/0", "raft_followers_stopped": "True"}, - ) - - harness.charm._raft_reinitialisation() - _reinitialise_raft_data.assert_called_once_with() - - # Cleanup - with harness.hooks_disabled(): - harness.set_leader() - harness.update_relation_data( - rel_id, - harness.charm.unit.name, - {"raft_stuck": "", "raft_candidate": "True", "raft_stopped": "True"}, - ) - harness.update_relation_data(rel_id, harness.charm.app.name, {"raft_rejoin": "True"}) - harness.charm._raft_reinitialisation() - _stuck_raft_cluster_cleanup.assert_called_once_with() - _update_config.assert_called_once_with() - - -# -# Secrets -# - - -@pytest.mark.parametrize("scope,field", [("app", "operator-password"), ("unit", "csr")]) -def test_get_secret_secrets(harness, scope, field): - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - harness.set_leader() - - assert harness.charm.get_secret(scope, field) is None - harness.charm.set_secret(scope, field, "test") - assert harness.charm.get_secret(scope, field) == "test" - - -@pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) -def test_set_reset_new_secret(harness, scope, is_leader): - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - """NOTE: currently ops.testing seems to allow for non-leader to set secrets too!""" - # App has to be leader, unit can be either - harness.set_leader(is_leader) - # Getting current password - harness.charm.set_secret(scope, "new-secret", "bla") - assert harness.charm.get_secret(scope, "new-secret") == "bla" - - # Reset new secret - harness.charm.set_secret(scope, "new-secret", "blablabla") - assert harness.charm.get_secret(scope, "new-secret") == "blablabla" - - # Set another new secret - harness.charm.set_secret(scope, "new-secret2", "blablabla") - assert harness.charm.get_secret(scope, "new-secret2") == "blablabla" - - -@pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) -def test_invalid_secret(harness, scope, is_leader): - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - # App has to be leader, unit can be either - harness.set_leader(is_leader) - - with pytest.raises((RelationDataTypeError, TypeError)): - harness.charm.set_secret(scope, "somekey", 1) - - harness.charm.set_secret(scope, "somekey", "") - assert harness.charm.get_secret(scope, "somekey") is None - - -def test_delete_password(harness, caplog): - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - """NOTE: currently ops.testing seems to allow for non-leader to remove secrets too!""" - harness.set_leader(True) - harness.charm.set_secret("app", "operator-password", "somepw") - harness.charm.remove_secret("app", "operator-password") - assert harness.charm.get_secret("app", "operator-password") is None - - harness.set_leader(False) - harness.charm.set_secret("unit", "operator-password", "somesecret") - harness.charm.remove_secret("unit", "operator-password") - assert harness.charm.get_secret("unit", "operator-password") is None - - harness.set_leader(True) - with caplog.at_level(logging.DEBUG): - error_message = "Non-existing secret operator-password was attempted to be removed." - - harness.charm.remove_secret("app", "operator-password") - assert error_message in caplog.text - - harness.charm.remove_secret("unit", "operator-password") - assert error_message in caplog.text - - harness.charm.remove_secret("app", "non-existing-secret") - assert ( - "Non-existing field 'non-existing-secret' was attempted to be removed" - in caplog.text - ) - - harness.charm.remove_secret("unit", "non-existing-secret") - assert ( - "Non-existing field 'non-existing-secret' was attempted to be removed" - in caplog.text - ) - - -@pytest.mark.parametrize("scope,is_leader", [("app", True), ("unit", True), ("unit", False)]) -def test_migration_from_single_secret(harness, scope, is_leader): - """Check if we're moving on to use secrets when live upgrade from databag to Secrets usage. - - Since it checks for a migration from databag to juju secrets, it's specific to juju3. - """ - with ( - patch("charm.PostgresqlOperatorCharm._on_leader_elected"), - ): - rel_id = harness.model.get_relation(PEER_RELATION).id - - # App has to be leader, unit can be either - harness.set_leader(is_leader) - - secret = harness.charm.app.add_secret({"operator-password": "bla"}) - - # Getting current password - entity = getattr(harness.charm, scope) - harness.update_relation_data(rel_id, entity.name, {SECRET_INTERNAL_LABEL: secret.id}) - assert harness.charm.get_secret(scope, "operator-password") == "bla" - - # Reset new secret - # Only the leader can set app secret content. - with harness.hooks_disabled(): - harness.set_leader(True) - harness.charm.set_secret(scope, "operator-password", "blablabla") - with harness.hooks_disabled(): - harness.set_leader(is_leader) - assert harness.charm.model.get_secret(label=f"{PEER_RELATION}.postgresql.{scope}") - assert harness.charm.get_secret(scope, "operator-password") == "blablabla" - assert SECRET_INTERNAL_LABEL not in harness.get_relation_data( - rel_id, getattr(harness.charm, scope).name - ) - - -def test_on_peer_relation_departed(harness): - with ( - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", new_callable=PropertyMock - ) as _primary_endpoint, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch( - "charm.PostgresqlOperatorCharm._remove_from_members_ips" - ) as _remove_from_members_ips, - patch("charm.PatroniManager.are_all_members_ready") as _are_all_members_ready, - patch("charm.PostgresqlOperatorCharm._get_ips_to_remove") as _get_ips_to_remove, - patch( - "charm.PostgresqlOperatorCharm.updated_synchronous_node_count" - ) as _updated_synchronous_node_count, - patch("charm.RaftManager.remove_raft_member") as _remove_raft_member, - patch("single_kernel_postgresql.core.state.CharmState.unit_ip") as _unit_ip, - patch("charm.PatroniManager.get_member_ip") as _get_member_ip, - ): - rel_id = harness.model.get_relation(PEER_RELATION).id - # Test when the current unit is the departing unit. - harness.charm.unit.status = ActiveStatus() - event = Mock() - event.departing_unit = harness.charm.unit - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_not_called() - event.defer.assert_not_called() - _updated_synchronous_node_count.assert_not_called() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the current unit is not the departing unit, but removing - # the member from the raft cluster fails. - _remove_raft_member.side_effect = RemoveRaftMemberFailedError - event.departing_unit = Unit( - f"{harness.charm.app.name}/1", None, harness.charm.app._backend, {} - ) - mock_ip_address = "1.1.1.1" - _get_member_ip.return_value = mock_ip_address - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_called_once() - _updated_synchronous_node_count.assert_not_called() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the member is successfully removed from the raft cluster, - # but the unit is not the leader. - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _remove_raft_member.side_effect = None - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_not_called() - _updated_synchronous_node_count.assert_not_called() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the unit is the leader, but the cluster hasn't initialized yet, - # or it was unable to set synchronous_node_count. - _remove_raft_member.reset_mock() - with harness.hooks_disabled(): - harness.set_leader() - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_called_once() - _updated_synchronous_node_count.assert_not_called() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _updated_synchronous_node_count.return_value = False - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, harness.charm.app.name, {"cluster_initialised": "True"} - ) - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when there is more units in the cluster. - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _updated_synchronous_node_count.reset_mock() - harness.add_relation_unit(rel_id, f"{harness.charm.app.name}/2") - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_not_called() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the cluster is initialised, and it could set synchronous_node_count, - # but there is no IPs to be removed from the members list. - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _updated_synchronous_node_count.reset_mock() - _updated_synchronous_node_count.return_value = True - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_called_once() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when there are IPs to be removed from the members list, but not all - # the members are ready yet. - _remove_raft_member.reset_mock() - _updated_synchronous_node_count.reset_mock() - _get_ips_to_remove.reset_mock() - ips_to_remove = ["2.2.2.2", "3.3.3.3"] - _get_ips_to_remove.return_value = ips_to_remove - _are_all_members_ready.return_value = False - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_called_once() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_called_once() - _remove_from_members_ips.assert_not_called() - _update_config.assert_not_called() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when all members are ready. - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _updated_synchronous_node_count.reset_mock() - _get_ips_to_remove.reset_mock() - _are_all_members_ready.return_value = True - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_called_once() - _remove_from_members_ips.assert_has_calls([call(ips_to_remove[0]), call(ips_to_remove[1])]) - assert _update_config.call_count == 2 - assert _update_relation_endpoints.call_count == 2 - assert isinstance(harness.charm.unit.status, ActiveStatus) - - # Test when the primary is not reachable yet. - _remove_raft_member.reset_mock() - event.defer.reset_mock() - _updated_synchronous_node_count.reset_mock() - _get_ips_to_remove.reset_mock() - _remove_from_members_ips.reset_mock() - _update_config.reset_mock() - _update_relation_endpoints.reset_mock() - _primary_endpoint.return_value = None - harness.charm._on_peer_relation_departed(event) - _remove_raft_member.assert_called_once_with(f"{mock_ip_address}:2222") - event.defer.assert_not_called() - _updated_synchronous_node_count.assert_called_once_with() - _get_ips_to_remove.assert_called_once() - _remove_from_members_ips.assert_called_once() - _update_config.assert_called_once() - _update_relation_endpoints.assert_not_called() - assert isinstance(harness.charm.unit.status, WaitingStatus) - - -def test_update_new_unit_status(harness): - with ( - patch("charm.PostgreSQLAsyncReplication.handle_read_only_mode") as handle_read_only_mode, - patch( - "charm.PostgresqlOperatorCharm._update_relation_endpoints" - ) as _update_relation_endpoints, - patch( - "charm.PostgresqlOperatorCharm.primary_endpoint", new_callable=PropertyMock - ) as _primary_endpoint, - ): - # Test when the primary endpoint is reachable. - _primary_endpoint.return_value = "endpoint" - harness.charm.unit.status = MaintenanceStatus("fake status") - harness.charm._update_new_unit_status() - _update_relation_endpoints.assert_called_once() - handle_read_only_mode.assert_called_once() - assert not isinstance(harness.charm.unit.status, WaitingStatus) - - # Test when the primary endpoint is not reachable yet. - _update_relation_endpoints.reset_mock() - handle_read_only_mode.reset_mock() - _primary_endpoint.return_value = None - harness.charm._update_new_unit_status() - _update_relation_endpoints.assert_not_called() - handle_read_only_mode.assert_not_called() - assert isinstance(harness.charm.unit.status, WaitingStatus) - - -@pytest.mark.parametrize("is_leader", [True, False]) -def test_set_primary_status_message(harness, is_leader): - with ( - patch("charm.RaftManager.has_raft_quorum", return_value=True), - patch("charm.PatroniManager.get_running_cluster_members", return_value=["test"]), - patch("charm.PatroniManager.member_started", new_callable=PropertyMock) as _member_started, - patch( - "charm.PostgresqlOperatorCharm.is_standby_leader", new_callable=PropertyMock - ) as _is_standby_leader, - patch("charm.PatroniManager.get_primary") as _get_primary, - ): - for values in itertools.product( - [ - RetryError(last_attempt=1), - ConnectionError, - harness.charm.unit.name, - f"{harness.charm.app.name}/2", - ], - [ - RetryError(last_attempt=1), - ConnectionError, - True, - False, - ], - [True, False], - ): - harness.charm.unit.status = MaintenanceStatus("fake status") - _member_started.return_value = values[2] - if isinstance(values[0], str): - _get_primary.side_effect = None - _get_primary.return_value = values[0] - if values[0] != harness.charm.unit.name and not isinstance(values[1], bool): - _is_standby_leader.side_effect = values[1] - _is_standby_leader.return_value = None - harness.charm._set_primary_status_message() - assert isinstance(harness.charm.unit.status, MaintenanceStatus) - else: - _is_standby_leader.side_effect = None - _is_standby_leader.return_value = ( - values[0] != harness.charm.unit.name and values[1] - ) - harness.charm._set_primary_status_message() - assert isinstance( - harness.charm.unit.status, - ActiveStatus - if values[0] == harness.charm.unit.name or values[1] or values[2] - else MaintenanceStatus, - ) - status = ( - "Primary" - if values[0] == harness.charm.unit.name - else ("Standby" if values[1] else "" if values[2] else "fake status") - ) - assert harness.charm.unit.status.message == status - else: - _get_primary.side_effect = values[0] - _get_primary.return_value = None - harness.charm._set_primary_status_message() - assert isinstance(harness.charm.unit.status, MaintenanceStatus) - - -def test_restart_services_after_reboot(harness): - with ( - patch( - "charm.BackupManager.start_stop_pgbackrest_service" - ) as _start_stop_pgbackrest_service, - patch("charm.PatroniManager.start_patroni") as _start_patroni, - patch( - "single_kernel_postgresql.core.state.CharmState.unit_ip", - new_callable=PropertyMock(return_value="1.1.1.1"), - ) as _unit_ip, - ): - with harness.hooks_disabled(): - harness.update_relation_data( - harness.model.get_relation(PEER_RELATION).id, - harness.charm.app.name, - {"members_ips": json.dumps([])}, - ) - harness.charm._restart_services_after_reboot() - _start_patroni.assert_not_called() - _start_stop_pgbackrest_service.assert_not_called() - - with harness.hooks_disabled(): - harness.update_relation_data( - harness.model.get_relation(PEER_RELATION).id, - harness.charm.app.name, - {"members_ips": json.dumps([_unit_ip])}, - ) - harness.charm._restart_services_after_reboot() - _start_patroni.assert_called_once() - _start_stop_pgbackrest_service.assert_called_once() - - -def test_get_plugins(harness): - with patch("charm.PostgresqlOperatorCharm._on_config_changed"): - # Test when the charm has no plugins enabled. - assert harness.charm.get_plugins() == ["pgaudit"] - - # Test when the charm has some plugins enabled. - harness.update_config({ - "plugin_audit_enable": True, - "plugin_citext_enable": True, - "plugin_spi_enable": True, - }) - del harness.charm.config - assert harness.charm.get_plugins() == [ - "pgaudit", - "citext", - "refint", - "autoinc", - "insert_username", - "moddatetime", - ] - - # Test when the charm has the pgAudit plugin disabled. - harness.update_config({"plugin_audit_enable": False}) - del harness.charm.config - assert harness.charm.get_plugins() == [ - "citext", - "refint", - "autoinc", - "insert_username", - "moddatetime", - ] - - -def test_on_promote_to_primary(harness): - with ( - patch("charm.PostgresqlOperatorCharm._raft_reinitialisation") as _raft_reinitialisation, - patch("charm.PostgreSQLAsyncReplication.promote_to_primary") as _promote_to_primary, - patch("charm.PatroniManager.switchover") as _switchover, - ): - event = Mock() - event.params = {"scope": "cluster"} - - # Cluster - harness.charm._on_promote_to_primary(event) - _promote_to_primary.assert_called_once_with(event) - - # Unit, no force, regular promotion - event.params = {"scope": "unit"} - - harness.charm._on_promote_to_primary(event) - - _switchover.assert_called_once_with("postgresql-0") - - # Unit, no force, switchover failed - event.params = {"scope": "unit"} - _switchover.side_effect = SwitchoverFailedError - - harness.charm._on_promote_to_primary(event) - - event.fail.assert_called_once_with( - "Switchover failed or timed out, check the logs for details" - ) - event.fail.reset_mock() - - # Unit, no force, not sync - event.params = {"scope": "unit"} - _switchover.side_effect = SwitchoverNotSyncError - - harness.charm._on_promote_to_primary(event) - - event.fail.assert_called_once_with("Unit is not sync standby") - event.fail.reset_mock() - - # Unit, no force, raft stuck - event.params = {"scope": "unit"} - rel_id = harness.model.get_relation(PEER_RELATION).id - with harness.hooks_disabled(): - harness.update_relation_data(rel_id, harness.charm.unit.name, {"raft_stuck": "True"}) - - harness.charm._on_promote_to_primary(event) - event.fail.assert_called_once_with( - "Raft is stuck. Set force to reinitialise with new primary" - ) - - # Unit, raft reinit - event.params = {"scope": "unit", "force": "true"} - with harness.hooks_disabled(): - harness.set_leader() - harness.charm._on_promote_to_primary(event) - _raft_reinitialisation.assert_called_once_with() - assert harness.charm.unit_peer_data["raft_candidate"] == "True" - - -def test_get_ldap_parameters(harness): - with ( - patch("single_kernel_postgresql.events.ldap.LDAP.get_relation_data") as _get_relation_data, - patch( - target="single_kernel_postgresql.core.peer_relation.PostgreSQLApplication.is_cluster_initialised", - new_callable=PropertyMock, - return_value=True, - ) as _cluster_initialised, - ): - with harness.hooks_disabled(): - harness.update_relation_data( - harness.model.get_relation(PEER_RELATION).id, - harness.charm.app.name, - {"ldap_enabled": "False"}, - ) - - harness.charm.ldap.get_ldap_parameters() - _get_relation_data.assert_not_called() - _get_relation_data.reset_mock() - - with harness.hooks_disabled(): - harness.update_relation_data( - harness.model.get_relation(PEER_RELATION).id, - harness.charm.app.name, - {"ldap_enabled": "True"}, - ) - - harness.charm.ldap.get_ldap_parameters() - _get_relation_data.assert_called_once() - _get_relation_data.reset_mock() - - -def test_handle_processes_failures(harness): - _now = datetime.now(UTC) - with ( - patch( - "charm.PatroniManager.member_inactive", - new_callable=PropertyMock, - return_value=False, - ) as _member_inactive, - patch( - "charm.PatroniManager.restart_patroni", - ) as _restart_patroni, - patch("charm.os.listdir", return_value=["other_dirs", "pg_wal"]) as _listdir, - patch("charm.os.path.exists", return_value=True) as _exists, - patch("charm.os.rename") as _rename, - patch("charm.datetime") as _datetime, - ): - _datetime.now.return_value = _now - rel_id = harness.model.get_relation(PEER_RELATION).id - with harness.hooks_disabled(): - harness.update_relation_data( - rel_id, - harness.charm.app.name, - {"cluster_initialised": "True", "members_ips": '["192.0.2.0"]'}, - ) - - # Does nothing if member is inactive - assert not harness.charm._handle_processes_failures() - assert not _restart_patroni.called - - # Will not remove pg_wal if dir look right - _member_inactive.return_value = True - assert harness.charm._handle_processes_failures() - _restart_patroni.assert_called_once_with() - _restart_patroni.reset_mock() - - # Will move pg_wal if there's only a pg_wal dir - _listdir.return_value = ["pg_wal"] - assert harness.charm._handle_processes_failures() - assert not _restart_patroni.called - _rename.assert_called_once_with( - os.path.join(POSTGRESQL_DATA_DIR, "pg_wal"), - os.path.join(POSTGRESQL_DATA_DIR, f"pg_wal-{_now.isoformat()}"), - ) - _rename.reset_mock() - - # Will not move pg_wal if there's only a pg_wal dir or other moved dirs - _listdir.return_value = ["pg_wal", f"pg_wal-{_now.isoformat()}"] - assert harness.charm._handle_processes_failures() - _restart_patroni.assert_called_once_with() - assert not _rename.called - _restart_patroni.reset_mock() - - # Does nothing if the data directory does not exist yet (member not bootstrapped) - _listdir.reset_mock() - _exists.return_value = False - assert not harness.charm._handle_processes_failures() - assert not _restart_patroni.called - assert not _rename.called - assert not _listdir.called - _exists.return_value = True - - -def test_relations_user_databases_map(harness): - with ( - patch.object(harness.charm, "postgresql") as _postgresql, - patch("charm.PatroniManager.member_started", new_callable=PropertyMock) as _member_started, - patch( - "charm.PostgresqlOperatorCharm.is_cluster_initialised", new_callable=PropertyMock - ) as _is_cluster_initialised, - ): - # Initial empty results from the functions used in the property that's being tested. - _postgresql.list_users_from_relation.return_value = set() - _postgresql.list_accessible_databases_for_user.return_value = set() - _postgresql.list_access_groups.return_value = { - "identity_access", - "internal_access", - "relation_access", - } - - # Test when the cluster isn't initialised yet. - _is_cluster_initialised.return_value = False - _member_started.return_value = True - assert harness.charm.relations_user_databases_map == { - "operator": "all", - "replication": "all", - "rewind": "all", - } - - # Test when the cluster is initialised but the cluster member hasn't started yet. - _is_cluster_initialised.return_value = True - _member_started.return_value = False - assert harness.charm.relations_user_databases_map == { - "operator": "all", - "replication": "all", - "rewind": "all", - } - - # Test when there are no relation users in the database. - _member_started.return_value = True - assert harness.charm.relations_user_databases_map == {} - - # Test when there are relation users in the database. - _postgresql.list_users.return_value = ["user1", "user2"] - _postgresql.list_accessible_databases_for_user.side_effect = [["db1", "db2"], ["db3"]] - assert harness.charm.relations_user_databases_map == {"user1": "db1,db2", "user2": "db3"} - - # Test when the access groups where not created yet. - _postgresql.list_accessible_databases_for_user.side_effect = [["db1", "db2"], ["db3"]] - _postgresql.list_access_groups.return_value = set() - assert harness.charm.relations_user_databases_map == { - "user1": "db1,db2", - "user2": "db3", - "operator": "all", - "replication": "all", - "rewind": "all", - } - - -def test_on_secret_remove(harness): - with ( - patch("ops.model.Model.juju_version", new_callable=PropertyMock) as _juju_version, - ): - event = Mock() - - # New juju - _juju_version.return_value = JujuVersion("3.6.11") - harness.charm._on_secret_remove(event) - event.remove_revision.assert_called_once_with() - event.reset_mock() - - # Old juju - _juju_version.return_value = JujuVersion("3.6.9") - harness.charm._on_secret_remove(event) - assert not event.remove_revision.called - event.reset_mock() - - # No secret - event.secret.label = None - harness.charm._on_secret_remove(event) - assert not event.remove_revision.called - - -def test_planned_units_returns_app_value_normally(harness): - charm = harness.charm - with patch.object(charm.app, "planned_units", return_value=5): - assert charm._planned_units == 5 - - -def test_planned_units_survives_goal_state_failure(harness): - # DPE-10203 Issue B: after a cross-model SAAS is force-removed, the goal-state hook - # command fails ("saas application ... not found"), so app.planned_units() raises - # ModelError. _planned_units must fall back to the current unit count instead of - # crashing the _patroni property (and every hook that touches it). - charm = harness.charm - with patch.object( - charm.app, - "planned_units", - side_effect=ModelError('ERROR saas application "db1" not found'), - ): - assert charm._planned_units == len(charm._hosts) diff --git a/tests/unit/test_cluster.py b/tests/unit/test_cluster.py deleted file mode 100644 index fffa8eb7117..00000000000 --- a/tests/unit/test_cluster.py +++ /dev/null @@ -1,323 +0,0 @@ -# Copyright 2021 Canonical Ltd. -# See LICENSE file for licensing details. - -from pathlib import Path -from unittest.mock import Mock, PropertyMock, patch, sentinel - -import pytest -from ops.testing import Harness -from pysyncobj.utility import UtilityException -from single_kernel_postgresql.config.exceptions import ( - AddRaftMemberFailedError, - RemoveRaftMemberFailedError, -) -from tenacity import wait_fixed - -from charm import PostgresqlOperatorCharm -from cluster import Patroni -from constants import PATRONI_CONF_PATH, RAFT_PARTNER_PREFIX - -PATRONI_SERVICE = "patroni" -CREATE_CLUSTER_CONF_PATH = "/var/snap/charmed-postgresql/current/etc/postgresql/postgresql.conf" - - -@pytest.fixture() -def harness(): - harness = Harness(PostgresqlOperatorCharm) - harness.begin() - yield harness - harness.cleanup() - - -@pytest.fixture(autouse=True) -def patroni(harness): - patroni = Patroni(harness.charm, "fake-raft-password") - yield patroni - - -def test_cleanup_raft_cluster(patroni): - with ( - patch("cluster.TcpUtility") as _tcp_utility, - patch("cluster.Patroni.remove_raft_member", return_value=True) as _remove_raft_member, - patch( - "charm.PostgresqlOperatorCharm._units_ips", - new_callable=PropertyMock, - return_value={"1.1.1.1"}, - ), - patch( - "charm.PostgresqlOperatorCharm._remove_from_members_ips" - ) as _remove_from_members_ips, - patch("charm.VMWorkload.is_patroni_running", return_value=True), - ): - # Error connecting to raft - _tcp_utility.side_effect = Exception - - assert not patroni.cleanup_raft_cluster() - - _tcp_utility.assert_called_once_with(password="fake-raft-password", timeout=3) - _tcp_utility.reset_mock() - - # No status - _tcp_utility.side_effect = None - _tcp_utility.return_value.executeCommand.return_value = {} - - assert not patroni.cleanup_raft_cluster() - - _tcp_utility.return_value.executeCommand.assert_called_once_with( - "127.0.0.1:2222", ["status"] - ) - - # All members active - _tcp_utility.return_value.executeCommand.return_value = { - f"{RAFT_PARTNER_PREFIX}1.1.1.1:2222": 2 - } - - assert patroni.cleanup_raft_cluster() - - assert not _remove_raft_member.called - - # Filter by unit ips - _tcp_utility.return_value.executeCommand.return_value = { - f"{RAFT_PARTNER_PREFIX}1.1.1.1:2222": 0, - f"{RAFT_PARTNER_PREFIX}2.2.2.2:2222": 0, - } - - assert patroni.cleanup_raft_cluster() - - _remove_raft_member.assert_called_once_with("2.2.2.2:2222") - _remove_from_members_ips.assert_called_once_with("2.2.2.2") - - -def test_add_raft_member(patroni): - with patch("cluster.TcpUtility") as _tcp_utility: - # Member already removed - _tcp_utility.return_value.executeCommand.return_value = { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": True, - "leader": sentinel.raft_leader, - } - - patroni.add_raft_member("1.2.3.4:2222") - - _tcp_utility.assert_called_once_with(password="fake-raft-password", timeout=3) - _tcp_utility.return_value.executeCommand.assert_called_once_with( - "127.0.0.1:2222", ["status"] - ) - _tcp_utility.reset_mock() - - # Add member - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - "SUCCESS", - ] - - patroni.add_raft_member("1.2.3.4:2222") - - _tcp_utility.assert_called_once_with(password="fake-raft-password", timeout=3) - assert _tcp_utility.return_value.executeCommand.call_count == 2 - _tcp_utility.return_value.executeCommand.assert_any_call("127.0.0.1:2222", ["status"]) - _tcp_utility.return_value.executeCommand.assert_any_call( - "127.0.0.1:2222", ["add", "1.2.3.4:2222"] - ) - _tcp_utility.reset_mock() - - # Raises on failed status - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - "FAIL", - ] - - with pytest.raises(AddRaftMemberFailedError): - patroni.add_raft_member("1.2.3.4:2222") - assert False - - # Raises on add error - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - UtilityException, - ] - - with pytest.raises(AddRaftMemberFailedError): - patroni.add_raft_member("1.2.3.4:2222") - assert False - - # Raises on status error - _tcp_utility.return_value.executeCommand.side_effect = [UtilityException] - - with pytest.raises(AddRaftMemberFailedError): - patroni.add_raft_member("1.2.3.4:2222") - assert False - - -def test_remove_raft_member(patroni): - with patch("cluster.TcpUtility") as _tcp_utility: - # Member already removed - _tcp_utility.return_value.executeCommand.return_value = "Response message" - - patroni.remove_raft_member("1.2.3.4:2222") - - _tcp_utility.assert_called_once_with(password="fake-raft-password", timeout=3) - _tcp_utility.return_value.executeCommand.assert_called_once_with( - "127.0.0.1:2222", ["status"] - ) - _tcp_utility.reset_mock() - - # Removing member - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - "SUCCESS", - ] - - patroni.remove_raft_member("1.2.3.4:2222") - - _tcp_utility.assert_called_once_with(password="fake-raft-password", timeout=3) - assert _tcp_utility.return_value.executeCommand.call_count == 2 - _tcp_utility.return_value.executeCommand.assert_any_call("127.0.0.1:2222", ["status"]) - _tcp_utility.return_value.executeCommand.assert_any_call( - "127.0.0.1:2222", ["remove", "1.2.3.4:2222"] - ) - _tcp_utility.reset_mock() - - # Raises on failed status - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - "FAIL", - ] - - with pytest.raises(RemoveRaftMemberFailedError): - patroni.remove_raft_member("1.2.3.4:2222") - assert False - - # Raises on remove error - _tcp_utility.return_value.executeCommand.side_effect = [ - { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": True, - "leader": sentinel.raft_leader, - }, - UtilityException, - ] - - with pytest.raises(RemoveRaftMemberFailedError): - patroni.remove_raft_member("1.2.3.4:2222") - assert False - - # Raises on status error - _tcp_utility.return_value.executeCommand.side_effect = [UtilityException] - - with pytest.raises(RemoveRaftMemberFailedError): - patroni.remove_raft_member("1.2.3.4:2222") - assert False - - -def test_remove_raft_member_no_quorum(patroni, harness): - with ( - patch("cluster.TcpUtility") as _tcp_utility, - patch( - "charm.PostgresqlOperatorCharm.unit_peer_data", new_callable=PropertyMock - ) as _unit_peer_data, - ): - # Async replica - _unit_peer_data.return_value = {} - _tcp_utility.return_value.executeCommand.return_value = { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": False, - "leader": None, - } - - patroni.remove_raft_member("1.2.3.4:2222") - assert harness.charm.unit_peer_data == {"raft_stuck": "True"} - - # No health - _unit_peer_data.return_value = {} - _tcp_utility.return_value.executeCommand.return_value = { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": False, - "leader": None, - } - - patroni.remove_raft_member("1.2.3.4:2222") - - assert harness.charm.unit_peer_data == {"raft_stuck": "True"} - - # Sync replica - _unit_peer_data.return_value = {} - leader_mock = Mock() - leader_mock.address = "1.2.3.4:2222" - _tcp_utility.return_value.executeCommand.return_value = { - "partner_node_status_server_1.2.3.4:2222": 0, - "has_quorum": False, - "leader": leader_mock, - } - - patroni.remove_raft_member("1.2.3.4:2222") - - assert harness.charm.unit_peer_data == {"raft_stuck": "True"} - - -def test_remove_raft_data(patroni): - with ( - patch("charm.PatroniManager.stop_patroni") as _stop_patroni, - patch("cluster.psutil") as _psutil, - patch("cluster.wait_fixed", return_value=wait_fixed(0)), - patch("shutil.rmtree") as _rmtree, - patch("pathlib.Path.is_dir") as _is_dir, - patch("pathlib.Path.exists") as _exists, - ): - mock_proc_pg = Mock() - mock_proc_not_pg = Mock() - mock_proc_pg.name.return_value = "postgres" - mock_proc_not_pg.name.return_value = "something_else" - _psutil.process_iter.side_effect = [[mock_proc_not_pg, mock_proc_pg], [mock_proc_not_pg]] - - patroni.remove_raft_data() - - _stop_patroni.assert_called_once_with() - assert _psutil.process_iter.call_count == 2 - _psutil.process_iter.assert_any_call(["name"]) - _rmtree.assert_called_once_with(Path(f"{PATRONI_CONF_PATH}/raft")) - - -def test_reinitialise_raft_data(patroni): - with ( - patch("charm.PatroniManager.get_patroni_health") as _get_patroni_health, - patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, - patch("charm.PatroniManager.start_patroni") as _start_patroni, - patch("charm.PatroniManager.restart_patroni") as _restart_patroni, - patch("cluster.psutil") as _psutil, - patch("cluster.wait_fixed", return_value=wait_fixed(0)), - ): - mock_proc_pg = Mock() - mock_proc_not_pg = Mock() - mock_proc_pg.name.return_value = "postgres" - mock_proc_not_pg.name.return_value = "something_else" - _psutil.process_iter.side_effect = [[mock_proc_not_pg], [mock_proc_not_pg, mock_proc_pg]] - _get_patroni_health.side_effect = [ - {"role": "replica", "state": "streaming"}, - {"role": "leader", "state": "running"}, - ] - - patroni.reinitialise_raft_data() - - _update_config.assert_called_once_with(no_peers=True) - _start_patroni.assert_called_once_with() - _restart_patroni.assert_called_once_with() - assert _psutil.process_iter.call_count == 2 - _psutil.process_iter.assert_any_call(["name"]) diff --git a/tests/unit/test_cluster_topology_observer.py b/tests/unit/test_cluster_topology_observer.py deleted file mode 100644 index 473b99ed6a7..00000000000 --- a/tests/unit/test_cluster_topology_observer.py +++ /dev/null @@ -1,71 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. -import signal -import sys -from unittest.mock import Mock, PropertyMock, mock_open, patch, sentinel - -import pytest -from jinja2 import Template -from ops.model import ActiveStatus, WaitingStatus -from ops.testing import Harness -from pysyncobj.utility import UtilityException -from single_kernel_postgresql.config.enums import Substrates -from single_kernel_postgresql.config.literals import PEER_RELATION - -from charm import PostgresqlOperatorCharm -from cluster_topology_observer import start_raft_observer -from scripts.cluster_topology_observer import ( - UnreachableUnitsError, - check_for_database_changes, - dispatch, - main, -) -from scripts.raft_observer import check_raft_connection - - -@pytest.fixture(autouse=True) -def harness(): - harness = Harness(PostgresqlOperatorCharm) - harness.add_relation(PEER_RELATION, "postgresql") - harness.begin() - yield harness - harness.cleanup() - - -def test_start_observer(harness): - with ( - patch("builtins.open") as _open, - patch("subprocess.Popen") as _popen, - patch( - "single_kernel_postgresql.core.state.CharmState.unit_ip", new_callable=PropertyMock - ) as _unit_ip, - ): - harness.charm.unit.status = ActiveStatus() - _unit_ip.return_value = "1.2.3.4" - - # Test that nothing is done if the PID is already recorded and alive. - with patch("single_kernel_postgresql.managers.observer.os.kill") as _kill: - harness.charm.state.peer.observer_pid = 1 - harness.charm.observer_manager.start_observer() - _popen.assert_not_called() - _kill.assert_called_once_with(1, 0) - harness.charm.state.peer.observer_pid = None - - # Test that nothing is done if the charm is not in an active status. - harness.charm.unit.status = WaitingStatus() - harness.charm.observer_manager.start_observer() - _popen.assert_not_called() - - # Test that nothing is done if there is no unit IP yet. - harness.charm.unit.status = ActiveStatus() - _unit_ip.return_value = None - harness.charm.observer_manager.start_observer() - _popen.assert_not_called() - - # Happy path: no PID, active, IP set -> the process spawns. - _unit_ip.return_value = "1.2.3.4" - _popen.return_value = Mock(pid=1) - harness.charm.observer_manager.start_observer() - _popen.assert_called_once() - - diff --git a/tests/unit/test_rotate_logs.py b/tests/unit/test_rotate_logs.py deleted file mode 100644 index e399e9d2d43..00000000000 --- a/tests/unit/test_rotate_logs.py +++ /dev/null @@ -1,129 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. -import signal -from unittest.mock import Mock, PropertyMock, patch - -import pytest -from ops.charm import CharmBase -from ops.model import ActiveStatus, Relation, WaitingStatus -from ops.testing import Harness - -from rotate_logs import RotateLogs - - -class MockCharm(CharmBase): - def __init__(self, *args): - super().__init__(*args) - - self.rotate_logs = RotateLogs(self) - - @property - def _peers(self) -> Relation | None: - return None - - @property - def unit_peer_data(self) -> dict: - """Unit peer relation data object.""" - if self._peers is None: - return {} - - return self._peers.data[self.unit] - - -@pytest.fixture(autouse=True) -def harness(): - harness = Harness(MockCharm, meta="name: test-charm") - harness.begin() - yield harness - harness.cleanup() - - -def test_start_log_rotation(harness): - with ( - patch("builtins.open") as _open, - patch("subprocess.Popen") as _popen, - patch("os.path.exists") as _exists, - patch.object(MockCharm, "_peers", new_callable=PropertyMock) as _peers, - ): - # Test that nothing is done if there is already a running process. - _peers.return_value = Mock(data={harness.charm.unit: {"rotate-logs-pid": "1"}}) - _exists.return_value = True - harness.charm.rotate_logs.start_log_rotation() - _popen.assert_not_called() - - # Test that nothing is done if the charm is not in an active status. - harness.charm.unit.status = WaitingStatus() - _peers.return_value = Mock(data={harness.charm.unit: {}}) - harness.charm.rotate_logs.start_log_rotation() - _popen.assert_not_called() - - # Test that nothing is done if peer relation is not available yet. - harness.charm.unit.status = ActiveStatus() - _peers.return_value = None - harness.charm.rotate_logs.start_log_rotation() - _popen.assert_not_called() - - # Test that nothing is done if the logrotate file does not exist. - _peers.return_value = Mock(data={harness.charm.unit: {}}) - _exists.return_value = False - harness.charm.rotate_logs.start_log_rotation() - _popen.assert_not_called() - - # Test that nothing is done if there is already a running process. - _popen.return_value = Mock(pid=1) - _exists.return_value = True - harness.charm.rotate_logs.start_log_rotation() - _popen.assert_called_once() - - -def test_start_log_rotation_already_running(harness): - with ( - patch("builtins.open") as _open, - patch("subprocess.Popen") as _popen, - patch("os.kill") as _kill, - patch("os.path.exists") as _exists, - patch.object(MockCharm, "_peers", new_callable=PropertyMock) as _peers, - ): - harness.charm.unit.status = ActiveStatus() - _peers.return_value = Mock(data={harness.charm.unit: {"rotate-logs-pid": "1234"}}) - _exists.return_value = True - harness.charm.rotate_logs.start_log_rotation() - _kill.assert_called_once_with(1234, 0) - assert not _popen.called - _kill.reset_mock() - - # If process is already dead, it should restart. - _kill.side_effect = OSError - harness.charm.rotate_logs.start_log_rotation() - _kill.assert_called_once_with(1234, 0) - _popen.assert_called_once() - - -def test_stop_log_rotation(harness): - with ( - patch("os.kill") as _kill, - patch.object(MockCharm, "_peers", new_callable=PropertyMock) as _peers, - ): - # Nothing is done when the peer relation (and so the stored PID) is gone. - _peers.return_value = None - harness.charm.rotate_logs.stop_log_rotation() - _kill.assert_not_called() - - # Nothing is done when there is no process running. - _peers.return_value = Mock(data={harness.charm.unit: {}}) - harness.charm.rotate_logs.stop_log_rotation() - _kill.assert_not_called() - - # The process is killed and the stored PID is cleared. - peer_data = {"rotate-logs-pid": "1"} - _peers.return_value = Mock(data={harness.charm.unit: peer_data}) - harness.charm.rotate_logs.stop_log_rotation() - _kill.assert_called_once_with(1, signal.SIGINT) - assert peer_data["rotate-logs-pid"] == "" - _kill.reset_mock() - - # A dead process doesn't break the script. - _peers.return_value = Mock(data={harness.charm.unit: {"rotate-logs-pid": "1"}}) - _kill.side_effect = OSError - harness.charm.rotate_logs.stop_log_rotation() - _kill.assert_called_once_with(1, signal.SIGINT) diff --git a/tests/unit/test_tls.py b/tests/unit/test_tls.py index 97bcf3edf25..69ddb210c00 100644 --- a/tests/unit/test_tls.py +++ b/tests/unit/test_tls.py @@ -43,16 +43,6 @@ def test_tls_handler_is_lib_backed(harness): assert not hasattr(charm, "push_tls_files_to_workload") -def test_is_tls_enabled_reflects_tls_manager(harness): - """is_tls_enabled is driven by TLSManager.get_client_tls_files(), not the handler.""" - with patch("charm.TLSManager.get_client_tls_files") as _get_client_tls_files: - _get_client_tls_files.return_value = (None, None, None) - assert harness.charm.is_tls_enabled is False - - _get_client_tls_files.return_value = ("key", "ca", "cert") - assert harness.charm.is_tls_enabled is True - - def test_reload_bridge_calls_update_config(harness): """_reload_tls_after_push delegates to update_config (the event guarantees readiness).""" with patch("charm.PostgresqlOperatorCharm.update_config") as _update_config: @@ -99,14 +89,18 @@ def test_reload_bridge_observes_tls_files_pushed(harness): def test_internal_cert_path_pushes_and_reloads(harness): """check_and_update_internal_cert generates, pushes, and reloads on CN mismatch.""" with ( - patch("charm.CharmState.unit_ip", new_callable=PropertyMock) as _unit_ip, patch( - "charm.CharmState.get_secret", + "single_kernel_postgresql.core.state.CharmState.unit_ip", new_callable=PropertyMock + ) as _unit_ip, + patch( + "single_kernel_postgresql.core.state.CharmState.get_secret", return_value="-----BEGIN CERTIFICATE-----", ), patch("single_kernel_postgresql.managers.refresh.load_pem_x509_certificate") as _load_cert, - patch("charm.TLSManager.generate_internal_peer_cert") as _generate, - patch("charm.TLSManager.push_tls_files") as _push, + patch( + "single_kernel_postgresql.managers.tls.TLSManager.generate_internal_peer_cert" + ) as _generate, + patch("single_kernel_postgresql.managers.tls.TLSManager.push_tls_files") as _push, patch("charm.PostgresqlOperatorCharm.update_config") as _update_config, ): _unit_ip.return_value = "1.2.3.4" From 82dda771929a4b670f9670ed2953cb464ab09943 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Sun, 11 Oct 2026 17:55:12 -0300 Subject: [PATCH 02/13] fix(charm): complete the thin-charm end state Drop the raft-observer/cluster/rotate-logs modules and their templates; construct TLSTransfer from the lib; import charm_tracing_config from the lib; drop the charm-level config plumbing in favor of CharmState.config. --- scripts/__init__.py | 4 - scripts/cluster_topology_observer.py | 177 ----------------- scripts/raft_observer.py | 81 -------- scripts/rotate_logs.py | 23 --- src/charm.py | 48 +---- src/cluster.py | 281 --------------------------- src/cluster_topology_observer.py | 193 ------------------ src/rotate_logs.py | 70 ------- templates/raft-observer.service.j2 | 13 -- templates/raft-observer.timer.j2 | 10 - tests/unit/test_charm.py | 22 +-- 11 files changed, 16 insertions(+), 906 deletions(-) delete mode 100644 scripts/__init__.py delete mode 100644 scripts/cluster_topology_observer.py delete mode 100644 scripts/raft_observer.py delete mode 100644 scripts/rotate_logs.py delete mode 100644 src/cluster.py delete mode 100644 src/cluster_topology_observer.py delete mode 100644 src/rotate_logs.py delete mode 100644 templates/raft-observer.service.j2 delete mode 100644 templates/raft-observer.timer.j2 diff --git a/scripts/__init__.py b/scripts/__init__.py deleted file mode 100644 index df36fbc2df7..00000000000 --- a/scripts/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Charm script utilities.""" diff --git a/scripts/cluster_topology_observer.py b/scripts/cluster_topology_observer.py deleted file mode 100644 index f0f16cec711..00000000000 --- a/scripts/cluster_topology_observer.py +++ /dev/null @@ -1,177 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Cluster topology changes observer.""" - -import subprocess -import sys -from asyncio import as_completed, create_task, run, wait -from contextlib import suppress -from ssl import create_default_context -from time import sleep -from urllib.parse import urljoin - -import psycopg2 -import yaml -from httpx import AsyncClient - -API_REQUEST_TIMEOUT = 5 -PATRONI_CLUSTER_STATUS_ENDPOINT = "cluster" -TLS_CA_BUNDLE_FILE = "peer_ca_bundle.pem" -SNAP_CURRENT_PATH = "/var/snap/charmed-postgresql/current" -SNAP_CONF_PATH = f"{SNAP_CURRENT_PATH}/etc" -PATRONI_CONF_PATH = f"{SNAP_CONF_PATH}/patroni" -PATRONI_CONF_FILE_PATH = f"{PATRONI_CONF_PATH}/patroni.yaml" - -# File path for the spawned cluster topology observer process to write logs. -LOG_FILE_PATH = "/var/log/cluster_topology_observer.log" - - -class UnreachableUnitsError(Exception): - """Cannot reach any known cluster member.""" - - -async def _httpx_get_request(url: str): - ssl_ctx = create_default_context() - with suppress(FileNotFoundError): - ssl_ctx.load_verify_locations(cafile=f"{PATRONI_CONF_PATH}/{TLS_CA_BUNDLE_FILE}") - async with AsyncClient(timeout=API_REQUEST_TIMEOUT, verify=ssl_ctx) as client: - try: - return (await client.get(url)).json() - except Exception as e: - print(f"Failed to contact {url} with {e}") - return None - - -def check_for_cluster_topology_changes( - run_cmd, unit, charm_dir, previous_cluster_topology, current_cluster_topology -): - """Check for changes in the cluster topology. - - If changes are detected, dispatch an event to handle them. - """ - # If it's the first time the cluster topology was retrieved, then store it and use - # it for subsequent checks. - if not previous_cluster_topology: - previous_cluster_topology = current_cluster_topology - # If the cluster topology changed, dispatch a charm event to handle this change. - elif current_cluster_topology != previous_cluster_topology: - previous_cluster_topology = current_cluster_topology - dispatch(run_cmd, unit, charm_dir, "cluster_topology_change") - return previous_cluster_topology - - -def check_for_database_changes(run_cmd, unit, charm_dir, previous_databases): - """Check for changes in the databases. - - If changes are detected, dispatch an event to handle them. - """ - with open(PATRONI_CONF_FILE_PATH) as conf_file: - conf_file_contents = yaml.safe_load(conf_file) - password = conf_file_contents["postgresql"]["authentication"]["superuser"]["password"] - listen_addr = conf_file_contents["postgresql"]["listen"] - if not listen_addr or ":" not in listen_addr: - with open(LOG_FILE_PATH, "a") as log_file: - log_file.write("Failed to retrieve databases: Host not set yet.\n") - return previous_databases - - connection = None - try: - # Input is generated by the charm - with ( - psycopg2.connect( - f"dbname='postgres' user='operator' host='/tmp/snap-private-tmp/snap.charmed-postgresql/tmp/' " - f"password='{password}' connect_timeout=1" - ) as connection, - connection.cursor() as cursor, - ): - cursor.execute("SELECT datname, datacl FROM pg_database;") - current_databases = cursor.fetchall() - cursor.execute( - "SELECT oid, rolname FROM pg_roles WHERE pg_has_role(oid, 'relation_access', 'member');" - ) - current_databases.append(cursor.fetchall()) - except psycopg2.Error as e: - with open(LOG_FILE_PATH, "a") as log_file: - log_file.write(f"Failed to retrieve databases: {e}\n") - return previous_databases - else: - # If it's the first time the databases were retrieved, then store it and use - # it for subsequent checks. - if not previous_databases: - previous_databases = current_databases - # If the databases changed, dispatch a charm event to handle this change. - elif current_databases != previous_databases: - previous_databases = current_databases - dispatch(run_cmd, unit, charm_dir, "databases_change") - return previous_databases - finally: - if connection: - connection.close() - - -def dispatch(run_cmd, unit, charm_dir, custom_event): - """Use the input juju-run command to dispatch a custom event.""" - dispatch_sub_cmd = "JUJU_DISPATCH_PATH=hooks/{} {}/dispatch" - # Input is generated by the charm - subprocess.run([run_cmd, "-u", unit, dispatch_sub_cmd.format(custom_event, charm_dir)]) # noqa: S603 - - -async def main(): - """Main watch and dispatch loop. - - Watch the Patroni API cluster info. When changes are detected, dispatch the change event. - """ - patroni_urls, run_cmd, unit, charm_dir = sys.argv[1:] - - previous_cluster_topology = {} - previous_databases = None - urls = [urljoin(url, PATRONI_CLUSTER_STATUS_ENDPOINT) for url in patroni_urls.split(",")] - member_name = unit.replace("/", "-") - while True: - # Disable TLS chain verification - context = create_default_context() - with suppress(FileNotFoundError): - context.load_verify_locations(cafile=f"{PATRONI_CONF_PATH}/{TLS_CA_BUNDLE_FILE}") - - cluster_status = None - tasks = [create_task(_httpx_get_request(url)) for url in urls] - for task in as_completed(tasks): - if result := await task: - for task in tasks: - task.cancel() - await wait(tasks) - cluster_status = result - break - if not cluster_status: - raise UnreachableUnitsError("Unable to reach cluster members") - current_cluster_topology = {} - urls = [] - is_primary = False - for member in cluster_status["members"]: - current_cluster_topology[member["name"]] = member["role"] - member_url = urljoin(member["api_url"], PATRONI_CLUSTER_STATUS_ENDPOINT) - # Call the current unit first - if member["name"] == member_name: - urls.insert(0, member_url) - # Check if the current member is the primary. - if member["role"] == "leader": - is_primary = True - else: - urls.append(member_url) - - previous_cluster_topology = check_for_cluster_topology_changes( - run_cmd, unit, charm_dir, previous_cluster_topology, current_cluster_topology - ) - - if is_primary: - previous_databases = check_for_database_changes( - run_cmd, unit, charm_dir, previous_databases - ) - - # Wait some time before checking again for a cluster topology change. - sleep(10) - - -if __name__ == "__main__": - run(main()) diff --git a/scripts/raft_observer.py b/scripts/raft_observer.py deleted file mode 100644 index d43ce724778..00000000000 --- a/scripts/raft_observer.py +++ /dev/null @@ -1,81 +0,0 @@ -# Copyright 2026 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Raft connectivity observer.""" - -from os import environ -from subprocess import run - -from charmlibs.snap import SnapCache -from pysyncobj.utility import TcpUtility -from yaml import safe_load - -from constants import PATRONI_CONF_PATH, RAFT_PARTNER_PREFIX, RAFT_PORT - -PATRONI_CONF_FILE_PATH = f"{PATRONI_CONF_PATH}/patroni.yaml" - - -def dispatch(custom_event): - """Use the input juju-run command to dispatch a custom event.""" - dispatch_sub_cmd = "JUJU_DISPATCH_PATH=hooks/{} {}/dispatch" - # Input is generated by the charm - run([ # noqa: S603 - "/usr/bin/juju-exec", - "-u", - environ["JUJU_UNIT_NAME"], - dispatch_sub_cmd.format(custom_event, environ["JUJU_CHARM_DIR"]), - ]) - - -def get_raft_password() -> str | None: - """Get the raft password from the patroni config file.""" - try: - with open(PATRONI_CONF_FILE_PATH) as conf_file: - conf_file_contents = safe_load(conf_file) - return conf_file_contents["raft"]["password"] - except Exception as e: - print(f"Unable to get password: {e}") - - -def check_raft_connection(password: str) -> None: - """Verify that the watcher has joined the Raft cluster.""" - # Get the status of the raft cluster. - syncobj_util = TcpUtility(password=password, timeout=3) - - try: - raft_status = syncobj_util.executeCommand(f"127.0.0.1:{RAFT_PORT}", ["status"]) - except Exception: - print("Cannot connect to local Raft") - return - if not raft_status: - print("No Raft status") - return - print(f"Local raft: {raft_status}") - if not raft_status["leader"] or not raft_status["has_quorum"]: - print("No leader or quorum") - return - for key in raft_status: - if key.startswith(RAFT_PARTNER_PREFIX) and raft_status[key] != 2: - addr = key.split(RAFT_PARTNER_PREFIX)[-1] - try: - print(f"{addr} Raft: {syncobj_util.executeCommand(addr, ['status'])}") - except Exception: - print(f"Unable to connect to {addr}") - continue - print(f"Potentially stuck connection with {addr}") - dispatch("raft_reconnect") - return - - -def main() -> None: - """Main watch and dispatch.""" - if not SnapCache()["charmed-postgresql"].services["patroni"]["active"]: - print("Patroni is not running") - return - if not (password := get_raft_password()): - return - check_raft_connection(password) - - -if __name__ == "__main__": - main() diff --git a/scripts/rotate_logs.py b/scripts/rotate_logs.py deleted file mode 100644 index 7fa617c94f0..00000000000 --- a/scripts/rotate_logs.py +++ /dev/null @@ -1,23 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Background process for rotating logs.""" - -import subprocess -from time import sleep - -PGBACKREST_LOGROTATE_FILE = "/etc/logrotate.d/pgbackrest.logrotate" - - -def main(): - """Main loop that calls logrotate.""" - while True: - # Input is constant - subprocess.run(["/usr/sbin/logrotate", "-f", PGBACKREST_LOGROTATE_FILE]) # noqa: S603 - - # Wait 60 seconds before executing logrotate again. - sleep(60) - - -if __name__ == "__main__": - main() diff --git a/src/charm.py b/src/charm.py index 16cf3484078..ef1d62e5e6a 100755 --- a/src/charm.py +++ b/src/charm.py @@ -4,13 +4,12 @@ """Charmed Machine Operator for PostgreSQL.""" import logging -from functools import cached_property import charm_refresh import ops -from charms.grafana_agent.v0.cos_agent import COSAgentProvider, ProtocolNotFoundError +from charms.grafana_agent.v0.cos_agent import COSAgentProvider from charms.rolling_ops.v0.rollingops import RollingOpsManager -from ops_tracing import Tracing, set_destination +from ops_tracing import Tracing from single_kernel_postgresql.charms.vm_charm import PostgreSQLVMCharm from single_kernel_postgresql.config.literals import ( METRICS_PORT, @@ -19,53 +18,15 @@ TRACING_PROTOCOL, TRACING_RELATION_NAME, ) -from single_kernel_postgresql.core.config import CharmConfig +from single_kernel_postgresql.events.tls_transfer import TLSTransfer +from single_kernel_postgresql.utils.tracing import charm_tracing_config logger = logging.getLogger(__name__) -def charm_tracing_config(endpoint_requirer) -> None: - """Utility function to set tracing destination.""" - if not endpoint_requirer.is_ready(): - return - - try: - if not (endpoint := endpoint_requirer.get_tracing_endpoint(TRACING_PROTOCOL)): - return - except ProtocolNotFoundError: - logger.warning( - "Endpoint for tracing wasn't provided as tracing backend isn't ready yet. If grafana-agent isn't connected to a tracing backend, integrate it. Otherwise this issue should resolve itself in a few events." - ) - return - - endpoint = f"{endpoint}/v1/traces" - if endpoint.startswith("https://"): - # if endpoint is https BUT we don't have a server_cert yet: - # disable charm tracing until we do to prevent tls errors - logger.warning("Cannot send traces to an https endpoint without a certificate.") - return - set_destination(endpoint, None) - - class PostgresqlOperatorCharm(PostgreSQLVMCharm): """Charmed Operator for the PostgreSQL database.""" - config_type = CharmConfig - - # Prefer the dash (-) option name with the underscore (_) fallback; the lib - # CharmState.config reads the same convention through its own cached property. - @cached_property - def config(self) -> CharmConfig: - """Return a config instance validated and parsed using the provided pydantic class.""" - config = { - config_option: self.model.config.get( - config_option.replace("_", "-"), self.model.config.get(config_option) - ) - for config_option in self.config_type.keys() # noqa: SIM118 - } - config = {k: v for k, v in config.items() if v is not None} - return self.config_type(**config) - def __init__(self, *args): super().__init__( *args, @@ -105,6 +66,7 @@ def __init__(self, *args): tracing_protocols=[TRACING_PROTOCOL], ) self.tracing = Tracing(self, tracing_relation_name=TRACING_RELATION_NAME) + self.tls_transfer = TLSTransfer(self, PEER_RELATION) charm_tracing_config(self._grafana_agent) diff --git a/src/cluster.py b/src/cluster.py deleted file mode 100644 index 7005bf42c26..00000000000 --- a/src/cluster.py +++ /dev/null @@ -1,281 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2021 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Helper class used to manage cluster lifecycle.""" - -import logging -import shutil -from contextlib import suppress -from pathlib import Path -from typing import TYPE_CHECKING - -import psutil -from ops import BlockedStatus -from pysyncobj.utility import TcpUtility, UtilityException -from single_kernel_postgresql.config.exceptions import ( - AddRaftMemberFailedError, - RaftNotPromotedError, - RaftPostgresqlNotUpError, - RaftPostgresqlStillUpError, - RemoveRaftMemberFailedError, -) -from single_kernel_postgresql.config.literals import ( - PEER_RELATION, -) -from tenacity import ( - Retrying, - wait_fixed, -) - -from constants import ( - PATRONI_CONF_PATH, - POSTGRESQL_CONF_PATH, - RAFT_PARTNER_PREFIX, - RAFT_PORT, -) - -logger = logging.getLogger(__name__) - -PG_BASE_CONF_PATH = f"{POSTGRESQL_CONF_PATH}/postgresql.conf" - -STARTED_STATES = ["running", "streaming"] -RUNNING_STATES = [*STARTED_STATES, "starting"] - -PATRONI_TIMEOUT = 10 - -if TYPE_CHECKING: - from charm import PostgresqlOperatorCharm - - -class Patroni: - """This class handles the bootstrap of a PostgreSQL database through Patroni.""" - - def __init__( - self, - charm: "PostgresqlOperatorCharm", - raft_password: str | None, - ): - """Initialize the Patroni class. - - Args: - charm: PostgreSQL charm instance. - raft_password: password for raft - """ - self.charm = charm - self.raft_password = raft_password - - def has_raft_quorum(self) -> bool: - """Check if raft cluster has quorum.""" - # Get the status of the raft cluster. - syncobj_util = TcpUtility(password=self.raft_password, timeout=3) - - raft_host = "127.0.0.1:2222" - try: - raft_status = syncobj_util.executeCommand(raft_host, ["status"]) - except UtilityException: - logger.warning("Has raft quorum: Cannot connect to raft cluster") - return False - if not raft_status: - logger.warning("Has raft quorum: No status reported") - return False - return raft_status["has_quorum"] - - def remove_raft_data(self) -> None: - """Stops Patroni and removes the raft journals.""" - logger.info("Stopping patroni") - self.charm.patroni_manager.stop_patroni() - - logger.info("Wait for postgresql to stop") - for attempt in Retrying(wait=wait_fixed(5)): - with attempt: - for proc in psutil.process_iter(["name"]): - if proc.name() == "postgres": - raise RaftPostgresqlStillUpError() - - logger.info("Removing raft data") - try: - path = Path(f"{PATRONI_CONF_PATH}/raft") - if path.exists() and path.is_dir(): - shutil.rmtree(path) - except OSError as e: - raise Exception( - f"Failed to remove previous cluster information with error: {e!s}" - ) from e - logger.info("Raft ready to reinitialise") - - def reinitialise_raft_data(self) -> None: - """Reinitialise the raft journals and promoting the unit to leader. Should only be run on sync replicas.""" - logger.info("Rerendering patroni config without peers") - self.charm.update_config(no_peers=True) - logger.info("Starting patroni") - self.charm.patroni_manager.start_patroni() - - logger.info("Waiting for new raft cluster to initialise") - for attempt in Retrying(wait=wait_fixed(5)): - with attempt: - health_status = self.charm.patroni_manager.get_patroni_health() - if ( - health_status["role"] not in ["leader", "master"] - or health_status["state"] != "running" - ): - raise RaftNotPromotedError() - - logger.info("Restarting patroni") - self.charm.patroni_manager.restart_patroni() - for attempt in Retrying(wait=wait_fixed(5)): - with attempt: - found_postgres = False - for proc in psutil.process_iter(["name"]): - if proc.name() == "postgres": - found_postgres = True - break - if not found_postgres: - raise RaftPostgresqlNotUpError() - logger.info("Raft should be unstuck") - - def cleanup_raft_cluster(self) -> bool: - """Cleanup RAFT members not belonging to the current cluster or not a related watcher.""" - # Get Raft cluster status to find all members - try: - if not self.charm.workload.is_patroni_running(): - logger.warning("Raft cleanup: Patroni service not running.") - return True - syncobj_util = TcpUtility(password=self.raft_password, timeout=3) - if raft_status := syncobj_util.executeCommand(f"127.0.0.1:{RAFT_PORT}", ["status"]): - # Find all partner nodes in the Raft cluster - # Keys look like: partner_node_status_server_10.131.50.142:2222 - for key in raft_status: - if key.startswith(RAFT_PARTNER_PREFIX) and raft_status[key] != 2: - member_addr = key.replace(RAFT_PARTNER_PREFIX, "") - member_ip = member_addr.split(":")[0] - - # Check if this is a stale watcher (not a PostgreSQL node and not current watcher) - if ( - member_ip not in self.charm._units_ips - and member_addr != self.charm.watcher_handler.watcher_raft_address - ): - logger.info(f"Removing stale Raft member: {member_addr}") - self.remove_raft_member(member_addr) - self.charm._remove_from_members_ips(member_ip) - return True - return False - except Exception as e: - logger.debug(f"Error during Raft cleanup: {e}") - return False - - def _set_stuck_raft_flag(self) -> None: - self.charm.set_unit_status(BlockedStatus("Raft majority loss, run: promote-to-primary")) - logger.warning("Remove raft member: Stuck raft cluster detected") - data_flags = {"raft_stuck": "True"} - self.charm.unit_peer_data.update(data_flags) - - # Leader doesn't always trigger when changing it's own peer data. - if self.charm.unit.is_leader(): - self.charm.on[PEER_RELATION].relation_changed.emit( - unit=self.charm.unit, - app=self.charm.app, - relation=self.charm.model.get_relation(PEER_RELATION), - ) - - def get_raft_status(self) -> dict | None: - """Get local raft status.""" - # Get the status of the raft cluster. - syncobj_util = TcpUtility(password=self.raft_password, timeout=3) - - raft_host = f"127.0.0.1:{RAFT_PORT}" - with suppress(UtilityException): - return syncobj_util.executeCommand(raft_host, ["status"]) - - def remove_raft_member( - self, member_address: str, remote_address: str | None = None, set_raft_flags: bool = True - ) -> None: - """Remove a member from the raft cluster. - - The raft cluster is a different cluster from the Patroni cluster. - It is responsible for defining which Patroni member can update - the primary member in the DCS. - - Raises: - RaftMemberNotFoundError: if the member to be removed - is not part of the raft cluster. - """ - if self.charm.has_raft_keys(): - logger.debug("Remove raft member: Raft already in recovery") - return - - # Get the status of the raft cluster. - syncobj_util = TcpUtility(password=self.raft_password, timeout=3) - - raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}" - try: - raft_status = syncobj_util.executeCommand(raft_host, ["status"]) - except UtilityException as e: - logger.warning("Remove raft member: Cannot connect to raft cluster") - raise RemoveRaftMemberFailedError() from e - if not raft_status: - logger.warning("Remove raft member: No raft status") - raise RemoveRaftMemberFailedError() from None - - # Check whether the member is still part of the raft cluster. - if f"{RAFT_PARTNER_PREFIX}{member_address}" not in raft_status: - logger.debug("Remove raft member: Address already removed") - return - - # If there's no quorum and the leader left raft cluster is stuck - if raft_status["has_quorum"] and not raft_status["leader"]: - logger.warning("Remove raft member: No raft leader") - raise RemoveRaftMemberFailedError() from None - if ( - not raft_status["has_quorum"] - and (not raft_status["leader"] or raft_status["leader"].address == member_address) - and set_raft_flags - ): - self._set_stuck_raft_flag() - return - - # Remove the member from the raft cluster. - try: - result = syncobj_util.executeCommand(raft_host, ["remove", member_address]) - except UtilityException as e: - logger.debug("Remove raft member: Remove call failed") - raise RemoveRaftMemberFailedError() from e - - if not result or not result.startswith("SUCCESS"): - logger.debug(f"Remove raft member: Remove call not successful with {result}") - raise RemoveRaftMemberFailedError() from None - - def add_raft_member(self, member_address: str, remote_address: str | None = None) -> None: - """Add a member to the raft cluster.""" - if self.charm.has_raft_keys(): - logger.debug("Add raft member: Raft already in recovery") - return - - # Get the status of the raft cluster. - syncobj_util = TcpUtility(password=self.raft_password, timeout=3) - - raft_host = remote_address if remote_address else f"127.0.0.1:{RAFT_PORT}" - try: - raft_status = syncobj_util.executeCommand(raft_host, ["status"]) - except UtilityException as e: - logger.warning("Add raft member: Cannot connect to raft cluster") - raise AddRaftMemberFailedError() from e - - if not raft_status: - logger.warning("Add raft member: No raft status") - raise AddRaftMemberFailedError() from None - - # Check whether the member is still part of the raft cluster. - if f"{RAFT_PARTNER_PREFIX}{member_address}" in raft_status: - logger.debug("Add raft member: Address already added") - return - - try: - result = syncobj_util.executeCommand(raft_host, ["add", member_address]) - except UtilityException as e: - logger.debug("Add raft member: Remove call failed") - raise AddRaftMemberFailedError() from e - - if not result or not result.startswith("SUCCESS"): - logger.debug(f"Add raft member: Remove call not successful with {result}") - raise AddRaftMemberFailedError() from None diff --git a/src/cluster_topology_observer.py b/src/cluster_topology_observer.py deleted file mode 100644 index da2cd4678d5..00000000000 --- a/src/cluster_topology_observer.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright 2023 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Cluster topology changes observer.""" - -import logging -import os -import signal -import subprocess -from pathlib import Path -from sys import version_info -from typing import TYPE_CHECKING - -from charmlibs.systemd import daemon_reload, service_enable -from jinja2 import Template -from ops import ActiveStatus, CharmEvents, EventBase, EventSource, Object -from single_kernel_postgresql.config.enums import Substrates - -from single_kernel_postgresql.utils import render_file - -if TYPE_CHECKING: - from charm import PostgresqlOperatorCharm -logger = logging.getLogger(__name__) - -# File path for the spawned cluster topology observer process to write logs. -LOG_FILE_PATH = "/var/log/cluster_topology_observer.log" - - -class ClusterTopologyChangeEvent(EventBase): - """A custom event for cluster topology changes.""" - - -class DatabasesChangeEvent(EventBase): - """A custom event for databases changes.""" - - -class RaftReconnectEvent(EventBase): - """A custom event for databases changes.""" - - -class ClusterTopologyChangeCharmEvents(CharmEvents): - """A CharmEvents extension for cluster topology changes. - - Includes :class:`ClusterTopologyChangeEvent` in those that can be handled. - """ - - cluster_topology_change = EventSource(ClusterTopologyChangeEvent) - databases_change = EventSource(DatabasesChangeEvent) - raft_reconnect = EventSource(RaftReconnectEvent) - - -class ClusterTopologyObserver(Object): - """Observes changing topology in the cluster. - - Observed cluster topology changes cause :class"`ClusterTopologyChangeEvent` to be emitted. - """ - - def __init__(self, charm: "PostgresqlOperatorCharm", run_cmd: str) -> None: - """Constructor for ClusterTopologyObserver. - - Args: - charm: the charm that is instantiating the library. - run_cmd: run command to use to dispatch events. - """ - super().__init__(charm, "cluster-topology-observer") - - self._charm = charm - self._run_cmd = run_cmd - - def start_observer(self) -> None: - """Start the cluster topology observer running in a new process.""" - if not isinstance(self._charm.unit.status, ActiveStatus) or self._charm._peers is None: - return - if "observer-pid" in self._charm.state.peer.data: - # Double check that the PID exists - pid = int(self._charm.state.peer.data["observer-pid"]) - try: - os.kill(pid, 0) - return - except OSError: - pass - if not self._charm.state.unit_ip: - logging.info("Cannot start cluster topology observer process: No IP set") - return - - logging.info("Starting cluster topology observer process") - - # We need to trick Juju into thinking that we are not running - # in a hook context, as Juju will disallow use of juju-run. - new_env = os.environ.copy() - if "JUJU_CONTEXT_ID" in new_env: - new_env.pop("JUJU_CONTEXT_ID") - # Generate the venv path based on the existing lib path - for loc in new_env["PYTHONPATH"].split(":"): - path = Path(loc) - venv_path = ( - path - / ".." - / "venv" - / "lib" - / f"python{version_info.major}.{version_info.minor}" - / "site-packages" - ) - if path.stem == "lib": - new_env["PYTHONPATH"] = f"{venv_path.resolve()}:{new_env['PYTHONPATH']}" - break - - urls = [self._charm.state.patroni_url] + [ - self._charm.state.patroni_url.replace(self._charm.state.unit_ip, peer) - for peer in list(self._charm._peer_members_ips) - ] - - # Input is generated by the charm - pid = subprocess.Popen( # noqa: S603 - [ - "/usr/bin/python3", - "scripts/cluster_topology_observer.py", - ",".join(urls), - self._run_cmd, - self._charm.unit.name, - self._charm.charm_dir, - ], - # File shouldn't close - stdout=open(LOG_FILE_PATH, "a"), # noqa: SIM115 - stderr=subprocess.STDOUT, - env=new_env, - ).pid - - self._charm.state.peer.data.update({"observer-pid": f"{pid}"}) - logging.info(f"Started cluster topology observer process with PID {pid}") - - def stop_observer(self): - """Stop the running observer process if we have previously started it.""" - if self._charm._peers is None or "observer-pid" not in self._charm.state.peer.data: - return - - if stored_pid := self._charm.state.peer.data.get("observer-pid"): - observer_pid = int(stored_pid) - - try: - os.kill(observer_pid, signal.SIGINT) - msg = "Stopped running cluster topology observer process with PID {}" - logging.info(msg.format(observer_pid)) - self._charm.state.peer.data.update({"observer-pid": ""}) - except OSError: - pass - - -def copy_environment() -> dict[str, str]: - """Environment variables to pass to a script.""" - new_env = os.environ.copy() - if "JUJU_CONTEXT_ID" in new_env: - new_env.pop("JUJU_CONTEXT_ID") - # Generate the venv path based on the existing lib path - for loc in new_env["PYTHONPATH"].split(":"): - path = Path(loc) - venv_path = ( - path - / ".." - / "venv" - / "lib" - / f"python{version_info.major}.{version_info.minor}" - / "site-packages" - ) - if path.stem == "lib": - new_env["PYTHONPATH"] = f"{venv_path.resolve()}:{new_env['PYTHONPATH']}" - break - return {var: val for var, val in new_env.items() if "JUJU" in var or "PATH" in var} - - -def start_raft_observer() -> None: - """Render systemd units and start the observer.""" - timer_service_file = "/etc/systemd/system/raft-observer.timer" - oneshot_service_file = "/etc/systemd/system/raft-observer.service" - - with open("templates/raft-observer.service.j2") as file: - template = Template(file.read()) - - rendered = template.render( - envvars=copy_environment(), script=f"{os.getcwd()}/scripts/raft_observer.py" - ) - render_file(Substrates.VM, oneshot_service_file, rendered, 0o644, change_owner=False) - - with open("templates/raft-observer.timer.j2") as file: - template = Template(file.read()) - - rendered = template.render() - render_file(Substrates.VM, timer_service_file, rendered, 0o644, change_owner=False) - - # Reload systemd to pick up the new service - daemon_reload() - service_enable(timer_service_file, "--now") - logger.info("Installed and enabled raft observer timer") diff --git a/src/rotate_logs.py b/src/rotate_logs.py deleted file mode 100644 index f6dd621e268..00000000000 --- a/src/rotate_logs.py +++ /dev/null @@ -1,70 +0,0 @@ -# Copyright 2024 Canonical Ltd. -# See LICENSE file for licensing details. - -"""Background process for rotating logs.""" - -import logging -import os -import signal -import subprocess -from typing import TYPE_CHECKING - -from ops.framework import Object -from ops.model import ActiveStatus -from single_kernel_postgresql.config.literals import PGBACKREST_LOGROTATE_FILE - -if TYPE_CHECKING: - from charm import PostgresqlOperatorCharm - -logger = logging.getLogger(__name__) - -# File path for the spawned rotate logs process to write logs. -LOG_FILE_PATH = "/var/log/rotate_logs.log" - - -class RotateLogs(Object): - """Rotate logs every minute.""" - - def __init__(self, charm: "PostgresqlOperatorCharm"): - super().__init__(charm, "rotate-logs") - self._charm = charm - - def start_log_rotation(self): - """Start the rotate logs running in a new process.""" - if ( - not isinstance(self._charm.unit.status, ActiveStatus) - or self._charm._peers is None - or not os.path.exists(PGBACKREST_LOGROTATE_FILE) - ): - return - if "rotate-logs-pid" in self._charm.unit_peer_data: - # Double check that the PID exists. - pid = int(self._charm.unit_peer_data["rotate-logs-pid"]) - try: - os.kill(pid, 0) - return - except OSError: - pass - - logging.info("Starting rotate logs process") - - pid = subprocess.Popen( - ["/usr/bin/python3", "scripts/rotate_logs.py"], - # File should not close - stdout=open(LOG_FILE_PATH, "a"), # noqa: SIM115 - stderr=subprocess.STDOUT, - ).pid - - self._charm.unit_peer_data.update({"rotate-logs-pid": f"{pid}"}) - logging.info(f"Started rotate logs process with PID {pid}") - - def stop_log_rotation(self) -> None: - """Stop the running rotate logs process if we have previously started it.""" - if stored_pid := self._charm.unit_peer_data.get("rotate-logs-pid"): - pid = int(stored_pid) - try: - os.kill(pid, signal.SIGINT) - logging.info(f"Stopped rotate logs process with PID {pid}") - self._charm.unit_peer_data.update({"rotate-logs-pid": ""}) - except OSError: - pass diff --git a/templates/raft-observer.service.j2 b/templates/raft-observer.service.j2 deleted file mode 100644 index 8f8debdcbf6..00000000000 --- a/templates/raft-observer.service.j2 +++ /dev/null @@ -1,13 +0,0 @@ -[Unit] -Description=Patroni Raft observer service -After=snap.charmed-postgresql.patroni.service -Wants=network.target - -[Service] -Type=oneshot -{% for var, val in envvars.items() -%} -Environment="{{ var }}={{ val }}" -{% endfor -%} -ExecStart=python3 {{ script }} -StandardOutput=journal -StandardError=journal diff --git a/templates/raft-observer.timer.j2 b/templates/raft-observer.timer.j2 deleted file mode 100644 index f7a73f0b1dd..00000000000 --- a/templates/raft-observer.timer.j2 +++ /dev/null @@ -1,10 +0,0 @@ -[Unit] -Description=Patroni Raft observer timer - -[Timer] -OnActiveSec=10s -OnUnitInactiveSec=10s -AccuracySec=1s - -[Install] -WantedBy=timers.target diff --git a/tests/unit/test_charm.py b/tests/unit/test_charm.py index 868734e9a53..d0ae46d0b20 100644 --- a/tests/unit/test_charm.py +++ b/tests/unit/test_charm.py @@ -50,24 +50,24 @@ def test_config_fallback(harness): harness.disable_hooks() charm: PostgresqlOperatorCharm = harness.charm - assert charm.config.connection_authentication_timeout == 60 + assert charm.state.config.connection_authentication_timeout == 60 harness.update_config({"connection_authentication_timeout": 50}) - del harness.charm.config - assert charm.config.connection_authentication_timeout == 50 + del charm.state.config + assert charm.state.config.connection_authentication_timeout == 50 harness.update_config({"connection-authentication-timeout": 90}) - del harness.charm.config - assert charm.config.connection_authentication_timeout == 90 + del charm.state.config + assert charm.state.config.connection_authentication_timeout == 90 harness.update_config(unset=["connection-authentication-timeout"]) - del harness.charm.config - assert charm.config.connection_authentication_timeout == 50 + del charm.state.config + assert charm.state.config.connection_authentication_timeout == 50 harness.update_config(unset=["connection_authentication_timeout"]) - del harness.charm.config - assert charm.config.connection_authentication_timeout == 60 + del charm.state.config + assert charm.state.config.connection_authentication_timeout == 60 harness.update_config({"connection-authentication-timeout": 120}) - del harness.charm.config - assert charm.config.connection_authentication_timeout == 120 + del charm.state.config + assert charm.state.config.connection_authentication_timeout == 120 From 297afd9ecb8f9e37e5aedb92bcbadffcb7e3f637 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Mon, 12 Oct 2026 12:54:45 -0300 Subject: [PATCH 03/13] fix(charm): import ops.log for the JujuLogHandler isinstance check --- src/charm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index ef1d62e5e6a..083c1297a8f 100755 --- a/src/charm.py +++ b/src/charm.py @@ -6,7 +6,7 @@ import logging import charm_refresh -import ops +import ops.log from charms.grafana_agent.v0.cos_agent import COSAgentProvider from charms.rolling_ops.v0.rollingops import RollingOpsManager from ops_tracing import Tracing From 976f46f0ec6afa47d6e44cf25220392c33606776 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Mon, 12 Oct 2026 13:01:26 -0300 Subject: [PATCH 04/13] fix(charm): drop the orphaned plugin_pg_cron_enable stub from config.yaml The union resolution left a second default/description block under plugin_orafce_enable (the pg-cron block whose header the merge lost); upstream 16/edge dropped plugin_pg_cron_enable entirely, so the stub is removed rather than re-headed. charmcraft pack fails on the duplicate 'default' key. --- config.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/config.yaml b/config.yaml index 70c8e095e8f..fd45e70be30 100644 --- a/config.yaml +++ b/config.yaml @@ -1577,12 +1577,6 @@ options: Deprecated: Use plugin-orafce-enable instead. Enable orafce extension - default: false - type: boolean - description: | - Deprecated: Use plugin-pg-cron-enable instead. - - Enable pg_cron extension plugin_pg_freespacemap_enable: default: false type: boolean From 00eeb60f7f2145de272a3ba27a115d679f15ef7a Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Mon, 12 Oct 2026 13:20:56 -0300 Subject: [PATCH 05/13] fix(charm): give rolling-ops a named _restart delegate on the charm The lib resolves its callback by name (getattr(charm, callback.__name__)), so a bound method on the events handler raised AttributeError on every restart flow. --- src/charm.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/charm.py b/src/charm.py index 083c1297a8f..3cf0ecfec68 100755 --- a/src/charm.py +++ b/src/charm.py @@ -41,12 +41,13 @@ def __init__(self, *args): if isinstance(handler, ops.log.JujuLogHandler): handler.setFormatter(logging.Formatter("{name}:{message}", style="{")) - # The rolling-ops manager is a charm lib; it stays charm-side and its - # callback lands in the lib events handler. + # The rolling-ops manager is a charm lib; it stays charm-side. It resolves + # its callback BY NAME on the charm (getattr(charm, callback.__name__)), so + # the charm must carry a named method delegating to the lib handler. self.restart_manager = RollingOpsManager( charm=self, relation="restart", - callback=self.postgresql_events_handler._restart, + callback=self._restart, ) # COS requirers (charm libs stay charm-side). @@ -69,6 +70,10 @@ def __init__(self, *args): self.tls_transfer = TLSTransfer(self, PEER_RELATION) charm_tracing_config(self._grafana_agent) + def _restart(self, event) -> None: + """Delegate to the lib events handler (rolling-ops resolves the callback by name).""" + self.postgresql_events_handler._restart(event) + if __name__ == "__main__": from ops.main import main From 673cbc0c16cc11b819a9077b07afdd1b43b380b8 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 20:36:34 -0300 Subject: [PATCH 06/13] fix(deps): repin the single-kernel library to 99d3e4d4 (failure fixes) --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 882e07d2b25..35b99150127 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "7cd947ee.tar.gz", hash = "sha256:8a9f77deda5366f1b55ae58c1084f6980a5d131994bc4629ff34700ae5c8a888"}, + {file = "99d3e4d4.tar.gz", hash = "sha256:317b87b9df6d4860e32e32a199b43e12e091d99cd66e08ea81c95a17e4bcc1a3"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7cd947ee.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/99d3e4d4.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "dbcd117b8b27eff89c63447d4ba2ab28c71c57b3369c2cdc74e55a644211effa" +content-hash = "e4b28854bf4b9bc7d62affaef337897912d4e58cf4ad69dbbc49f71cf1e4e0eb" diff --git a/pyproject.toml b/pyproject.toml index f583c2a0ed8..b77faf66826 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7cd947ee.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/99d3e4d4.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py From 723f66558a9c1edfcd816c1a487dbf560cdc8938 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 20:36:34 -0300 Subject: [PATCH 07/13] fix(charm): restore the wrong-architecture boot guard The thin charm dropped trunk's first-platform-import try/except; wrong-arch deploys died at import time with an error unit instead of the blocked warning charm (test_wrong_arch). --- src/charm.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/charm.py b/src/charm.py index 3cf0ecfec68..6c4b30d56d2 100755 --- a/src/charm.py +++ b/src/charm.py @@ -5,12 +5,26 @@ import logging +# First platform-specific import; on wrong architectures fall back to a +# blocked warning charm (the wheel for this arch is unavailable). +try: + from single_kernel_postgresql.charms.vm_charm import PostgreSQLVMCharm +except ModuleNotFoundError: + from ops.main import main + from single_kernel_postgresql.utils.arch import ( + WrongArchitectureWarningCharm, + is_wrong_architecture, + ) + + if is_wrong_architecture() and __name__ == "__main__": + main(WrongArchitectureWarningCharm) + raise + import charm_refresh import ops.log from charms.grafana_agent.v0.cos_agent import COSAgentProvider from charms.rolling_ops.v0.rollingops import RollingOpsManager from ops_tracing import Tracing -from single_kernel_postgresql.charms.vm_charm import PostgreSQLVMCharm from single_kernel_postgresql.config.literals import ( METRICS_PORT, PEER_RELATION, From 4058bf3ee91ba8f12fffd0eb0138bf68a7c3f003 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 21:38:23 -0300 Subject: [PATCH 08/13] fix(deps): repin the single-kernel library to c687c9cc (watcher fix) --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 35b99150127..9ce3c21c12f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "99d3e4d4.tar.gz", hash = "sha256:317b87b9df6d4860e32e32a199b43e12e091d99cd66e08ea81c95a17e4bcc1a3"}, + {file = "c687c9cc.tar.gz", hash = "sha256:1be39df0c749f0e3495b43e3dbd36ad785a37dbb1b2ec10850b544c2fe23c301"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/99d3e4d4.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/c687c9cc.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "e4b28854bf4b9bc7d62affaef337897912d4e58cf4ad69dbbc49f71cf1e4e0eb" +content-hash = "e1062a37750b33a6c88c14f9d2c4cd33fb8d8c9c0235c210bb4e10d562d410a3" diff --git a/pyproject.toml b/pyproject.toml index b77faf66826..e5c610a21d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/99d3e4d4.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/c687c9cc.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py From 628abd62ee7adb474789132cf7b06122e2e04d43 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 22:26:49 -0300 Subject: [PATCH 09/13] fix(deps): repin the single-kernel library to 1c605328 (tls watcher fix) --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 9ce3c21c12f..4857c6a98a7 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "c687c9cc.tar.gz", hash = "sha256:1be39df0c749f0e3495b43e3dbd36ad785a37dbb1b2ec10850b544c2fe23c301"}, + {file = "1c605328.tar.gz", hash = "sha256:83a7b92ea74663400216524ca21b944c001a8e7e775cc4d3b7a6bfd37dc22a5b"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/c687c9cc.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/1c605328.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "e1062a37750b33a6c88c14f9d2c4cd33fb8d8c9c0235c210bb4e10d562d410a3" +content-hash = "128de7d4a5c8a880dd563ae322421cfdbc292ff56b2981c0f7f4f7b3cbe68c76" diff --git a/pyproject.toml b/pyproject.toml index e5c610a21d5..304d79ca592 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/c687c9cc.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/1c605328.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py From 38c3d534b6edb532b91b44b3fdb657b7a6a44926 Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 22:30:39 -0300 Subject: [PATCH 10/13] fix(deps): repin the single-kernel library to 7ac4833e (patroni alias fix) --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 4857c6a98a7..ae51974da75 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "1c605328.tar.gz", hash = "sha256:83a7b92ea74663400216524ca21b944c001a8e7e775cc4d3b7a6bfd37dc22a5b"}, + {file = "7ac4833e.tar.gz", hash = "sha256:64e16ec93b1b0f96f5321cc3750fa24502afa567d61e7f4d379fe402914f15dd"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/1c605328.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7ac4833e.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "128de7d4a5c8a880dd563ae322421cfdbc292ff56b2981c0f7f4f7b3cbe68c76" +content-hash = "d382b91e8732f1768cefbae8dd6c0c5ddfebf759749149f020761887638f184b" diff --git a/pyproject.toml b/pyproject.toml index 304d79ca592..17b109e3267 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/1c605328.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7ac4833e.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py From ca1f42624164fe8975b3716ee1c9644eb6e040ae Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Tue, 13 Oct 2026 23:59:31 -0300 Subject: [PATCH 11/13] chore: pin the lib to the migration-integration SHA b17e5c50 --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index ae51974da75..5c11f3cd4c9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "7ac4833e.tar.gz", hash = "sha256:64e16ec93b1b0f96f5321cc3750fa24502afa567d61e7f4d379fe402914f15dd"}, + {file = "b17e5c50.tar.gz", hash = "sha256:c7f5e2c39753a04cab10f33b4fd6d465e2f1e1a392229218a206ecdd9ad6c2f7"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7ac4833e.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/b17e5c50.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "d382b91e8732f1768cefbae8dd6c0c5ddfebf759749149f020761887638f184b" +content-hash = "a7349f4627407750959082cd421693d4ff5ac1d831988b3c32225dae33d1b2f7" diff --git a/pyproject.toml b/pyproject.toml index 17b109e3267..66053967bf5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/7ac4833e.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/b17e5c50.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py From 86f413a67444966446e90c5132b679a2b90a602b Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Wed, 14 Oct 2026 00:16:17 -0300 Subject: [PATCH 12/13] test(terraform): handle both local-charm refresh outcomes Ports the test_refresh_to_local_charm rewrite from #2036: wait for the "Refresh incompatible" block (unreleased PR builds always block), force the refresh with a 30-minute budget, and gate the final active+idle wait on the statuses' since timestamps so the pre-refresh state cannot pass vacuously. --- tests/integration/jubilant_helpers.py | 40 ++++++++++++++++ .../terraform/test_terraform_deploy.py | 48 +++++++++++++------ 2 files changed, 73 insertions(+), 15 deletions(-) diff --git a/tests/integration/jubilant_helpers.py b/tests/integration/jubilant_helpers.py index 0d0bc427798..ec42057e732 100644 --- a/tests/integration/jubilant_helpers.py +++ b/tests/integration/jubilant_helpers.py @@ -1709,3 +1709,43 @@ def instance_ip(juju: JujuFixture, instance: str) -> str: for line in output.splitlines(): if instance in line: return line.split()[2] + + +def blocked_on_incompatible_refresh(status: jubilant.Status, app_name: str) -> bool: + """Whether the app, refreshed to a local charm, is blocked as "Refresh incompatible". + + charm_refresh blocks when the packed charm's charm version is unreleased (`…postN.dev0+ + gSHA`), which is the case for every PR CI build. Released-style builds—a nightly run when + the branch tip equals the last release tag—are accepted instead and never block. + """ + app = status.apps[app_name] + return ( + app.charm.startswith("local:") + and app.is_blocked + and "Refresh incompatible" in app.app_status.message + ) + + +def parse_juju_status_timestamp(timestamp: str) -> datetime: + """Parse a juju status timestamp, e.g. `12 Oct 2026 17:36:58-03:00`.""" + return datetime.strptime(timestamp, "%d %b %Y %H:%M:%S%z") + + +def units_status_reset_after( + status: jubilant.Status, app_name: str, stale_sinces: dict[str, str] +) -> bool: + """Whether every unit's workload status was re-set after the statuses in `stale_sinces`. + + Right after `juju refresh` flips the charm origin to `local:`, the app still shows the + pre-refresh statuses (the charm's hook starts later)—which are also active and idle—so + the statuses' `since` timestamps are what tells a completed refresh apart from the state + the old charm left behind. + """ + for unit_name, unit in status.apps[app_name].units.items(): + since = unit.workload_status.since + stale = stale_sinces.get(unit_name, "") + if not since or not stale: + return False + if parse_juju_status_timestamp(since) <= parse_juju_status_timestamp(stale): + return False + return True diff --git a/tests/integration/terraform/test_terraform_deploy.py b/tests/integration/terraform/test_terraform_deploy.py index 4a068f92ac5..1c2d9a91c6c 100644 --- a/tests/integration/terraform/test_terraform_deploy.py +++ b/tests/integration/terraform/test_terraform_deploy.py @@ -13,6 +13,7 @@ """ import json +import logging import os import shutil import subprocess @@ -22,6 +23,7 @@ import pytest from .. import architecture +from ..jubilant_helpers import blocked_on_incompatible_refresh, units_status_reset_after _JUJU_PROVIDER = "registry.terraform.io/juju/juju" # Major admitted by the module's `required_providers` constraint (`~> 1.0`). Kept in sync with @@ -32,6 +34,9 @@ TERRAFORM_MODULE = REPO_ROOT / "terraform" APP = "postgresql" TIMEOUT = 20 * 60 +# The forced (or accepted) refresh fetches the pinned snap (~7.5 min on cold runners) and then +# settles replication (~10 min) — the deploy budget is not enough for that flow. +REFRESH_TIMEOUT = 30 * 60 # `terraform apply` blocks until the charm's units are created, so give it the deploy budget. TF_TIMEOUT = 15 * 60 TF_BINARY = os.getenv("TF_BINARY") or "terraform" @@ -118,18 +123,25 @@ def test_refresh_to_local_charm(juju: jubilant.Juju, charm: str) -> None: juju.wait(lambda status: jubilant.all_agents_idle(status, APP), timeout=TIMEOUT) juju.refresh(app=APP, path=charm) - # Check the charm origin so the wait cannot succeed before the refresh has started. - juju.wait( - lambda status: ( - status.apps[APP].charm.startswith("local:") - and jubilant.all_agents_idle(status, APP) - and (status.apps[APP].is_active or status.apps[APP].is_blocked) - ), - error=lambda status: jubilant.any_error(status, APP), - timeout=TIMEOUT, - ) - app_status = juju.status().apps[APP] - if app_status.is_blocked: + # The compatibility check runs on the charm's first hook after the refresh: unreleased + # charm builds block ("Refresh incompatible"), released-style ones don't. + # Snapshot the statuses so the final wait can tell the post-refresh state from this + # pre-refresh one (also active and idle). + stale_sinces = { + unit_name: unit.workload_status.since + for unit_name, unit in juju.status().apps[APP].units.items() + } + try: + juju.wait( + lambda status: blocked_on_incompatible_refresh(status, APP), + error=lambda status: jubilant.any_error(status, APP), + # The block is set and re-asserted on the ~5 min update-status cadence. + timeout=TIMEOUT, + ) + except TimeoutError: + logging.info("No incompatibility block: waiting for the accepted refresh to settle") + else: + app_status = juju.status().apps[APP] # A single unit never pauses for `resume-refresh`; it only blocks on incompatibility. assert "Refresh incompatible" in app_status.app_status.message, ( f"unexpected blocked status: {app_status.app_status.message!r}" @@ -138,11 +150,17 @@ def test_refresh_to_local_charm(juju: jubilant.Juju, charm: str) -> None: unit=unit, action="force-refresh-start", params={"check-compatibility": False}, - wait=TIMEOUT, + wait=REFRESH_TIMEOUT, ) + # The `since` gate rules out the pre-refresh state (also active and idle): a refresh + # that never progressed keeps the app blocked or waiting and fails the waits above. juju.wait( - lambda status: jubilant.all_active(status, APP) and jubilant.all_agents_idle(status, APP), + lambda status: ( + jubilant.all_active(status, APP) + and jubilant.all_agents_idle(status, APP) + and units_status_reset_after(status, APP, stale_sinces) + ), error=lambda status: jubilant.any_error(status, APP), - timeout=TIMEOUT, + timeout=REFRESH_TIMEOUT, ) From 59535087300ad85b54bb854251e96b4425985c7a Mon Sep 17 00:00:00 2001 From: Marcelo Neppel Date: Wed, 14 Oct 2026 10:44:56 -0300 Subject: [PATCH 13/13] chore: pin the lib to the migration-integration SHA 3600e902 --- poetry.lock | 6 +++--- pyproject.toml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 5c11f3cd4c9..3c812c4e3d8 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2193,7 +2193,7 @@ optional = false python-versions = ">=3.8,<4.0" groups = ["main", "integration"] files = [ - {file = "b17e5c50.tar.gz", hash = "sha256:c7f5e2c39753a04cab10f33b4fd6d465e2f1e1a392229218a206ecdd9ad6c2f7"}, + {file = "3600e902.tar.gz", hash = "sha256:4f60fa0373cd0d1cc7704b4e95c757e2fdb4552bcb4bd29f05b1dcc3b94dfe8c"}, ] [package.dependencies] @@ -2225,7 +2225,7 @@ vm = ["charmlibs-snap (>=1.0.1,<2) ; python_version >= \"3.12\"", "charmlibs-sys [package.source] type = "url" -url = "https://github.com/canonical/postgresql-single-kernel-library/archive/b17e5c50.tar.gz" +url = "https://github.com/canonical/postgresql-single-kernel-library/archive/3600e902.tar.gz" [[package]] name = "prompt-toolkit" @@ -3683,4 +3683,4 @@ h11 = ">=0.16.0,<1" [metadata] lock-version = "2.1" python-versions = ">=3.12,<4.0" -content-hash = "a7349f4627407750959082cd421693d4ff5ac1d831988b3c32225dae33d1b2f7" +content-hash = "07bf2367967a3602a37c2a3f8a4cb49f5938609ef1bd6773128c0ebe9e586679" diff --git a/pyproject.toml b/pyproject.toml index 66053967bf5..8ed81bb1352 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,7 +21,7 @@ charm-refresh = "^3.1.0.2" charmlibs-snap = "^1.0.1" charmlibs-systemd = "^1.0.0.post0" charmlibs-interfaces-tls-certificates = "^1.12.0" -postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/b17e5c50.tar.gz"} +postgresql-charms-single-kernel = {extras = ["postgresql", "vm"], url = "https://github.com/canonical/postgresql-single-kernel-library/archive/3600e902.tar.gz"} [tool.poetry.group.charm-libs.dependencies] # data_platform_libs/v0/data_interfaces.py