From 590b3e1c6877f8a77fee887119cd216d5ab3b7aa Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 15:53:12 +0800 Subject: [PATCH 01/27] fix(cluster): make qvotec markers async for lmon --- src/backend/cluster/cluster_clean_leave.c | 248 +++++-- src/backend/cluster/cluster_debug.c | 10 + src/backend/cluster/cluster_guc.c | 10 + src/backend/cluster/cluster_inject.c | 1 + src/backend/cluster/cluster_lmon.c | 119 ++++ src/backend/cluster/cluster_node_remove.c | 166 ++++- src/backend/cluster/cluster_qvotec.c | 5 + src/backend/cluster/cluster_reconfig.c | 641 +++++++++++++++--- src/backend/cluster/cluster_write_fence.c | 34 + src/include/cluster/cluster_clean_leave.h | 11 + src/include/cluster/cluster_guc.h | 1 + src/include/cluster/cluster_lmon.h | 9 + src/include/cluster/cluster_marker_async.h | 208 ++++++ src/include/cluster/cluster_node_remove.h | 11 + src/include/cluster/cluster_reconfig.h | 18 + src/include/cluster/cluster_write_fence.h | 11 + src/test/cluster_tap/t/015_inject.pl | 8 +- src/test/cluster_tap/t/017_debug.pl | 12 +- src/test/cluster_tap/t/018_shared_fs.pl | 4 +- src/test/cluster_tap/t/020_shmem_registry.pl | 4 +- src/test/cluster_tap/t/021_block_format.pl | 4 +- src/test/cluster_tap/t/022_itl_slot.pl | 4 +- .../cluster_tap/t/023_buffer_descriptor.pl | 10 +- src/test/cluster_tap/t/024_pcm_lock.pl | 10 +- src/test/cluster_tap/t/030_acceptance.pl | 10 +- ...358_cluster_2_29a_marker_async_liveness.pl | 176 +++++ src/test/cluster_unit/Makefile | 2 +- src/test/cluster_unit/test_cluster_debug.c | 25 + src/test/cluster_unit/test_cluster_lmon.c | 24 +- .../cluster_unit/test_cluster_marker_async.c | 236 +++++++ src/test/cluster_unit/test_cluster_qvotec.c | 8 + src/test/cluster_unit/test_cluster_reconfig.c | 20 + 32 files changed, 1864 insertions(+), 196 deletions(-) create mode 100644 src/include/cluster/cluster_marker_async.h create mode 100644 src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl create mode 100644 src/test/cluster_unit/test_cluster_marker_async.c diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index d6152a4e28..78a95163d9 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -59,6 +59,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ #include "cluster/cluster_reconfig.h" /* apply_clean_leave + record/is_clean_departed + join_in_progress */ @@ -745,6 +746,126 @@ cluster_clean_leave_ic_send_ack(int32 dest_node_id, int32 leaving_node_id, uint6 /* qvotec-side per-process handshake cursors (mirror the fence-marker ones). */ static uint64 cl_qvotec_last_processed_marker_seq = 0; static uint64 cl_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync cl_lmon_marker_async; +static ClusterLeaveIntentMarker cl_lmon_marker; +static int cl_lmon_marker_phase = 0; +static int32 cl_lmon_marker_leaving = -1; +static uint64 cl_lmon_marker_epoch = 0; +static bool cl_lmon_marker_submitted = false; + +static void cl_build_marker(ClusterLeaveIntentMarker *m, uint8 marker_phase, int32 leaving, + uint64 epoch); + +typedef enum ClusterLeaveAsyncMarkerResult { + CL_LEAVE_ASYNC_PENDING = 0, + CL_LEAVE_ASYNC_ACKED, + CL_LEAVE_ASYNC_FAILED +} ClusterLeaveAsyncMarkerResult; + +static ClusterMarkerAsyncKind +cl_marker_phase_kind(int phase) +{ + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) + return CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED; + return CLUSTER_MARKER_KIND_UNKNOWN; +} + +static void +cl_release_lmon_marker_stage(void) +{ + cluster_marker_async_release_stage(&cl_lmon_marker_async); + memset(&cl_lmon_marker, 0, sizeof(cl_lmon_marker)); + cl_lmon_marker_phase = 0; + cl_lmon_marker_leaving = -1; + cl_lmon_marker_epoch = 0; + cl_lmon_marker_submitted = false; +} + +static bool +cl_start_lmon_marker_stage(const ClusterLeaveIntentMarker *m, int phase, int32 leaving, + uint64 epoch) +{ + if (cl_lmon_marker_async.has_staged_event) + return false; + cl_lmon_marker = *m; + cl_lmon_marker_phase = phase; + cl_lmon_marker_leaving = leaving; + cl_lmon_marker_epoch = epoch; + cl_lmon_marker_async.has_staged_event = true; + cl_lmon_marker_submitted = false; + return true; +} + +static ClusterLeaveAsyncMarkerResult +cl_poll_lmon_marker_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_LEAVE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + if (!cl_lmon_marker_async.has_staged_event) + return CL_LEAVE_ASYNC_FAILED; + + now = GetCurrentTimestamp(); + kind = cl_marker_phase_kind(cl_lmon_marker_phase); + if (!cl_lmon_marker_submitted) { + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, + kind, cl_lmon_marker_leaving, now)) + return CL_LEAVE_ASYNC_PENDING; + cl_lmon_marker_submitted = true; + return CL_LEAVE_ASYNC_PENDING; + } + + pr = cluster_clean_leave_poll_marker_async(&cl_lmon_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return CL_LEAVE_ASYNC_PENDING; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, cl_lmon_marker_leaving, elapsed_us); + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + + cluster_reconfig_note_marker_slow_ack(kind, cl_lmon_marker_leaving, elapsed_us); + if (result != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { + cl_release_lmon_marker_stage(); + return CL_LEAVE_ASYNC_FAILED; + } + return CL_LEAVE_ASYNC_ACKED; +} + +static void +cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) +{ + ClusterLeaveIntentMarker cm; + ClusterLeaveAsyncMarkerResult ar; + + if (cl_lmon_marker_async.has_staged_event) { + if (cl_lmon_marker_phase != CLUSTER_LEAVE_MARKER_PHASE_COMMITTED + || cl_lmon_marker_leaving != leaving || cl_lmon_marker_epoch != committed_epoch) + return; + } else { + cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); + (void)cl_start_lmon_marker_stage(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + committed_epoch); + } + + ar = cl_poll_lmon_marker_stage(); + if (ar == CL_LEAVE_ASYNC_ACKED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, committed_epoch); + cl_release_lmon_marker_stage(); + } else if (ar == CL_LEAVE_ASYNC_FAILED) { + ereport(LOG, + (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); + } +} ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) @@ -787,6 +908,38 @@ cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m) } } +bool +cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (cl_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cl_state->marker_request_seq, + &cl_state->marker_completion_seq)) + return false; + + cl_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &cl_state->marker_request_seq, + &cl_state->marker_completion_seq, cl_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (cl_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cl_state->marker_completion_seq, &cl_state->marker_result, + now, out_result, out_elapsed_us); +} + bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512) { @@ -818,6 +971,7 @@ cluster_clean_leave_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&cl_state->marker_completion_seq, cl_qvotec_inflight_marker_seq); cl_qvotec_last_processed_marker_seq = cl_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -1640,6 +1794,44 @@ cl_coordinator_commit(int32 leaving) uint64 new_epoch; ClusterLeaveIntentMarker m; + if (cl_lmon_marker_async.has_staged_event) { + ClusterLeaveAsyncMarkerResult ar = cl_poll_lmon_marker_stage(); + int phase = cl_lmon_marker_phase; + uint64 staged_epoch = cl_lmon_marker_epoch; + + if (ar == CL_LEAVE_ASYNC_PENDING) + return; + if (ar == CL_LEAVE_ASYNC_FAILED) + return; + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { + cl_release_lmon_marker_stage(); + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); + if (new_epoch == 0) { + ereport(LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); + return; + } + cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, + new_epoch); + (void)cl_poll_lmon_marker_stage(); + ereport(LOG, + (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", + leaving, (unsigned long long)new_epoch))); + return; + } + if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTED) { + pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); + cl_send_committed(leaving, staged_epoch); + cl_release_lmon_marker_stage(); + return; + } + cl_release_lmon_marker_stage(); + return; + } + /* CL-I3 pre-check: refuse to commit a clean leave on a version a real death * already bumped — the leaving node will then observe a non-CLEAN_LEAVE event * and escalate. This is a cheap early-out; the authoritative guard is the @@ -1668,51 +1860,9 @@ cl_coordinator_commit(int32 leaving) /* (1) COMMITTING(E) marker (coordinator's own slot, before the bump; NOT a * trust basis). Not durable -> do not commit. */ cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, baseline_epoch + 1); - if (cluster_clean_leave_submit_marker(&m) != CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster clean-leave: COMMITTING marker not durable for node %d; " - "not committing (the leaving node will retry/escalate)", - leaving))); - return; - } - - /* (2) the real commit point: guarded epoch bump (only if still baseline) + - * publish CLEAN_LEAVE + record clean-departed. Returns 0 if a concurrent - * reconfig moved the epoch off the baseline after the COMMITTING marker — - * fail-closed, do NOT write the COMMITTED marker (CL-I3). */ - new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); - if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); - return; /* CL-I3: stale baseline (or cluster disabled / bad id) */ - } - - /* - * (3) COMMITTED(E) marker (after the bump; the ONLY rebuild trust basis). - * Post-commit it MUST become majority-durable — the durable truth-source must - * exist BEFORE the leaving node departs (P1-V0.7 exit gate; §2.5). Try once - * here. If it reaches majority, mark durable + tell the leaving node it may - * exit (LEAVE_COMMITTED). If NOT, do not give up: committed_marker_durable - * stays 0, cl_survivor_tick retries the marker every tick, and the leaving - * node stays in BARRIER_WAIT (no LEAVE_COMMITTED) until it is durable — it - * never departs without a durable truth-source. The leave is already - * committed (epoch bumped); we never revert. - */ - cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); - if (cluster_clean_leave_submit_marker(&m) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) { - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - cl_send_committed(leaving, new_epoch); - } else { - ereport( - LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the COMMITTED " - "marker is not yet majority-durable; retrying each tick (leaving node waits)", - leaving, (unsigned long long)new_epoch))); - } - - ereport(LOG, (errmsg("cluster clean-leave: committed departure of node %d at epoch %llu", - leaving, (unsigned long long)new_epoch))); + (void)cl_start_lmon_marker_stage(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTING, leaving, + baseline_epoch + 1); + (void)cl_poll_lmon_marker_stage(); } /* survivor (incl. coordinator) side of another node's leave. */ @@ -1773,13 +1923,9 @@ cl_survivor_tick(int32 leaving) if (coordinator == cluster_node_id && cluster_reconfig_is_clean_departed(leaving)) { uint64 committed_epoch = cluster_reconfig_get_clean_departed_epoch(leaving); - if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) { - ClusterLeaveIntentMarker cm; + if (!pg_atomic_read_u32(&cl_state->committed_marker_durable)) + cl_drive_committed_marker_stage(leaving, committed_epoch); - cl_build_marker(&cm, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, committed_epoch); - if (cluster_clean_leave_submit_marker(&cm) == CLUSTER_LEAVE_MARKER_SUBMIT_ACK) - pg_atomic_write_u32(&cl_state->committed_marker_durable, 1); - } /* Re-send LEAVE_COMMITTED every tick once durable until the leaver is gone * (best-effort IC): the leaving node will not depart until it receives one, * and step 3 holds the slot until it is CSSD-dead, so delivery is assured. */ diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..6ece9182d4 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -544,6 +544,12 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", + fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", + fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_slow_iter_count", + fmt_int64((int64)cluster_lmon_slow_iter_count())); } /* @@ -1475,6 +1481,10 @@ dump_reconfig_join(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_reconfig_get_join_timeout_count())); emit_row(rsinfo, "reconfig_join", "clean_departed_cleared_count", fmt_int64((int64)cluster_reconfig_get_clean_departed_cleared_count())); + emit_row(rsinfo, "reconfig", "marker_slow_ack_count", + fmt_int64((int64)cluster_reconfig_get_marker_slow_ack_count())); + emit_row(rsinfo, "reconfig", "marker_timeout_count", + fmt_int64((int64)cluster_reconfig_get_marker_timeout_count())); } /* diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..869e7c2866 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -307,6 +307,7 @@ int cluster_phase4_timeout = 30; * LMON main-loop tick / WaitLatch timeout in milliseconds. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; /* spec-1.12 Sprint B D8: cluster.lck_main_loop_interval (mirror). */ int cluster_lck_main_loop_interval = 1000; @@ -2573,6 +2574,15 @@ cluster_init_guc(void) &cluster_lmon_main_loop_interval, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.lmon_slow_iteration_warn_ms", + gettext_noop("LMON main-loop slow-iteration warning threshold in milliseconds."), + gettext_noop("A value greater than zero logs LMON main-loop iterations above this " + "duration; the lmon_slow_iter_count counter is maintained even when " + "logging is disabled with 0."), + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( "cluster.lck_main_loop_interval", gettext_noop("LCK main-loop tick interval in milliseconds."), diff --git a/src/backend/cluster/cluster_inject.c b/src/backend/cluster/cluster_inject.c index 89127fe949..332a4e97ff 100644 --- a/src/backend/cluster/cluster_inject.c +++ b/src/backend/cluster/cluster_inject.c @@ -199,6 +199,7 @@ static ClusterInjectPoint cluster_injection_points[] = { /* spec-2.6 Sprint A Step 4 D14 — 5 qvotec / quorum-lite injects. */ { .name = "cluster-qvotec-poll-pre" }, { .name = "cluster-qvotec-poll-post" }, + { .name = "cluster-qvotec-marker-service-hold" }, { .name = "cluster-voting-disk-write-fail" }, { .name = "cluster-quorum-loss-broadcast" }, { .name = "cluster-collision-detect" }, diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 73e816a0c7..51884d0a80 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -681,6 +681,61 @@ cluster_lmon_main_loop_iters(void) return result; } +uint64 +cluster_lmon_last_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->last_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_max_iter_us(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->max_iter_us; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +uint64 +cluster_lmon_slow_iter_count(void) +{ + uint64 result; + + if (cluster_lmon_state == NULL) + return 0; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_SHARED); + result = cluster_lmon_state->slow_iter_count; + LWLockRelease(&cluster_lmon_state->lwlock); + return result; +} + +void +cluster_lmon_marker_complete_wakeup(void) +{ + struct Latch *latch; + + if (cluster_lmon_state == NULL) + return; + + latch = cluster_lmon_state->lmon_latch; + if (latch != NULL) + SetLatch(latch); +} + /* ============================================================ * LMON main entry (AuxiliaryProcessMain dispatch target). @@ -720,12 +775,30 @@ lmon_publish_status(ClusterLmonStatus status) cluster_lmon_state->ready_at = 0; cluster_lmon_state->last_liveness_tick_at = 0; cluster_lmon_state->main_loop_iters = 0; + cluster_lmon_state->last_iter_us = 0; + cluster_lmon_state->max_iter_us = 0; + cluster_lmon_state->slow_iter_count = 0; + cluster_lmon_state->lmon_latch = NULL; } else if (status == CLUSTER_LMON_READY) { cluster_lmon_state->ready_at = now; + cluster_lmon_state->lmon_latch = MyLatch; + } else if (status == CLUSTER_LMON_EXITED) { + cluster_lmon_state->lmon_latch = NULL; } LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_clear_latch(int code, Datum arg) +{ + if (cluster_lmon_state == NULL) + return; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->lmon_latch = NULL; + LWLockRelease(&cluster_lmon_state->lwlock); +} + static bool lmon_shutdown_requested(void) @@ -760,6 +833,45 @@ lmon_advance_liveness_tick(void) LWLockRelease(&cluster_lmon_state->lwlock); } +static void +lmon_record_iteration(TimestampTz iter_started_at) +{ + TimestampTz now = GetCurrentTimestamp(); + uint64 elapsed_us; + bool slow; + bool should_log = false; + static TimestampTz last_slow_log_at = 0; + + if (now <= iter_started_at) + elapsed_us = 0; + else + elapsed_us = (uint64)(now - iter_started_at); + + slow = (cluster_lmon_slow_iteration_warn_ms >= 0) + && elapsed_us >= (uint64)cluster_lmon_slow_iteration_warn_ms * 1000ULL; + + LWLockAcquire(&cluster_lmon_state->lwlock, LW_EXCLUSIVE); + cluster_lmon_state->last_iter_us = elapsed_us; + if (elapsed_us > cluster_lmon_state->max_iter_us) + cluster_lmon_state->max_iter_us = elapsed_us; + if (slow) + cluster_lmon_state->slow_iter_count++; + LWLockRelease(&cluster_lmon_state->lwlock); + + if (slow && cluster_lmon_slow_iteration_warn_ms > 0 + && (last_slow_log_at == 0 + || now - last_slow_log_at >= INT64CONST(60) * INT64CONST(1000000))) { + last_slow_log_at = now; + should_log = true; + } + + if (should_log) + ereport(LOG, + (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); +} + void LmonMain(void) @@ -811,6 +923,7 @@ LmonMain(void) /* Publish SPAWNING (records pid + spawned_at). */ lmon_publish_status(CLUSTER_LMON_SPAWNING); + on_shmem_exit(lmon_clear_latch, (Datum)0); /* Sprint B inject: ready-publish (test slow startup / phase 1 wait timeout). */ CLUSTER_INJECTION_POINT("cluster-lmon-ready-publish"); @@ -938,6 +1051,7 @@ LmonMain(void) int n_events; long wait_ms; TimestampTz now; + TimestampTz iter_started_at; int32 i; CHECK_FOR_INTERRUPTS(); @@ -950,6 +1064,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1414,6 +1529,7 @@ LmonMain(void) wait_ms = 1; } + lmon_record_iteration(iter_started_at); n_events = WaitEventSetWait(wes, wait_ms, ev, lengthof(ev), WAIT_EVENT_CLUSTER_IC_HEARTBEAT_WAIT); @@ -1588,6 +1704,7 @@ LmonMain(void) /* Stub / mock / disabled mode -- preserve spec-1.11 simple loop. */ for (;;) { int rc; + TimestampTz iter_started_at; CHECK_FOR_INTERRUPTS(); @@ -1599,6 +1716,7 @@ LmonMain(void) if (ShutdownRequestPending || lmon_shutdown_requested()) break; + iter_started_at = GetCurrentTimestamp(); lmon_advance_liveness_tick(); /* @@ -1677,6 +1795,7 @@ LmonMain(void) CLUSTER_INJECTION_POINT("cluster-lmon-main-loop-iter"); + lmon_record_iteration(iter_started_at); rc = WaitLatch(MyLatch, WL_LATCH_SET | WL_TIMEOUT | WL_EXIT_ON_PM_DEATH, cluster_lmon_main_loop_interval, WAIT_EVENT_CLUSTER_BGPROC_LMON_MAIN_LOOP); diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 8825ac6828..830a7a8d66 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -57,6 +57,7 @@ #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" #include "cluster/cluster_inject.h" +#include "cluster/cluster_lmon.h" #include "cluster/cluster_membership.h" #include "cluster/cluster_node_remove.h" #include "cluster/cluster_pcm_lock.h" @@ -162,6 +163,14 @@ cluster_node_remove_get_state(ClusterNodeRemoveState *out) static uint64 nr_qvotec_last_processed_marker_seq = 0; static uint64 nr_qvotec_inflight_marker_seq = 0; +static ClusterMarkerAsync nr_marker_async; +static ClusterRemovalMarker nr_marker_stage; +static int nr_marker_phase = 0; +static int32 nr_marker_node_id = -1; +static uint64 nr_marker_epoch = 0; +static uint64 nr_marker_removed_incarnation = 0; +static uint64 nr_marker_removal_event_id = 0; +static bool nr_marker_submitted = false; ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) @@ -195,6 +204,38 @@ cluster_node_remove_submit_marker(const ClusterRemovalMarker *m) } } +bool +cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + int wait_ms; + + if (nr_state == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&nr_state->marker_request_seq, + &nr_state->marker_completion_seq)) + return false; + + nr_state->pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit(a, &nr_state->marker_request_seq, + &nr_state->marker_completion_seq, nr_state->qvotec_latch, + now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (nr_state == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, + &nr_state->marker_result, now, out_result, out_elapsed_us); +} + bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out) { @@ -223,6 +264,7 @@ cluster_node_remove_qvotec_complete(bool acked) pg_write_barrier(); pg_atomic_write_u64(&nr_state->marker_completion_seq, nr_qvotec_inflight_marker_seq); nr_qvotec_last_processed_marker_seq = nr_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -241,24 +283,124 @@ cluster_node_remove_publish_qvotec_latch(struct Latch *latch) on_shmem_exit(nr_clear_qvotec_latch, (Datum)0); } -/* build + submit one durable removal marker; returns true iff majority-durable. */ +static ClusterMarkerAsyncKind +nr_marker_phase_kind(int phase) +{ + switch (phase) { + case CLUSTER_REMOVAL_MARKER_REMOVING: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING; + case CLUSTER_REMOVAL_MARKER_SHRUNK: + return CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK; + case CLUSTER_REMOVAL_MARKER_REMOVED: + return CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED; + default: + return CLUSTER_MARKER_KIND_UNKNOWN; + } +} + +static void +nr_release_marker_stage(void) +{ + cluster_marker_async_release_stage(&nr_marker_async); + memset(&nr_marker_stage, 0, sizeof(nr_marker_stage)); + nr_marker_phase = 0; + nr_marker_node_id = -1; + nr_marker_epoch = 0; + nr_marker_removed_incarnation = 0; + nr_marker_removal_event_id = 0; + nr_marker_submitted = false; +} + +static bool +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, + uint64 removed_incarnation, uint64 removal_event_id) +{ + return nr_marker_async.has_staged_event && nr_marker_phase == phase + && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch + && nr_marker_removed_incarnation == removed_incarnation + && nr_marker_removal_event_id == removal_event_id; +} + +/* build + poll one durable removal marker; returns true iff majority-durable. */ static bool nr_write_marker(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, uint64 removal_event_id) { ClusterRemovalMarker m; + TimestampTz now; + uint32 result = CLUSTER_REMOVAL_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + ClusterMarkerAsyncKind kind; + + now = GetCurrentTimestamp(); + kind = nr_marker_phase_kind(phase); + + if (nr_marker_async.has_staged_event + && !nr_marker_stage_matches(phase, node_id, remove_epoch, removed_incarnation, + removal_event_id)) { + if (!nr_marker_submitted) { + if (cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, + nr_marker_phase_kind(nr_marker_phase), + nr_marker_node_id, now)) + nr_marker_submitted = true; + return false; + } + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } else if (pr == CLUSTER_MARKER_POLL_ACKED) { + cluster_reconfig_note_marker_slow_ack(nr_marker_async.kind, nr_marker_node_id, + elapsed_us); + nr_release_marker_stage(); + } + return false; + } + + if (!nr_marker_async.has_staged_event) { + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; + m.version = CLUSTER_REMOVAL_MARKER_VERSION; + m.phase = (uint16)phase; + m.removed_node_id = node_id; + m.remove_epoch = remove_epoch; + m.removed_incarnation = removed_incarnation; + m.removal_event_id = removal_event_id; + cluster_removal_marker_compute_crc(&m); + + nr_marker_stage = m; + nr_marker_phase = phase; + nr_marker_node_id = node_id; + nr_marker_epoch = remove_epoch; + nr_marker_removed_incarnation = removed_incarnation; + nr_marker_removal_event_id = removal_event_id; + nr_marker_async.has_staged_event = true; + nr_marker_async.staged_expect_epoch = remove_epoch; + nr_marker_submitted = false; + } + + if (!nr_marker_submitted) { + if (!cluster_node_remove_submit_marker_async(&nr_marker_async, &nr_marker_stage, kind, + node_id, now)) + return false; + nr_marker_submitted = true; + return false; + } + + pr = cluster_node_remove_poll_marker_async(&nr_marker_async, now, &result, &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return false; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return false; + } - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_REMOVAL_MARKER_MAGIC; - m.version = CLUSTER_REMOVAL_MARKER_VERSION; - m.phase = (uint16)phase; - m.removed_node_id = node_id; - m.remove_epoch = remove_epoch; - m.removed_incarnation = removed_incarnation; - m.removal_event_id = removal_event_id; - cluster_removal_marker_compute_crc(&m); - - return cluster_node_remove_submit_marker(&m) == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; + cluster_reconfig_note_marker_slow_ack(kind, node_id, elapsed_us); + nr_release_marker_stage(); + return result == CLUSTER_REMOVAL_MARKER_SUBMIT_ACK; } diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 8eb25379a7..6fd071a17a 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -94,6 +94,7 @@ #include "cluster/cluster_elog.h" /* CLUSTER_LOG (best-effort logging) */ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ +#include "cluster/cluster_inject.h" #include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ #include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ @@ -883,6 +884,10 @@ qvotec_poll_once(void) * carried forward every poll like the fence marker (R12), and acked majority- * durable from the self-slot write tally. */ have_removal_submit = cluster_node_remove_qvotec_poll_pending(&removal_submit_marker); + if ((have_submit || have_leave_submit || have_join_submit || have_removal_submit) + && cluster_cr_injection_armed("cluster-qvotec-marker-service-hold", NULL)) + return; + memset(&apply_lease_request, 0, sizeof(apply_lease_request)); memset(&apply_lease_winner, 0, sizeof(apply_lease_winner)); apply_lease_winner.owner_node_id = -1; diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 1f79a6c9b9..931ea26b8d 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -77,6 +77,7 @@ #include "cluster/cluster_clean_leave.h" /* v1.0.4 — cluster_clean_leave_in_progress (serialize) */ #include "cluster/cluster_guc.h" /* cluster_enabled, cluster_online_join */ #include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_voting_disk_io.h" /* spec-5.15 D4 — region-3 join-marker slot I/O */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D4 — durable fence marker submit */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum */ @@ -114,6 +115,38 @@ StaticAssertDecl(CLUSTER_RECONFIG_TOUCH_KIND_COUNT == CLUSTER_TOUCH_KIND_COUNT, */ static ClusterReconfigState *ReconfigShmem = NULL; +typedef struct ClusterReconfigFenceStage { + ClusterMarkerAsync async; + ReconfigEvent event; + ClusterFenceMarker marker; + int32 node_id; + uint64 last_incarnation; + uint64 removal_event_id; + bool submitted; +} ClusterReconfigFenceStage; + +typedef struct ClusterReconfigJoinPrepareStage { + ClusterMarkerAsync async; + ReconfigEvent event; + uint8 join_bitmap[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 joiner_incarnations[CLUSTER_MAX_NODES]; + int next_node; + bool submitted; +} ClusterReconfigJoinPrepareStage; + +typedef struct ClusterReconfigJoinCommitStage { + ClusterMarkerAsync async; + ClusterJoinCommitMarker marker; + int32 node_id; + uint64 admitted_incarnation; + bool submitted; +} ClusterReconfigJoinCommitStage; + +static ClusterReconfigFenceStage failstop_fence_stage; +static ClusterReconfigFenceStage node_removed_fence_stage; +static ClusterReconfigJoinPrepareStage join_prepare_stage; +static ClusterReconfigJoinCommitStage join_commit_stage; + /* ============================================================ * Shmem region lifecycle. @@ -211,6 +244,8 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->join_reject_count, 0); pg_atomic_init_u64(&ReconfigShmem->join_timeout_count, 0); pg_atomic_init_u64(&ReconfigShmem->clean_departed_cleared_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_slow_ack_count, 0); + pg_atomic_init_u64(&ReconfigShmem->marker_timeout_count, 0); } /* @@ -1032,6 +1067,44 @@ cluster_reconfig_get_clean_departed_cleared_count(void) : pg_atomic_read_u64(&ReconfigShmem->clean_departed_cleared_count); } +void +cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem == NULL || elapsed_us < 1000000ULL) + return; + + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); + ereport(LOG, + (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +void +cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us) +{ + if (ReconfigShmem != NULL) + pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); + ereport(LOG, + (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); +} + +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_slow_ack_count); +} + +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return ReconfigShmem == NULL ? 0 : pg_atomic_read_u64(&ReconfigShmem->marker_timeout_count); +} + /* * cluster_reconfig_join_in_progress -- Hardening v1.0.4 (spec-5.13 clean-leave x * spec-5.15 online-join serialization, P1-1/P2): is a membership JOIN currently in @@ -1148,6 +1221,383 @@ cluster_reconfig_join_publish_proven(uint64 admitted_epoch) return advanced >= ((members / 2u) + 1u); } +static void +cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) +{ + cluster_marker_async_release_stage(&stage->async); + stage->submitted = false; + memset(&stage->event, 0, sizeof(stage->event)); + memset(&stage->marker, 0, sizeof(stage->marker)); + stage->node_id = -1; + stage->last_incarnation = 0; + stage->removal_event_id = 0; +} + +static bool +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (stage->submitted) + return true; + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, + target_node, now)) + return false; + stage->submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_failstop_fence_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!failstop_fence_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, now)) + return true; + + pr = cluster_write_fence_poll_marker_async(&failstop_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + failstop_fence_stage.event.coordinator_node_id, + elapsed_us); + if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_broadcast_local_procsig(); + } else { + ereport(LOG, + (errmsg("cluster reconfig: fence marker did not reach a voting-disk majority " + "for epoch %llu; not publishing reconfig event (write-fenced, will retry)", + (unsigned long long)failstop_fence_stage.event.new_epoch))); + } + cluster_reconfig_release_fence_stage(&failstop_fence_stage); + return true; +} + +static uint64 +cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 removal_event_id, + uint64 last_incarnation) +{ + TimestampTz now; + uint32 result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + + if (!node_removed_fence_stage.async.has_staged_event) + return 0; + if (node_removed_fence_stage.node_id != removed_node_id + || node_removed_fence_stage.removal_event_id != removal_event_id) + return 0; + + now = GetCurrentTimestamp(); + if (!cluster_reconfig_submit_fence_stage(&node_removed_fence_stage, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, now)) + return 0; + + pr = cluster_write_fence_poll_marker_async(&node_removed_fence_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return 0; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + removed_node_id, elapsed_us); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " + "voting-disk majority for epoch %llu; not publishing removal " + "(write-fenced, will retry)", + removed_node_id, + (unsigned long long)node_removed_fence_stage.event.new_epoch))); + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return 0; + } + + CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); + cluster_reconfig_publish_event(&node_removed_fence_stage.event); + cluster_reconfig_record_removed(removed_node_id, node_removed_fence_stage.event.new_epoch, + false); + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_shrink_to_removed(removed_node_id, last_incarnation); + LWLockRelease(&ReconfigShmem->lock); + + { + uint64 new_epoch = node_removed_fence_stage.event.new_epoch; + + cluster_reconfig_release_fence_stage(&node_removed_fence_stage); + return new_epoch; + } +} + +static void +cluster_reconfig_release_join_prepare_stage(void) +{ + cluster_marker_async_release_stage(&join_prepare_stage.async); + memset(&join_prepare_stage.event, 0, sizeof(join_prepare_stage.event)); + memset(join_prepare_stage.join_bitmap, 0, sizeof(join_prepare_stage.join_bitmap)); + memset(join_prepare_stage.joiner_incarnations, 0, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; +} + +static void +cluster_reconfig_abort_join_prepare_stage(void) +{ + int i; + + if (ReconfigShmem != NULL) { + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (!dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + continue; + ReconfigShmem->pending_join_bitmap[i / 8] &= (uint8) ~(1u << (i % 8)); + if (cluster_membership_get_state(i) == CLUSTER_MEMBER_JOINING) + cluster_membership_set_state(i, CLUSTER_MEMBER_DEAD); + } + LWLockRelease(&ReconfigShmem->lock); + } + cluster_reconfig_release_join_prepare_stage(); +} + +static bool +cluster_reconfig_submit_join_prepare_current(TimestampTz now) +{ + ClusterJoinCommitMarker m; + int i; + + for (i = join_prepare_stage.next_node; i < CLUSTER_MAX_NODES; i++) { + if (dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) + break; + } + join_prepare_stage.next_node = i; + if (i >= CLUSTER_MAX_NODES) { + cluster_reconfig_publish_event(&join_prepare_stage.event); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + cluster_reconfig_broadcast_local_procsig(); + cluster_reconfig_release_join_prepare_stage(); + return true; + } + + if (join_prepare_stage.submitted) + return true; + + memset(&m, 0, sizeof(m)); + m.magic = CLUSTER_JCMK_MAGIC; + m.version = CLUSTER_JCMK_VERSION; + m.node_id = i; + m.phase = CLUSTER_JCMK_PHASE_PREPARE; + m.admitted_incarnation = join_prepare_stage.joiner_incarnations[i]; + m.generation = join_prepare_stage.joiner_incarnations[i]; + m.admitted_epoch = join_prepare_stage.event.new_epoch; + cluster_join_marker_compute_crc(&m); + + if (!cluster_reconfig_submit_join_marker_async(&join_prepare_stage.async, i, &m, + CLUSTER_MARKER_KIND_JOIN_PREPARE, now)) + return false; + join_prepare_stage.submitted = true; + return true; +} + +static bool +cluster_reconfig_poll_join_prepare_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + ClusterMarkerPollResult pr; + int target; + + if (!join_prepare_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_prepare_stage.submitted) { + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; + } + + target = join_prepare_stage.next_node; + pr = cluster_reconfig_poll_join_marker_async(&join_prepare_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, + elapsed_us); + cluster_reconfig_abort_join_prepare_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, + elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach a " + "voting-disk majority; not publishing JOIN_PENDING (will retry)", + target))); + cluster_reconfig_abort_join_prepare_stage(); + return true; + } + + join_prepare_stage.submitted = false; + join_prepare_stage.next_node++; + (void)cluster_reconfig_submit_join_prepare_current(now); + return true; +} + +static void +cluster_reconfig_release_join_commit_stage(void) +{ + cluster_marker_async_release_stage(&join_commit_stage.async); + memset(&join_commit_stage.marker, 0, sizeof(join_commit_stage.marker)); + join_commit_stage.node_id = -1; + join_commit_stage.admitted_incarnation = 0; + join_commit_stage.submitted = false; +} + +static bool +cluster_reconfig_publish_join_commit(int32 node_id, uint64 admitted_incarnation, + uint64 expected_epoch) +{ + uint64 old_epoch, new_epoch; + XLogRecPtr lsn; + ReconfigEvent evt; + uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; + uint64 incs[CLUSTER_MAX_NODES]; + + if (cluster_epoch_get_current() + 1 != expected_epoch) + return false; + + CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); + + cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); + if (new_epoch != expected_epoch) + return false; + lsn = GetXLogInsertRecPtr(); + cluster_epoch_set_changed_at_lsn((uint64)lsn); + cluster_gcs_block_on_epoch_advance(new_epoch); + cluster_sinval_reset_all_on_reconfig(); + cluster_tt_status_flush_all((uint32)new_epoch); + + LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); + cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); + cluster_membership_record_admitted(node_id, admitted_incarnation); + ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); + LWLockRelease(&ReconfigShmem->lock); + + if (cluster_reconfig_is_clean_departed(node_id)) + pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); + cluster_reconfig_clear_clean_departed(node_id); + + dead_bitmap_set_bit(jb, node_id); + memset(incs, 0, sizeof(incs)); + incs[node_id] = admitted_incarnation; + + memset(&evt, 0, sizeof(evt)); + evt.event_id = cluster_reconfig_compute_event_id_v2( + RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); + evt.coordinator_node_id = cluster_node_id; + evt.old_epoch = old_epoch; + evt.new_epoch = new_epoch; + memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + evt.applied_at = GetCurrentTimestamp(); + evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); + evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; + cluster_reconfig_publish_event(&evt); + pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); + + return true; +} + +static bool +cluster_reconfig_poll_join_commit_stage(void) +{ + TimestampTz now; + uint32 result = CLUSTER_JOIN_MARKER_SUBMIT_FAILED; + uint64 elapsed_us = 0; + uint64 admitted_incarnation = 0; + uint64 admitted_generation = 0; + ClusterMarkerPollResult pr; + + if (!join_commit_stage.async.has_staged_event) + return false; + + now = GetCurrentTimestamp(); + if (!join_commit_stage.submitted) { + if (!cluster_reconfig_submit_join_marker_async( + &join_commit_stage.async, join_commit_stage.node_id, &join_commit_stage.marker, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, now)) + return true; + join_commit_stage.submitted = true; + return true; + } + + pr = cluster_reconfig_poll_join_marker_async(&join_commit_stage.async, now, &result, + &elapsed_us); + if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) + return true; + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_COMMITTED, + join_commit_stage.node_id, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { + ereport(LOG, + (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " + "voting-disk majority; not committing (will retry)", + join_commit_stage.node_id))); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + if (!cluster_reconfig_get_observed_slot(join_commit_stage.node_id, &admitted_incarnation, + &admitted_generation) + || admitted_incarnation != join_commit_stage.admitted_incarnation + || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, + admitted_generation) + != CLUSTER_JOIN_ACCEPT + || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, + join_commit_stage.admitted_incarnation, + join_commit_stage.async.staged_expect_epoch)) { + pg_atomic_fetch_add_u64(&ReconfigShmem->join_reject_count, 1); + cluster_reconfig_release_join_commit_stage(); + return true; + } + + cluster_reconfig_release_join_commit_stage(); + return true; +} + /* * spec-5.15 D4 — coordinator-side join driver (called from the tick when * online_join is on and self is the min-MEMBER coordinator). Phase-1: fresh @@ -1166,6 +1616,10 @@ cluster_reconfig_drive_joins(int coordinator) if (state == NULL) return; + if (cluster_reconfig_poll_join_prepare_stage()) + return; + if (cluster_reconfig_poll_join_commit_stage()) + return; /* Phase-1 detection + a snapshot of the current pending set, under the lock * (compute_join_bitmap reads membership_state). */ @@ -1182,8 +1636,7 @@ cluster_reconfig_drive_joins(int coordinator) (void)cluster_reconfig_get_observed_slot(i, &joiner_incarnations[i], NULL); } cluster_reconfig_apply_join_as_coordinator(join_bitmap, coordinator, joiner_incarnations); - /* enter the JOIN_PENDING transition fail-closed on every local backend. */ - cluster_reconfig_broadcast_local_procsig(); + return; } /* Phase-2: commit pending joins that have converged. The just-added Phase-1 @@ -1646,6 +2099,7 @@ cluster_reconfig_lmon_tick(void) uint64 cssd_dead_generation; uint64 event_id; int i; + bool failstop_stage_handled = false; /* L20: runtime feature flag check first line. */ if (!cluster_enabled) @@ -1665,6 +2119,7 @@ cluster_reconfig_lmon_tick(void) self_id = cluster_node_id; if (self_id < 0 || self_id >= CLUSTER_MAX_NODES) return; /* defensive: bad self id, cannot participate */ + failstop_stage_handled = cluster_reconfig_poll_failstop_fence_stage(); /* * §3.1 + F11: build the raw CSSD DEAD bitmap, filtering out un-declared @@ -1886,7 +2341,7 @@ cluster_reconfig_lmon_tick(void) * stabilizing the survivor base), THEN the join edge. Each is an independent * ReconfigEvent; neither early-returns past the other. */ - if (!dead_bitmap_is_zero(dead_bitmap)) { + if (!failstop_stage_handled && !dead_bitmap_is_zero(dead_bitmap)) { CLUSTER_INJECTION_POINT("cluster-reconfig-decide-coordinator"); /* §3.2 P1.2: event_id from dead_bitmap + dead_generation snapshot. */ @@ -2123,13 +2578,14 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " - "majority for epoch %llu; not publishing reconfig event " - "(write-fenced, will retry)", - (unsigned long long)new_epoch))); - return; /* fail-closed: epoch bumped, event NOT published, recovery NOT started */ - } + failstop_fence_stage.event = evt; + failstop_fence_stage.marker = marker; + failstop_fence_stage.node_id = coordinator_node_id; + failstop_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + coordinator_node_id, GetCurrentTimestamp()); + return; /* fail-closed until the staged marker is majority-durable */ } cluster_reconfig_publish_event(&evt); @@ -2287,6 +2743,9 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 return 0; if (removed_node_id < 0 || removed_node_id >= CLUSTER_MAX_NODES) return 0; + if (node_removed_fence_stage.async.has_staged_event) + return cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); CLUSTER_INJECTION_POINT("cluster-node-remove-shrink-committing"); @@ -2340,13 +2799,25 @@ cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_node_id, uint64 * a concurrent dead set, if any, is carried by its own fail-stop fence. */ memcpy(marker.fenced_dead_bitmap, removed_with_n, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - if (cluster_write_fence_submit_marker(&marker) != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { - ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " - "voting-disk majority for epoch %llu; not publishing removal " - "(write-fenced, will retry)", - removed_node_id, (unsigned long long)new_epoch))); - return 0; /* fail-closed: epoch bumped, removal NOT published, driver retries */ - } + node_removed_fence_stage.event.event_id + = cluster_reconfig_compute_removal_event_id(removed_with_n, removal_event_id); + node_removed_fence_stage.event.coordinator_node_id = cluster_node_id; + node_removed_fence_stage.event.old_epoch = old_epoch; + node_removed_fence_stage.event.new_epoch = new_epoch; + memcpy(node_removed_fence_stage.event.dead_bitmap, empty_dead, + CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); + node_removed_fence_stage.event.applied_at = GetCurrentTimestamp(); + node_removed_fence_stage.event.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; + node_removed_fence_stage.event.cssd_dead_generation = cssd_dead_generation; + node_removed_fence_stage.event.reconfig_kind = RECONFIG_KIND_NODE_REMOVED; + node_removed_fence_stage.marker = marker; + node_removed_fence_stage.node_id = removed_node_id; + node_removed_fence_stage.last_incarnation = last_incarnation; + node_removed_fence_stage.removal_event_id = removal_event_id; + node_removed_fence_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_node_removed_fence_stage(removed_node_id, removal_event_id, + last_incarnation); + return 0; /* fail-closed until the staged fence marker ACKs */ } CLUSTER_INJECTION_POINT("cluster-node-remove-fence-armed"); @@ -2431,6 +2902,42 @@ cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMa } } +bool +cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, TimestampTz now) +{ + int wait_ms; + + if (ReconfigShmem == NULL || m == NULL || a == NULL) + return false; + if (target_node < 0 || target_node >= CLUSTER_MAX_NODES) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&ReconfigShmem->join_marker_request_seq, + &ReconfigShmem->join_marker_completion_seq)) + return false; + + ReconfigShmem->join_marker_target_node_id = target_node; + ReconfigShmem->join_pending_marker = *m; + wait_ms = cluster_quorum_poll_interval_ms * 3 + 2000; + return cluster_marker_async_submit( + a, &ReconfigShmem->join_marker_request_seq, &ReconfigShmem->join_marker_completion_seq, + ReconfigShmem->join_qvotec_latch, now, (uint64)wait_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (ReconfigShmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &ReconfigShmem->join_marker_completion_seq, + &ReconfigShmem->join_marker_result, now, out_result, + out_elapsed_us); +} + bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512) { @@ -2464,6 +2971,7 @@ cluster_reconfig_join_qvotec_complete(bool acked) pg_atomic_write_u64(&ReconfigShmem->join_marker_completion_seq, join_qvotec_inflight_marker_seq); join_qvotec_last_processed_marker_seq = join_qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } static void @@ -2614,27 +3122,6 @@ cluster_reconfig_apply_join_as_coordinator( } LWLockRelease(&ReconfigShmem->lock); - /* Durable PREPARE marker per joiner (records the presented incarnation; does - * NOT seed — only COMMITTED is a basis). Best-effort: PREPARE failure does - * not block the JOIN_PENDING publish (the COMMITTED marker in Phase-2 is the - * commit point that must be majority-durable). */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - ClusterJoinCommitMarker m; - - if (!dead_bitmap_test_bit(join_bitmap, i)) - continue; - memset(&m, 0, sizeof(m)); - m.magic = CLUSTER_JCMK_MAGIC; - m.version = CLUSTER_JCMK_VERSION; - m.node_id = i; - m.phase = CLUSTER_JCMK_PHASE_PREPARE; - m.admitted_incarnation = joiner_incarnations[i]; - m.generation = joiner_incarnations[i]; /* monotonic per node (read-newest intent) */ - m.admitted_epoch = new_epoch; - cluster_join_marker_compute_crc(&m); - (void)cluster_reconfig_submit_join_marker(i, &m); - } - memset(&evt, 0, sizeof(evt)); evt.event_id = cluster_reconfig_compute_event_id_v2(RECONFIG_KIND_JOIN_PENDING, empty_dead, join_bitmap, @@ -2647,20 +3134,21 @@ cluster_reconfig_apply_join_as_coordinator( evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; evt.cssd_dead_generation = cssd_dead_generation; evt.reconfig_kind = RECONFIG_KIND_JOIN_PENDING; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_pending_count, 1); + + join_prepare_stage.event = evt; + memcpy(join_prepare_stage.join_bitmap, join_bitmap, sizeof(join_prepare_stage.join_bitmap)); + memcpy(join_prepare_stage.joiner_incarnations, joiner_incarnations, + sizeof(join_prepare_stage.joiner_incarnations)); + join_prepare_stage.next_node = 0; + join_prepare_stage.submitted = false; + join_prepare_stage.async.has_staged_event = true; + (void)cluster_reconfig_poll_join_prepare_stage(); } bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) { ClusterJoinCommitMarker m; - uint64 old_epoch, new_epoch; - XLogRecPtr lsn; - ReconfigEvent evt; - uint8 jb[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint8 empty_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES] = { 0 }; - uint64 incs[CLUSTER_MAX_NODES]; if (!cluster_enabled || ReconfigShmem == NULL) return false; @@ -2704,60 +3192,17 @@ cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarnation) m.commit_nonce = ((uint64)cluster_node_id << 56) ^ (uint64)GetCurrentTimestamp(); cluster_join_marker_compute_crc(&m); - if (cluster_reconfig_submit_join_marker(node_id, &m) != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: COMMITTED join marker for node %d did not reach a " - "voting-disk majority; not committing (will retry)", - node_id))); + if (join_commit_stage.async.has_staged_event) return false; - } - - CLUSTER_INJECTION_POINT("cluster-reconfig-join-commit-marker-durable"); - - /* - * ② publish: bump JOIN_COMMITTED epoch + state MEMBER + last_admitted + clear - * pending + clear clean_departed[node] (INV-J10). Strict order — the publish - * (epoch + state MEMBER) precedes the joiner opening its write gate (D5). - */ - cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); - lsn = GetXLogInsertRecPtr(); - cluster_epoch_set_changed_at_lsn((uint64)lsn); - cluster_gcs_block_on_epoch_advance(new_epoch); - cluster_sinval_reset_all_on_reconfig(); - cluster_tt_status_flush_all((uint32)new_epoch); - - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - cluster_membership_set_state(node_id, CLUSTER_MEMBER_MEMBER); - cluster_membership_record_admitted(node_id, admitted_incarnation); - ReconfigShmem->pending_join_bitmap[node_id / 8] &= (uint8) ~(1u << (node_id % 8)); - LWLockRelease(&ReconfigShmem->lock); + join_commit_stage.marker = m; + join_commit_stage.node_id = node_id; + join_commit_stage.admitted_incarnation = admitted_incarnation; + join_commit_stage.async.has_staged_event = true; + join_commit_stage.async.staged_expect_epoch = m.admitted_epoch; + join_commit_stage.submitted = false; + (void)cluster_reconfig_poll_join_commit_stage(); - /* INV-J10: clear the in-shmem clean_departed suppression so a clean-left node - * that just rejoined has its later real fail-stop honored again (the durable - * supersede across restart is resolved by the seed, RC-5). */ - if (cluster_reconfig_is_clean_departed(node_id)) - pg_atomic_fetch_add_u64(&ReconfigShmem->clean_departed_cleared_count, 1); - cluster_reconfig_clear_clean_departed(node_id); - - dead_bitmap_set_bit(jb, node_id); - memset(incs, 0, sizeof(incs)); - incs[node_id] = admitted_incarnation; - - memset(&evt, 0, sizeof(evt)); - evt.event_id = cluster_reconfig_compute_event_id_v2( - RECONFIG_KIND_JOIN_COMMITTED, empty_dead, jb, incs, cluster_cssd_get_dead_generation()); - evt.coordinator_node_id = cluster_node_id; - evt.old_epoch = old_epoch; - evt.new_epoch = new_epoch; - memcpy(evt.join_bitmap, jb, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES); - evt.applied_at = GetCurrentTimestamp(); - evt.observer_role = CLUSTER_RECONFIG_OBSERVER_COORDINATOR; - evt.cssd_dead_generation = cluster_cssd_get_dead_generation(); - evt.reconfig_kind = RECONFIG_KIND_JOIN_COMMITTED; - cluster_reconfig_publish_event(&evt); - pg_atomic_fetch_add_u64(&ReconfigShmem->join_apply_count, 1); - - return true; + return false; } diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index 2baebb13bd..d4988b042d 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -40,6 +40,7 @@ #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_guc.h" /* cluster_node_id, cluster_voting_disks */ +#include "cluster/cluster_lmon.h" /* cluster_lmon_marker_complete_wakeup */ #include "cluster/cluster_qvotec.h" /* ClusterVotingSlot (marker layout asserts) */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_voting_disk_io.h" /* D6 direct durable marker read */ @@ -769,6 +770,38 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m) } } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, int32 target_node, + TimestampTz now) +{ + if (cluster_write_fence_shmem == NULL || m == NULL || a == NULL) + return false; + if (cluster_marker_async_is_submitted(a)) + return true; + if (cluster_marker_async_mailbox_busy(&cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq)) + return false; + + memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); + return cluster_marker_async_submit( + a, &cluster_write_fence_shmem->marker_request_seq, + &cluster_write_fence_shmem->marker_completion_seq, + cluster_write_fence_shmem->qvotec_latch, now, + (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); +} + +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + if (cluster_write_fence_shmem == NULL || a == NULL) + return CLUSTER_MARKER_POLL_IDLE; + return cluster_marker_async_poll(a, &cluster_write_fence_shmem->marker_completion_seq, + &cluster_write_fence_shmem->marker_result, now, out_result, + out_elapsed_us); +} + /* * cluster_write_fence_clear_qvotec_latch / _publish_qvotec_latch -- qvotec publishes * its MyLatch at startup so LMON can wake it; auto-cleared at proc_exit so a stale @@ -835,6 +868,7 @@ cluster_write_fence_qvotec_complete(bool acked) pg_atomic_write_u64(&cluster_write_fence_shmem->marker_completion_seq, qvotec_inflight_marker_seq); qvotec_last_processed_marker_seq = qvotec_inflight_marker_seq; + cluster_lmon_marker_complete_wakeup(); } #endif /* USE_PGRAC_CLUSTER */ diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index e49ecfc194..2d75de8e2a 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -65,6 +65,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ #define CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES 16 /* Default drain deadline; aligns with the feature-082 30s barrier. */ @@ -408,6 +410,15 @@ extern void cluster_clean_leave_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); +extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, + const ClusterLeaveIntentMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_clean_leave_qvotec_poll_pending(void *out_slot512); extern void cluster_clean_leave_qvotec_complete(bool acked); extern void cluster_clean_leave_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..1b3a06016f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -469,6 +469,7 @@ extern int cluster_phase4_timeout; * range: [100, 60000] (millisecond) */ extern int cluster_lmon_main_loop_interval; +extern int cluster_lmon_slow_iteration_warn_ms; /* diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 63efc4e045..1b6a516f4b 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -100,6 +100,7 @@ #include "datatype/timestamp.h" #include "storage/lwlock.h" +struct Latch; /* * ClusterLmonStatus -- HC2 SSOT for LMON lifecycle state. @@ -137,6 +138,10 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 max_iter_us; /* high-water iteration wall time */ + uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ + struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ bool shutdown_requested; /* postmaster sets; LMON main loop polls + exits */ } ClusterLmonSharedState; @@ -188,6 +193,10 @@ extern TimestampTz cluster_lmon_spawned_at(void); extern TimestampTz cluster_lmon_ready_at(void); extern TimestampTz cluster_lmon_last_liveness_tick_at(void); extern int64 cluster_lmon_main_loop_iters(void); +extern uint64 cluster_lmon_last_iter_us(void); +extern uint64 cluster_lmon_max_iter_us(void); +extern uint64 cluster_lmon_slow_iter_count(void); +extern void cluster_lmon_marker_complete_wakeup(void); /* * Status enum -> canonical lowercase string ("not_started", "spawning", diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h new file mode 100644 index 0000000000..792e583b47 --- /dev/null +++ b/src/include/cluster/cluster_marker_async.h @@ -0,0 +1,208 @@ +/*------------------------------------------------------------------------- + * + * cluster_marker_async.h + * Small process-local async FSM for qvotec marker submit mailboxes. + * + * The mailbox remains the existing request_seq/completion_seq/result slot. + * This wrapper changes only the waiting shape: submit publishes a request and + * returns; the owning LMON tick polls completion without sleeping. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_MARKER_ASYNC_H +#define CLUSTER_MARKER_ASYNC_H + +#include "c.h" + +#include + +#include "datatype/timestamp.h" +#include "port/atomics.h" +#include "storage/latch.h" + +typedef enum ClusterMarkerAsyncState { + CLUSTER_MARKER_ASYNC_IDLE = 0, + CLUSTER_MARKER_ASYNC_SUBMITTED +} ClusterMarkerAsyncState; + +typedef enum ClusterMarkerPollResult { + CLUSTER_MARKER_POLL_IDLE = 0, + CLUSTER_MARKER_POLL_PENDING, + CLUSTER_MARKER_POLL_ACKED, + CLUSTER_MARKER_POLL_TIMEOUT +} ClusterMarkerPollResult; + +typedef enum ClusterMarkerAsyncKind { + CLUSTER_MARKER_KIND_UNKNOWN = 0, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, + CLUSTER_MARKER_KIND_JOIN_PREPARE, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED +} ClusterMarkerAsyncKind; + +typedef struct ClusterMarkerAsync { + ClusterMarkerAsyncState state; + uint64 inflight_seq; + uint64 deadline_us; + TimestampTz submitted_at; + ClusterMarkerAsyncKind kind; + int32 target_node; + + /* + * Caller-owned staged publish/commit state. While true, the owner must not + * re-enter the pre-bump path; it either submits/polls this record, publishes + * after ACK, or releases it after failure/timeout. + */ + bool has_staged_event; + uint64 staged_expect_epoch; +} ClusterMarkerAsync; + +static inline const char * +cluster_marker_async_kind_name(ClusterMarkerAsyncKind kind) +{ + switch (kind) { + case CLUSTER_MARKER_KIND_FENCE_FAILSTOP: + return "fence_failstop"; + case CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED: + return "fence_node_removed"; + case CLUSTER_MARKER_KIND_JOIN_PREPARE: + return "join_prepare"; + case CLUSTER_MARKER_KIND_JOIN_COMMITTED: + return "join_committed"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTING: + return "clean_leave_committing"; + case CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED: + return "clean_leave_committed"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVING: + return "node_remove_removing"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK: + return "node_remove_shrunk"; + case CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED: + return "node_remove_removed"; + case CLUSTER_MARKER_KIND_UNKNOWN: + default: + return "unknown"; + } +} + +static inline void +cluster_marker_async_init(ClusterMarkerAsync *a) +{ + memset(a, 0, sizeof(*a)); + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->target_node = -1; +} + +static inline bool +cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) +{ + return a != NULL && a->state == CLUSTER_MARKER_ASYNC_SUBMITTED; +} + +static inline bool +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq) +{ + return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); +} + +static inline bool +cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, + TimestampTz now, uint64 timeout_us, + ClusterMarkerAsyncKind kind, int32 target_node) +{ + uint64 seq; + + Assert(a != NULL); + Assert(request_seq != NULL); + Assert(completion_seq != NULL); + + if (a->state == CLUSTER_MARKER_ASYNC_SUBMITTED) + return true; + if (cluster_marker_async_mailbox_busy(request_seq, completion_seq)) + return false; + + pg_write_barrier(); + seq = pg_atomic_add_fetch_u64(request_seq, 1); + if (qvotec_latch != NULL) + SetLatch(qvotec_latch); + + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->inflight_seq = seq; + a->deadline_us = (uint64)now + timeout_us; + a->submitted_at = now; + a->kind = kind; + a->target_node = target_node; + return true; +} + +static inline ClusterMarkerPollResult +cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot, TimestampTz now, + uint32 *out_result, uint64 *out_elapsed_us) +{ + uint64 elapsed; + + Assert(a != NULL); + Assert(completion_seq != NULL); + Assert(result_slot != NULL); + + if (out_result != NULL) + *out_result = 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (a->state != CLUSTER_MARKER_ASYNC_SUBMITTED) + return CLUSTER_MARKER_POLL_IDLE; + + if (pg_atomic_read_u64(completion_seq) == a->inflight_seq) { + pg_read_barrier(); + if (out_result != NULL) + *out_result = pg_atomic_read_u32(result_slot); + elapsed = ((uint64)now > (uint64)a->submitted_at) + ? ((uint64)now - (uint64)a->submitted_at) + : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_ACKED; + } + + if ((uint64)now >= a->deadline_us) { + elapsed = ((uint64)now > (uint64)a->submitted_at) + ? ((uint64)now - (uint64)a->submitted_at) + : 0; + if (out_elapsed_us != NULL) + *out_elapsed_us = elapsed; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return CLUSTER_MARKER_POLL_TIMEOUT; + } + + return CLUSTER_MARKER_POLL_PENDING; +} + +static inline void +cluster_marker_async_release_stage(ClusterMarkerAsync *a) +{ + if (a == NULL) + return; + a->state = CLUSTER_MARKER_ASYNC_IDLE; + a->inflight_seq = 0; + a->deadline_us = 0; + a->submitted_at = 0; + a->kind = CLUSTER_MARKER_KIND_UNKNOWN; + a->target_node = -1; + a->has_staged_event = false; + a->staged_expect_epoch = 0; +} + +#endif /* CLUSTER_MARKER_ASYNC_H */ diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index ce6bbd7b19..1df3d8c868 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -74,6 +74,8 @@ #include "port/atomics.h" #include "storage/lwlock.h" +#include "cluster/cluster_marker_async.h" + #include "cluster/cluster_write_fence.h" /* CLUSTER_FENCE_MARKER_BYTES (marker offset) */ /* 128 nodes, same width as ReconfigEvent.dead_bitmap. */ @@ -415,6 +417,15 @@ extern void cluster_node_remove_shmem_register(void); * ------------------------------------------------------------------ */ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); +extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, + const ClusterRemovalMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_node_remove_qvotec_poll_pending(ClusterRemovalMarker *out); extern void cluster_node_remove_qvotec_complete(bool acked); extern void cluster_node_remove_publish_qvotec_latch(struct Latch *latch); diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index e34c3a1942..3e6abadc16 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -57,6 +57,7 @@ #include "storage/lwlock.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ struct Latch; /* spec-5.15 D4 — join-marker qvotec mailbox latch (pointer only) */ @@ -316,6 +317,8 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 join_reject_count; pg_atomic_uint64 join_timeout_count; pg_atomic_uint64 clean_departed_cleared_count; + pg_atomic_uint64 marker_slow_ack_count; + pg_atomic_uint64 marker_timeout_count; /* * spec-5.15 D1 — per declared node, the freshest voting-slot incarnation + @@ -580,10 +583,25 @@ extern bool cluster_reconfig_commit_member(int32 node_id, uint64 admitted_incarn */ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); +extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, + const ClusterJoinCommitMarker *m, + ClusterMarkerAsyncKind kind, + TimestampTz now); +extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern bool cluster_reconfig_join_qvotec_poll_pending(int32 *out_target_node, void *out_slot512); extern void cluster_reconfig_join_qvotec_complete(bool acked); extern void cluster_reconfig_publish_join_qvotec_latch(struct Latch *latch); +extern void cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern void cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_node, + uint64 elapsed_us); +extern uint64 cluster_reconfig_get_marker_slow_ack_count(void); +extern uint64 cluster_reconfig_get_marker_timeout_count(void); + /* ============================================================ * spec-5.15 D5 — joiner-side write gate + admission (INV-J9 / §2.4 / §2.7). * ============================================================ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index 7543664e72..dad13ff00d 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -39,6 +39,8 @@ #ifndef CLUSTER_WRITE_FENCE_H #define CLUSTER_WRITE_FENCE_H +#include "cluster/cluster_marker_async.h" + /* * cluster_write_fence_decide -- the PURE cooperative write-fence judge (spec-4.12 * D3). A shared-storage write is allowed iff enforcement is on AND the local @@ -517,6 +519,15 @@ extern void cluster_write_fence_note_baseline_stale(void); */ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); +extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, + const ClusterFenceMarker *m, + ClusterMarkerAsyncKind kind, + int32 target_node, + TimestampTz now); +extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, + TimestampTz now, + uint32 *out_result, + uint64 *out_elapsed_us); extern void cluster_write_fence_publish_qvotec_latch(struct Latch *latch); extern bool cluster_write_fence_qvotec_poll_pending(ClusterFenceMarker *out); extern void cluster_write_fence_qvotec_complete(bool acked); diff --git a/src/test/cluster_tap/t/015_inject.pl b/src/test/cluster_tap/t/015_inject.pl index 772ef599f4..2035b40072 100644 --- a/src/test/cluster_tap/t/015_inject.pl +++ b/src/test/cluster_tap/t/015_inject.pl @@ -57,8 +57,8 @@ # ---------- is( $node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'pg_stat_cluster_injections returns 158 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'pg_stat_cluster_injections returns 159 rows (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- @@ -86,8 +86,8 @@ 'postgres', 'SELECT string_agg(name, \',\' ORDER BY name) FROM pg_stat_cluster_injections' ), - 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', - '158 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + 'cluster-catalog-services-ready-force-closed,cluster-clean-leave-barrier-complete,cluster-clean-leave-escalate-to-failstop,cluster-clean-leave-gcs-flushed,cluster-clean-leave-ges-drained,cluster-clean-leave-quiesce-pre,cluster-clean-leave-request,cluster-clean-leave-survivor-suppress-preflight-ack,cluster-clean-xfer-stale-holder,cluster-collision-detect,cluster-conf-load-success,cluster-conf-parse-fail,cluster-conf-shmem-init,cluster-cr-resolver-memo-suspect,cluster-cr-skip-epoch-bump,cluster-cssd-main-loop-pre-tick,cluster-cssd-mark-peer-dead,cluster-cssd-post-spawn,cluster-cssd-pre-spawn,cluster-cssd-ready-publish,cluster-cssd-shutdown-post,cluster-cssd-shutdown-pre,cluster-debug-dump-entry,cluster-diag-main-loop-iter,cluster-diag-post-spawn,cluster-diag-pre-spawn,cluster-diag-ready-publish,cluster-diag-shutdown-post,cluster-diag-shutdown-pre,cluster-fence-post-thaw-broadcast,cluster-fence-pre-freeze-broadcast,cluster-fence-pre-self-fence-shutdown,cluster-gcs-block-bast-nudge,cluster-gcs-block-drop-reply-before-send,cluster-gcs-block-evict-holder-before-ship,cluster-gcs-block-force-epoch-stale-reply,cluster-gcs-block-forward-master-side,cluster-gcs-block-invalidate-drop-broadcast,cluster-gcs-block-invalidate-stall-ack,cluster-gcs-block-remote-downgrade,cluster-gcs-block-stale-ship,cluster-gcs-block-starvation-force-denied,cluster-gcs-block-x-forward-master-side,cluster-grd-redeclare-skip,cluster-guc-init-pre-define,cluster-ic-mock-send-pre-enqueue,cluster-ic-tier-selected,cluster-init-post-shmem,cluster-init-pre-shmem,cluster-init-top,cluster-ko-peer-skip-ack,cluster-lck-main-loop-iter,cluster-lck-post-spawn,cluster-lck-pre-spawn,cluster-lck-ready-publish,cluster-lck-shutdown-post,cluster-lck-shutdown-pre,cluster-lmd-force-partial-round,cluster-lmon-main-loop-iter,cluster-lmon-post-spawn,cluster-lmon-pre-spawn,cluster-lmon-ready-publish,cluster-lmon-shutdown-post,cluster-lmon-shutdown-pre,cluster-lms-cr-construct,cluster-lms-undo-fetch,cluster-node-remove-cleanup-done,cluster-node-remove-escalate,cluster-node-remove-fence-armed,cluster-node-remove-precheck,cluster-node-remove-request,cluster-node-remove-shrink-committed,cluster-node-remove-shrink-committing,cluster-pcm-acquire-entry,cluster-pcm-convert-pre,cluster-pcm-downgrade-pre,cluster-pcm-release-pre,cluster-pgstat-mirror-sync,cluster-quorum-loss-broadcast,cluster-qvotec-marker-service-hold,cluster-qvotec-poll-post,cluster-qvotec-poll-pre,cluster-reconfig-broadcast-procsig-pre,cluster-reconfig-decide-coordinator,cluster-reconfig-epoch-bump-pre,cluster-reconfig-join-commit-marker-durable,cluster-reconfig-tick-entry,cluster-recovery-anchor-force-failclosed,cluster-relmap-crash-after-stage,cluster-relmap-crash-before-publish,cluster-run-shutdown-top,cluster-run-startup-top,cluster-scn-abort-post-advance,cluster-scn-abort-pre-advance,cluster-scn-advance-post,cluster-scn-advance-pre,cluster-scn-boc-sweep-post,cluster-scn-boc-sweep-pre,cluster-scn-commit-post-advance,cluster-scn-commit-pre-advance,cluster-scn-observe-bump-pre,cluster-scn-observe-entry,cluster-scn-replay-observe-pre,cluster-scn-wal-write-pre,cluster-scn-wraparound-warning,cluster-shared-fs-backend-register,cluster-shared-fs-init-top,cluster-shared-fs-local-open,cluster-shmem-region-init-post,cluster-shmem-region-init-pre,cluster-shmem-register-region,cluster-shmem-request,cluster-shmem-views-srf-entry,cluster-shutdown-top,cluster-sinval-ack-drop-send,cluster-sinval-ack-skip-validate,cluster-sinval-broadcast-drop-send,cluster-sinval-receive-skip-validate,cluster-smgr-create-top,cluster-smgr-open-top,cluster-smgr-which-decision,cluster-startup-phase-0-enter,cluster-startup-phase-0-exit,cluster-startup-phase-0-fail,cluster-startup-phase-1-enter,cluster-startup-phase-1-exit,cluster-startup-phase-1-fail,cluster-startup-phase-2-enter,cluster-startup-phase-2-exit,cluster-startup-phase-2-fail,cluster-startup-phase-3-enter,cluster-startup-phase-3-exit,cluster-startup-phase-3-fail,cluster-startup-phase-4-enter,cluster-startup-phase-4-exit,cluster-startup-phase-4-fail,cluster-stats-main-loop-iter,cluster-stats-post-spawn,cluster-stats-pre-spawn,cluster-stats-ready-publish,cluster-stats-shutdown-post,cluster-stats-shutdown-pre,cluster-thread-recovery-drive,cluster-views-srf-entry,cluster-voting-disk-write-fail,cluster-wal-page-init-thread-id,cluster-wal-state-ensure-pre,cluster-wal-state-write-fail,cluster-wal-thread-claim-create-fail,cluster-wal-thread-validate-pre,cluster-xid-herding-stall,cluster-xid-window-hard-limit,cr_construct_delay_us,cr_corruption,cr_cross_instance,cr_force_read_scn,cr_snapshot_too_old,undo-force-wal-before-data-violation,undo-skip-checkpoint-flush-one', + '159 injection point names match the full registry (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 cluster-clean-leave-survivor-suppress-preflight-ack; spec-5.53 +1 cluster-cr-skip-epoch-bump; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.8 +1 cluster-lmd-force-partial-round; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/017_debug.pl b/src/test/cluster_tap/t/017_debug.pl index 1f24fab0a2..6d53bad1df 100644 --- a/src/test/cluster_tap/t/017_debug.pl +++ b/src/test/cluster_tap/t/017_debug.pl @@ -63,8 +63,8 @@ 'postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'all 55 categories appear (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'all 56 categories appear (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; L122 alphabetic verify)'); # ---------- @@ -123,15 +123,15 @@ 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.fault_type'}), - '158', - 'all 158 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .fault_type entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is( $node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='inject' AND key LIKE '%.hits'}), - '158', - 'all 158 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'all 159 injection points have a .hits entry under inject category (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2; spec-5.13 H1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/018_shared_fs.pl b/src/test/cluster_tap/t/018_shared_fs.pl index 676dfc9e1a..42f8817b91 100644 --- a/src/test/cluster_tap/t/018_shared_fs.pl +++ b/src/test/cluster_tap/t/018_shared_fs.pl @@ -131,8 +131,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L9 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L9 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); # ---------- diff --git a/src/test/cluster_tap/t/020_shmem_registry.pl b/src/test/cluster_tap/t/020_shmem_registry.pl index cb4c48513e..eab706fd4c 100644 --- a/src/test/cluster_tap/t/020_shmem_registry.pl +++ b/src/test/cluster_tap/t/020_shmem_registry.pl @@ -280,8 +280,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L15 total injection registry size is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; full breakdown in t/015)'); + '159', + 'L15 total injection registry size is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 cluster-xid-herding-stall + cluster-xid-window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/021_block_format.pl b/src/test/cluster_tap/t/021_block_format.pl index a1c589de51..57a321a3a6 100644 --- a/src/test/cluster_tap/t/021_block_format.pl +++ b/src/test/cluster_tap/t/021_block_format.pl @@ -188,8 +188,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- diff --git a/src/test/cluster_tap/t/022_itl_slot.pl b/src/test/cluster_tap/t/022_itl_slot.pl index 871ee4a7d0..f2cac828a8 100644 --- a/src/test/cluster_tap/t/022_itl_slot.pl +++ b/src/test/cluster_tap/t/022_itl_slot.pl @@ -204,8 +204,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L12a pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L12a pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); is($node->safe_psql( 'postgres', diff --git a/src/test/cluster_tap/t/023_buffer_descriptor.pl b/src/test/cluster_tap/t/023_buffer_descriptor.pl index d5aa071872..2c41642a72 100644 --- a/src/test/cluster_tap/t/023_buffer_descriptor.pl +++ b/src/test/cluster_tap/t/023_buffer_descriptor.pl @@ -157,8 +157,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L11 pg_stat_cluster_injections is 158 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; full breakdown in t/015)'); + '159', + 'L11 pg_stat_cluster_injections is 159 (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 xid herding/hard-limit; spec-6.12 waves a/b/i +3; spec-2.29a +1 cluster-qvotec-marker-service-hold; full breakdown in t/015)'); # ---------- @@ -168,8 +168,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12 pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12 pg_cluster_state has 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -232,7 +232,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L16 cluster_smoke surface integrates buffer_format + pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + xid_stripe + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..62f66a1aa8 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -162,8 +162,8 @@ is($node->safe_psql( 'postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', - 'L6a pg_stat_cluster_injections has 158 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', + 'L6a pg_stat_cluster_injections has 159 entries (spec-6.14 D5+D8 +3; spec-5.6a +1; spec-5.13 +6 cluster-clean-leave-* + Hardening v1.0.3 +1 suppress-preflight-ack) (spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.7 +1 cluster-ko-peer-skip-ack; spec-2.41 +1 cluster-gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql( 'postgres', @@ -189,8 +189,8 @@ is($node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L7b pg_cluster_state has 55 distinct categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L7b pg_cluster_state has 56 distinct categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); # ---------- @@ -209,7 +209,7 @@ my $smoke_categories = $node->safe_psql( 'postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}); -is($smoke_categories, '55', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion categories (55 categories; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); +is($smoke_categories, '56', 'L9 cluster_smoke surface integrates pcm + gcs + tt_status + tt_status_hint + tt_2pc + tt_recovery + undo_record + visibility + wal_thread + dl + hw + ir + ko + ts + smart_fusion + reconfig categories (56 categories; spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.14 adds catalog)'); # ---------- diff --git a/src/test/cluster_tap/t/030_acceptance.pl b/src/test/cluster_tap/t/030_acceptance.pl index 6d163b4552..3672d99bce 100644 --- a/src/test/cluster_tap/t/030_acceptance.pl +++ b/src/test/cluster_tap/t/030_acceptance.pl @@ -330,7 +330,7 @@ is($node->safe_psql('postgres', 'SELECT count(*) FROM pg_stat_cluster_injections'), - '158', 'M1 158 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + '159', 'M1 159 injection points (spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.18 +7 cluster-node-remove-*; spec-5.13 +6 cluster-clean-leave-*; spec-5.13 Hardening v1.0.3 +1 clean-leave-survivor-suppress-preflight-ack; spec-2.41 +1 gcs-block-stale-ship; spec-5.7 D6 +1 ko-peer-skip-ack; spec-5.2a +1 clean-xfer stale-holder; spec-4.8ab +2 undo boundary guards; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->safe_psql('postgres', q{SELECT string_agg(name, ',' ORDER BY name) FROM pg_stat_cluster_injections WHERE name LIKE 'cluster-init-%'}), @@ -360,8 +360,8 @@ 'postgres', q{SELECT count(DISTINCT key) FROM pg_cluster_state WHERE category='inject' AND (key LIKE '%.fault_type' OR key LIKE '%.hits')} - ) eq '316', - 'M5 inject category has 158×2 = 316 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable)'); + ) eq '318', + 'M5 inject category has 159×2 = 318 sub-keys (.fault_type + .hits; spec-6.14 D5 +2 cluster-relmap-crash-*; spec-5.6a +1 cluster-recovery-anchor-force-failclosed; spec-6.14 D8 +1 cluster-catalog-services-ready-force-closed; spec-6.12e2 +1 cluster-gcs-block-bast-nudge; spec-6.15 D3 +2 herding-stall + window-hard-limit; spec-6.12i +1 cluster-lms-undo-fetch; spec-6.12b +1 cluster-lms-cr-construct; spec-6.12a ㉕ +1 cluster-gcs-block-remote-downgrade; spec-5.13 +6 cluster-clean-leave-*; spec-2.41 +1 gcs-block-stale-ship; spec-5.55 Hardening v1.1 +1 cluster-cr-resolver-memo-suspect; spec-5.15 Hardening v1.1 +1 cluster-reconfig-join-commit-marker-durable; spec-2.29a +1 cluster-qvotec-marker-service-hold)'); is($node->get_cluster_state_value('inject', 'armed_count'), '0', 'M6 inject.armed_count starts at 0 in fresh backend'); @@ -395,8 +395,8 @@ is($node->safe_psql('postgres', q{SELECT string_agg(DISTINCT category, ',' ORDER BY category) FROM pg_cluster_state}), - 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', - 'O2 pg_cluster_state has all 55 categories (spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); + 'advisory,block_format,buffer_format,catalog,cf,cluster_cssd,cluster_stats,conf,cr,cr_coord,cr_pool,diag,dl,gcs,gcs_recovery,ges,grd,grd_recovery,guc,hang,hw,ic,inject,ir,ko,lck,lmd,lmon,lms,pcm,pgstat,phase,reconfig,reconfig_join,reconfig_touched,recovery,resolver_cache,scn,sequence,shared_fs,shmem,sinval,smart_fusion,ts,tt_2pc,tt_recovery,tt_status,tt_status_hint,undo,undo_cleaner,visibility,wal_thread,write_fence,xid_stripe,xnode_lever,xnode_profile', + 'O2 pg_cluster_state has all 56 categories (spec-2.29a adds reconfig marker telemetry; spec-6.15 adds xid_stripe; spec-6.12 adds xnode_lever; spec-6.2 adds smart_fusion; spec-6.14 adds catalog)'); is($node->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state WHERE value IS NULL}), diff --git a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl new file mode 100644 index 0000000000..e2f19d6454 --- /dev/null +++ b/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl @@ -0,0 +1,176 @@ +#!/usr/bin/env perl +#------------------------------------------------------------------------- +# +# 358_cluster_2_29a_marker_async_liveness.pl +# BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. +# +# The hold injection freezes qvotec marker completion until released. Before +# BUG-C1, LMON synchronously waited inside marker submit, stopped sending HELLOs, +# and peers could false-DEAD it during a long marker write. This TAP holds the +# marker service for >=90s while an online join wants a PREPARE marker and checks: +# L1 hold injection is registered. +# L2 a true leave/restart creates a join marker backlog. +# L3 during a 95s hold, LMON keeps iterating and both live nodes keep each +# other CSSD-alive (no false-DEAD). +# L4 marker timeout/backlog is observable and does not publish admission. +# L5 releasing the hold lets the join converge. +# L6 an actual stopped node still becomes DEAD (no true-DEAD regression). +# +# Portions Copyright (c) 2026, pgrac contributors +# +# IDENTIFICATION +# src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterPair; +use Test::More; +use Time::HiRes qw(usleep); + +sub poll_until +{ + my ($node, $query, $expected, $timeout_s, $label) = @_; + $expected //= 't'; + $timeout_s //= 30; + my $deadline = time + $timeout_s; + my $last = ''; + while (time < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) } // ''; + return 1 if defined $last && $last eq $expected; + usleep(200_000); + } + diag("poll_until timeout ($label): last='$last' expected='$expected'"); + return 0; +} + +sub state_int +{ + my ($node, $category, $key) = @_; + my $v = $node->safe_psql('postgres', + "SELECT COALESCE((SELECT value::bigint FROM pg_cluster_state " + . "WHERE category = '$category' AND key = '$key'), 0)"); + return $v + 0; +} + +sub set_injection +{ + my ($node, $value) = @_; + $node->safe_psql('postgres', "ALTER SYSTEM SET cluster.injection_points = '$value'"); + $node->safe_psql('postgres', 'SELECT pg_reload_conf()'); + usleep(700_000); + return; +} + +my @conf = ( + 'autovacuum = off', + 'jit = off', + 'cluster.online_join = on', + 'cluster.join_convergence_timeout_ms = 180000', + 'cluster.quorum_poll_interval_ms = 500', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 8', + 'cluster.ges_request_timeout_ms = 30000', + 'cluster.lmon_slow_iteration_warn_ms = 0', +); + +my $pair = PostgreSQL::Test::ClusterPair->new_pair( + 'marker_async_liveness', + quorum_voting_disks => 3, + shared_data => 1, + extra_conf => [@conf]); +$pair->start_pair; +usleep(3_000_000); + +my $node0 = $pair->node0; +my $node1 = $pair->node1; + +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_stat_cluster_injections + WHERE name = 'cluster-qvotec-marker-service-hold'}), + '1', + 'L1 hold injection point is registered'); + +ok($pair->wait_for_peer_state(0, 1, 'connected', 30), 'L1 node0 sees node1 connected'); +ok($pair->wait_for_peer_state(1, 0, 'connected', 30), 'L1 node1 sees node0 connected'); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 30, 'node1 initially member'), + 'L1 node1 initially MEMBER'); + +my $epoch0 = $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; + +$node1->stop; +ok(poll_until($node0, + q{SELECT state IN ('suspected','dead') FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead'), + 'L2 node0 detects the stopped node1 as dead'); +ok(poll_until($node0, + q{SELECT state <> 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'node1 demoted from member'), + 'L2 node1 is demoted from MEMBER before rejoin'); + +set_injection($node0, 'cluster-qvotec-marker-service-hold'); +my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); + +$node1->start; +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 CSSD alive after restart'), + 'L2 node1 is live again while marker service is held'); + +sleep 95; + +my $lmon_iters_after = state_int($node0, 'lmon', 'lmon_slow_iter_count'); +my $timeouts_after = state_int($node0, 'reconfig', 'marker_timeout_count'); + +cmp_ok($lmon_iters_after, '>', $lmon_iters_before + 20, + 'L3 node0 LMON keeps iterating during >=90s qvotec marker hold'); +ok(poll_until($node1, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 0}, + 't', 5, 'node1 still sees node0 alive after hold'), + 'L3 node1 still sees node0 CSSD-alive after the hold (no false-DEAD)'); +ok(poll_until($node0, + q{SELECT state = 'alive' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 5, 'node0 still sees node1 alive after hold'), + 'L3 node0 still sees node1 CSSD-alive after the hold'); + +cmp_ok($timeouts_after, '>', $timeouts_before, + 'L4 marker timeout/backlog counter advances while qvotec marker service is held'); +isnt($node0->safe_psql('postgres', + q{SELECT state FROM pg_cluster_membership WHERE node_id = 1}), + 'member', + 'L4 held/timeout marker does not publish node1 admission'); + +set_injection($node0, ''); +ok(poll_until($node0, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 joins after hold release'), + 'L5 node0 admits node1 after hold release'); +ok(poll_until($node1, + q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 90, 'node1 self member after hold release'), + 'L5 node1 self-state reaches MEMBER after hold release'); +cmp_ok($node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0, + '>', $epoch0, + 'L5 membership epoch advances after rejoin'); + +$node1->stop; +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_cssd_peers WHERE node_id = 1}, + 't', 60, 'node1 true dead after final stop'), + 'L6 true DEAD detection still works after async marker liveness fix'); +ok(poll_until($node0, + q{SELECT state = 'dead' FROM pg_cluster_membership WHERE node_id = 1}, + 't', 60, 'membership dead after final stop'), + 'L6 membership fail-stop path still demotes a truly stopped peer'); + +$pair->stop_pair; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..0d753ce2b6 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -41,7 +41,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_buffer_desc test_cluster_pcm_lock test_cluster_bufmgr_pcm_hook test_cluster_gcs_dispatch test_cluster_gcs_block test_cluster_gcs_block_retransmit test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_sinval test_cluster_sinval_ack test_cluster_stage2_acceptance test_cluster_tt_status test_cluster_tt_status_hint test_cluster_visibility_fork test_cluster_visibility_decide_scn test_cluster_snapshot_source test_cluster_itl_touch test_cluster_itl_wal test_cluster_uba \ test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision \ test_cluster_xlog test_cluster_tt_slot test_cluster_undo_segment \ - test_cluster_epoch test_cluster_fence test_cluster_reconfig \ + test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_marker_async \ test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory \ test_cluster_tt_slot_allocator test_cluster_itl_reader_real_triple \ test_cluster_itl_cleanout test_cluster_visibility_inject test_cluster_itl_cleanout_perf \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..b0a4fe5746 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -2638,6 +2638,21 @@ cluster_lmon_main_loop_iters(void) { return 0; } +uint64 +cluster_lmon_last_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_max_iter_us(void) +{ + return 0; +} +uint64 +cluster_lmon_slow_iter_count(void) +{ + return 0; +} int cluster_lmon_status(void) { @@ -3377,6 +3392,16 @@ cluster_reconfig_get_clean_departed_cleared_count(void) { return 0; } +uint64 +cluster_reconfig_get_marker_slow_ack_count(void) +{ + return 0; +} +uint64 +cluster_reconfig_get_marker_timeout_count(void) +{ + return 0; +} void cluster_touched_peers_self_hex(char *buf, Size buflen) { diff --git a/src/test/cluster_unit/test_cluster_lmon.c b/src/test/cluster_unit/test_cluster_lmon.c index d5b86a9bfd..15bc7894b1 100644 --- a/src/test/cluster_unit/test_cluster_lmon.c +++ b/src/test/cluster_unit/test_cluster_lmon.c @@ -178,6 +178,7 @@ cluster_postmaster_start_lmon(void) * MyLatch + cluster_inject framework. Stubs cover them all -- * runtime LmonMain is not exercised. */ int cluster_lmon_main_loop_interval = 1000; +int cluster_lmon_slow_iteration_warn_ms = 1000; #include "cluster/cluster_inject.h" int cluster_injection_armed_count = 0; @@ -590,6 +591,9 @@ pg_usleep(long microsec pg_attribute_unused()) /* Sprint B: Latch / WaitLatch / ResetLatch stubs (LmonMain runtime * is not invoked at unit-test level). */ struct Latch *MyLatch = NULL; +void +SetLatch(struct Latch *latch pg_attribute_unused()) +{} int WaitLatch(struct Latch *latch pg_attribute_unused(), int wakeEvents pg_attribute_unused(), long timeout pg_attribute_unused(), uint32 wait_event_info pg_attribute_unused()) @@ -600,6 +604,11 @@ void ResetLatch(struct Latch *latch pg_attribute_unused()) {} +#include "storage/ipc.h" +void +on_shmem_exit(pg_on_exit_callback function pg_attribute_unused(), Datum arg pg_attribute_unused()) +{} + /* cluster_lmon.c references MyBackendType (set by LmonMain). */ #include "miscadmin.h" BackendType MyBackendType = B_INVALID; @@ -804,9 +813,21 @@ UT_TEST(test_lmon_public_symbols_linkable) UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_size); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_init); UT_ASSERT_NOT_NULL((void *)cluster_lmon_shmem_register); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_last_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_max_iter_us); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_slow_iter_count); + UT_ASSERT_NOT_NULL((void *)cluster_lmon_marker_complete_wakeup); UT_ASSERT_NOT_NULL((void *)LmonMain); } +UT_TEST(test_lmon_iteration_counters_null_safe) +{ + UT_ASSERT_EQ((unsigned long long)cluster_lmon_last_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_max_iter_us(), 0ULL); + UT_ASSERT_EQ((unsigned long long)cluster_lmon_slow_iter_count(), 0ULL); + cluster_lmon_marker_complete_wakeup(); +} + /* ============================================================ * Test runner @@ -815,12 +836,13 @@ UT_TEST(test_lmon_public_symbols_linkable) int main(void) { - UT_PLAN(5); + UT_PLAN(6); UT_RUN(test_lmon_status_enum_values_frozen); UT_RUN(test_lmon_shared_state_size_under_4kb); UT_RUN(test_lmon_status_to_string_lookup); UT_RUN(test_lmon_status_unknown_returns_unknown); UT_RUN(test_lmon_public_symbols_linkable); + UT_RUN(test_lmon_iteration_counters_null_safe); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c new file mode 100644 index 0000000000..1618c8ea2d --- /dev/null +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -0,0 +1,236 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_marker_async.c + * Unit tests for the process-local qvotec marker async FSM. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_marker_async.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_marker_async.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +static int ut_set_latch_count = 0; + +void +SetLatch(Latch *latch pg_attribute_unused()) +{ + ut_set_latch_count++; +} + +static void +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, + pg_atomic_uint64 *completion_seq, pg_atomic_uint32 *result_slot) +{ + cluster_marker_async_init(a); + pg_atomic_init_u64(request_seq, 0); + pg_atomic_init_u64(completion_seq, 0); + pg_atomic_init_u32(result_slot, 0); + ut_set_latch_count = 0; +} + +UT_TEST(test_init_defaults_idle) +{ + ClusterMarkerAsync a; + + cluster_marker_async_init(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); +} + +UT_TEST(test_submit_publishes_one_request) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 7); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(pg_atomic_read_u64(&completion_seq), 0); + UT_ASSERT_EQ(ut_set_latch_count, 1); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.deadline_us, 6000); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_FAILSTOP); + UT_ASSERT_EQ(a.target_node, 7); +} + +UT_TEST(test_poll_pending_before_completion) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 99; + uint64 elapsed = 99; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_PREPARE, 2)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 1500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_PENDING); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_SUBMITTED); + UT_ASSERT_EQ(result, 0); + UT_ASSERT_EQ(elapsed, 0); +} + +UT_TEST(test_poll_ack_returns_result_and_elapsed) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint32 result = 0; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_JOIN_COMMITTED, 3)); + pg_atomic_write_u32(&result_slot, 42); + pg_atomic_write_u64(&completion_seq, 1); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2500, &result, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_ACKED); + UT_ASSERT_EQ(result, 42); + UT_ASSERT_EQ(elapsed, 1500); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_poll_timeout_releases_submit_state) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + ClusterMarkerPollResult pr; + uint64 elapsed = 0; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 1000, + CLUSTER_MARKER_KIND_CLEAN_LEAVE_COMMITTED, 4)); + + pr = cluster_marker_async_poll(&a, &completion_seq, &result_slot, 2000, NULL, &elapsed); + + UT_ASSERT_EQ(pr, CLUSTER_MARKER_POLL_TIMEOUT); + UT_ASSERT_EQ(elapsed, 1000); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_mailbox_busy_does_not_publish) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + Latch latch; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + pg_atomic_write_u64(&request_seq, 2); + pg_atomic_write_u64(&completion_seq, 1); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, &latch, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_SHRUNK, 5); + + UT_ASSERT(!ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 2); + UT_ASSERT_EQ(ut_set_latch_count, 0); + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); +} + +UT_TEST(test_reentrant_submit_does_not_bump_again) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + bool ok; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, 6)); + + ok = cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1200, 5000, + CLUSTER_MARKER_KIND_FENCE_FAILSTOP, 99); + + UT_ASSERT(ok); + UT_ASSERT_EQ(pg_atomic_read_u64(&request_seq), 1); + UT_ASSERT_EQ(a.inflight_seq, 1); + UT_ASSERT_EQ(a.kind, CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED); + UT_ASSERT_EQ(a.target_node, 6); +} + +UT_TEST(test_release_stage_clears_staged_record) +{ + ClusterMarkerAsync a; + pg_atomic_uint64 request_seq; + pg_atomic_uint64 completion_seq; + pg_atomic_uint32 result_slot; + + ut_reset(&a, &request_seq, &completion_seq, &result_slot); + a.has_staged_event = true; + a.staged_expect_epoch = 123; + UT_ASSERT(cluster_marker_async_submit(&a, &request_seq, &completion_seq, NULL, 1000, 5000, + CLUSTER_MARKER_KIND_NODE_REMOVE_REMOVED, 8)); + + cluster_marker_async_release_stage(&a); + + UT_ASSERT_EQ(a.state, CLUSTER_MARKER_ASYNC_IDLE); + UT_ASSERT_EQ(a.inflight_seq, 0); + UT_ASSERT_EQ(a.target_node, -1); + UT_ASSERT(!a.has_staged_event); + UT_ASSERT_EQ(a.staged_expect_epoch, 0); +} + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_init_defaults_idle); + UT_RUN(test_submit_publishes_one_request); + UT_RUN(test_poll_pending_before_completion); + UT_RUN(test_poll_ack_returns_result_and_elapsed); + UT_RUN(test_poll_timeout_releases_submit_state); + UT_RUN(test_mailbox_busy_does_not_publish); + UT_RUN(test_reentrant_submit_does_not_bump_again); + UT_RUN(test_release_stage_clears_staged_record); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_qvotec.c b/src/test/cluster_unit/test_cluster_qvotec.c index 62b2099932..3400d68962 100644 --- a/src/test/cluster_unit/test_cluster_qvotec.c +++ b/src/test/cluster_unit/test_cluster_qvotec.c @@ -212,6 +212,14 @@ void cluster_elog_init(void) {} +#include "cluster/cluster_inject.h" +bool +cluster_cr_injection_armed(const char *name pg_attribute_unused(), + uint64 *out_param pg_attribute_unused()) +{ + return false; +} + /* Step 3 D7 stubs: signal/ps_display/procsignal symbols not linked * here (cluster_qvotec.c references them for ClusterQvotecMain; * unit test never invokes Main, just address-takes for T-6). */ diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 573406761c..5860c19831 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -357,6 +357,26 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +bool +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), + const ClusterFenceMarker *m pg_attribute_unused(), + ClusterMarkerAsyncKind kind pg_attribute_unused(), + int32 target_node pg_attribute_unused(), + TimestampTz now pg_attribute_unused()) +{ + return false; +} +ClusterMarkerPollResult +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), + TimestampTz now pg_attribute_unused(), + uint32 *out_result pg_attribute_unused(), + uint64 *out_elapsed_us pg_attribute_unused()) +{ + return CLUSTER_MARKER_POLL_IDLE; +} +void +cluster_lmon_marker_complete_wakeup(void) +{} /* spec-3.1 D7 stub: cluster_reconfig_apply_epoch_bump_as_coordinator * calls cluster_tt_status_flush_all. Fixture has no TT overlay shmem; From 3eeeea1db625bf585d9e5ae7d1882f3f749bdea3 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:01:32 +0800 Subject: [PATCH 02/27] ci: include marker async TAP in nightly --- .github/workflows/nightly.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..307a387f2e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,12 +176,13 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/358 spec-2.29a marker-async LMON liveness. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-358", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 From c7a2c9fc25884e97f1ceaf12478f78958f342dd0 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:10:03 +0800 Subject: [PATCH 03/27] test: stub assertions in marker async unit --- src/test/cluster_unit/test_cluster_marker_async.c | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c index 1618c8ea2d..9823fd69bf 100644 --- a/src/test/cluster_unit/test_cluster_marker_async.c +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -33,6 +33,14 @@ UT_DEFINE_GLOBALS(); static int ut_set_latch_count = 0; +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + void SetLatch(Latch *latch pg_attribute_unused()) { From 8924f6e732966db954e39bc105cf9a4a27a5f086 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 15:54:48 +0800 Subject: [PATCH 04/27] fix(cluster): shared catalog native-era XID authority Publish and seal native-era XID authority after seed shutdown, adopt prehistory from per-node anchors, and max-merge startup nextXid before ShmemVariableCache seeding. Spec: spec-6.15b-xid-authority-native-era.md --- .github/workflows/nightly.yml | 4 +- docs/reference/system-views.md | 3 + docs/user-guide/configuration.md | 47 ++ src/backend/access/transam/xlog.c | 57 ++ src/backend/access/transam/xlogrecovery.c | 10 +- src/backend/cluster/Makefile | 2 + .../cluster/cluster_catalog_bootstrap.c | 180 ++++++- src/backend/cluster/cluster_catalog_migrate.c | 5 +- src/backend/cluster/cluster_debug.c | 14 + src/backend/cluster/cluster_shmem.c | 16 + src/backend/cluster/cluster_xid_authority.c | 340 ++++++++++++ src/backend/cluster/cluster_xid_prehistory.c | 450 ++++++++++++++++ src/backend/utils/errcodes.txt | 11 + src/include/cluster/cluster_catalog_migrate.h | 8 +- src/include/cluster/cluster_xid_authority.h | 194 +++++++ .../t/337_shared_catalog_ddl_2node.pl | 2 + .../t/339_shared_catalog_faults_2node.pl | 26 +- .../346_shared_catalog_relmap_crash_2node.pl | 14 +- .../t/361_xid_authority_native_era_2node.pl | 410 +++++++++++++++ src/test/cluster_unit/Makefile | 18 +- src/test/cluster_unit/test_cluster_debug.c | 16 + .../cluster_unit/test_cluster_pi_shadow.c | 2 +- .../cluster_unit/test_cluster_xid_authority.c | 488 ++++++++++++++++++ 23 files changed, 2290 insertions(+), 27 deletions(-) create mode 100644 src/backend/cluster/cluster_xid_authority.c create mode 100644 src/backend/cluster/cluster_xid_prehistory.c create mode 100644 src/include/cluster/cluster_xid_authority.h create mode 100644 src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl create mode 100644 src/test/cluster_unit/test_cluster_xid_authority.c diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..8422820ee2 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,11 +177,11 @@ jobs: - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family + t/357 multi-xmax alias floor. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/docs/reference/system-views.md b/docs/reference/system-views.md index 649aa954c8..9ceab9b723 100644 --- a/docs/reference/system-views.md +++ b/docs/reference/system-views.md @@ -210,6 +210,9 @@ must not be read as evidence that early transfer is active. | `smart_fusion` | `origin_suspect_count` | Pending dependencies associated with an origin suspected dead or unavailable. | | `smart_fusion` | `dep_lost_failclosed_count` | Missing or malformed dependency evidence rejected fail-closed. | | `smart_fusion` | `retry_failclosed_count` | Retryable Smart Fusion fail-closed outcomes, primarily commit-brake timeout. | +| `catalog` | `xid_authority_native_hw` | Sealed native-era xid high-water for shared-catalog formation; `0` when the authority is inactive or unreadable. | +| `catalog` | `xid_authority_sealed` | `t` when the seed completed a clean native-era shutdown and published adoptable XID prehistory. | +| `catalog` | `xid_prehistory_adopted` | Process-local marker showing whether this postmaster boot adopted native-era pg_xact prehistory. | Example: diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 007c2cb90e..76d6cc8f73 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -776,6 +776,9 @@ Requirements and behaviour: * `cluster.node_id` must be between 0 and 15; startup is refused otherwise. Striped clusters are therefore limited to **16 nodes**. * `cluster.online_join` must be `on`; startup is refused otherwise. +* `cluster.shared_catalog = on` with more than one declared node requires + `cluster.xid_striping = on`; startup fails closed with SQLSTATE `53RB5` + if a multi-node shared-catalog cluster is formed without striping. * All nodes must run with the same `cluster.xid_striping` setting. The cluster records its striping state durably on the voting disk; a node whose configuration disagrees with that recorded state (or @@ -791,6 +794,50 @@ cluster.xid_striping = on cluster.online_join = on ``` +### Shared-catalog native-era XID authority + +A shared-catalog cluster that is formed from a single seed node has a +short native era: the seed runs with `cluster.enabled = off` while it +creates initial catalog rows, roles, schemas, and seed data. Those +transactions consume the seed node's local xids before other nodes have +joined. During the seed node's clean shutdown, pgrac publishes two +formation files under `cluster.shared_data_dir/global`: +`pgrac_xid_authority` (the sealed native xid high-water) and +`pgrac_xid_prehistory` (the seed node's pg_xact truth for those xids). + +Provisioning sequence: + +1. Create or clone joiner data directories before the seed load if they + must share the seed node's system identifier. +2. Start the seed with `cluster.shared_catalog = on`, + `cluster.controlfile_shared_authority = on`, `cluster.enabled = off`, + and a fixed `cluster.node_id`; perform the seed load. +3. Stop the seed cleanly. Do not use immediate shutdown for the final + native-era stop; joiners refuse an unsealed authority with SQLSTATE + `53RB5`. +4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, + and `cluster.xid_striping = on`. Joiners adopt the prehistory before + WAL startup seeds `nextXid`, so seed tuples and roles are judged from + first-hand pg_xact truth rather than hint bits. + +Do not delete or re-create the authority files to recover a failed join. +A missing, unsealed, or corrupt authority/prehistory is intentionally +fail-closed (`53RB5`); restore the shared tree files from backup or +re-provision the cluster. After the first `cluster.enabled = on` boot, +the native era is permanently closed and a later `cluster.enabled = off` +boot against the same shared tree is refused. + +Observe formation state with: + +```sql +SELECT key, value + FROM pg_cluster_state + WHERE category = 'catalog' + AND key IN ('xid_authority_native_hw', + 'xid_authority_sealed', + 'xid_prehistory_adopted'); +``` + ### `cluster.xid_herding_slack` | | | diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 670985611f..03d5719b89 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -201,6 +201,7 @@ #include "cluster/cluster_guc.h" /* PGRAC: spec-5.6 cluster_controlfile_shared_authority */ #include "cluster/cluster_hw_snapshot.h" /* PGRAC: spec-5.7 D3 HW authority checkpoint snapshot */ #include "cluster/cluster_xid_stripe_xlog.h" /* PGRAC: spec-6.15 D5d checkpoint re-emit */ +#include "cluster/cluster_xid_authority.h" /* PGRAC: spec-6.15b native-era XID authority */ #include "cluster/cluster_recovery_anchor.h" /* PGRAC: spec-5.6a per-node recovery anchor */ #include "cluster/cluster_lms.h" /* PGRAC: spec-5.6 GES-ready boundary for CF X */ #endif @@ -5593,6 +5594,35 @@ StartupXLOG(void) */ if (cluster_recovery_anchor_active() && !InRecovery) checkPoint = cluster_recovery_anchor_get()->checkPointCopy; + + /* + * PGRAC (spec-6.15b D5): shared_catalog nodes must reconcile their + * transaction horizon with the sealed native-era XID authority before + * ShmemVariableCache is seeded and before StartupCLOG computes its latest + * page. This deliberately consumes a dedicated never-lowered authority, + * not the shared pg_control CheckPoint copy (spec-5.6a keeps that + * per-writer). + */ + if (cluster_shared_catalog && cluster_enabled) + { + ClusterXidAuthorityHeader auth; + + if (!cluster_xid_authority_read(&auth) || + (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable during WAL startup"), + errdetail("StartupXLOG cannot seed nextXid without a sealed native-era high-water."), + errhint("Complete a clean native-era seed shutdown before starting shared_catalog nodes."))); + + if (U64FromFullTransactionId(checkPoint.nextXid) < auth.native_hw_full) + { + uint64 own_next = U64FromFullTransactionId(checkPoint.nextXid); + + checkPoint.nextXid = FullTransactionIdFromU64(auth.native_hw_full); + elog(LOG, "cluster shared_catalog: merged nextXid with XID authority native high-water %llu (own was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } + } #endif /* initialize shared memory variables from the checkpoint record */ @@ -7549,6 +7579,33 @@ CreateCheckPoint(int flags) */ END_CRIT_SECTION(); +#ifdef USE_PGRAC_CLUSTER + /* + * PGRAC (spec-6.15b D3): after the shutdown checkpoint record, + * control-file update, CLOG flush, and per-node recovery anchor publish + * are complete, publish the native-era pg_xact prehistory and seal the + * shared XID authority. This does durable file I/O and may fail closed, + * so it must stay outside the checkpoint critical section. + */ + if (shutdown && cluster_shared_catalog && !cluster_enabled) + { + uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); + + if (checkPoint.nextMulti > FirstMultiXactId) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era seed loads that create MultiXacts are not supported"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Recreate the seed without MultiXact-producing operations, or move the load in-protocol."))); + + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long)native_hw); + } + +#endif + /* * Let smgr do post-checkpoint cleanup (eg, deleting old files). */ diff --git a/src/backend/access/transam/xlogrecovery.c b/src/backend/access/transam/xlogrecovery.c index 7c3daf86a5..c0b7a009c7 100644 --- a/src/backend/access/transam/xlogrecovery.c +++ b/src/backend/access/transam/xlogrecovery.c @@ -3273,8 +3273,16 @@ ApplyWalRecord(XLogReaderState *xlogreader, XLogRecord *record, TimeLineID *repl /* * ShmemVariableCache->nextXid must be beyond record's xid. + * + * PGRAC: a merged-recovery FOREIGN record belongs to a peer's xid + * authority, not this node's allocator. Advancing the local nextXid here + * would make StartupCLOG/TrimCLOG expect local pg_xact pages for peer + * striped xids, while the peer outcome truth lives in pg_xact_remote. */ - AdvanceNextFullTransactionIdPastXid(record->xl_xid); +#ifdef USE_PGRAC_CLUSTER + if (!(cluster_recmerge_window_active && cluster_recmerge_apply_foreign)) +#endif + AdvanceNextFullTransactionIdPastXid(record->xl_xid); /* * Before replaying this record, check if this record causes the current diff --git a/src/backend/cluster/Makefile b/src/backend/cluster/Makefile index 5ea41237f7..1a21622e4d 100644 --- a/src/backend/cluster/Makefile +++ b/src/backend/cluster/Makefile @@ -220,6 +220,8 @@ OBJS = \ cluster_views.o \ cluster_wal_state.o \ cluster_wal_thread.o \ + cluster_xid_authority.o \ + cluster_xid_prehistory.o \ cluster_xid_stripe.o \ cluster_xid_stripe_boot.o \ cluster_xid_stripe_xlog.o \ diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 9fee3e481c..96dd7912c6 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -35,6 +35,10 @@ */ #include "postgres.h" +#include +#include + +#include "access/transam.h" #include "access/xlog.h" #include "catalog/pg_control.h" #include "cluster/cluster_catalog_bootstrap.h" @@ -44,13 +48,184 @@ #include "cluster/cluster_inject.h" #include "cluster/cluster_mode.h" #include "cluster/cluster_oid_lease.h" +#include "cluster/cluster_recovery_anchor.h" #include "cluster/cluster_startup_phase.h" +#include "cluster/cluster_xid_authority.h" #include "miscadmin.h" +#include "storage/fd.h" + +static bool +cluster_catalog_backup_label_present(void) +{ + char label_path[MAXPGPATH]; + struct stat st; + + snprintf(label_path, sizeof(label_path), "%s/%s", DataDir, BACKUP_LABEL_FILE); + return stat(label_path, &st) == 0; +} + +static void +cluster_catalog_xid_authority_corrupt_fatal(const char *detail) +{ + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is unavailable at catalog bootstrap"), + errdetail("%s", detail), + errhint("Restore \"%s/%s\" (or its .bak) from a backup of the shared tree; " + "do not delete or re-seed it from a stale xid high-water.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH))); +} + + +/* + * Early D6 topology sniff: cluster_conf_load() runs later when shmem exists, + * but shared_catalog bootstrap must reject multi-node xid_striping=off before + * it seeds/adopts any authority. This deliberately counts only [node.N] + * section headers; the real parser remains the startup SSOT and will still + * validate syntax, required fields, and node_id consistency later. + */ +static int +cluster_catalog_declared_node_count_early(void) +{ + const char *path = (cluster_config_file != NULL && cluster_config_file[0] != '\0') + ? cluster_config_file + : "pgrac.conf"; + FILE *f; + char line[1024]; + int nodes = 0; + + f = AllocateFile(path, "r"); + if (f == NULL) + return 1; + + while (fgets(line, sizeof(line), f) != NULL) { + const char *p = line; + + while (*p != '\0' && isspace((unsigned char)*p)) + p++; + if (strncmp(p, "[node.", 6) == 0) + nodes++; + } + FreeFile(f); + + return nodes > 0 ? nodes : 1; +} + +static void +cluster_catalog_vet_xid_striping_for_shared_catalog(void) +{ + int nodes; + + if (!cluster_enabled || cluster_xid_striping) + return; + + nodes = cluster_catalog_declared_node_count_early(); + if (nodes > 1) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", nodes), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); +} + +static void +cluster_catalog_prepare_xid_authority(const ControlFileData *cf, + ClusterCatalogMigrateResult migrate_result) +{ + ClusterXidAuthorityHeader auth; + bool have_auth; + + have_auth = cluster_xid_authority_read(&auth); + if (!have_auth && cluster_xid_authority_present()) + cluster_catalog_xid_authority_corrupt_fatal( + "Neither the primary shared XID authority nor its .bak fallback passes validation."); + + if (!have_auth) { + if (migrate_result != CLUSTER_CATALOG_MIGRATE_SEEDED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is absent for an existing shared catalog tree"), + errdetail("The shared catalog marker exists, but \"%s/%s\" does not.", + cluster_shared_data_dir, CLUSTER_XID_AUTHORITY_REL_PATH), + errhint("Re-seed the shared tree with a build carrying spec-6.15b."))); + + if (cluster_xid_authority_seed_if_absent( + U64FromFullTransactionId(cf->checkPointCopy.nextXid))) + elog(LOG, "cluster shared_catalog: seeded XID authority native high-water at %llu", + (unsigned long long)U64FromFullTransactionId(cf->checkPointCopy.nextXid)); + + if (!cluster_xid_authority_read(&auth)) + cluster_catalog_xid_authority_corrupt_fatal( + "The shared XID authority was just seeded but cannot be read back."); + } + + if (!cluster_enabled) { + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"), + errdetail("The shared XID authority is already marked cluster-era started."), + errhint( + "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + return; + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is not sealed"), + errdetail("The seed node has not completed a clean native-era shutdown."), + errhint( + "Start the seed with cluster.enabled=off, stop it cleanly, then start joiners."))); + + if (cluster_catalog_backup_label_present()) + elog(LOG, "cluster shared_catalog: skipped XID prehistory adopt on backup_label boot"); + else { + ClusterRecoveryAnchor ra; + uint64 own_next; + + if (!cluster_recovery_anchor_read(cf->system_identifier, &ra, NULL)) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("per-node recovery anchor is unavailable for XID prehistory adopt"), + errdetail("The joiner cannot determine its own pre-adopt nextXid without \"%s\".", + cluster_recovery_anchor_path() != NULL ? cluster_recovery_anchor_path() + : "(unset)"), + errhint( + "Re-provision this node or verify cluster.shared_data_dir before startup."))); + + own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + if (own_next < auth.native_hw_full) { + cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); + elog(LOG, + "cluster shared_catalog: adopted XID prehistory through native high-water %llu " + "(own nextXid was %llu)", + (unsigned long long)auth.native_hw_full, (unsigned long long)own_next); + } else + elog(LOG, + "cluster shared_catalog: skipped XID prehistory adopt; own nextXid %llu >= native " + "high-water %llu", + (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); + } + + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) { + cluster_xid_authority_mark_cluster_era(); + elog(LOG, + "cluster shared_catalog: marked XID authority cluster era started at native " + "high-water %llu", + (unsigned long long)auth.native_hw_full); + } +} void cluster_catalog_startup_prepare(void) { ControlFileData cf; + ClusterCatalogMigrateResult migrate_result; /* Postmaster-once: only the postmaster seeds; forked backends inherit. */ if (IsUnderPostmaster) @@ -66,6 +241,8 @@ cluster_catalog_startup_prepare(void) return; /* off: stock per-node catalog */ } + cluster_catalog_vet_xid_striping_for_shared_catalog(); + /* * shared_catalog=on requires the shared pg_control authority (D1 vet), so * it is the single source of the cluster-wide next-OID high-water. Reading @@ -113,7 +290,8 @@ cluster_catalog_startup_prepare(void) * the (D3-flipped) shared smgr route. system_identifier is the shared * pg_control's cluster-wide value both seed and join agree on. */ - cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + migrate_result = cluster_catalog_migrate_tree(DataDir, cf.system_identifier); + cluster_catalog_prepare_xid_authority(&cf, migrate_result); } /* diff --git a/src/backend/cluster/cluster_catalog_migrate.c b/src/backend/cluster/cluster_catalog_migrate.c index 961179b008..bb17838229 100644 --- a/src/backend/cluster/cluster_catalog_migrate.c +++ b/src/backend/cluster/cluster_catalog_migrate.c @@ -588,7 +588,7 @@ vet_seed_no_tablespaces(const char *local_pgdata) FreeDir(dir); } -void +ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) { ClusterCatalogAuthorityMarker existing; @@ -624,7 +624,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) elog(LOG, "cluster shared_catalog: adopted shared catalog authority (sysid " UINT64_FORMAT ")", system_identifier); - return; + return CLUSTER_CATALOG_MIGRATE_ADOPTED; } /* @@ -645,6 +645,7 @@ cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier) "cluster shared_catalog: seeded shared catalog authority under \"%s\" " "(sysid " UINT64_FORMAT ")", cluster_shared_data_dir, system_identifier); + return CLUSTER_CATALOG_MIGRATE_SEEDED; } /* diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..187ce5f79e 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -130,6 +130,7 @@ PG_FUNCTION_INFO_V1(cluster_dump_state); #include "cluster/cluster_catalog_stats.h" /* catalog counters (spec-6.14 D10b) */ #include "cluster/cluster_oid_lease.h" /* catalog category (spec-6.14 D10) */ #include "cluster/cluster_relmap_authority.h" /* relmap authority state (spec-6.14 D5) */ +#include "cluster/cluster_xid_authority.h" /* XID authority state (spec-6.15b D7) */ #include "cluster/cluster_remote_xact.h" /* remote outcome counters (spec-4.5a D11) */ #include "cluster/cluster_ic.h" /* ClusterICOps_Active, ClusterICTier */ #include "cluster/cluster_ic_tier1.h" /* listener metadata accessors (Hardening v1.0.1 F3) */ @@ -3035,6 +3036,19 @@ dump_catalog(ReturnSetInfo *rsinfo) emit_row(rsinfo, "catalog", "relmap_shared_pending_owner_node", fmt_int64(rmok ? (int64)rmhdr.owner_node : -1)); + /* spec-6.15b D7: native-era XID authority and this-boot adopt marker. */ + { + ClusterXidAuthorityHeader xahdr; + bool xaok = cluster_shared_catalog && cluster_xid_authority_read(&xahdr); + + emit_row(rsinfo, "catalog", "xid_authority_native_hw", + fmt_int64(xaok ? (int64)xahdr.native_hw_full : 0)); + emit_row(rsinfo, "catalog", "xid_authority_sealed", + fmt_bool(xaok && (xahdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) != 0)); + emit_row(rsinfo, "catalog", "xid_prehistory_adopted", + fmt_bool(cluster_xid_prehistory_was_adopted())); + } + /* * D10b shared-catalog data-plane counters: cluster-path visibility * resolutions of catalog-page tuples, their fail-closed (53R97-family) diff --git a/src/backend/cluster/cluster_shmem.c b/src/backend/cluster/cluster_shmem.c index fa4b2db725..5444528540 100644 --- a/src/backend/cluster/cluster_shmem.c +++ b/src/backend/cluster/cluster_shmem.c @@ -1077,6 +1077,22 @@ cluster_init_shmem(void) errhint("Enable cluster.online_join on every node, or disable " "cluster.xid_striping."))); + /* + * spec-6.15b D6: shared_catalog puts catalog heap pages on one shared + * storage image, so a multi-node cluster must not allocate dense native + * xids from every node. Without striping, two nodes can reuse the seed + * native-era xid range and poison shared catalog visibility. + */ + if (cluster_enabled && cluster_shared_catalog && cluster_conf_node_count() > 1 + && !cluster_xid_striping) + ereport( + FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("cluster.shared_catalog on a multi-node cluster requires cluster.xid_striping"), + errdetail("The configured cluster declares %d nodes.", cluster_conf_node_count()), + errhint("Set cluster.xid_striping=on (and cluster.online_join=on) on every " + "shared_catalog node."))); + /* * Stage 0.18: bind the cluster_ic vtable for the configured tier. * Stub mode allocates no shmem; tier1+ (Stage 2+) will piggyback diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c new file mode 100644 index 0000000000..0d73ad4552 --- /dev/null +++ b/src/backend/cluster/cluster_xid_authority.c @@ -0,0 +1,340 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.c + * Shared XID authority: never-lowered native-era xid high-water plus + * one-way lifecycle flags, as a torn-safe file under + * cluster.shared_data_dir (spec-6.15b D1). + * + * The authority records the first xid NOT consumed by the native era + * (the pre-formation cluster.enabled=off single-writer window of the + * seed node) and two one-way flags: SEALED (a clean native-era + * shutdown published a complete high-water together with the pg_xact + * prehistory blob, so joiners may adopt) and CLUSTER_ERA (a + * cluster.enabled=on boot closed the native era forever). Writers are + * single by construction: the seed node's postmaster/checkpointer + * during the native era, and the catalog bootstrap window afterwards. + * + * File discipline mirrors cluster_oid_lease.c: temp + fsync + .bak + * roll + durable_rename on write; primary-then-.bak fail-closed read; + * ENOENT-only-absent presence probe. The pure classify layer is + * standalone-linkable for cluster_unit. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1, §4 U1-U3/U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* On-disk image size: the fixed header, exactly. */ +#define CLUSTER_XID_AUTHORITY_FILE_SIZE ((int)sizeof(ClusterXidAuthorityHeader)) + +/* ============================================================ + * Pure layer (no elog/shmem/fd; standalone-linkable for cluster_unit). + * ============================================================ */ + +/* + * cluster_xid_authority_classify -- validate an authority image buffer. + */ +ClusterXidAuthorityValidity +cluster_xid_authority_classify(const char *buf, size_t len) +{ + ClusterXidAuthorityHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidAuthorityHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_AUTHORITY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * build_path -- join cluster_shared_data_dir + relpath into dst. Returns + * false when the shared root is not configured. + */ +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * write_durable -- write buf into tmp, fsync, then durable_rename over final. + * PANICs on any I/O failure (mirrors cluster_oid_lease write_durable). + */ +static void +write_durable(const char *tmp, const char *final, const char *buf) +{ + int fd; + + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + + errno = 0; + if (write(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE) != CLUSTER_XID_AUTHORITY_FILE_SIZE) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + if (durable_rename(tmp, final, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, final))); +} + +/* + * roll_primary_to_bak -- preserve an existing primary into .bak before a new + * primary write. A missing/short primary (first write) is skipped. + */ +static void +roll_primary_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char buf[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + int fd; + int r; + + fd = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (fd < 0) + return; /* first write: no prior primary */ + + r = read(fd, buf, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return; /* short/odd primary: don't manufacture a .bak */ + + write_durable(baktmp, bak, buf); +} + +/* + * read_image -- read a candidate authority file and classify it. Missing or + * short is INVALID_SHORT. + */ +static ClusterXidAuthorityValidity +read_image(const char *path, char *image) +{ + int fd; + int r; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + r = read(fd, image, CLUSTER_XID_AUTHORITY_FILE_SIZE); + CloseTransientFile(fd); + if (r != CLUSTER_XID_AUTHORITY_FILE_SIZE) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); +} + +/* + * cluster_xid_authority_read -- fail-closed read of the shared authority. + * Tries primary then .bak; returns false when neither is trustworthy. Never + * ereports (safe on the bootstrap early-read path). + */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { + if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; /* fail-closed: neither trustworthy */ + } + + memcpy(out, image, sizeof(*out)); + return true; +} + +/* + * cluster_xid_authority_present -- does any authority image (primary or .bak) + * exist on disk at all, trustworthy or not? Lets callers distinguish + * "absent: genuine first seed" from "present but corrupt: fail-closed" -- a + * corrupt authority must never be silently re-seeded (a re-seed from a + * checkpointed value could put already-consumed xids back "in the future"). + * Never ereports. + * + * Fail-closed errno discipline: only ENOENT proves absence (mirrors + * cluster_oid_authority_present; EIO/EACCES/ESTALE report present). + */ +bool +cluster_xid_authority_present(void) +{ + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + struct stat st; + + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) + return false; + + if (stat(primary_path, &st) == 0 || errno != ENOENT) + return true; + if (stat(bak_path, &st) == 0 || errno != ENOENT) + return true; + return false; +} + +/* + * write_header -- CRC-stamp and atomically install an authority header. + */ +static void +write_header(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + roll_primary_to_bak(primary, bak, baktmp); + write_durable(tmp, primary, buffer); +} + +/* + * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is + * already present (any trustworthy image) do nothing; otherwise create the + * unsealed initial image. Returns true when this call created it. + */ +bool +cluster_xid_authority_seed_if_absent(uint64 initial_native_hw) +{ + ClusterXidAuthorityHeader hdr; + + if (cluster_xid_authority_read(&hdr)) + return false; /* already seeded (join node / prior seed) */ + if (cluster_xid_authority_present()) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at seed"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "do not re-seed from a stale xid high-water.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + + memset(&hdr, 0, sizeof(hdr)); + hdr.flags = 0; + hdr.native_hw_full = initial_native_hw; + hdr.next_multi = 0; + write_header(&hdr); + return true; +} + +/* + * cluster_xid_authority_publish_native -- monotone raise of the native-era + * high-water; seal=true additionally stamps SEALED. Flags are never + * cleared and the high-water is never lowered (spec §3.1). An absent + * authority is created outright (crash between bootstrap-seed and the + * first publish); a present-but-corrupt one PANICs rather than re-seed. + */ +void +cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) { + if (cluster_xid_authority_present()) + ereport(PANIC, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is present but corrupt at publish"), + errhint("Restore \"%s\" (or its .bak) from a backup of the shared tree; " + "re-seeding from a stale high-water could re-issue consumed xids.", + CLUSTER_XID_AUTHORITY_REL_PATH))); + memset(&hdr, 0, sizeof(hdr)); + } + + if (native_hw_full > hdr.native_hw_full) + hdr.native_hw_full = native_hw_full; + if (next_multi > hdr.next_multi) + hdr.next_multi = next_multi; + if (seal) + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_SEALED; + + write_header(&hdr); +} + +/* + * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first + * cluster.enabled=on boot; spec §3.1). No-op when already stamped. A + * missing/corrupt authority PANICs: the catalog bootstrap seeds it before + * any caller can reach this point, so absence here is real damage. + */ +void +cluster_xid_authority_mark_cluster_era(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + return; + + hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; + write_header(&hdr); +} diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c new file mode 100644 index 0000000000..9333c1eaba --- /dev/null +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -0,0 +1,450 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_prehistory.c + * Native-era pg_xact prehistory: publish the seed node's CLOG page run + * [0, native_hw) into a CRC-guarded blob under cluster.shared_data_dir, + * and adopt it into a joiner's local pg_xact (spec-6.15b D2). + * + * The native era (pre-formation, cluster.enabled=off) is a single- + * writer window: the seed node's local pg_xact is the complete commit- + * status truth for every xid below the native high-water, including + * aborted ones. Publishing that page run with the shared tree lets a + * pre-seed-lineage joiner adopt first-hand truth instead of trusting + * hint bits (false-invisible / poison-stamp hazard; spec §0). This is + * a one-shot formation-time carry of pre-cluster history -- the same + * bytes a post-seed clone would have carried -- NOT a runtime mirror of + * cluster-era foreign commit bits (the rejected CLOG-overlay design; + * spec §1.4.1 boundary note). + * + * Both publish and adopt stream page-at-a-time (no large allocations): + * a CRC pass validates or stamps the whole image, then a second pass + * moves the bytes. Their callers run in single-writer windows (the + * seed's shutdown checkpoint; the joiner's postmaster-once bootstrap, + * strictly before StartupCLOG), so the two passes see stable bytes. + * Adopt fsyncs every touched segment and the pg_xact directory before + * returning (spec P2). + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/backend/cluster/cluster_xid_prehistory.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D2, §4 U1/U4) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include + +#include "cluster/cluster_guc.h" /* cluster_shared_data_dir */ +#include "cluster/cluster_xid_authority.h" +#include "storage/fd.h" + +/* + * CLOG geometry, restated locally so the module stays standalone-linkable + * for cluster_unit: 2 status bits per xact -> 4 xacts per byte -> BLCKSZ*4 + * xacts per page (clog.c CLOG_XACTS_PER_PAGE, private to clog.c), and 32 + * pages per SLRU segment file (slru.h SLRU_PAGES_PER_SEGMENT). The unit + * truth table locks this arithmetic against drift. + */ +#define PREHISTORY_XACTS_PER_PAGE ((uint32)(BLCKSZ * 4)) +#define PREHISTORY_PAGES_PER_SEGMENT 32 + +static bool prehistory_adopted_this_boot = false; + +/* ============================================================ + * Pure layer. + * ============================================================ */ + +/* + * cluster_xid_prehistory_payload_bytes -- whole-page payload size covering + * xids [0, native_hw_full); 0 for an empty or over-cap native era (callers + * treat 0 as "nothing to carry" / refuse respectively). + */ +uint32 +cluster_xid_prehistory_payload_bytes(uint64 native_hw_full) +{ + uint64 pages; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + return 0; + + pages = (native_hw_full + PREHISTORY_XACTS_PER_PAGE - 1) / PREHISTORY_XACTS_PER_PAGE; + return (uint32)(pages * BLCKSZ); +} + +/* + * cluster_xid_prehistory_classify -- validate a full in-memory blob image + * (header + payload). Used by the unit truth table; the streaming adopt + * path performs the same checks incrementally. + */ +ClusterXidAuthorityValidity +cluster_xid_prehistory_classify(const char *buf, size_t len) +{ + ClusterXidPrehistoryHeader hdr; + pg_crc32c crc; + + if (buf == NULL || len < sizeof(ClusterXidPrehistoryHeader)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + memcpy(&hdr, buf, sizeof(hdr)); + + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC) + return CLUSTER_XID_AUTHORITY_INVALID_MAGIC; + if (len != sizeof(hdr) + hdr.payload_len + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(hdr.native_hw_full)) + return CLUSTER_XID_AUTHORITY_INVALID_SHORT; + + INIT_CRC32C(crc); + COMP_CRC32C(crc, buf, offsetof(ClusterXidPrehistoryHeader, crc)); + COMP_CRC32C(crc, buf + sizeof(hdr), hdr.payload_len); + FIN_CRC32C(crc); + if (!EQ_CRC32C(crc, hdr.crc)) + return CLUSTER_XID_AUTHORITY_INVALID_CRC; + + return CLUSTER_XID_AUTHORITY_VALID; +} + +/* ============================================================ + * Streaming file plumbing. + * ============================================================ */ + +static bool +build_path(char *dst, size_t dstlen, const char *relpath) +{ + if (cluster_shared_data_dir == NULL || cluster_shared_data_dir[0] == '\0') + return false; + snprintf(dst, dstlen, "%s/%s", cluster_shared_data_dir, relpath); + return true; +} + +/* + * read_clog_page -- read local pg_xact page `pageno` into buf. Fail-closed: + * a missing or short segment is FATAL -- the caller's window (post + * CheckPointGuts / clean shutdown) guarantees every allocated page is on + * disk, so a hole would mean fabricating "aborted" truth for real xids. + */ +static void +read_clog_page(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not open pg_xact segment \"%s\" for prehistory publish: %m", seg), + errhint("The native-era CLOG must be complete on disk; publish runs only " + "after a CLOG flush."))); + if (lseek(fd, offset, SEEK_SET) != offset || read(fd, buf, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of pg_xact segment \"%s\" page %u for prehistory publish", seg, + pageno), + errhint("The native-era CLOG must be complete on disk; a hole would fabricate " + "commit-status truth."))); + CloseTransientFile(fd); +} + +/* + * roll_blob_to_bak -- preserve an existing primary blob into .bak (streamed + * in page-sized chunks; variable length). Missing primary is skipped. + */ +static void +roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) +{ + char chunk[BLCKSZ]; + int src; + int dst; + int r; + + src = OpenTransientFile(primary, O_RDONLY | PG_BINARY); + if (src < 0) + return; /* first write: no prior primary */ + + dst = OpenTransientFile(baktmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (dst < 0) { + CloseTransientFile(src); + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", baktmp))); + } + while ((r = read(src, chunk, sizeof(chunk))) > 0) { + if (write(dst, chunk, r) != r) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); + } + CloseTransientFile(src); + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", baktmp))); + CloseTransientFile(dst); + if (durable_rename(baktmp, bak, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", baktmp, bak))); +} + +/* ============================================================ + * Publish (seed side). + * ============================================================ */ + +/* + * cluster_xid_prehistory_publish -- snapshot local pg_xact [0, native_hw) + * into the shared blob, torn-safe. Two passes over the source pages: one + * to compute the image CRC, one to write; the caller's single-writer + * window (seed shutdown checkpoint / catalog-seed StartupXLOG) keeps the + * bytes stable between the passes. + */ +void +cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + char page[BLCKSZ]; + uint32 payload_len; + uint32 pages; + uint32 p; + int fd; + + if (native_hw_full == 0 || native_hw_full > CLUSTER_XID_PREHISTORY_MAX_XID) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native-era xid high-water " UINT64_FORMAT + " is outside the prehistory publish range", + native_hw_full), + errhint("Native-era seed loads are capped at " UINT64_FORMAT + " xids; split the seed load or move it in-protocol.", + CLUSTER_XID_PREHISTORY_MAX_XID))); + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_PREHISTORY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + payload_len = cluster_xid_prehistory_payload_bytes(native_hw_full); + pages = payload_len / BLCKSZ; + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_PREHISTORY_MAGIC; + hdr.version = CLUSTER_XID_PREHISTORY_VERSION; + hdr.native_hw_full = native_hw_full; + hdr.payload_len = payload_len; + hdr.reserved = 0; + + /* pass 1: CRC over header fields + source pages */ + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + COMP_CRC32C(hdr.crc, page, BLCKSZ); + } + FIN_CRC32C(hdr.crc); + + /* pass 2: stream header + pages into tmp, then install torn-safe */ + fd = OpenTransientFile(tmp, O_RDWR | O_CREAT | O_TRUNC | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", tmp))); + errno = 0; + if (write(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + for (p = 0; p < pages; p++) { + read_clog_page(local_pgdata, p, page); + errno = 0; + if (write(fd, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", tmp))); + } + } + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not fsync file \"%s\": %m", tmp))); + if (CloseTransientFile(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not close file \"%s\": %m", tmp))); + + roll_blob_to_bak(primary, bak, baktmp); + if (durable_rename(tmp, primary, PANIC) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not rename file \"%s\" to \"%s\": %m", tmp, primary))); +} + +/* ============================================================ + * Adopt (joiner side). + * ============================================================ */ + +/* + * verify_blob -- streaming validation of one candidate blob file against + * the sealed authority's native_hw. Returns true (and leaves *out_pages + * set) only when magic/version/hw/length/CRC all hold. + */ +static bool +verify_blob(const char *path, uint64 native_hw_full, uint32 *out_pages) +{ + ClusterXidPrehistoryHeader hdr; + char page[BLCKSZ]; + pg_crc32c crc; + uint32 pages; + uint32 p; + int fd; + + fd = OpenTransientFile(path, O_RDONLY | PG_BINARY); + if (fd < 0) + return false; + if (read(fd, &hdr, sizeof(hdr)) != sizeof(hdr)) { + CloseTransientFile(fd); + return false; + } + if (hdr.magic != CLUSTER_XID_PREHISTORY_MAGIC || hdr.version != CLUSTER_XID_PREHISTORY_VERSION + || hdr.native_hw_full != native_hw_full + || hdr.payload_len != cluster_xid_prehistory_payload_bytes(native_hw_full)) { + CloseTransientFile(fd); + return false; + } + + pages = hdr.payload_len / BLCKSZ; + INIT_CRC32C(crc); + COMP_CRC32C(crc, (char *)&hdr, offsetof(ClusterXidPrehistoryHeader, crc)); + for (p = 0; p < pages; p++) { + if (read(fd, page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(fd); + return false; + } + COMP_CRC32C(crc, page, BLCKSZ); + } + FIN_CRC32C(crc); + CloseTransientFile(fd); + + if (!EQ_CRC32C(crc, hdr.crc)) + return false; + + *out_pages = pages; + return true; +} + +/* + * cluster_xid_prehistory_adopt -- decode the shared blob into local pg_xact + * page files. Validates the whole image (primary, then .bak) BEFORE the + * first local write -- a joiner must never take half a truth. Writes are + * whole-page, sequential from page 0, fsynced per touched segment, then the + * pg_xact directory is fsynced (spec P2: complete and durable strictly + * before StartupCLOG). Idempotent: re-running overwrites the same bytes. + */ +void +cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char seg[MAXPGPATH]; + char dir[MAXPGPATH]; + char page[BLCKSZ]; + const char *src_path = NULL; + uint32 pages = 0; + uint32 p; + int src; + int dst = -1; + int cur_segno = -1; + int fd; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is missing, corrupt, or does not match the " + "sealed authority high-water " UINT64_FORMAT, + native_hw_full), + errhint("The seed node must complete a clean native-era shutdown before " + "joiners can adopt; restore \"%s\" from the shared tree's backup " + "if it was damaged.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + + /* pass 2: stream the validated pages into local pg_xact segments */ + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) + ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("could not re-open shared XID prehistory \"%s\": %m", src_path))); + + for (p = 0; p < pages; p++) { + int segno = (int)(p / PREHISTORY_PAGES_PER_SEGMENT); + off_t offset = (off_t)(p % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + if (read(src, page, BLCKSZ) != BLCKSZ) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("short read of shared XID prehistory \"%s\" page %u", src_path, p))); + + if (segno != cur_segno) { + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, segno); + dst = OpenTransientFile(seg, O_RDWR | O_CREAT | PG_BINARY); + if (dst < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact segment \"%s\": %m", seg))); + cur_segno = segno; + } + errno = 0; + if (lseek(dst, offset, SEEK_SET) != offset || write(dst, page, BLCKSZ) != BLCKSZ) { + if (errno == 0) + errno = ENOSPC; + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not write pg_xact segment \"%s\": %m", seg))); + } + } + if (dst >= 0) { + if (pg_fsync(dst) != 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not fsync pg_xact segment: %m"))); + CloseTransientFile(dst); + } + CloseTransientFile(src); + + /* P2: directory fsync makes the adopted truth durable before StartupCLOG */ + snprintf(dir, sizeof(dir), "%s/pg_xact", local_pgdata); + fd = OpenTransientFile(dir, O_RDONLY | PG_BINARY); + if (fd < 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not open pg_xact directory \"%s\": %m", dir))); + if (pg_fsync(fd) != 0) + ereport(PANIC, (errcode_for_file_access(), + errmsg("could not fsync pg_xact directory \"%s\": %m", dir))); + CloseTransientFile(fd); + prehistory_adopted_this_boot = true; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return prehistory_adopted_this_boot; +} diff --git a/src/backend/utils/errcodes.txt b/src/backend/utils/errcodes.txt index be8af8e758..ac8fe1c784 100644 --- a/src/backend/utils/errcodes.txt +++ b/src/backend/utils/errcodes.txt @@ -875,6 +875,17 @@ Section: Class 53 - Insufficient Resources (pgrac extension) # the first own checkpoint). 53RB3 E ERRCODE_CLUSTER_RECOVERY_ANCHOR_UNAVAILABLE cluster_recovery_anchor_unavailable +# spec-6.15b: shared XID authority / native-era prehistory unavailable. +# Under cluster.shared_catalog=on a joiner must reconcile its transaction +# horizon against the sealed shared XID authority and adopt the native-era +# prehistory (pg_xact truth for pre-formation seed xids); a missing, +# unsealed or corrupt authority/prehistory fails closed rather than judge +# seed rows through a stale horizon or a truthless local CLOG. Also raised +# when a cluster.enabled=off boot tries to re-enter the native seed era on +# a formed cluster's shared tree. (53RB4 is claimed by the spec-7.1 lane's +# ERRCODE_CLUSTER_MXID_HALFSPACE_LIMIT; do not reuse.) +53RB5 E ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE cluster_xid_authority_unavailable + Section: Class 54 - Program Limit Exceeded # this is for wired-in limits, not resource exhaustion problems (class borrowed from DB2) diff --git a/src/include/cluster/cluster_catalog_migrate.h b/src/include/cluster/cluster_catalog_migrate.h index aa17cae1cb..18e3c640ec 100644 --- a/src/include/cluster/cluster_catalog_migrate.h +++ b/src/include/cluster/cluster_catalog_migrate.h @@ -61,6 +61,11 @@ typedef struct ClusterCatalogAuthorityMarker { pg_crc32c crc; /* over [0, offsetof(crc)) */ } ClusterCatalogAuthorityMarker; +typedef enum ClusterCatalogMigrateResult { + CLUSTER_CATALOG_MIGRATE_SEEDED = 0, + CLUSTER_CATALOG_MIGRATE_ADOPTED +} ClusterCatalogMigrateResult; + /* * cluster_catalog_migrate_tree -- establish the shared catalog relation tree * for this node. Postmaster-once, only meaningful when @@ -74,7 +79,8 @@ typedef struct ClusterCatalogAuthorityMarker { * Fail-closed: FATAL 53RB0 on a foreign/mismatched or unreadable authority. * system_identifier is the cluster-wide value from the shared pg_control. */ -extern void cluster_catalog_migrate_tree(const char *local_pgdata, uint64 system_identifier); +extern ClusterCatalogMigrateResult cluster_catalog_migrate_tree(const char *local_pgdata, + uint64 system_identifier); /* * cluster_catalog_vet_no_unlogged -- spec-6.14 Q12 enable-time vet: FATAL diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h new file mode 100644 index 0000000000..ed9471907a --- /dev/null +++ b/src/include/cluster/cluster_xid_authority.h @@ -0,0 +1,194 @@ +/*------------------------------------------------------------------------- + * + * cluster_xid_authority.h + * Shared XID authority + native-era prehistory (spec-6.15b D1/D2). + * + * Under cluster.shared_catalog the pre-formation ("native era", + * cluster.enabled=off single-writer) xid consumption of the seed node is + * invisible to every other node: a joiner's per-node recovery anchor was + * seeded from its pre-seed clone-era control file, so its transaction + * horizon (nextXid) stays below the seed's committed xids and MVCC + * silently judges seed rows "in the future" (false-invisible), while its + * local pg_xact carries no commit bits for them (poison-hint hazard on + * the shared catalog pages). + * + * Two small durable files under cluster.shared_data_dir close this: + * + * global/pgrac_xid_authority -- never-lowered native-era xid + * high-water (first xid NOT consumed by the native era) plus two + * one-way lifecycle flags: SEALED (a clean native-era shutdown + * published a complete high-water + prehistory; joiners may adopt) + * and CLUSTER_ERA (a cluster.enabled=on boot happened; the native + * era is closed forever and may not be re-entered). + * + * global/pgrac_xid_prehistory -- the raw local pg_xact page run + * covering [0, native_hw), CRC-guarded. A joiner whose own + * nextXid proves it is a pre-seed lineage adopts these bytes into + * its local pg_xact before StartupCLOG, giving it first-hand + * commit-status truth for every native-era xid (no hint-bit + * trust, no CLOG overlay of cluster-era foreign bits -- see the + * spec's no-clog-overlay boundary note). + * + * Torn-safe file discipline mirrors cluster_oid_lease.c: temp + fsync + + * .bak roll + durable_rename on write; primary-then-.bak fail-closed + * read; ENOENT-only-absent presence probe. + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_xid_authority.h + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2) + * + *------------------------------------------------------------------------- + */ +#ifndef CLUSTER_XID_AUTHORITY_H +#define CLUSTER_XID_AUTHORITY_H + +#include "port/pg_crc32c.h" + +/* ============================================================ + * On-disk authority image (spec-6.15b §2). + * ============================================================ */ + +#define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 +#define CLUSTER_XID_AUTHORITY_VERSION 1 + +/* A clean native-era shutdown published a complete hw + prehistory. */ +#define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 +/* A cluster.enabled=on boot closed the native era forever (one-way). */ +#define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 + +typedef struct ClusterXidAuthorityHeader { + uint32 magic; + uint32 version; + uint32 flags; + uint32 reserved; /* zero */ + uint64 native_hw_full; /* FullTransactionId value: first xid NOT + * consumed by the native era; never lowered */ + uint64 next_multi; /* native-era nextMulti at publish; vet-only + * (spec Q5: >FirstMultiXactId refuses seed) */ + pg_crc32c crc; /* over all preceding bytes */ +} ClusterXidAuthorityHeader; + +/* Relative paths under cluster.shared_data_dir. */ +#define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" +#define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" +#define CLUSTER_XID_AUTHORITY_TMP_REL_PATH "global/pgrac_xid_authority.tmp" +#define CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH "global/pgrac_xid_authority.bak.tmp" + +typedef enum ClusterXidAuthorityValidity { + CLUSTER_XID_AUTHORITY_VALID = 0, + CLUSTER_XID_AUTHORITY_INVALID_SHORT, + CLUSTER_XID_AUTHORITY_INVALID_MAGIC, + CLUSTER_XID_AUTHORITY_INVALID_CRC, +} ClusterXidAuthorityValidity; + +/* ============================================================ + * On-disk prehistory image (spec-6.15b D2). + * ============================================================ */ + +#define CLUSTER_XID_PREHISTORY_MAGIC 0x0142D617 +#define CLUSTER_XID_PREHISTORY_VERSION 1 + +/* + * Refusal cap on the native era (spec §3.4): 8M xids = 256 CLOG pages = + * 2MB of payload. A seed load that consumes more must be split or moved + * in-protocol; the cap keeps the blob bounded and the adopt O(small). + */ +#define CLUSTER_XID_PREHISTORY_MAX_XID UINT64CONST(8388608) + +typedef struct ClusterXidPrehistoryHeader { + uint32 magic; + uint32 version; + uint64 native_hw_full; /* payload covers xids [0, native_hw_full) */ + uint32 payload_len; /* raw pg_xact bytes following this header */ + uint32 reserved; /* zero */ + pg_crc32c crc; /* over header fields above + payload */ +} ClusterXidPrehistoryHeader; + +#define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" +#define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" +#define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" +#define CLUSTER_XID_PREHISTORY_BAK_TMP_REL_PATH "global/pgrac_xid_prehistory.bak.tmp" + +/* ============================================================ + * Pure layer (standalone-linkable; exercised by cluster_unit). + * ============================================================ */ + +extern ClusterXidAuthorityValidity cluster_xid_authority_classify(const char *buf, size_t len); + +/* + * cluster_xid_prehistory_payload_bytes -- whole-CLOG-page payload size + * covering xids [0, native_hw_full). 0 when native_hw_full is 0 (nothing + * to carry); the per-page constant mirrors CLOG_XACTS_PER_PAGE without + * dragging clog.c internals into the unit build. + */ +extern uint32 cluster_xid_prehistory_payload_bytes(uint64 native_hw_full); + +extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *buf, size_t len); + +/* ============================================================ + * Torn-safe authority file I/O (mirrors cluster_oid_lease.c). + * ============================================================ */ + +/* + * Fail-closed read: primary then .bak; false when neither validates. + * Never ereports (safe on the bootstrap early-read path). + */ +extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); + +/* + * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() + * failure other than ENOENT reports present, so a transiently unreadable + * authority is never re-seeded over. + */ +extern bool cluster_xid_authority_present(void); + +/* + * Seed the authority (unsealed) when neither image exists. Returns true + * when this call created it. A trustworthy existing image is a no-op. + */ +extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); + +/* + * Monotone raise of native_hw_full + set SEALED (never lowers, never + * clears flags). seal=false publishes a crash-safe interim high-water + * during the native era without opening adoption. PANICs on I/O error. + */ +extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, + bool seal); + +/* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ +extern void cluster_xid_authority_mark_cluster_era(void); + +/* ============================================================ + * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and + * fsynced strictly before StartupCLOG -- both run under the postmaster / + * shutdown-checkpoint single-writer windows, file-level only). + * ============================================================ */ + +/* + * Seed side: snapshot local pg_xact pages [0, native_hw_full) into the + * shared prehistory file (torn-safe). PANICs on I/O error; FATALs when + * the native era exceeds CLUSTER_XID_PREHISTORY_MAX_XID. + */ +extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 native_hw_full); + +/* + * Joiner side: decode the shared prehistory into local pg_xact page files, + * pg_fsync each touched segment and the pg_xact directory before returning. + * Idempotent (same-byte overwrite). FATALs (53RB5) on a missing/corrupt + * blob or a native_hw mismatch against the sealed authority. + */ +extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); +extern bool cluster_xid_prehistory_was_adopted(void); + +#endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..1d45da20c0 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -161,6 +161,8 @@ # ---------- my $cluster_conf = <safe_psql('postgres', 'SELECT 1'), '1', 'R0: node0 is up'); +# Immediately after online_join + striping activation, a backend's bootstrap +# catalog read can transiently hit a peer whose block view is still rebuilding. +# Poll both nodes with the same tolerant shape used by t/339. +my $n0_up = 0; +for (1 .. 60) +{ + my ($rc) = $node0->psql('postgres', 'SELECT 1'); + if (defined $rc && $rc == 0) { $n0_up = 1; last; } + usleep(500_000); +} +is($n0_up, 1, 'R0: node0 is up'); is($n1_up, 1, 'R0: node1 is up'); $node0->safe_psql('postgres', diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl new file mode 100644 index 0000000000..bacd49ac69 --- /dev/null +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -0,0 +1,410 @@ +#------------------------------------------------------------------------- +# +# 361_xid_authority_native_era_2node.pl +# spec-6.15b BUG-A1/A2 -- shared_catalog native-era XID authority. +# +# node1 is a cold pre-seed clone of node0: it has the same sysid and a +# pre-seed pg_control/pg_xact horizon, but it does not contain node0's +# cluster.enabled=off seed transactions. node0 then seeds shared_catalog, +# consumes native xids, cleanly shuts down, and publishes the sealed XID +# authority plus pg_xact prehistory. On first cluster boot node1 must use +# its per-node recovery anchor's pre-seed nextXid as the adoption proof, +# copy the native-era prehistory into local pg_xact before StartupCLOG, and +# max-merge nextXid with the sealed authority before ShmemVariableCache is +# seeded. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::Cluster; +use PostgreSQL::Test::RecursiveCopy; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared catalog requires --enable-cluster'; +} + +sub make_shared_root +{ + my $root = PostgreSQL::Test::Utils::tempdir(); + mkdir "$root/global" or die "mkdir $root/global: $!"; + return $root; +} + +sub make_voting_disks +{ + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); + my @disks; + for my $i (0 .. 2) + { + my $p = "$disk_dir/disk$i"; + open(my $fh, '>', $p) or die "open $p: $!"; + binmode $fh; + print $fh ("\0" x (128 * 512)); + close $fh; + push @disks, $p; + } + return join(',', @disks); +} + +sub cold_clone_data_dir +{ + my ($src, $dst) = @_; + + PostgreSQL::Test::RecursiveCopy::copypath( + $src->data_dir, + $dst->data_dir, + filterfn => sub { + my $rel = shift; + return ($rel ne 'log' && $rel ne 'postmaster.pid'); + }); + chmod(0700, $dst->data_dir) or die "chmod clone data dir: $!"; + + # The cold copy carries node0's port; append node1's port so the last + # setting wins, matching init_from_backup's post-copy rewrite. + $dst->append_conf('postgresql.conf', 'port = ' . $dst->port . "\n"); +} + +sub append_pgrac_conf +{ + my ($node, $name, $ic0, $ic1) = @_; + my $pgrac_conf = <data_dir . '/pgrac.conf', $pgrac_conf); +} + +sub start_background +{ + my ($node) = @_; + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $node->data_dir, + '-l', $node->logfile, '-o', '--cluster-name=' . $node->name, 'start'); +} + +sub wait_sql_eq +{ + my ($node, $sql, $want, $name, $attempts) = @_; + $attempts //= 80; + my $got = ''; + + for (1 .. $attempts) + { + my ($rc, $out) = $node->psql('postgres', $sql); + if (defined $rc && $rc == 0 && defined $out) + { + $got = $out; + last if $got eq $want; + } + usleep(250_000); + } + is($got, $want, $name); +} + +sub read_xid_authority +{ + my ($shared_root) = @_; + my $path = "$shared_root/global/pgrac_xid_authority"; + open(my $fh, '<:raw', $path) or die "open $path: $!"; + read($fh, my $buf, 40) or die "read $path: $!"; + close $fh; + my ($magic, $version, $flags, $reserved, $native_hw, $next_multi, $crc) = + unpack('L< L< L< L< Q< Q< L<', $buf); + return { + magic => $magic, + version => $version, + flags => $flags, + native_hw => $native_hw, + next_multi => $next_multi, + crc => $crc, + }; +} + +sub corrupt_file +{ + my ($path) = @_; + open(my $fh, '>:raw', $path) or die "open $path: $!"; + print $fh "not-a-valid-xid-authority"; + close $fh; +} + +sub append_common_shared_catalog_conf +{ + my ($node, $shared_root) = @_; + $node->append_conf('postgresql.conf', <append_conf('postgresql.conf', <new('sxid_no_stripe'); + + $n->init; + append_common_shared_catalog_conf($n, $shared_root); + $n->append_conf('postgresql.conf', <start(fail_ok => 1); + ok($start_failed, 'D6: shared_catalog multi-node refuses cluster.xid_striping=off'); + like(PostgreSQL::Test::Utils::slurp_file($n->logfile), + qr/requires cluster\.xid_striping/, + 'D6: startup log names the xid_striping requirement'); +} + +# ------------------------------------------------------------------------- +# Main green path: pre-seed clone adopts node0 native-era pg_xact truth. +# ------------------------------------------------------------------------- +my $shared_root = make_shared_root(); +my $disks_csv = make_voting_disks(); +my $ic0 = PostgreSQL::Test::Cluster::get_free_port(); +my $ic1 = PostgreSQL::Test::Cluster::get_free_port(); + +my $node0 = PostgreSQL::Test::Cluster->new('sxid_node0'); +$node0->init(allows_streaming => 1); +$node0->start; +$node0->safe_psql('postgres', 'CHECKPOINT'); +$node0->stop; + +my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); +cold_clone_data_dir($node0, $node1); + +append_common_shared_catalog_conf($node0, $shared_root); +$node0->append_conf('postgresql.conf', <start; +ok(-e "$shared_root/global/pgrac_catalog_authority", + 'G1: node0 seeded the shared catalog authority marker'); +ok(-e "$shared_root/global/pg_control", + 'G1: node0 seeded the shared pg_control authority'); +ok(-e "$shared_root/global/pgrac_xid_authority", + 'G1: node0 seeded the shared XID authority before native-era load'); + +my $abort_xid = $node0->safe_psql('postgres', q{ +BEGIN; +SELECT txid_current(); +CREATE SCHEMA sxid_abort_shadow; +ROLLBACK; +}); + +$node0->safe_psql('postgres', q{ +CREATE SCHEMA sxid; +CREATE ROLE sxid_login LOGIN PASSWORD 'sxidpass'; +}); + +my $seed_schema_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_namespace WHERE nspname = 'sxid'}); +my $seed_role_xid = $node0->safe_psql('postgres', + q{SELECT xmin::text FROM pg_authid WHERE rolname = 'sxid_login'}); + +is($node0->safe_psql('postgres', "SELECT txid_status($seed_schema_xid)"), + 'committed', 'G1: seed schema catalog xid is committed on the seed'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G1: native-era aborted xid is aborted on the seed'); + +$node0->stop; + +my $auth = read_xid_authority($shared_root); +ok(($auth->{flags} & 0x0001) != 0, 'G2: clean seed shutdown sealed the XID authority'); +cmp_ok($auth->{native_hw}, '>', $seed_schema_xid, + 'G2: native high-water is above the seed schema catalog xid'); +cmp_ok($auth->{native_hw}, '>', $seed_role_xid, + 'G2: native high-water is above the seed role xid'); +ok(-e "$shared_root/global/pgrac_xid_prehistory", + 'G2: clean seed shutdown published XID prehistory'); + +append_strict_two_node_conf($node0, $disks_csv, undef); +$node0->append_conf('postgresql.conf', "cluster.node_id = 0\n"); + +append_common_shared_catalog_conf($node1, $shared_root); +append_strict_two_node_conf($node1, $disks_csv, undef); +$node1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + +append_pgrac_conf($node0, 'sxid', $ic0, $ic1); +append_pgrac_conf($node1, 'sxid', $ic0, $ic1); + +start_background($node1); +$node0->start; +$node1->_update_pid(1); + +wait_sql_eq($node1, 'SELECT 1', '1', 'G3: node1 is up after XID prehistory adopt'); +wait_sql_eq($node0, + "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " + . "FROM pg_cluster_ic_peers WHERE node_id = 1", + 't', 'G3: node0 sees node1 heartbeats'); + +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}, + '1', 'G4: node1 sees the native-era seed schema catalog row'); +wait_sql_eq($node1, + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}, + '1', 'G4: node1 sees the seed role catalog row'); +wait_sql_eq($node1, + q{SET ROLE sxid_login; SELECT current_user}, + 'sxid_login', 'G4: node1 can resolve and SET ROLE to the seed role'); + +wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed schema xid'); +wait_sql_eq($node1, "SELECT txid_status($seed_role_xid)", 'committed', + 'G5: node1 txid_status sees the native-era committed role xid'); +wait_sql_eq($node1, "SELECT txid_status($abort_xid)", 'aborted', + 'G5: node1 txid_status sees the native-era aborted xid'); +is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), + 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); + +my $state_hw = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_native_hw'}); +my $state_sealed = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_authority_sealed'}); +my $state_adopted = $node1->safe_psql('postgres', q{ +SELECT value FROM pg_cluster_state + WHERE category = 'catalog' AND key = 'xid_prehistory_adopted'}); + +is($state_hw, "$auth->{native_hw}", 'G6: pg_cluster_state exposes authority native high-water'); +is($state_sealed, 't', 'G6: pg_cluster_state exposes sealed XID authority'); +is($state_adopted, 't', 'G6: pg_cluster_state exposes node1 prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/adopted XID prehistory through native high-water/, + 'G6: node1 log records XID prehistory adoption'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/merged nextXid with XID authority native high-water/, + 'G6: node1 log records StartupXLOG nextXid max-merge'); + +$node1->stop; +$node0->stop; + +# ------------------------------------------------------------------------- +# Negative lifecycle legs on the formed shared tree. +# ------------------------------------------------------------------------- +$node0->append_conf('postgresql.conf', <start(fail_ok => 1); +ok($reentry_failed, 'N1: cluster.enabled=off cannot re-enter native seed era after formation'); +like(PostgreSQL::Test::Utils::slurp_file($node0->logfile), + qr/native seed era cannot be re-entered/, + 'N1: startup log names native-era re-entry refusal'); + +corrupt_file("$shared_root/global/pgrac_xid_authority"); +corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); +$node1->append_conf('postgresql.conf', "# t/361 force new logfile generation after corrupt authority\n"); +my $corrupt_failed = !$node1->start(fail_ok => 1); +ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, + 'N2: startup log names corrupt/unavailable XID authority'); + +# ------------------------------------------------------------------------- +# Unsealed authority negative: seed startup wrote the authority, but the seed +# crashed before clean shutdown could publish prehistory and seal it. +# ------------------------------------------------------------------------- +{ + my $u_shared = make_shared_root(); + my $u_disks = make_voting_disks(); + my $u_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $u_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $u0 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node0'); + my $u1 = PostgreSQL::Test::Cluster->new('sxid_unsealed_node1'); + + $u0->init(allows_streaming => 1); + $u0->start; + $u0->safe_psql('postgres', 'CHECKPOINT'); + $u0->stop; + cold_clone_data_dir($u0, $u1); + + append_common_shared_catalog_conf($u0, $u_shared); + $u0->append_conf('postgresql.conf', <start; + ok(-e "$u_shared/global/pgrac_xid_authority", + 'N3: unsealed fixture created the authority during seed startup'); + $u0->stop('immediate'); + + append_common_shared_catalog_conf($u1, $u_shared); + append_strict_two_node_conf($u1, $u_disks, undef); + $u1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($u1, 'sxid_unsealed', $u_ic0, $u_ic1); + + my $unsealed_failed = !$u1->start(fail_ok => 1); + ok($unsealed_failed, 'N3: joiner refuses an unsealed XID authority'); + like(PostgreSQL::Test::Utils::slurp_file($u1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N3: startup log names the unsealed authority'); +} + +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..b7a6ab9c2c 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -63,6 +63,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_sequence \ test_cluster_shared_catalog \ test_cluster_oid_lease \ + test_cluster_xid_authority \ test_cluster_relmap_authority \ test_cluster_hw \ test_cluster_dl \ @@ -178,7 +179,7 @@ test_cluster_backup: test_cluster_backup.c unit_test.h $(CLUSTER_VERSION_O) \ # separate rules because they also link additional cluster_*.o # objects (the test files stub the PG backend symbols those # objects reference). -SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) +SIMPLE_TESTS = $(filter-out test_cluster_guc test_cluster_shmem test_cluster_signal test_cluster_views test_cluster_gviews test_cluster_ic test_cluster_conf test_cluster_ic_mock test_cluster_inject test_cluster_pgstat test_cluster_debug test_cluster_shared_fs test_cluster_shared_fs_sharedfs test_cluster_shared_fs_block_device test_cluster_smgr test_cluster_startup_phase test_cluster_lmon test_cluster_lck test_cluster_diag test_cluster_stats test_cluster_cssd test_cluster_qvotec test_cluster_voting_disk_io test_cluster_quorum_decision test_cluster_scn test_cluster_adg test_cluster_epoch test_cluster_fence test_cluster_reconfig test_cluster_ges test_cluster_grd test_cluster_grd_starvation test_cluster_lmd test_cluster_lmd_graph test_cluster_lmd_wait_state test_cluster_cancel_token test_cluster_lmd_probe_collector test_cluster_lock_acquire test_cluster_advisory test_cluster_terminal_authority test_cluster_retention test_cluster_visibility_variants test_cluster_tt_2pc test_cluster_stage3_acceptance test_cluster_undo_buf test_cluster_block_apply test_cluster_thread_apply test_cluster_thread_replay test_cluster_thread_driver test_cluster_thread_orchestrator test_cluster_write_fence test_cluster_stage4_acceptance test_cluster_stage5_integrated_acceptance test_cluster_stage5_beta_acceptance test_cluster_ges_mode test_cluster_sequence test_cluster_shared_catalog test_cluster_hw test_cluster_dl test_cluster_extend_gate test_cluster_ir test_cluster_ts test_cluster_ko test_cluster_hw_snapshot test_cluster_cf_authority test_cluster_cf_storage test_cluster_cf_enqueue test_cluster_cf_phase2 test_cluster_cf_stats test_cluster_hang test_cluster_hang_resolve test_cluster_cr_server_policy test_cluster_touched_peers test_cluster_clean_leave test_cluster_membership test_cluster_node_remove test_cluster_resolver_cache test_cluster_backup test_cluster_hang_acceptance test_cluster_gcs_reqid test_cluster_runtime_visibility test_cluster_xid_stripe test_cluster_bufmgr_pcm_hook test_cluster_cr test_cluster_cr_admit test_cluster_cr_admit_stat test_cluster_cr_cache test_cluster_cr_coordinator test_cluster_cr_key test_cluster_cr_lifecycle test_cluster_cr_pool test_cluster_cr_tuple test_cluster_cr_tuple_stat test_cluster_gcs_block test_cluster_gcs_block_2way test_cluster_gcs_block_3way test_cluster_gcs_block_lost_write test_cluster_gcs_block_retransmit test_cluster_gcs_dispatch test_cluster_ges_handoff test_cluster_heap_lock_tuple test_cluster_hw_lease test_cluster_ic_envelope test_cluster_ic_router test_cluster_itl_cleanout test_cluster_itl_cleanout_perf test_cluster_itl_reader_real_triple test_cluster_itl_touch test_cluster_itl_wal test_cluster_multixact test_cluster_pcm_lock test_cluster_perf_gates test_cluster_recovery_merge test_cluster_recovery_plan test_cluster_recovery_worker test_cluster_reverse_key test_cluster_sinval test_cluster_sinval_ack test_cluster_snapshot_source test_cluster_stage2_acceptance test_cluster_stage5_5_cr_acceptance test_cluster_subtrans test_cluster_tt_durable test_cluster_tt_slot_allocator test_cluster_tt_status test_cluster_tt_status_hint test_cluster_uba test_cluster_undo_format test_cluster_undo_lifecycle test_cluster_undo_record test_cluster_visibility_decide_scn test_cluster_visibility_fork test_cluster_visibility_inject test_cluster_wal_state test_cluster_wal_thread test_cluster_xnode_lever test_cluster_xnode_profile test_cluster_pi_shadow test_cluster_oid_lease test_cluster_xid_authority test_cluster_recovery_anchor test_cluster_relmap_authority,$(TESTS)) # spec-2.4 D16: test_cluster_epoch links cluster_epoch.o standalone. # cluster_epoch.c references ShmemInitStruct + cluster_shmem_register_region @@ -443,6 +444,21 @@ test_cluster_oid_lease: test_cluster_oid_lease.c unit_test.h \ $(CLUSTER_OID_LEASE_O) \ $(top_builddir)/src/port/libpgport_srv.a -o $@ +# spec-6.15b D1/D2: test_cluster_xid_authority links cluster_xid_authority.o +# (the shared XID authority + native-era prehistory: torn-safe file I/O, +# seal/flag monotonicity, prehistory publish/adopt byte round trip). fd.c +# openers, pg_fsync, durable_rename and the ereport machinery are stubbed +# locally; CRC32C comes from libpgport. The startup wiring (bootstrap adopt, +# StartupXLOG horizon merge, shutdown-checkpoint publish) is TAP territory. +CLUSTER_XID_AUTHORITY_O = $(top_builddir)/src/backend/cluster/cluster_xid_authority.o \ + $(top_builddir)/src/backend/cluster/cluster_xid_prehistory.o +test_cluster_xid_authority: test_cluster_xid_authority.c unit_test.h \ + $(CLUSTER_XID_AUTHORITY_O) + $(CC) $(CFLAGS) $(CPPFLAGS) $< \ + $(CLUSTER_XID_AUTHORITY_O) \ + $(top_builddir)/src/port/libpgport_srv.a -o $@ + + # spec-6.14 D5: test_cluster_relmap_authority links cluster_relmap_authority.o # (the two-phase pending/committed authority substrate). fd.c + ereport # stubbed locally; CRC32C from libpgport. The write-path activation (relmapper diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..98998a0f78 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -53,6 +53,7 @@ #include "cluster/cluster_xnode_lever.h" /* spec-6.12 lever counter stub */ #include "cluster/cluster_hw_lease.h" /* spec-6.12d lease counter stub */ #include "cluster/cluster_relmap_authority.h" /* spec-6.14 D5 header-read stub */ +#include "cluster/cluster_xid_authority.h" /* spec-6.15b XID authority dump stubs */ #undef printf #undef fprintf @@ -342,6 +343,21 @@ cluster_relmap_authority_read_header(bool shared_map, Oid dbid, ClusterRelmapAut return false; } +/* spec-6.15b D7 stubs: dump_catalog reads the XID authority observation + * keys; this standalone debug unit does not link the file-I/O authority + * objects. */ +bool +cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return false; +} + +bool +cluster_xid_prehistory_was_adopted(void) +{ + return false; +} + /* spec-4.12 D7 + spec-4.12b D6 stubs: dump_write_fence (cluster_debug.o) reads 8 * counters now, and cluster_startup_phase.o (linked here) references the rejoin * self-fence gate. cluster_write_fence.o is not linked -- provide stubs returning diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..60b4da16e6 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,7 +207,7 @@ UT_TEST(test_gate_raw_compare_traps) } int -main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) +main(void) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ cluster_pi_shadow_shmem_init(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c new file mode 100644 index 0000000000..66a9c3db32 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -0,0 +1,488 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_xid_authority.c + * Runtime unit tests for the shared XID authority + native-era + * prehistory module (spec-6.15b D1/D2): on-disk layout lock, the pure + * classification and payload-sizing logic, the torn-safe read/write + * round trip with .bak fallback, seal/flag monotonicity, and a real + * prehistory publish -> adopt byte round trip between two fake PGDATA + * pg_xact trees. + * + * Like test_cluster_oid_lease.c this exercises the module for REAL + * against temp directories: fd.c openers map onto open(2)/close(2), + * pg_fsync onto fsync(2), durable_rename onto rename(2). The errstart + * stub aborts on elevel >= ERROR, so every FATAL/PANIC leg (corrupt + * authority at adopt, cap refusal, era re-entry) is TAP territory + * (t/361), not unit territory; reaching an abort here is a real bug. + * + * Covers (spec §4): + * U1 classify truth table (short / magic / CRC / valid), both files + * U2 publish monotonicity (native_hw never lowered) + * U3 flags one-way (SEALED / CLUSTER_ERA never cleared) + * U4 prehistory payload sizing + publish/adopt byte round trip + * U5 header layout locked by offset assertions + * + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_xid_authority.c + * + * NOTES + * This is a pgrac-original file. + * Spec: spec-6.15b-xid-authority-native-era.md (D1/D2, §4 U1-U5) + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include +#include +#include +#include +#include +#include + +#include "cluster/cluster_xid_authority.h" +#include "port/pg_crc32c.h" +#include "storage/fd.h" +#include "utils/elog.h" + +#undef printf +#undef fprintf +#undef snprintf +#undef sprintf +#undef vsnprintf +#undef vfprintf +#undef vprintf +#undef vsprintf +#undef strerror +#undef strerror_r + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +/* Global read by cluster_xid_authority.o's file I/O. */ +char *cluster_shared_data_dir = NULL; + +/* ---- Assert + ereport machinery (aborts on ERROR; the read paths never + * ereport, the write paths only PANIC on real I/O failure). ---- */ +void +ExceptionalCondition(const char *conditionName, const char *fileName, int lineNumber) +{ + printf("# Assert failed: %s at %s:%d\n", conditionName, fileName, lineNumber); + abort(); +} +bool +errstart(int elevel, const char *domain pg_attribute_unused()) +{ + if (elevel >= ERROR) { + printf("# unexpected ereport(elevel=%d) -- aborting\n", elevel); + abort(); + } + return false; +} +bool +errstart_cold(int elevel, const char *domain) +{ + return errstart(elevel, domain); +} +void +errfinish(const char *filename pg_attribute_unused(), int lineno pg_attribute_unused(), + const char *funcname pg_attribute_unused()) +{} +int +errcode(int sqlerrcode pg_attribute_unused()) +{ + return 0; +} +int +errcode_for_file_access(void) +{ + return 0; +} +int +errmsg(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errdetail(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errhint(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} + +/* ---- functional fd.c stubs: map onto the kernel ---- */ +int +OpenTransientFile(const char *fileName, int fileFlags) +{ + return open(fileName, fileFlags, 0600); +} +int +CloseTransientFile(int fd) +{ + return close(fd); +} +int +pg_fsync(int fd) +{ + return fsync(fd); +} +int +durable_rename(const char *oldfile, const char *newfile, int elevel pg_attribute_unused()) +{ + return (rename(oldfile, newfile) != 0) ? -1 : 0; +} + +/* ---- temp dir scaffolding ---- */ +static char test_root[MAXPGPATH]; + +static void +setup_shared_dir(void) +{ + char globaldir[MAXPGPATH]; + + snprintf(test_root, sizeof(test_root), "/tmp/pgrac_xid_auth_ut_%d", (int)getpid()); + mkdir(test_root, 0700); + snprintf(globaldir, sizeof(globaldir), "%s/global", test_root); + mkdir(globaldir, 0700); + cluster_shared_data_dir = test_root; +} + +static void +unlink_files(void) +{ + const char *rels[] = { CLUSTER_XID_AUTHORITY_REL_PATH, CLUSTER_XID_AUTHORITY_BAK_REL_PATH, + CLUSTER_XID_PREHISTORY_REL_PATH, CLUSTER_XID_PREHISTORY_BAK_REL_PATH }; + char p[MAXPGPATH]; + int i; + + for (i = 0; i < (int)lengthof(rels); i++) { + snprintf(p, sizeof(p), "%s/%s", test_root, rels[i]); + unlink(p); + } +} + +/* Build a fake PGDATA with a pg_xact/0000 of n_pages, filled with byte. */ +static void +make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigned char byte) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd; + int i; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + snprintf(seg, sizeof(seg), "%s/0000", dir); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + memset(page, byte, sizeof(page)); + for (i = 0; i < n_pages; i++) + (void)!write(fd, page, sizeof(page)); + close(fd); +} + +/* ============================================================ + * U5: on-disk layout locked + * ============================================================ */ + +UT_TEST(test_layout_offsets_locked) +{ + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, flags), 8); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, reserved), 12); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, native_hw_full), 16); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, next_multi), 24); + UT_ASSERT_EQ(offsetof(ClusterXidAuthorityHeader, crc), 32); + + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, magic), 0); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, version), 4); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, native_hw_full), 8); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, payload_len), 16); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, reserved), 20); + UT_ASSERT_EQ(offsetof(ClusterXidPrehistoryHeader, crc), 24); +} + +/* ============================================================ + * U1: classify truth table + * ============================================================ */ + +UT_TEST(test_classify_short_magic_crc_valid) +{ + ClusterXidAuthorityHeader hdr; + char buf[sizeof(ClusterXidAuthorityHeader)]; + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_authority_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, 3), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = 0xDEADBEEF; + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); + + memset(&hdr, 0, sizeof(hdr)); + hdr.magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr.version = CLUSTER_XID_AUTHORITY_VERSION; + hdr.native_hw_full = 816; + INIT_CRC32C(hdr.crc); + COMP_CRC32C(hdr.crc, (char *)&hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr.crc); + memcpy(buf, &hdr, sizeof(hdr)); + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), CLUSTER_XID_AUTHORITY_VALID); + + buf[16] ^= 0x01; /* flip a native_hw bit -> CRC mismatch */ + UT_ASSERT_EQ(cluster_xid_authority_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_CRC); +} + +/* ============================================================ + * U4: prehistory payload sizing + * ============================================================ */ + +UT_TEST(test_payload_bytes_boundaries) +{ + const uint32 xids_per_page = BLCKSZ * 4; /* CLOG: 2 bits/xact */ + + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(0), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(1), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(816), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page), (uint32)BLCKSZ); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(xids_per_page + 1), (uint32)(2 * BLCKSZ)); + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID), + (uint32)(CLUSTER_XID_PREHISTORY_MAX_XID / 4)); + /* over the refusal cap: 0 = "no valid payload" (publish FATALs) */ + UT_ASSERT_EQ(cluster_xid_prehistory_payload_bytes(CLUSTER_XID_PREHISTORY_MAX_XID + 1), 0); +} + +/* ============================================================ + * torn-safe authority round trips + * ============================================================ */ + +UT_TEST(test_seed_if_absent_creates_unsealed) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0); + + /* second seed is a no-op and keeps the existing value */ + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(123456), false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_publish_monotone_and_seal) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + + /* interim (unsealed) publish raises the high-water */ + cluster_xid_authority_publish_native(810, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 810); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* sealing publish raises further and stamps SEALED */ + cluster_xid_authority_publish_native(816, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); + + /* a lower publish never lowers, and never clears SEALED */ + cluster_xid_authority_publish_native(500, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + +UT_TEST(test_mark_cluster_era_one_way) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + cluster_xid_authority_mark_cluster_era(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + /* later publishes keep both flags (never cleared) */ + cluster_xid_authority_publish_native(900, 1, false); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags, + CLUSTER_XID_AUTHORITY_FLAG_SEALED | CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_primary_corrupt_falls_back_to_bak) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + /* second write rolls the 791 image into .bak */ + cluster_xid_authority_publish_native(816, 1, true); + + /* corrupt the primary in place */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + /* falls back to the rolled 791 image */ + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 791); +} + +UT_TEST(test_present_distinguishes_corrupt_from_absent) +{ + ClusterXidAuthorityHeader got; + char p[MAXPGPATH]; + int fd; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_present(), false); + + /* a lone corrupt primary: present=true, read=false (fail-closed) */ + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(p, O_RDWR | O_CREAT | O_TRUNC, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_present(), true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), false); +} + +/* ============================================================ + * U4: prehistory publish -> adopt byte round trip + * ============================================================ */ + +UT_TEST(test_prehistory_publish_adopt_round_trip) +{ + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char seed_page[BLCKSZ]; + char join_page[BLCKSZ]; + int fd; + + unlink_files(); + + /* seed: one clog page of 0xAA; joiner: same-size page of zeros */ + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "join", 1, 0x00); + + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), false); + cluster_xid_prehistory_adopt(pgdata_join, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_was_adopted(), true); + + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_seed); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, seed_page, BLCKSZ), BLCKSZ); + close(fd); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); + + /* adopt is idempotent: run again, bytes unchanged */ + cluster_xid_prehistory_adopt(pgdata_join, 816); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT_EQ(read(fd, join_page, BLCKSZ), BLCKSZ); + close(fd); + UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); +} + +UT_TEST(test_prehistory_classify_corrupt) +{ + char buf[64]; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(NULL, 0), CLUSTER_XID_AUTHORITY_INVALID_SHORT); + + /* publish a real blob, then flip a payload bit -> CRC catches it */ + { + char pgdata_seed[MAXPGPATH]; + char p[MAXPGPATH]; + char *image; + struct stat st; + int fd; + int r; + + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "seed2", 1, 0x55); + cluster_xid_prehistory_publish(pgdata_seed, 816); + + snprintf(p, sizeof(p), "%s/%s", test_root, CLUSTER_XID_PREHISTORY_REL_PATH); + UT_ASSERT_EQ(stat(p, &st), 0); + image = malloc(st.st_size); + fd = open(p, O_RDONLY, 0); + r = (int)read(fd, image, st.st_size); + close(fd); + UT_ASSERT_EQ(r, (int)st.st_size); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_VALID); + + image[sizeof(ClusterXidPrehistoryHeader) + 100] ^= 0x01; + UT_ASSERT_EQ(cluster_xid_prehistory_classify(image, st.st_size), + CLUSTER_XID_AUTHORITY_INVALID_CRC); + free(image); + } + + memset(buf, 0, sizeof(buf)); + UT_ASSERT_EQ(cluster_xid_prehistory_classify(buf, sizeof(buf)), + CLUSTER_XID_AUTHORITY_INVALID_MAGIC); +} + +int +main(void) +{ + setup_shared_dir(); + + UT_PLAN(10); + UT_RUN(test_layout_offsets_locked); + UT_RUN(test_classify_short_magic_crc_valid); + UT_RUN(test_payload_bytes_boundaries); + UT_RUN(test_seed_if_absent_creates_unsealed); + UT_RUN(test_publish_monotone_and_seal); + UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_primary_corrupt_falls_back_to_bak); + UT_RUN(test_present_distinguishes_corrupt_from_absent); + UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_classify_corrupt); + UT_DONE(); + return ut_failed_count == 0 ? 0 : 1; +} From adf62e01a02ad47716f5468e886c38194fbe3f2d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:33:17 +0800 Subject: [PATCH 05/27] fix(cluster): unseal XID authority when a follow-up native-era run boots A second cluster.enabled=off seed pass on a SEALED authority re-opens the native era: clear SEALED at bootstrap so a crash of that pass leaves joiners fail-closed (53RB5, unsealed) instead of silently adopting the previous pass's stale high-water -- false-invisible for every xid the crashed pass consumed. The pass's own clean shutdown re-publishes and re-seals monotonically. Adds cluster_xid_authority_begin_native_run(), the bootstrap enabled=off arm call, unit coverage (seal kept-hw/cleared-flag/idempotent/re-seal), and t/361 N4 (sealed pass-1 -> unsealing pass-2 boot -> immediate-kill -> joiner refuses adoption). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 13 ++++ src/backend/cluster/cluster_xid_authority.c | 41 +++++++++++++ src/include/cluster/cluster_xid_authority.h | 13 +++- .../t/361_xid_authority_native_era_2node.pl | 59 +++++++++++++++++++ .../cluster_unit/test_cluster_xid_authority.c | 29 ++++++++- 5 files changed, 153 insertions(+), 2 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 96dd7912c6..453b78038d 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -169,6 +169,19 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, errdetail("The shared XID authority is already marked cluster-era started."), errhint( "Keep cluster.enabled=on for this shared tree, or destroy and re-form it."))); + + /* + * A follow-up native run on a SEALED authority re-opens the era: the + * seal is cleared up front so a crash of this run leaves joiners + * fail-closed (unsealed) instead of adopting the previous pass's + * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The + * clean shutdown of this run re-publishes and re-seals. + */ + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) { + cluster_xid_authority_begin_native_run(); + elog(LOG, "cluster shared_catalog: re-opened native seed era " + "(XID authority unsealed for this run)"); + } return; } diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 0d73ad4552..5d242c315f 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -338,3 +338,44 @@ cluster_xid_authority_mark_cluster_era(void) hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; write_header(&hdr); } + +/* + * cluster_xid_authority_begin_native_run -- re-open the native era for a + * follow-up cluster.enabled=off seed run (spec-6.15b §3.1 multi-pass arm). + * + * A sealed authority means "the high-water and prehistory are complete as + * of a clean native shutdown". A NEW native run invalidates that + * completeness the moment it allocates its first xid, so the seal is + * cleared up front: if this run crashes, joiners fail closed (unsealed, + * 53RB5) instead of adopting the previous pass's stale high-water -- + * false-invisible for every xid the crashed pass consumed (rule 8.A). + * The clean shutdown of this run re-publishes and re-seals monotonically. + * + * Only legal while CLUSTER_ERA is unset (the caller FATALs re-entry + * first; re-checked here defensively). Formation recipes keep native-era + * boots and first cluster boots strictly ordered; if a racing first + * cluster boot stamps CLUSTER_ERA inside this read-modify-write window, + * the stamp is re-applied by the next cluster boot (mark is re-issued + * whenever unset) and this authority is left unsealed -- which only ever + * blocks adoption, never yields a wrong answer. + */ +void +cluster_xid_authority_begin_native_run(void) +{ + ClusterXidAuthorityHeader hdr; + + if (!cluster_xid_authority_read(&hdr)) + ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID authority is missing or corrupt at native-run open"))); + + if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); + + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) + return; /* already open (first run, or a prior pass crashed) */ + + hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; + write_header(&hdr); +} diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index ed9471907a..7676d89303 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -61,7 +61,10 @@ #define CLUSTER_XID_AUTHORITY_MAGIC 0x0141D617 #define CLUSTER_XID_AUTHORITY_VERSION 1 -/* A clean native-era shutdown published a complete hw + prehistory. */ +/* A clean native-era shutdown published a complete hw + prehistory. + * Cleared again -- only while CLUSTER_ERA is unset -- when a follow-up + * native-era run boots, so a crash of that run leaves joiners + * fail-closed instead of adopting the previous pass's stale hw. */ #define CLUSTER_XID_AUTHORITY_FLAG_SEALED 0x0001 /* A cluster.enabled=on boot closed the native era forever (one-way). */ #define CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA 0x0002 @@ -169,6 +172,14 @@ extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 n /* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ extern void cluster_xid_authority_mark_cluster_era(void); +/* + * A follow-up native-era (cluster.enabled=off) boot on a sealed authority + * re-opens the era: clears SEALED so a crash of this run never exposes the + * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA + * first (re-entry is FATAL); no-op when already unsealed. + */ +extern void cluster_xid_authority_begin_native_run(void); + /* ============================================================ * Prehistory publish / adopt (spec-6.15b D2; P2: adopt is complete and * fsynced strictly before StartupCLOG -- both run under the postmaster / diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl index bacd49ac69..f4652c83ca 100644 --- a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -407,4 +407,63 @@ sub append_strict_two_node_conf 'N3: startup log names the unsealed authority'); } +# ------------------------------------------------------------------------- +# Multi-pass seed negative: a SECOND native-era run unseals the authority at +# boot, so crashing that run leaves joiners fail-closed instead of silently +# adopting the first pass's stale high-water (spec-6.15b §3.1 multi-pass arm). +# ------------------------------------------------------------------------- +{ + my $m_shared = make_shared_root(); + my $m_disks = make_voting_disks(); + my $m_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $m_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $m0 = PostgreSQL::Test::Cluster->new('sxid_multipass_node0'); + my $m1 = PostgreSQL::Test::Cluster->new('sxid_multipass_node1'); + + $m0->init(allows_streaming => 1); + $m0->start; + $m0->safe_psql('postgres', 'CHECKPOINT'); + $m0->stop; + cold_clone_data_dir($m0, $m1); + + append_common_shared_catalog_conf($m0, $m_shared); + $m0->append_conf('postgresql.conf', < sealed + $m0->start; + $m0->safe_psql('postgres', 'CREATE SCHEMA sxid_pass1;'); + $m0->stop; + my $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) != 0, + 'N4: first native pass sealed the XID authority'); + + # pass 2: booting a follow-up native run re-opens (unseals) the era + $m0->start; + $m_auth = read_xid_authority($m_shared); + ok(($m_auth->{flags} & 0x0001) == 0, + 'N4: follow-up native pass unsealed the XID authority at boot'); + like(PostgreSQL::Test::Utils::slurp_file($m0->logfile), + qr/re-opened native seed era/, + 'N4: seed log names the re-opened native era'); + + # crash pass 2: the stale pass-1 high-water must NOT become adoptable + $m0->stop('immediate'); + append_common_shared_catalog_conf($m1, $m_shared); + append_strict_two_node_conf($m1, $m_disks, undef); + $m1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($m1, 'sxid_multipass', $m_ic0, $m_ic1); + + my $multipass_failed = !$m1->start(fail_ok => 1); + ok($multipass_failed, + 'N4: joiner refuses adoption after a crashed follow-up native pass'); + like(PostgreSQL::Test::Utils::slurp_file($m1->logfile), + qr/shared XID authority is not sealed|shared XID authority is unavailable/, + 'N4: startup log fails closed on the unsealed authority'); +} + done_testing(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 66a9c3db32..3129bd73e3 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -467,18 +467,45 @@ UT_TEST(test_prehistory_classify_corrupt) CLUSTER_XID_AUTHORITY_INVALID_MAGIC); } +UT_TEST(test_begin_native_run_unseals_before_cluster_era) +{ + ClusterXidAuthorityHeader got; + + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* a follow-up native run clears SEALED but keeps the high-water */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* idempotent while open */ + cluster_xid_authority_begin_native_run(); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* the run's clean shutdown re-publishes and re-seals monotonically */ + cluster_xid_authority_publish_native(900, 1, true); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 900); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, CLUSTER_XID_AUTHORITY_FLAG_SEALED); +} + int main(void) { setup_shared_dir(); - UT_PLAN(10); + UT_PLAN(11); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); UT_RUN(test_seed_if_absent_creates_unsealed); UT_RUN(test_publish_monotone_and_seal); UT_RUN(test_mark_cluster_era_one_way); + UT_RUN(test_begin_native_run_unseals_before_cluster_era); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); From 964e587500a15e0baef7184278954d9c81c5bd29 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 16:44:43 +0800 Subject: [PATCH 06/27] test(cluster): settle first-contact probes in t/337/t/339 for the join-gate window spec-6.15b D6 flipped both tests to online_join+xid_striping: formation now passes through the join gate, whose home-block rebuild window transiently answers 'block master is recovering ... retry is safe' -- even at connection time, via the pg_authid read. Retry the first-contact probes and first DDLs (bounded, 60s); every semantic assert stays strict. Spec: spec-6.15b-xid-authority-native-era.md --- .../t/337_shared_catalog_ddl_2node.pl | 29 ++++++++++++++--- .../t/339_shared_catalog_faults_2node.pl | 32 +++++++++++++++---- 2 files changed, 50 insertions(+), 11 deletions(-) diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index 1d45da20c0..0f3073e4fd 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -202,6 +202,24 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time, via the pg_authid read). First-contact probes therefore +# retry; every semantic assert below stays strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + # node1 must actually be up. my $n1_up = 0; for (1 .. 60) @@ -214,7 +232,7 @@ # ---------- # L1: both alive + IC-connected on the shared-sysid cluster. # ---------- -my $a0 = $node0->safe_psql('postgres', 'SELECT 1'); +my ($n0_up, $a0) = psql_retry_ok($node0, 'SELECT 1'); is($a0, '1', 'L1: node0 is up on the shared-catalog 2-node cluster'); is($n1_up, 1, 'L1: node1 is up on the shared-catalog 2-node cluster'); @@ -222,10 +240,10 @@ my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L1: node0 sees node1 heartbeats (IC connected)'); @@ -235,9 +253,10 @@ # node1 ever running the DDL. This is the Stage 2 exit debt # (spec-2.0:135) closure. # ---------- -$node0->safe_psql('postgres', +my ($ddl_ok) = psql_retry_ok($node0, 'CREATE TABLE a1_shared_cat (id int PRIMARY KEY, note text);' . "INSERT INTO a1_shared_cat VALUES (1, 'from-node0');"); +die 'first cross-node DDL never succeeded' unless $ddl_ok; # Poll node1 up to 1s (10 x 100ms). "retry is safe" per the GCS remaster # transient; a settled shared catalog resolves well within the budget. diff --git a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl index c37a1d6225..fd97003b23 100644 --- a/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl +++ b/src/test/cluster_tap/t/339_shared_catalog_faults_2node.pl @@ -243,6 +243,23 @@ $node0->start; $node1->_update_pid(1); +# spec-6.15b D6 flipped this test to online_join+xid_striping: formation now +# passes through the join gate, whose home-block rebuild window answers +# transiently with "block master is recovering ... retry is safe" (even at +# connection time). First-contact probes retry; semantic asserts stay strict. +sub psql_retry_ok +{ + my ($node, $sql, $tries) = @_; + $tries //= 120; + for (1 .. $tries) + { + my ($rc, $out) = $node->psql('postgres', $sql); + return (1, $out) if defined $rc && $rc == 0; + usleep(500_000); + } + return (0, undef); +} + my $n1_up = 0; for (1 .. 60) { @@ -254,16 +271,17 @@ # ---------- # L0: both alive + IC-connected. # ---------- -is($node0->safe_psql('postgres', 'SELECT 1'), '1', 'L0: node0 is up'); +my ($n0_up_rc, $n0_up_out) = psql_retry_ok($node0, 'SELECT 1'); +is($n0_up_out, '1', 'L0: node0 is up'); is($n1_up, 1, 'L0: node1 is up'); my $peer_ok = 0; for (1 .. 40) { - my $s = $node0->safe_psql('postgres', + my ($prc, $s) = psql_retry_ok($node0, "SELECT COALESCE(bool_or(heartbeat_recv_count > 0), false) " - . "FROM pg_cluster_ic_peers WHERE node_id = 1"); - if (defined $s && $s eq 't') { $peer_ok = 1; last; } + . "FROM pg_cluster_ic_peers WHERE node_id = 1", 1); + if ($prc && defined $s && $s eq 't') { $peer_ok = 1; last; } usleep(500_000); } is($peer_ok, 1, 'L0: node0 sees node1 heartbeats (IC connected)'); @@ -279,15 +297,17 @@ # CREATE t_crash_uncommitted (+ row, NO commit) -> 8.A leg # -> node1's cold merge must divert all of these. # ---------- -$node1->safe_psql('postgres', +my ($survivor_ok) = psql_retry_ok($node1, "CREATE TABLE t_survivor (id int PRIMARY KEY, note text);" . "INSERT INTO t_survivor VALUES (1, 'alive');"); +die 'node1 first DDL never succeeded' unless $survivor_ok; -$node0->safe_psql('postgres', +my ($keep_ok) = psql_retry_ok($node0, "CREATE TABLE t_keep (id int PRIMARY KEY, note text);" . "INSERT INTO t_keep VALUES (1, 'keep');" . "CREATE TABLE t_dropme (id int PRIMARY KEY, note text);" . "INSERT INTO t_dropme VALUES (1, 'doomed');"); +die 'node0 first DDL never succeeded' unless $keep_ok; # Shared-tree relfile of t_dropme, captured BEFORE the drop. my $drop_relpath = $node0->safe_psql('postgres', From a76d08c8ee185958af8a9c97019c523b33dabc65 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:06:21 +0800 Subject: [PATCH 07/27] =?UTF-8?q?fix(cluster):=20marker-async=20review=20r?= =?UTF-8?q?1=20=E2=80=94=20PREPARE=20best-effort=20drain,=20CL-I3=20rechec?= =?UTF-8?q?k,=20P1-1/P1-2=20hard=20assertions,=20t/363=20rename?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review r1 dispositions (0 P0 / 3 P1 / 2 P2, all fixed): - P1: join Phase-1 PREPARE TIMEOUT/non-ACK now drains best-effort and still publishes the staged JOIN_PENDING (the pre-async code ignored PREPARE results; aborting reverted the joiner to DEAD and the next tick re-detected it and re-bumped the epoch once per deadline period — the exact bump storm the staged record forbids). The revert helper is removed; t/363 gains a bounded epoch-advance assertion across the 95s hold window. - P1: clean-leave staged-ACK handoff re-runs the CL-I3 dead_gen-aware version_coherent check before apply (the COMMITTING marker wait now spans ticks; the guarded CAS alone misses a dead_gen-only move). - P1: unit hard assertions — fail-stop fence stage bumps the epoch exactly once while the marker is PENDING and publishes only on ACK; node-remove re-entry while PENDING reports no false contest and no re-bump. - P2: node-remove FENCE_ARMING skips REMOVING-marker/stripe-retire pre-work while the staged fence marker is in flight (new cluster_reconfig_node_removed_fence_stage_pending predicate). - P2: PG-style banners (Author/IDENTIFICATION/NOTES) on the new files. - TAP renamed t/358 -> t/363 (358-362 reserved by parallel lanes); nightly shard split accordingly. Unit fixture gains MyBackendType stub (B_INVALID) and controllable async-marker stubs. Local gates (cassert build): cluster_unit 158 binaries, t/363 all legs, smoke+touched-baseline TAP 13 files/327 tests, cluster_regress 13, PG 219/219, clang-format 0 violations, scn-cmp + no-clog-overlay clean. cppcheck reports one finding in test_cluster_pi_shadow.c:210 — byte-identical to the base commit, local cppcheck 2.20 version drift, not introduced by this branch. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- .github/workflows/nightly.yml | 9 +- src/backend/cluster/cluster_clean_leave.c | 52 +++-- src/backend/cluster/cluster_debug.c | 6 +- src/backend/cluster/cluster_guc.c | 4 +- src/backend/cluster/cluster_lmon.c | 7 +- src/backend/cluster/cluster_node_remove.c | 68 ++++--- src/backend/cluster/cluster_qvotec.c | 4 +- src/backend/cluster/cluster_reconfig.c | 174 ++++++++++------ src/backend/cluster/cluster_write_fence.c | 9 +- src/include/cluster/cluster_clean_leave.h | 3 +- src/include/cluster/cluster_lmon.h | 2 +- src/include/cluster/cluster_marker_async.h | 42 ++-- src/include/cluster/cluster_node_remove.h | 3 +- src/include/cluster/cluster_reconfig.h | 11 +- src/include/cluster/cluster_write_fence.h | 3 +- ...63_cluster_2_29a_marker_async_liveness.pl} | 19 +- .../cluster_unit/test_cluster_marker_async.c | 6 +- src/test/cluster_unit/test_cluster_reconfig.c | 188 +++++++++++++++++- 18 files changed, 442 insertions(+), 168 deletions(-) rename src/test/cluster_tap/t/{358_cluster_2_29a_marker_async_liveness.pl => 363_cluster_2_29a_marker_async_liveness.pl} (86%) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 307a387f2e..e1b41e3ae8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -177,12 +177,15 @@ jobs: - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), - # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and - # t/358 spec-2.29a marker-async LMON liveness. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-358", unit: false, regress: false } + - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by + # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy + # verified against origin branches 2026-07-08). + - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 78a95163d9..56b4b7fc62 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -58,7 +58,7 @@ #include "cluster/cluster_guc.h" #include "cluster/cluster_ic_envelope.h" #include "cluster/cluster_ic_router.h" -#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ +#include "cluster/cluster_inject.h" /* CLUSTER_INJECTION_POINT (D12) */ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" /* PCM release-all-self + no-leftover verify (D5) */ #include "cluster/cluster_qvotec.h" /* cluster_qvotec_in_quorum (request gate) */ @@ -813,8 +813,8 @@ cl_poll_lmon_marker_stage(void) now = GetCurrentTimestamp(); kind = cl_marker_phase_kind(cl_lmon_marker_phase); if (!cl_lmon_marker_submitted) { - if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, - kind, cl_lmon_marker_leaving, now)) + if (!cluster_clean_leave_submit_marker_async(&cl_lmon_marker_async, &cl_lmon_marker, kind, + cl_lmon_marker_leaving, now)) return CL_LEAVE_ASYNC_PENDING; cl_lmon_marker_submitted = true; return CL_LEAVE_ASYNC_PENDING; @@ -859,11 +859,10 @@ cl_drive_committed_marker_stage(int32 leaving, uint64 committed_epoch) cl_send_committed(leaving, committed_epoch); cl_release_lmon_marker_stage(); } else if (ar == CL_LEAVE_ASYNC_FAILED) { - ereport(LOG, - (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " - "COMMITTED marker is not yet majority-durable; retrying each tick " - "(leaving node waits)", - leaving, (unsigned long long)committed_epoch))); + ereport(LOG, (errmsg("cluster clean-leave: committed node %d at epoch %llu but the " + "COMMITTED marker is not yet majority-durable; retrying each tick " + "(leaving node waits)", + leaving, (unsigned long long)committed_epoch))); } } @@ -931,8 +930,8 @@ cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeav } ClusterMarkerPollResult -cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (cl_state == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; @@ -1805,12 +1804,37 @@ cl_coordinator_commit(int32 leaving) return; if (phase == CLUSTER_LEAVE_MARKER_PHASE_COMMITTING) { cl_release_lmon_marker_stage(); + + /* + * spec-2.29a review r1 P1 — CL-I3 re-check at the staged-ACK + * handoff. The COMMITTING marker wait now spans LMON ticks, so a + * real death can bump CSSD dead_generation INSIDE the wait window + * while the fail-stop epoch has not yet advanced (the >=3-node + * window of Hardening v1.0.1 P1-2). The guarded CAS inside + * apply_clean_leave_as_coordinator only catches the epoch move; + * re-run the same dead_gen-aware coherence check the non-staged + * pre-check uses, at the last observable point before the commit + * applies. On failure the leave does not commit and the leaving + * node escalates to fail-stop (identical to the pre-check path). + */ + if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), + cl_state->leave_baseline_dead_gen, + cluster_cssd_get_dead_generation())) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or dead_generation) " + "across the COMMITTING marker wait for node %d; not committing " + "(escalate to fail-stop, CL-I3)", + leaving))); + return; + } + new_epoch = cluster_reconfig_apply_clean_leave_as_coordinator(leaving, baseline_epoch); if (new_epoch == 0) { - ereport(LOG, - (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " - "for node %d; not committing (the leaving node escalates to fail-stop)", - (unsigned long long)baseline_epoch, leaving))); + ereport( + LOG, + (errmsg("cluster clean-leave: epoch moved off baseline %llu before commit " + "for node %d; not committing (the leaving node escalates to fail-stop)", + (unsigned long long)baseline_epoch, leaving))); return; } cl_build_marker(&m, CLUSTER_LEAVE_MARKER_PHASE_COMMITTED, leaving, new_epoch); diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 6ece9182d4..bcebe9a058 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -544,10 +544,8 @@ dump_lmon(ReturnSetInfo *rsinfo) iters = cluster_lmon_main_loop_iters(); emit_row(rsinfo, "lmon", "lmon_main_loop_iters", fmt_int64(iters)); - emit_row(rsinfo, "lmon", "lmon_last_iter_us", - fmt_int64((int64)cluster_lmon_last_iter_us())); - emit_row(rsinfo, "lmon", "lmon_max_iter_us", - fmt_int64((int64)cluster_lmon_max_iter_us())); + emit_row(rsinfo, "lmon", "lmon_last_iter_us", fmt_int64((int64)cluster_lmon_last_iter_us())); + emit_row(rsinfo, "lmon", "lmon_max_iter_us", fmt_int64((int64)cluster_lmon_max_iter_us())); emit_row(rsinfo, "lmon", "lmon_slow_iter_count", fmt_int64((int64)cluster_lmon_slow_iter_count())); } diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 869e7c2866..5a53b070a9 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -2580,8 +2580,8 @@ cluster_init_guc(void) gettext_noop("A value greater than zero logs LMON main-loop iterations above this " "duration; the lmon_slow_iter_count counter is maintained even when " "logging is disabled with 0."), - &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, - NULL, NULL); + &cluster_lmon_slow_iteration_warn_ms, 1000, 0, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, + NULL); DefineCustomIntVariable( "cluster.lck_main_loop_interval", diff --git a/src/backend/cluster/cluster_lmon.c b/src/backend/cluster/cluster_lmon.c index 51884d0a80..1af59978cc 100644 --- a/src/backend/cluster/cluster_lmon.c +++ b/src/backend/cluster/cluster_lmon.c @@ -866,10 +866,9 @@ lmon_record_iteration(TimestampTz iter_started_at) } if (should_log) - ereport(LOG, - (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", - (unsigned long long)(elapsed_us / 1000ULL), - cluster_lmon_slow_iteration_warn_ms))); + ereport(LOG, (errmsg("cluster lmon: main loop iteration took %llu ms (threshold %d ms)", + (unsigned long long)(elapsed_us / 1000ULL), + cluster_lmon_slow_iteration_warn_ms))); } diff --git a/src/backend/cluster/cluster_node_remove.c b/src/backend/cluster/cluster_node_remove.c index 830a7a8d66..39e31c4b72 100644 --- a/src/backend/cluster/cluster_node_remove.c +++ b/src/backend/cluster/cluster_node_remove.c @@ -227,13 +227,13 @@ cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemo } ClusterMarkerPollResult -cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (nr_state == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; - return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, - &nr_state->marker_result, now, out_result, out_elapsed_us); + return cluster_marker_async_poll(a, &nr_state->marker_completion_seq, &nr_state->marker_result, + now, out_result, out_elapsed_us); } bool @@ -312,8 +312,8 @@ nr_release_marker_stage(void) } static bool -nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, - uint64 removed_incarnation, uint64 removal_event_id) +nr_marker_stage_matches(int phase, int32 node_id, uint64 remove_epoch, uint64 removed_incarnation, + uint64 removal_event_id) { return nr_marker_async.has_staged_event && nr_marker_phase == phase && nr_marker_node_id == node_id && nr_marker_epoch == remove_epoch @@ -685,31 +685,43 @@ cluster_node_remove_drive(void) (void)baseline_dead_gen; /* contest is now signalled by out_contest, not derived */ - /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ - (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, - last_incarnation, removal_event_id); - /* - * spec-6.15 D5c (appendix B.3): durably retire the removed node's - * xid stripe slot BEFORE the removal point of no return below. - * Ordering is the identity-reuse defence: once removal commits, a - * later fresh join of the same node_id (new incarnation = a new - * durable owner) must land on a retired slot and refuse (53RB1), - * never resume the old owner's congruence class. Not durable yet - * -> stay FENCE_ARMING and retry next tick (fail-closed; same - * shape as a transient fence/quorum failure). A never-activated - * cluster returns true (nothing to retire). + * spec-2.29a review r1 P2: while the staged fence marker from a + * previous FENCE_ARMING tick is still in flight, skip the pre-work + * below — the epoch was already self-bumped at stage-entry, so + * re-running it would key a SECOND REMOVING marker to the post-bump + * epoch and re-submit the stripe retire during the very contention + * window the async stage relieves. Fall through straight to + * apply(), which polls the stage. */ - if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { - ereport(LOG, (errmsg("cluster node removal: stripe slot retire for node %d not durable " - "yet — retrying before the removal commit", - node_id))); - return; + if (!cluster_reconfig_node_removed_fence_stage_pending()) { + /* §2.5: durable REMOVING marker (pre-commit; not a trust source). */ + (void)nr_write_marker(CLUSTER_REMOVAL_MARKER_REMOVING, node_id, baseline_epoch, + last_incarnation, removal_event_id); + + /* + * spec-6.15 D5c (appendix B.3): durably retire the removed node's + * xid stripe slot BEFORE the removal point of no return below. + * Ordering is the identity-reuse defence: once removal commits, a + * later fresh join of the same node_id (new incarnation = a new + * durable owner) must land on a retired slot and refuse (53RB1), + * never resume the old owner's congruence class. Not durable yet + * -> stay FENCE_ARMING and retry next tick (fail-closed; same + * shape as a transient fence/quorum failure). A never-activated + * cluster returns true (nothing to retire). + */ + if (!cluster_xid_stripe_submit_retire(node_id, last_incarnation)) { + ereport(LOG, + (errmsg("cluster node removal: stripe slot retire for node %d not durable " + "yet — retrying before the removal commit", + node_id))); + return; + } + /* D5d: carry the retire to WAL consumers (standby stops gap- + * filling the dead class). The voting-disk retire above is the + * correctness anchor; emission failure is tolerated (logged). */ + cluster_xid_stripe_emit_retire_wal(node_id); } - /* D5d: carry the retire to WAL consumers (standby stops gap- - * filling the dead class). The voting-disk retire above is the - * correctness anchor; emission failure is tolerated (logged). */ - cluster_xid_stripe_emit_retire_wal(node_id); /* * The commit point: guarded epoch bump + fence-arm (majority-durable, at diff --git a/src/backend/cluster/cluster_qvotec.c b/src/backend/cluster/cluster_qvotec.c index 6fd071a17a..916a629399 100644 --- a/src/backend/cluster/cluster_qvotec.c +++ b/src/backend/cluster/cluster_qvotec.c @@ -95,8 +95,8 @@ #include "cluster/cluster_epoch.h" /* spec-4.12b D2/D5: current-epoch upper-bound Assert */ #include "cluster/cluster_guc.h" /* cluster_enabled */ #include "cluster/cluster_inject.h" -#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ -#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ +#include "cluster/cluster_pgstat.h" /* cluster.qvotec.* counters */ +#include "cluster/cluster_reconfig.h" /* spec-4.12b D2: applied-membership snapshot */ #include "cluster/cluster_xid_stripe_boot.h" /* spec-6.15 D5b: region-5 scan + seed */ #include "cluster/cluster_shmem.h" /* cluster_shmem_register_region */ #include "cluster/cluster_write_fence.h" /* spec-4.12 D2: fence marker scan + token refresh */ diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 931ea26b8d..cf03d3e8b7 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -341,6 +341,28 @@ cluster_reconfig_publish_event(const ReconfigEvent *evt) } +static void +cluster_reconfig_log_failstop_epoch_bump(const ReconfigEvent *evt) +{ + char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; + int off = 0; + int n; + + Assert(evt != NULL); + + dead[0] = '\0'; + for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { + if (evt->dead_bitmap[n / 8] & (1 << (n % 8))) + off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); + } + + ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " + "(coordinator node %d, dead node(s) {%s})", + (unsigned long long)evt->old_epoch, (unsigned long long)evt->new_epoch, + (int)evt->coordinator_node_id, dead))); +} + + /* ============================================================ * Counter accessors (Step 2 + Step 3 SRF support). * ============================================================ @@ -1075,10 +1097,9 @@ cluster_reconfig_note_marker_slow_ack(ClusterMarkerAsyncKind kind, int32 target_ return; pg_atomic_fetch_add_u64(&ReconfigShmem->marker_slow_ack_count, 1); - ereport(LOG, - (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", - cluster_marker_async_kind_name(kind), target_node, - (unsigned long long)(elapsed_us / 1000ULL)))); + ereport(LOG, (errmsg("cluster marker: slow qvotec ACK for %s target node %d took %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); } void @@ -1087,10 +1108,9 @@ cluster_reconfig_note_marker_timeout(ClusterMarkerAsyncKind kind, int32 target_n { if (ReconfigShmem != NULL) pg_atomic_fetch_add_u64(&ReconfigShmem->marker_timeout_count, 1); - ereport(LOG, - (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", - cluster_marker_async_kind_name(kind), target_node, - (unsigned long long)(elapsed_us / 1000ULL)))); + ereport(LOG, (errmsg("cluster marker: qvotec marker %s target node %d timed out after %llu ms", + cluster_marker_async_kind_name(kind), target_node, + (unsigned long long)(elapsed_us / 1000ULL)))); } uint64 @@ -1234,14 +1254,13 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) } static bool -cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, - ClusterMarkerAsyncKind kind, int32 target_node, - TimestampTz now) +cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, + int32 target_node, TimestampTz now) { if (stage->submitted) return true; - if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, - target_node, now)) + if (!cluster_write_fence_submit_marker_async(&stage->async, &stage->marker, kind, target_node, + now)) return false; stage->submitted = true; return true; @@ -1281,6 +1300,7 @@ cluster_reconfig_poll_failstop_fence_stage(void) elapsed_us); if (result == CLUSTER_FENCE_MARKER_SUBMIT_ACK) { cluster_reconfig_publish_event(&failstop_fence_stage.event); + cluster_reconfig_log_failstop_epoch_bump(&failstop_fence_stage.event); cluster_reconfig_broadcast_local_procsig(); } else { ereport(LOG, @@ -1324,8 +1344,8 @@ cluster_reconfig_poll_node_removed_fence_stage(int32 removed_node_id, uint64 rem return 0; } - cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, - removed_node_id, elapsed_us); + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_FENCE_NODE_REMOVED, removed_node_id, + elapsed_us); if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { ereport(LOG, (errmsg("cluster node removal: fence marker for node %d did not reach a " "voting-disk majority for epoch %llu; not publishing removal " @@ -1364,24 +1384,12 @@ cluster_reconfig_release_join_prepare_stage(void) join_prepare_stage.submitted = false; } -static void -cluster_reconfig_abort_join_prepare_stage(void) -{ - int i; - - if (ReconfigShmem != NULL) { - LWLockAcquire(&ReconfigShmem->lock, LW_EXCLUSIVE); - for (i = 0; i < CLUSTER_MAX_NODES; i++) { - if (!dead_bitmap_test_bit(join_prepare_stage.join_bitmap, i)) - continue; - ReconfigShmem->pending_join_bitmap[i / 8] &= (uint8) ~(1u << (i % 8)); - if (cluster_membership_get_state(i) == CLUSTER_MEMBER_JOINING) - cluster_membership_set_state(i, CLUSTER_MEMBER_DEAD); - } - LWLockRelease(&ReconfigShmem->lock); - } - cluster_reconfig_release_join_prepare_stage(); -} +/* (spec-2.29a review r1: the former abort_join_prepare_stage — revert + * JOINING→DEAD + clear pending on PREPARE failure — was removed. Q5=A makes + * PREPARE strictly best-effort: TIMEOUT / non-ACK advances the queue and the + * staged JOIN_PENDING still publishes, so no revert path exists; reverting + * would let compute_join_bitmap re-detect the joiner and re-bump the epoch + * every deadline period — the P1-1 bump storm.) */ static bool cluster_reconfig_submit_join_prepare_current(TimestampTz now) @@ -1422,6 +1430,22 @@ cluster_reconfig_submit_join_prepare_current(TimestampTz now) return true; } +/* + * spec-2.29a review r1 P2 — node-remove driver pre-work gate. + * + * While the staged node-removed fence marker from a previous FENCE_ARMING + * tick is still in flight, the driver must not re-run its pre-work + * (REMOVING marker write / stripe retire): the epoch has already been + * self-bumped by the stage-entry, so a re-run would key a SECOND REMOVING + * marker to the post-bump epoch and add voting-disk writes during the + * exact contention window the async stage exists to relieve. + */ +bool +cluster_reconfig_node_removed_fence_stage_pending(void) +{ + return node_removed_fence_stage.async.has_staged_event; +} + static bool cluster_reconfig_poll_join_prepare_stage(void) { @@ -1445,22 +1469,28 @@ cluster_reconfig_poll_join_prepare_stage(void) &elapsed_us); if (pr == CLUSTER_MARKER_POLL_PENDING || pr == CLUSTER_MARKER_POLL_IDLE) return true; - if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { - cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, - elapsed_us); - cluster_reconfig_abort_join_prepare_stage(); - return true; - } - cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, - elapsed_us); - if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) { - ereport(LOG, - (errmsg("cluster membership: PREPARE join marker for node %d did not reach a " - "voting-disk majority; not publishing JOIN_PENDING (will retry)", - target))); - cluster_reconfig_abort_join_prepare_stage(); - return true; + /* + * spec-2.29a Q5=A (review r1 P1): PREPARE is best-effort in OUTCOME — the + * pre-async code ignored the submit result and always published + * JOIN_PENDING (only the Phase-2 COMMITTED marker is a commit point, P1-r5). + * A TIMEOUT / non-ACK PREPARE therefore advances the queue instead of + * aborting: aborting would revert the joiner JOINING→DEAD, and the next + * tick's compute_join_bitmap would re-detect it CSSD-alive and RE-BUMP the + * epoch — exactly the per-tick epoch-bump storm the P1-1 staged record + * forbids. Draining to publish keeps the joiner on a stable JOINING + * (one Phase-1 bump total, regardless of PREPARE outcomes). + */ + if (pr == CLUSTER_MARKER_POLL_TIMEOUT) { + cluster_reconfig_note_marker_timeout(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + } else { + cluster_reconfig_note_marker_slow_ack(CLUSTER_MARKER_KIND_JOIN_PREPARE, target, elapsed_us); + if (result != CLUSTER_JOIN_MARKER_SUBMIT_ACK) + ereport(LOG, + (errmsg("cluster membership: PREPARE join marker for node %d did not reach " + "a voting-disk majority; continuing best-effort (COMMITTED is the " + "commit point)", + target))); } join_prepare_stage.submitted = false; @@ -1584,7 +1614,7 @@ cluster_reconfig_poll_join_commit_stage(void) &admitted_generation) || admitted_incarnation != join_commit_stage.admitted_incarnation || cluster_membership_vet_joiner(join_commit_stage.node_id, admitted_incarnation, - admitted_generation) + admitted_generation) != CLUSTER_JOIN_ACCEPT || !cluster_reconfig_publish_join_commit(join_commit_stage.node_id, join_commit_stage.admitted_incarnation, @@ -2578,6 +2608,32 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( marker.fenced_dead_bitmap[b] |= removed_bitmap[b]; } + /* + * The async stage is process-local by design and is driven only by LMON + * ticks. Test-only backend callers, such as + * cluster_reconfig_inject_dead_node_test(), would otherwise stage the marker + * in their own process and return with no LMON-visible pending publish + * record. Keep those non-LMON paths on the old bounded wait: they are not + * transport-liveness actors, so this does not reintroduce the BUG-C1 LMON + * park. + */ + if (MyBackendType != B_LMON) { + ClusterFenceMarkerSubmitResult result; + + result = cluster_write_fence_submit_marker(&marker); + if (result != CLUSTER_FENCE_MARKER_SUBMIT_ACK) { + ereport(LOG, (errmsg("cluster reconfig: fence marker did not reach a voting-disk " + "majority for epoch %llu; not publishing reconfig event " + "(write-fenced, will retry)", + (unsigned long long)new_epoch))); + return; + } + + cluster_reconfig_publish_event(&evt); + cluster_reconfig_log_failstop_epoch_bump(&evt); + return; + } + failstop_fence_stage.event = evt; failstop_fence_stage.marker = marker; failstop_fence_stage.node_id = coordinator_node_id; @@ -2599,21 +2655,7 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * fixed stack buffer (worst case: 128 node ids x 4 chars) keeps the LMON * tick free of allocator traffic. */ - { - char dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8 * 4 + 1]; - int off = 0; - int n; - - dead[0] = '\0'; - for (n = 0; n < CLUSTER_RECONFIG_DEAD_BITMAP_BYTES * 8; n++) { - if (dead_bitmap[n / 8] & (1 << (n % 8))) - off += snprintf(dead + off, sizeof(dead) - off, "%s%d", off ? "," : "", n); - } - ereport(LOG, (errmsg("cluster reconfig: fail-stop epoch bump %llu -> %llu published " - "(coordinator node %d, dead node(s) {%s})", - (unsigned long long)old_epoch, (unsigned long long)new_epoch, - (int)coordinator_node_id, dead))); - } + cluster_reconfig_log_failstop_epoch_bump(&evt); } @@ -2928,8 +2970,8 @@ cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_no } ClusterMarkerPollResult -cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (ReconfigShmem == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; diff --git a/src/backend/cluster/cluster_write_fence.c b/src/backend/cluster/cluster_write_fence.c index d4988b042d..8917959359 100644 --- a/src/backend/cluster/cluster_write_fence.c +++ b/src/backend/cluster/cluster_write_fence.c @@ -786,14 +786,13 @@ cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenc memcpy(&cluster_write_fence_shmem->pending_marker, m, sizeof(*m)); return cluster_marker_async_submit( a, &cluster_write_fence_shmem->marker_request_seq, - &cluster_write_fence_shmem->marker_completion_seq, - cluster_write_fence_shmem->qvotec_latch, now, - (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); + &cluster_write_fence_shmem->marker_completion_seq, cluster_write_fence_shmem->qvotec_latch, + now, (uint64)cluster_write_fence_lease_ms * 1000ULL, kind, target_node); } ClusterMarkerPollResult -cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { if (cluster_write_fence_shmem == NULL || a == NULL) return CLUSTER_MARKER_POLL_IDLE; diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 2d75de8e2a..1330a40490 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -412,8 +412,7 @@ extern ClusterLeaveMarkerSubmitResult cluster_clean_leave_submit_marker(const ClusterLeaveIntentMarker *m); extern bool cluster_clean_leave_submit_marker_async(ClusterMarkerAsync *a, const ClusterLeaveIntentMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_clean_leave_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/include/cluster/cluster_lmon.h b/src/include/cluster/cluster_lmon.h index 1b6a516f4b..e1099ece40 100644 --- a/src/include/cluster/cluster_lmon.h +++ b/src/include/cluster/cluster_lmon.h @@ -138,7 +138,7 @@ typedef struct ClusterLmonSharedState { TimestampTz ready_at; /* set by LMON in CLUSTER_LMON_READY */ TimestampTz last_liveness_tick_at; /* HC6: local liveness tick — NOT inter-node heartbeat */ int64 main_loop_iters; /* monotone counter; observable proof of liveness */ - uint64 last_iter_us; /* most recent main-loop iteration wall time */ + uint64 last_iter_us; /* most recent main-loop iteration wall time */ uint64 max_iter_us; /* high-water iteration wall time */ uint64 slow_iter_count; /* iterations over cluster.lmon_slow_iteration_warn_ms */ struct Latch *lmon_latch; /* qvotec completion wake target; LMON owns lifecycle */ diff --git a/src/include/cluster/cluster_marker_async.h b/src/include/cluster/cluster_marker_async.h index 792e583b47..56cd836bf5 100644 --- a/src/include/cluster/cluster_marker_async.h +++ b/src/include/cluster/cluster_marker_async.h @@ -3,14 +3,27 @@ * cluster_marker_async.h * Small process-local async FSM for qvotec marker submit mailboxes. * - * The mailbox remains the existing request_seq/completion_seq/result slot. - * This wrapper changes only the waiting shape: submit publishes a request and - * returns; the owning LMON tick polls completion without sleeping. - * * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 2026, pgrac contributors * + * Author: SqlRush + * + * IDENTIFICATION + * src/include/cluster/cluster_marker_async.h + * + * NOTES + * The mailbox remains the existing request_seq/completion_seq/result + * slot. This wrapper changes only the waiting shape: submit publishes + * a request and returns; the owning LMON tick polls completion without + * sleeping (spec-2.29a — the pre-async 2ms pg_usleep spin inside the + * LMON tick starved the CSSD heartbeat relay and caused false-DEAD + * storms during cold formation, BUG-C1). + * + * The staged has_staged_event flag is the P1-1 bump-once contract: a + * pre-bump caller (fail-stop fence / node-remove / join Phase-1) must + * not re-enter its epoch-bump path while a stage is live. + * *------------------------------------------------------------------------- */ #ifndef CLUSTER_MARKER_ASYNC_H @@ -109,8 +122,7 @@ cluster_marker_async_is_submitted(const ClusterMarkerAsync *a) } static inline bool -cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, - pg_atomic_uint64 *completion_seq) +cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq) { return pg_atomic_read_u64(request_seq) != pg_atomic_read_u64(completion_seq); } @@ -118,8 +130,8 @@ cluster_marker_async_mailbox_busy(pg_atomic_uint64 *request_seq, static inline bool cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, struct Latch *qvotec_latch, - TimestampTz now, uint64 timeout_us, - ClusterMarkerAsyncKind kind, int32 target_node) + TimestampTz now, uint64 timeout_us, ClusterMarkerAsyncKind kind, + int32 target_node) { uint64 seq; @@ -148,8 +160,8 @@ cluster_marker_async_submit(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq static inline ClusterMarkerPollResult cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_seq, - pg_atomic_uint32 *result_slot, TimestampTz now, - uint32 *out_result, uint64 *out_elapsed_us) + pg_atomic_uint32 *result_slot, TimestampTz now, uint32 *out_result, + uint64 *out_elapsed_us) { uint64 elapsed; @@ -168,9 +180,8 @@ cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_se pg_read_barrier(); if (out_result != NULL) *out_result = pg_atomic_read_u32(result_slot); - elapsed = ((uint64)now > (uint64)a->submitted_at) - ? ((uint64)now - (uint64)a->submitted_at) - : 0; + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; if (out_elapsed_us != NULL) *out_elapsed_us = elapsed; a->state = CLUSTER_MARKER_ASYNC_IDLE; @@ -178,9 +189,8 @@ cluster_marker_async_poll(ClusterMarkerAsync *a, pg_atomic_uint64 *completion_se } if ((uint64)now >= a->deadline_us) { - elapsed = ((uint64)now > (uint64)a->submitted_at) - ? ((uint64)now - (uint64)a->submitted_at) - : 0; + elapsed + = ((uint64)now > (uint64)a->submitted_at) ? ((uint64)now - (uint64)a->submitted_at) : 0; if (out_elapsed_us != NULL) *out_elapsed_us = elapsed; a->state = CLUSTER_MARKER_ASYNC_IDLE; diff --git a/src/include/cluster/cluster_node_remove.h b/src/include/cluster/cluster_node_remove.h index 1df3d8c868..ef722e3f62 100644 --- a/src/include/cluster/cluster_node_remove.h +++ b/src/include/cluster/cluster_node_remove.h @@ -419,8 +419,7 @@ extern ClusterRemovalMarkerSubmitResult cluster_node_remove_submit_marker(const ClusterRemovalMarker *m); extern bool cluster_node_remove_submit_marker_async(ClusterMarkerAsync *a, const ClusterRemovalMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_node_remove_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index 3e6abadc16..4ce28e13fd 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -56,7 +56,7 @@ #include "port/atomics.h" #include "storage/lwlock.h" -#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ +#include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES (spec-5.13 clean_departed_epoch) */ #include "cluster/cluster_marker_async.h" #include "cluster/cluster_membership.h" /* ClusterMembershipTable (spec-5.15 D2 SSOT) */ @@ -585,8 +585,7 @@ extern ClusterJoinMarkerSubmitResult cluster_reconfig_submit_join_marker(int32 target_node, const ClusterJoinCommitMarker *m); extern bool cluster_reconfig_submit_join_marker_async(ClusterMarkerAsync *a, int32 target_node, const ClusterJoinCommitMarker *m, - ClusterMarkerAsyncKind kind, - TimestampTz now); + ClusterMarkerAsyncKind kind, TimestampTz now); extern ClusterMarkerPollResult cluster_reconfig_poll_join_marker_async(ClusterMarkerAsync *a, TimestampTz now, uint32 *out_result, @@ -774,5 +773,11 @@ extern uint64 cluster_reconfig_apply_node_removed_as_coordinator(int32 removed_n uint64 last_incarnation, bool *out_contest); +/* spec-2.29a review r1 P2: true while the staged node-removed fence marker is + * in flight -- the node-remove driver skips its FENCE_ARMING pre-work + * (REMOVING marker / stripe retire) instead of re-keying it to the live + * (already self-bumped) epoch. */ +extern bool cluster_reconfig_node_removed_fence_stage_pending(void); + #endif /* CLUSTER_RECONFIG_H */ diff --git a/src/include/cluster/cluster_write_fence.h b/src/include/cluster/cluster_write_fence.h index dad13ff00d..ac5a965882 100644 --- a/src/include/cluster/cluster_write_fence.h +++ b/src/include/cluster/cluster_write_fence.h @@ -521,8 +521,7 @@ extern ClusterFenceMarkerSubmitResult cluster_write_fence_submit_marker(const ClusterFenceMarker *m); extern bool cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m, - ClusterMarkerAsyncKind kind, - int32 target_node, + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now); extern ClusterMarkerPollResult cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now, diff --git a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl similarity index 86% rename from src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl rename to src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl index e2f19d6454..005c0d4a4b 100644 --- a/src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +++ b/src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl @@ -1,7 +1,7 @@ #!/usr/bin/env perl #------------------------------------------------------------------------- # -# 358_cluster_2_29a_marker_async_liveness.pl +# 363_cluster_2_29a_marker_async_liveness.pl # BUG-C1 / spec-2.29a — qvotec marker backlog must not park LMON. # # The hold injection freezes qvotec marker completion until released. Before @@ -18,8 +18,10 @@ # # Portions Copyright (c) 2026, pgrac contributors # +# Author: SqlRush +# # IDENTIFICATION -# src/test/cluster_tap/t/358_cluster_2_29a_marker_async_liveness.pl +# src/test/cluster_tap/t/363_cluster_2_29a_marker_async_liveness.pl # #------------------------------------------------------------------------- @@ -119,6 +121,8 @@ sub set_injection set_injection($node0, 'cluster-qvotec-marker-service-hold'); my $lmon_iters_before = state_int($node0, 'lmon', 'lmon_slow_iter_count'); my $timeouts_before = state_int($node0, 'reconfig', 'marker_timeout_count'); +my $epoch_hold_start = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; $node1->start; ok(poll_until($node0, @@ -149,6 +153,17 @@ sub set_injection 'member', 'L4 held/timeout marker does not publish node1 admission'); +# P1-1 bump-once (spec-2.29a review r1): the whole >=95s hold must cost at most +# ONE join Phase-1 epoch bump (staged record; PREPARE timeouts drain best-effort +# and never revert the joiner, so compute_join_bitmap cannot re-detect + re-bump +# per deadline period). Pre-fix abort+revert behavior re-bumped ~once per 3.5s +# deadline (~27 bumps in this window); a <=2 bound separates cleanly without +# being flaky against one incidental episode. +my $epoch_hold_end = + $node0->safe_psql('postgres', 'SELECT new_epoch FROM pg_cluster_reconfig_state') + 0; +cmp_ok($epoch_hold_end - $epoch_hold_start, '<=', 2, + 'L4 epoch advance across the hold window is bounded (no per-deadline bump storm)'); + set_injection($node0, ''); ok(poll_until($node0, q{SELECT state = 'member' FROM pg_cluster_membership WHERE node_id = 1}, diff --git a/src/test/cluster_unit/test_cluster_marker_async.c b/src/test/cluster_unit/test_cluster_marker_async.c index 9823fd69bf..b90fa463a2 100644 --- a/src/test/cluster_unit/test_cluster_marker_async.c +++ b/src/test/cluster_unit/test_cluster_marker_async.c @@ -7,6 +7,8 @@ * Portions Copyright (c) 1994, Regents of the University of California * Portions Copyright (c) 2026, pgrac contributors * + * Author: SqlRush + * * IDENTIFICATION * src/test/cluster_unit/test_cluster_marker_async.c * @@ -48,8 +50,8 @@ SetLatch(Latch *latch pg_attribute_unused()) } static void -ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, - pg_atomic_uint64 *completion_seq, pg_atomic_uint32 *result_slot) +ut_reset(ClusterMarkerAsync *a, pg_atomic_uint64 *request_seq, pg_atomic_uint64 *completion_seq, + pg_atomic_uint32 *result_slot) { cluster_marker_async_init(a); pg_atomic_init_u64(request_seq, 0); diff --git a/src/test/cluster_unit/test_cluster_reconfig.c b/src/test/cluster_unit/test_cluster_reconfig.c index 5860c19831..6d6fcbfd0f 100644 --- a/src/test/cluster_unit/test_cluster_reconfig.c +++ b/src/test/cluster_unit/test_cluster_reconfig.c @@ -61,6 +61,14 @@ UT_DEFINE_GLOBALS(); +/* spec-2.29a: cluster_reconfig.c gates the async marker stage on + * MyBackendType == B_LMON. The fixture exercises the pre-2.29a bounded-wait + * semantics (write_fence submit stub returns FAILED synchronously), so run as + * a non-LMON backend; the async FSM itself is covered by + * test_cluster_marker_async.c. */ +#include "miscadmin.h" +BackendType MyBackendType = B_INVALID; + /* ============================================================ * Stubs — link cluster_reconfig.o + cluster_epoch.o standalone. @@ -357,22 +365,44 @@ cluster_write_fence_submit_marker(const ClusterFenceMarker *m pg_attribute_unuse { return CLUSTER_FENCE_MARKER_SUBMIT_FAILED; } +/* spec-2.29a review r1 P1-c: controllable async-marker stubs so the tick-level + * P1-1 invariants (bump-once while PENDING, node-remove zero-false-contest, + * publish deferred to ACK) can be hard-asserted. Defaults preserve the + * pre-existing inert behavior (submit rejected, poll idle). */ +static bool ut_fence_async_submit_ok = false; +static ClusterMarkerPollResult ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; +static uint32 ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +static int ut_fence_async_submit_calls = 0; +static int ut_fence_async_poll_calls = 0; + bool -cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), +cluster_write_fence_submit_marker_async(ClusterMarkerAsync *a, const ClusterFenceMarker *m pg_attribute_unused(), - ClusterMarkerAsyncKind kind pg_attribute_unused(), - int32 target_node pg_attribute_unused(), + ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now pg_attribute_unused()) { - return false; + ut_fence_async_submit_calls++; + if (!ut_fence_async_submit_ok) + return false; + /* mirror the real wrapper's FSM effect so is_submitted() sees SUBMITTED */ + a->state = CLUSTER_MARKER_ASYNC_SUBMITTED; + a->kind = kind; + a->target_node = target_node; + return true; } ClusterMarkerPollResult -cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a pg_attribute_unused(), - TimestampTz now pg_attribute_unused(), - uint32 *out_result pg_attribute_unused(), - uint64 *out_elapsed_us pg_attribute_unused()) -{ - return CLUSTER_MARKER_POLL_IDLE; +cluster_write_fence_poll_marker_async(ClusterMarkerAsync *a, TimestampTz now pg_attribute_unused(), + uint32 *out_result, uint64 *out_elapsed_us) +{ + ut_fence_async_poll_calls++; + if (out_result != NULL) + *out_result = ut_fence_async_poll_result; + if (out_elapsed_us != NULL) + *out_elapsed_us = 0; + if (ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_ACKED + || ut_fence_async_poll_pr == CLUSTER_MARKER_POLL_TIMEOUT) + a->state = CLUSTER_MARKER_ASYNC_IDLE; + return ut_fence_async_poll_pr; } void cluster_lmon_marker_complete_wakeup(void) @@ -982,6 +1012,142 @@ UT_TEST(test_reconfig_lmon_tick_survivor_does_not_advance_epoch) } +/* ============================================================ + * T-reconfig-2.29a — P1-1 staged-record hard assertions (spec-2.29a + * review r1 P1-c). + * + * (a) fail-stop fence stage: while the marker is PENDING across ticks + * the epoch is bumped EXACTLY ONCE and nothing publishes; the + * staged event publishes only on ACKED+ACK. + * (b) node-remove stage: a tick re-entry while the fence marker is + * PENDING returns 0 with *out_contest == false (no false contest + * against our own already-advanced baseline) and no re-bump. + * ============================================================ */ + +UT_TEST(test_reconfig_failstop_fence_stage_bump_once_while_pending) +{ + uint64 epoch_after_bump; + uint64 apply_before; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + ut_peer_state[1] = CLUSTER_CSSD_PEER_DEAD; + ut_dead_generation = 1; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + apply_before = cluster_reconfig_get_apply_counter(); + + /* tick #1: stage-entry — bump once, submit, DO NOT publish. */ + cluster_reconfig_lmon_tick(); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT(epoch_after_bump > 0); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* ticks #2/#3: marker PENDING — the bump path must NOT re-enter + * (P1-1 invariant 0: zero re-bump, zero publish, zero side-effects). */ + cluster_reconfig_lmon_tick(); + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)apply_before); + + /* tick #4: ACKED + ACK — the STAGED event publishes; still no re-bump. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + cluster_reconfig_lmon_tick(); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + UT_ASSERT_EQ((unsigned long long)cluster_reconfig_get_apply_counter(), + (unsigned long long)(apply_before + 1)); + + /* restore fixture defaults for the neighboring tests */ + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + +UT_TEST(test_reconfig_node_remove_stage_no_false_contest) +{ + uint64 baseline; + uint64 epoch_after_bump; + uint64 ret; + bool contest; + + ut_reset_mocks(); + reconfig_init_done = false; + epoch_init_done = false; + cluster_reconfig_shmem_init(); + cluster_epoch_shmem_init(); + + cluster_node_id = 0; + ut_in_quorum_value = true; + ut_declared_set[0] = true; + ut_declared_set[1] = true; + + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_ON; + MyBackendType = B_LMON; + ut_fence_async_submit_ok = true; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_PENDING; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; + + baseline = cluster_epoch_get_current(); + + /* call #1: guarded CAS advances baseline→baseline+1, stages the fence + * marker, returns 0 (not yet committed), contest MUST be false. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + epoch_after_bump = cluster_epoch_get_current(); + UT_ASSERT_EQ((unsigned long long)epoch_after_bump, (unsigned long long)(baseline + 1)); + + /* call #2 (tick re-entry with the SAME stale baseline while PENDING): + * without the staged record this would re-run the baseline CAS against + * our own advanced epoch and report a FALSE contest. P1-1: it must + * poll the stage instead — 0, contest==false, no second bump. */ + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, 0ULL); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + /* call #3: ACKED + ACK — removal publishes at the STAGED epoch. */ + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_ACKED; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_ACK; + contest = true; + ret = cluster_reconfig_apply_node_removed_as_coordinator(1, baseline, 42, 7, &contest); + UT_ASSERT_EQ((unsigned long long)ret, (unsigned long long)epoch_after_bump); + UT_ASSERT(!contest); + UT_ASSERT_EQ((unsigned long long)cluster_epoch_get_current(), + (unsigned long long)epoch_after_bump); + + MyBackendType = B_INVALID; + cluster_write_fence_enforcement = CLUSTER_WRITE_FENCE_ENFORCE_OFF; + ut_fence_async_submit_ok = false; + ut_fence_async_poll_pr = CLUSTER_MARKER_POLL_IDLE; + ut_fence_async_poll_result = CLUSTER_FENCE_MARKER_SUBMIT_FAILED; +} + + /* ============================================================ * T-reconfig-8 — ProcessInterrupts I6 guard (D4). * @@ -1595,6 +1761,8 @@ main(void) /* T-reconfig-3 — lmon_tick dedup. */ UT_RUN(test_reconfig_lmon_tick_dedups_same_event_id); UT_RUN(test_reconfig_lmon_tick_refires_on_dead_gen_bump); + UT_RUN(test_reconfig_failstop_fence_stage_bump_once_while_pending); + UT_RUN(test_reconfig_node_remove_stage_no_false_contest); /* T-reconfig-4 / 4b — Q2 A'' + I2 + L20 + F11 + empty-dead-set gates. */ UT_RUN(test_reconfig_lmon_tick_skips_when_not_in_quorum); From e71fc34315d614921e62c9321ca6e24a765de007 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:08:35 +0800 Subject: [PATCH 08/27] =?UTF-8?q?fix(cluster):=20review=20fixes=20?= =?UTF-8?q?=E2=80=94=20clean-shutdown-only=20seal,=20WARNING-skip=20on=20u?= =?UTF-8?q?nsupported=20seed,=20.bak=20read=20guard,=20multi-segment=20uni?= =?UTF-8?q?t=20leg?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review P2-1: the native-era publish+seal now runs only on a TRUE shutdown checkpoint (END_OF_RECOVERY excluded -- sealing mid-run after crash recovery would expose a stale high-water to joiners), and an unsupported native era (MultiXacts / past the prehistory cap) skips the seal with a WARNING instead of FATALing the checkpointer: the refusal still lands fail-closed on every joiner (53RB5, unsealed) but the seed can shut down cleanly instead of wedging in a FATAL loop. Review P3-4: roll_blob_to_bak PANICs on a read() error instead of treating it as EOF (a truncated .bak was already rejected downstream by CRC, but silently swallowing I/O errors broke the module's discipline). Review P3-3: unit multi-segment prehistory round trip (33 pages across SLRU segments 0000/0001, per-page byte pattern, short-clone extension). Spec: spec-6.15b-xid-authority-native-era.md --- src/backend/access/transam/xlog.c | 48 ++++++++++--- src/backend/cluster/cluster_xid_prehistory.c | 3 + .../cluster_unit/test_cluster_xid_authority.c | 71 ++++++++++++++++++- 3 files changed, 110 insertions(+), 12 deletions(-) diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 03d5719b89..1563ea5d3c 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -7586,22 +7586,48 @@ CreateCheckPoint(int flags) * are complete, publish the native-era pg_xact prehistory and seal the * shared XID authority. This does durable file I/O and may fail closed, * so it must stay outside the checkpoint critical section. + * + * Two deliberate exclusions (review P2-1): + * + * - END_OF_RECOVERY checkpoints carry `shutdown` but are NOT a clean + * close of the native run: the seed keeps consuming xids afterwards, + * so sealing here would let a joiner adopt a mid-run stale high-water + * (the same 8.A window the begin_native_run unseal closes for the + * multi-pass case). Only a true shutdown checkpoint seals. + * + * - An unsupported native era (MultiXacts created, or past the + * prehistory cap) SKIPS the seal with a WARNING instead of FATALing: + * the refusal still lands fail-closed on every joiner (53RB5, + * authority unsealed), but the seed itself can shut down cleanly + * instead of wedging every later shutdown/boot in a FATAL loop. */ - if (shutdown && cluster_shared_catalog && !cluster_enabled) + if (shutdown && !(flags & CHECKPOINT_END_OF_RECOVERY) + && cluster_shared_catalog && !cluster_enabled) { uint64 native_hw = U64FromFullTransactionId(checkPoint.nextXid); if (checkPoint.nextMulti > FirstMultiXactId) - ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), - errmsg("native-era seed loads that create MultiXacts are not supported"), - errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", - checkPoint.nextMulti, FirstMultiXactId), - errhint("Recreate the seed without MultiXact-producing operations, or move the load in-protocol."))); - - cluster_xid_prehistory_publish(DataDir, native_hw); - cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); - elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", - (unsigned long long)native_hw); + ereport(WARNING, + (errmsg("native-era seed loads that create MultiXacts are not supported; " + "leaving the shared XID authority unsealed"), + errdetail("Shutdown checkpoint nextMulti is %u, but only %u is supported.", + checkPoint.nextMulti, FirstMultiXactId), + errhint("Joiners will fail closed (53RB5); recreate the seed without " + "MultiXact-producing operations, or move the load in-protocol."))); + else if (cluster_xid_prehistory_payload_bytes(native_hw) == 0) + ereport(WARNING, + (errmsg("native-era xid high-water %llu is outside the prehistory publish " + "range; leaving the shared XID authority unsealed", + (unsigned long long) native_hw), + errhint("Joiners will fail closed (53RB5); split the seed load or move " + "it in-protocol."))); + else + { + cluster_xid_prehistory_publish(DataDir, native_hw); + cluster_xid_authority_publish_native(native_hw, checkPoint.nextMulti, true); + elog(LOG, "cluster shared_catalog: published and sealed native-era XID authority high-water %llu", + (unsigned long long) native_hw); + } } #endif diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index 9333c1eaba..0e5ed2018d 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -189,6 +189,9 @@ roll_blob_to_bak(const char *primary, const char *bak, const char *baktmp) ereport(PANIC, (errcode_for_file_access(), errmsg("could not write file \"%s\": %m", baktmp))); } + if (r < 0) + ereport(PANIC, + (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", primary))); CloseTransientFile(src); if (pg_fsync(dst) != 0) ereport(PANIC, diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 3129bd73e3..6178ebbfcd 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -200,6 +200,40 @@ make_fake_pgdata(char *pgdata, size_t len, const char *tag, int n_pages, unsigne close(fd); } +/* Build a fake PGDATA whose pg_xact spans SLRU segments: n_pages whole CLOG + * pages, 32 per segment file (0000, 0001, ...), page p filled with byte + * (p & 0xFF) so cross-segment ordering is byte-verifiable. */ +static void +make_fake_pgdata_multiseg(char *pgdata, size_t len, const char *tag, int n_pages) +{ + char dir[MAXPGPATH]; + char seg[MAXPGPATH]; + char page[BLCKSZ]; + int fd = -1; + int cur_seg = -1; + int p; + + snprintf(pgdata, len, "%s/pgdata_%s", test_root, tag); + mkdir(pgdata, 0700); + snprintf(dir, sizeof(dir), "%s/pg_xact", pgdata); + mkdir(dir, 0700); + for (p = 0; p < n_pages; p++) { + int segno = p / 32; + + if (segno != cur_seg) { + if (fd >= 0) + close(fd); + snprintf(seg, sizeof(seg), "%s/%04X", dir, segno); + fd = open(seg, O_RDWR | O_CREAT | O_TRUNC, 0600); + cur_seg = segno; + } + memset(page, p & 0xFF, sizeof(page)); + (void)!write(fd, page, sizeof(page)); + } + if (fd >= 0) + close(fd); +} + /* ============================================================ * U5: on-disk layout locked * ============================================================ */ @@ -427,6 +461,40 @@ UT_TEST(test_prehistory_publish_adopt_round_trip) UT_ASSERT_EQ(memcmp(seed_page, join_page, BLCKSZ), 0); } +UT_TEST(test_prehistory_multi_segment_round_trip) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; /* segment 0000 full (32 pages) + 0001 (1 page) */ + const uint64 hw = (uint64)n_pages * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + char want[BLCKSZ]; + char got[BLCKSZ]; + int p; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "seedms", n_pages); + /* short pre-seed clone: only one zeroed page in segment 0000 */ + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "joinms", 1); + + cluster_xid_prehistory_publish(pgdata_seed, hw); + cluster_xid_prehistory_adopt(pgdata_join, hw); + + for (p = 0; p < n_pages; p++) { + int fd; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", pgdata_join, p / 32); + fd = open(seg, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, (off_t)(p % 32) * BLCKSZ, SEEK_SET), (off_t)(p % 32) * BLCKSZ); + UT_ASSERT_EQ(read(fd, got, BLCKSZ), BLCKSZ); + close(fd); + memset(want, p & 0xFF, sizeof(want)); + UT_ASSERT_EQ(memcmp(got, want, BLCKSZ), 0); + } +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -498,7 +566,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(11); + UT_PLAN(12); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -509,6 +577,7 @@ main(void) UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); + UT_RUN(test_prehistory_multi_segment_round_trip); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From 51cdc674ffe7330429913c68e610f9a066e34085 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 17:50:02 +0800 Subject: [PATCH 09/27] fix(cluster): dual-copy flag writes, divergent-lineage guard, TAP/doc hardening F1: begin_native_run/mark_cluster_era install the transitioned image as BOTH primary and .bak (write_header_both) -- rolling the pre-transition image into .bak let a later single-copy corruption fall back to a stale SEALED (or pre-CLUSTER_ERA) authority, handing joiners the previous pass's high-water or re-opening era re-entry. F2: divergent-lineage guard -- before the adopt/skip decision the joiner byte-compares its local pg_xact prefix [0, min(own_next, native_hw)) against the sealed prehistory blob at 2-bit precision; any contradiction is FATAL (a same-sysid clone that ran standalone after cloning is not a pre-seed lineage; neither trusting nor overwriting its bits is sound). Bypassed only when local oldestXid has advanced past the native range. t/361: real scram TCP login leg (hba + listener), node0 shared-heap re-reads after joiner scans (poison proof on the heap, not just CLOG), a 53RB5 SQLSTATE pin (%e before %q -- %q stops expansion for the postmaster), the N5 divergent-lineage negative, and the L9 shared_catalog=off dormant leg. Also: on-disk header layouts locked with StaticAssertDecl; consumers LOG when the authority is served from the .bak fallback; user manual trimmed to user-visible effect. Spec: spec-6.15b-xid-authority-native-era.md --- docs/user-guide/configuration.md | 6 +- src/backend/access/transam/xlog.c | 9 +- .../cluster/cluster_catalog_bootstrap.c | 45 ++++++- src/backend/cluster/cluster_xid_authority.c | 63 ++++++++- src/backend/cluster/cluster_xid_prehistory.c | 120 +++++++++++++++++ src/include/cluster/cluster_xid_authority.h | 48 +++++++ .../t/361_xid_authority_native_era_2node.pl | 109 +++++++++++++++- .../cluster_unit/test_cluster_xid_authority.c | 121 +++++++++++++++++- 8 files changed, 512 insertions(+), 9 deletions(-) diff --git a/docs/user-guide/configuration.md b/docs/user-guide/configuration.md index 76d6cc8f73..5165f233d9 100644 --- a/docs/user-guide/configuration.md +++ b/docs/user-guide/configuration.md @@ -816,9 +816,9 @@ Provisioning sequence: native-era stop; joiners refuse an unsealed authority with SQLSTATE `53RB5`. 4. Start all nodes with `cluster.enabled = on`, `cluster.online_join = on`, - and `cluster.xid_striping = on`. Joiners adopt the prehistory before - WAL startup seeds `nextXid`, so seed tuples and roles are judged from - first-hand pg_xact truth rather than hint bits. + and `cluster.xid_striping = on`. Joiners adopt the seed's transaction + history at startup, so seed tables, roles, and data are visible on + every node. Do not delete or re-create the authority files to recover a failed join. A missing, unsealed, or corrupt authority/prehistory is intentionally diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 1563ea5d3c..850c468df5 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -5606,8 +5606,15 @@ StartupXLOG(void) if (cluster_shared_catalog && cluster_enabled) { ClusterXidAuthorityHeader auth; + bool auth_from_bak = false; + bool auth_ok; - if (!cluster_xid_authority_read(&auth) || + auth_ok = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (auth_ok && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + + if (!auth_ok || (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID authority is unavailable during WAL startup"), diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 453b78038d..f4c1718f6a 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -136,7 +136,14 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, ClusterXidAuthorityHeader auth; bool have_auth; - have_auth = cluster_xid_authority_read(&auth); + { + bool auth_from_bak = false; + + have_auth = cluster_xid_authority_read_checked(&auth, &auth_from_bak); + if (have_auth && auth_from_bak) + elog(LOG, "cluster shared_catalog: XID authority served from .bak fallback; " + "the primary copy failed validation"); + } if (!have_auth && cluster_xid_authority_present()) cluster_catalog_xid_authority_corrupt_fatal( "Neither the primary shared XID authority nor its .bak fallback passes validation."); @@ -212,6 +219,42 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, "Re-provision this node or verify cluster.shared_data_dir before startup."))); own_next = U64FromFullTransactionId(ra.checkPointCopy.nextXid); + + /* + * Review F2 (spec Q6 amendment): the sysid gate cannot tell a true + * pre-seed clone from a same-sysid clone that ran STANDALONE after + * cloning -- such a node's pg_xact holds its own outcomes in the + * native range, so skipping would trust divergent local truth and + * adopting would overwrite the node's own outcomes; both are + * silently wrong (8.A). Byte-compare the comparable prefix + * [0, min(own_next, native_hw)) against the sealed blob and fail + * closed on any contradiction. Bypass only when the local + * oldestXid has advanced past the native range (frozen+truncated: + * those bits are never consulted again). + */ + if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { + ClusterXidPrefixVerdict pv; + + pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, + Min(own_next, auth.native_hw_full)); + if (pv == CLUSTER_XID_PREFIX_DIVERGED) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("local pg_xact contradicts the sealed native-era prehistory"), + errdetail("divergent native-era lineage: this node's transaction " + "history overlaps the seed's native xid range with " + "different outcomes."), + errhint("This node is not a pre-seed clone of the shared tree; " + "destroy and re-provision it from the seed lineage."))); + if (pv == CLUSTER_XID_PREFIX_UNAVAILABLE) + ereport(FATAL, + (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), + errmsg("shared XID prehistory is unavailable for the lineage check"), + errhint("Restore \"%s\" (or its .bak) from the shared tree's " + "backup.", + CLUSTER_XID_PREHISTORY_REL_PATH))); + } + if (own_next < auth.native_hw_full) { cluster_xid_prehistory_adopt(DataDir, auth.native_hw_full); elog(LOG, diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 5d242c315f..08904b7445 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -178,11 +178,26 @@ read_image(const char *path, char *image) */ bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out) +{ + return cluster_xid_authority_read_checked(out, NULL); +} + +/* + * cluster_xid_authority_read_checked -- as above, additionally reporting + * whether the image came from the .bak fallback so consumers can surface a + * LOG-once operator signal (review F8; the read itself stays silent for + * the bootstrap early-read path). + */ +bool +cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak) { char primary_path[MAXPGPATH]; char bak_path[MAXPGPATH]; char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + if (used_bak != NULL) + *used_bak = false; + if (!build_path(primary_path, sizeof(primary_path), CLUSTER_XID_AUTHORITY_REL_PATH) || !build_path(bak_path, sizeof(bak_path), CLUSTER_XID_AUTHORITY_BAK_REL_PATH)) return false; @@ -190,6 +205,8 @@ cluster_xid_authority_read(ClusterXidAuthorityHeader *out) if (read_image(primary_path, image) != CLUSTER_XID_AUTHORITY_VALID) { if (read_image(bak_path, image) != CLUSTER_XID_AUTHORITY_VALID) return false; /* fail-closed: neither trustworthy */ + if (used_bak != NULL) + *used_bak = true; } memcpy(out, image, sizeof(*out)); @@ -257,6 +274,48 @@ write_header(ClusterXidAuthorityHeader *hdr) write_durable(tmp, primary, buffer); } +/* + * write_header_both -- CRC-stamp the header and install the SAME image as + * both primary and .bak (primary first). For FLAG transitions (unseal, + * CLUSTER_ERA stamp) the usual roll-primary-to-.bak is unsafe: it would + * preserve a pre-transition image that the fail-closed read falls back to + * when the primary is later damaged -- a stale SEALED .bak hands a joiner + * the previous pass's high-water (review F1, 8.A), and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash + * points: both copies old (transition not yet taken: state still + * consistent), primary new + .bak old (read prefers the valid primary), + * both new. A later single-copy corruption therefore never resurrects the + * pre-transition flags. + */ +static void +write_header_both(ClusterXidAuthorityHeader *hdr) +{ + char buffer[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char tmp[MAXPGPATH]; + char baktmp[MAXPGPATH]; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_AUTHORITY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_AUTHORITY_BAK_REL_PATH) + || !build_path(tmp, sizeof(tmp), CLUSTER_XID_AUTHORITY_TMP_REL_PATH) + || !build_path(baktmp, sizeof(baktmp), CLUSTER_XID_AUTHORITY_BAK_TMP_REL_PATH)) + ereport(PANIC, (errmsg("cluster shared_data_dir is not configured"))); + + hdr->magic = CLUSTER_XID_AUTHORITY_MAGIC; + hdr->version = CLUSTER_XID_AUTHORITY_VERSION; + hdr->reserved = 0; + INIT_CRC32C(hdr->crc); + COMP_CRC32C(hdr->crc, (char *)hdr, offsetof(ClusterXidAuthorityHeader, crc)); + FIN_CRC32C(hdr->crc); + + memset(buffer, 0, sizeof(buffer)); + memcpy(buffer, hdr, sizeof(*hdr)); + + write_durable(tmp, primary, buffer); + write_durable(baktmp, bak, buffer); +} + /* * cluster_xid_authority_seed_if_absent -- read-then-seed: if the authority is * already present (any trustworthy image) do nothing; otherwise create the @@ -336,7 +395,7 @@ cluster_xid_authority_mark_cluster_era(void) return; hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; - write_header(&hdr); + write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ } /* @@ -377,5 +436,5 @@ cluster_xid_authority_begin_native_run(void) return; /* already open (first run, or a prior pass crashed) */ hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; - write_header(&hdr); + write_header_both(&hdr); /* review F1: no copy may retain SEALED */ } diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index 0e5ed2018d..a61550e009 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -451,3 +451,123 @@ cluster_xid_prehistory_was_adopted(void) { return prehistory_adopted_this_boot; } + +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +/* + * read_local_clog_page_optional -- read local pg_xact page `pageno` into + * buf. Returns false when the segment file or the page does not exist + * (short clone: the comparable prefix ends there); PANICs on a real read + * error so an I/O fault is never mistaken for a short prefix. + */ +static bool +read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) +{ + char seg[MAXPGPATH]; + off_t offset; + int fd; + int r; + + snprintf(seg, sizeof(seg), "%s/pg_xact/%04X", local_pgdata, + pageno / PREHISTORY_PAGES_PER_SEGMENT); + offset = (off_t)(pageno % PREHISTORY_PAGES_PER_SEGMENT) * BLCKSZ; + + fd = OpenTransientFile(seg, O_RDONLY | PG_BINARY); + if (fd < 0) { + if (errno == ENOENT) + return false; + ereport(PANIC, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", seg))); + } + if (lseek(fd, offset, SEEK_SET) != offset) { + CloseTransientFile(fd); + return false; + } + errno = 0; + r = read(fd, buf, BLCKSZ); + CloseTransientFile(fd); + if (r < 0) + ereport(PANIC, (errcode_for_file_access(), errmsg("could not read file \"%s\": %m", seg))); + return r == BLCKSZ; +} + +/* + * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed + * blob (primary, then .bak) and byte-compares the local pg_xact prefix + * covering xids [0, limit_xid_full) at per-xact (2-bit) precision. + */ +ClusterXidPrefixVerdict +cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, + uint64 limit_xid_full) +{ + ClusterXidPrehistoryHeader hdr; + char primary[MAXPGPATH]; + char bak[MAXPGPATH]; + char blob_page[BLCKSZ]; + char local_page[BLCKSZ]; + const char *src_path; + uint32 pages = 0; + uint32 p; + uint64 limit; + int src; + + if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) + || !build_path(bak, sizeof(bak), CLUSTER_XID_PREHISTORY_BAK_REL_PATH)) + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + if (verify_blob(primary, native_hw_full, &pages)) + src_path = primary; + else if (verify_blob(bak, native_hw_full, &pages)) + src_path = bak; + else + return CLUSTER_XID_PREFIX_UNAVAILABLE; + + limit = Min(limit_xid_full, native_hw_full); + if (limit == 0) + return CLUSTER_XID_PREFIX_CONSISTENT; + + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); + if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { + if (src >= 0) + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + + for (p = 0; p < pages; p++) { + uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; + uint64 in_scope; + uint32 full_bytes; + uint32 partial_bits; + + if (read(src, blob_page, BLCKSZ) != BLCKSZ) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_UNAVAILABLE; + } + if (page_first_xid >= limit) + break; /* comparison scope ended on a page boundary */ + + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) + break; /* short clone: no local bits left to contradict */ + + in_scope = Min((uint64)PREHISTORY_XACTS_PER_PAGE, limit - page_first_xid); + full_bytes = (uint32)(in_scope / 4); + partial_bits = (uint32)(in_scope % 4) * 2; + + if (full_bytes > 0 && memcmp(blob_page, local_page, full_bytes) != 0) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + if (partial_bits > 0) { + unsigned char mask = (unsigned char)((1 << partial_bits) - 1); + + if (((unsigned char)blob_page[full_bytes] & mask) + != ((unsigned char)local_page[full_bytes] & mask)) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; + } + } + } + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_CONSISTENT; +} diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index 7676d89303..c9c51f7ab2 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -81,6 +81,19 @@ typedef struct ClusterXidAuthorityHeader { pg_crc32c crc; /* over all preceding bytes */ } ClusterXidAuthorityHeader; +/* On-disk ABI locked at compile time (review F7; mirrors the recovery + * anchor's precedent). The unit truth table re-checks these at runtime. */ +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, flags) == 8, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, native_hw_full) == 16, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, next_multi) == 24, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidAuthorityHeader, crc) == 32, + "xid authority header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidAuthorityHeader) == 40, + "xid authority header size is on-disk ABI"); + /* Relative paths under cluster.shared_data_dir. */ #define CLUSTER_XID_AUTHORITY_REL_PATH "global/pgrac_xid_authority" #define CLUSTER_XID_AUTHORITY_BAK_REL_PATH "global/pgrac_xid_authority.bak" @@ -117,6 +130,15 @@ typedef struct ClusterXidPrehistoryHeader { pg_crc32c crc; /* over header fields above + payload */ } ClusterXidPrehistoryHeader; +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, native_hw_full) == 8, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, payload_len) == 16, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(offsetof(ClusterXidPrehistoryHeader, crc) == 24, + "xid prehistory header layout is on-disk ABI"); +StaticAssertDecl(sizeof(ClusterXidPrehistoryHeader) == 32, + "xid prehistory header size is on-disk ABI"); + #define CLUSTER_XID_PREHISTORY_REL_PATH "global/pgrac_xid_prehistory" #define CLUSTER_XID_PREHISTORY_BAK_REL_PATH "global/pgrac_xid_prehistory.bak" #define CLUSTER_XID_PREHISTORY_TMP_REL_PATH "global/pgrac_xid_prehistory.tmp" @@ -148,6 +170,9 @@ extern ClusterXidAuthorityValidity cluster_xid_prehistory_classify(const char *b */ extern bool cluster_xid_authority_read(ClusterXidAuthorityHeader *out); +/* As read(), reporting .bak fallback for a consumer-side LOG (review F8). */ +extern bool cluster_xid_authority_read_checked(ClusterXidAuthorityHeader *out, bool *used_bak); + /* * ENOENT-only-absent presence probe (spec-6.14 §3.6 posture): any stat() * failure other than ENOENT reports present, so a transiently unreadable @@ -202,4 +227,27 @@ extern void cluster_xid_prehistory_publish(const char *local_pgdata, uint64 nati extern void cluster_xid_prehistory_adopt(const char *local_pgdata, uint64 native_hw_full); extern bool cluster_xid_prehistory_was_adopted(void); +/* ============================================================ + * Divergent-lineage guard (review F2; spec Q6 amendment). + * ============================================================ */ + +typedef enum ClusterXidPrefixVerdict { + CLUSTER_XID_PREFIX_CONSISTENT = 0, /* local prefix matches the blob */ + CLUSTER_XID_PREFIX_DIVERGED, /* local pg_xact contradicts the blob */ + CLUSTER_XID_PREFIX_UNAVAILABLE, /* no trustworthy blob to compare */ +} ClusterXidPrefixVerdict; + +/* + * Compare the local pg_xact bytes covering xids [0, limit_xid_full) with the + * sealed prehistory blob at 2-bit (per-xact) precision. A local segment or + * page that does not exist ends the comparable prefix (a shorter clone has + * no bits to contradict). Callers FATAL on DIVERGED/UNAVAILABLE: a joiner + * whose own native-era history contradicts the seed's is not a pre-seed + * lineage, and neither skipping (trusting local bits) nor adopting + * (overwriting the joiner's own outcomes) is sound for it. + */ +extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, + uint64 native_hw_full, + uint64 limit_xid_full); + #endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl index f4652c83ca..37f99b898f 100644 --- a/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl +++ b/src/test/cluster_tap/t/361_xid_authority_native_era_2node.pl @@ -232,6 +232,11 @@ sub append_strict_two_node_conf my $node1 = PostgreSQL::Test::Cluster->new('sxid_node1'); cold_clone_data_dir($node0, $node1); +# F3: real scram TCP login leg needs a host auth line AND a TCP listener on +# the joiner (test nodes default to unix sockets only). +PostgreSQL::Test::Utils::append_to_file($node1->data_dir . '/pg_hba.conf', + "host all sxid_login 127.0.0.1/32 scram-sha-256\n"); +$node1->append_conf('postgresql.conf', "listen_addresses = '127.0.0.1'\n"); append_common_shared_catalog_conf($node0, $shared_root); $node0->append_conf('postgresql.conf', <connect_ok( + 'host=127.0.0.1 port=' . $node1->port + . ' user=sxid_login password=sxidpass dbname=postgres', + 'G4: seed role logs in over scram TCP on node1'); wait_sql_eq($node1, "SELECT txid_status($seed_schema_xid)", 'committed', 'G5: node1 txid_status sees the native-era committed schema xid'); @@ -319,6 +330,15 @@ sub append_strict_two_node_conf 'G5: node1 txid_status sees the native-era aborted xid'); is($node0->safe_psql('postgres', "SELECT txid_status($abort_xid)"), 'aborted', 'G5: node0 still sees the aborted xid after node1 reads'); +# F4: the poison-stamp proof must re-read the shared HEAP rows on node0 -- +# a wrong HEAP_XMIN_INVALID hint written by node1 lives in the shared +# catalog block, not in node0's CLOG. +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_namespace WHERE nspname = 'sxid'}), + '1', 'G5: node0 still sees the seed schema row after node1 scans (no poison stamp)'); +is($node0->safe_psql('postgres', + q{SELECT count(*) FROM pg_authid WHERE rolname = 'sxid_login'}), + '1', 'G5: node0 still sees the seed role row after node1 scans (no poison stamp)'); my $state_hw = $node1->safe_psql('postgres', q{ SELECT value FROM pg_cluster_state @@ -359,12 +379,20 @@ sub append_strict_two_node_conf corrupt_file("$shared_root/global/pgrac_xid_authority"); corrupt_file("$shared_root/global/pgrac_xid_authority.bak"); -$node1->append_conf('postgresql.conf', "# t/361 force new logfile generation after corrupt authority\n"); +# F9: pin the 53RB5 SQLSTATE contract on one leg (startup FATALs surface in +# the server log only, so %e in log_line_prefix carries the SQLSTATE; the +# other negative legs deliberately match message text). +# %e must precede %q: the refusal FATAL comes from the postmaster (a +# non-session process), and %q stops expansion there. +$node1->append_conf('postgresql.conf', + "log_line_prefix = '%m [%p] %e %q%a '\n"); my $corrupt_failed = !$node1->start(fail_ok => 1); ok($corrupt_failed, 'N2: corrupt shared XID authority fails closed'); like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), qr/shared XID authority is unavailable|present but corrupt|cluster_xid_authority_unavailable/, 'N2: startup log names corrupt/unavailable XID authority'); +like(PostgreSQL::Test::Utils::slurp_file($node1->logfile), + qr/53RB5/, 'N2: the refusal carries SQLSTATE 53RB5'); # ------------------------------------------------------------------------- # Unsealed authority negative: seed startup wrote the authority, but the seed @@ -466,4 +494,83 @@ sub append_strict_two_node_conf 'N4: startup log fails closed on the unsealed authority'); } +# ------------------------------------------------------------------------- +# Divergent-lineage negative (review F2 / spec Q6 amendment): a same-sysid +# clone that ran STANDALONE after cloning holds its own outcomes in the +# native xid range; neither skipping nor adopting is sound -> FATAL 53RB5. +# ------------------------------------------------------------------------- +{ + my $d_shared = make_shared_root(); + my $d_disks = make_voting_disks(); + my $d_ic0 = PostgreSQL::Test::Cluster::get_free_port(); + my $d_ic1 = PostgreSQL::Test::Cluster::get_free_port(); + my $d0 = PostgreSQL::Test::Cluster->new('sxid_diverge_node0'); + my $d1 = PostgreSQL::Test::Cluster->new('sxid_diverge_node1'); + + $d0->init(allows_streaming => 1); + $d0->start; + $d0->safe_psql('postgres', 'CHECKPOINT'); + $d0->stop; + cold_clone_data_dir($d0, $d1); + + # d1 diverges: a plain standalone run committing 30 transactions writes + # d1's own outcomes into the xid range the seed is about to consume. + $d1->start; + $d1->safe_psql('postgres', + join('', map { "BEGIN; CREATE TABLE d_t$_ (i int); COMMIT;\n" } 1 .. 30)); + $d1->stop; + + # d0 seeds: one aborted xid guarantees a byte-level contradiction with + # d1's straight-committed overlap. + append_common_shared_catalog_conf($d0, $d_shared); + $d0->append_conf('postgresql.conf', <start; + $d0->safe_psql('postgres', q{ +BEGIN; +CREATE SCHEMA d_abort_shadow; +ROLLBACK; +}); + $d0->safe_psql('postgres', 'CREATE SCHEMA d_seed;'); + $d0->stop; + + append_common_shared_catalog_conf($d1, $d_shared); + append_strict_two_node_conf($d1, $d_disks, undef); + $d1->append_conf('postgresql.conf', "cluster.node_id = 1\n"); + append_pgrac_conf($d1, 'sxid_diverge', $d_ic0, $d_ic1); + + my $diverge_failed = !$d1->start(fail_ok => 1); + ok($diverge_failed, 'N5: divergent-lineage joiner is refused'); + like(PostgreSQL::Test::Utils::slurp_file($d1->logfile), + qr/divergent native-era lineage/, + 'N5: startup log names the divergent lineage'); +} + +# ------------------------------------------------------------------------- +# L9 dormant leg: with cluster.shared_catalog=off nothing of the XID +# authority machinery may touch the shared tree. +# ------------------------------------------------------------------------- +{ + my $o_shared = make_shared_root(); + my $o0 = PostgreSQL::Test::Cluster->new('sxid_off_node0'); + + $o0->init(allows_streaming => 1); + $o0->append_conf('postgresql.conf', <start; + $o0->safe_psql('postgres', 'CREATE SCHEMA off_mode_probe;'); + $o0->stop; + ok(!-e "$o_shared/global/pgrac_xid_authority", + 'L9: shared_catalog=off never creates the XID authority'); + ok(!-e "$o_shared/global/pgrac_xid_prehistory", + 'L9: shared_catalog=off never creates the XID prehistory'); +} + done_testing(); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 6178ebbfcd..4a646270e1 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -495,6 +495,122 @@ UT_TEST(test_prehistory_multi_segment_round_trip) } } +UT_TEST(test_unseal_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1: after begin_native_run, NO on-disk copy may still say SEALED -- + * a corrupt primary falling back to a stale SEALED .bak would hand a + * joiner the previous pass's high-water (8.A false-invisible). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_begin_native_run(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + /* read failing entirely (no trustworthy copy) is also acceptable: + * joiners fail closed on unavailable exactly like on unsealed. */ +} + +UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) +{ + ClusterXidAuthorityHeader got; + char path[MAXPGPATH]; + int fd; + + /* F1 variant: after mark_cluster_era, no on-disk copy may lack the + * one-way CLUSTER_ERA flag -- a stale .bak without it would let an + * enabled=off boot re-enter the native era on a formed tree. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + cluster_xid_authority_mark_cluster_era(); + + snprintf(path, sizeof(path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_REL_PATH); + fd = open(path, O_RDWR, 0600); + (void)!write(fd, "garbage!", 8); + close(fd); + + if (cluster_xid_authority_read(&got)) + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + +UT_TEST(test_prefix_check_divergence_truth_table) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + int fd; + + /* F2: identical prefix -> CONSISTENT (both trees one page of 0xAA) */ + unlink_files(); + make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); + make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); + cluster_xid_prehistory_publish(pgdata_seed, 816); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* flip one status byte INSIDE the limit -> DIVERGED */ + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_DIVERGED); + + /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: + * bytes at/after own_next belong to the seed alone) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401), + CLUSTER_XID_PREFIX_DIVERGED); + + /* missing local pg_xact segment -> comparable prefix is empty -> + * CONSISTENT (a shorter clone has no bits to contradict) */ + unlink(seg); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* no trustworthy blob -> UNAVAILABLE */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + CLUSTER_XID_PREFIX_UNAVAILABLE); + + /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ + { + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + CLUSTER_XID_PREFIX_CONSISTENT); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "Z", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + CLUSTER_XID_PREFIX_DIVERGED); + } +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -566,7 +682,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(12); + UT_PLAN(15); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -574,10 +690,13 @@ main(void) UT_RUN(test_publish_monotone_and_seal); UT_RUN(test_mark_cluster_era_one_way); UT_RUN(test_begin_native_run_unseals_before_cluster_era); + UT_RUN(test_unseal_survives_primary_corruption_via_bak); + UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); UT_RUN(test_prehistory_multi_segment_round_trip); + UT_RUN(test_prefix_check_divergence_truth_table); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From 0dca50fab30c6af7b273c48872197975bf20e881 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:16:22 +0800 Subject: [PATCH 10/27] =?UTF-8?q?fix(cluster):=20GRD=20recovery=20liveness?= =?UTF-8?q?=20=E2=80=94=20same-episode=20HW=20remaster=20retry,=20WAIT=5FC?= =?UTF-8?q?LUSTER=20watchdog,=20event-scoped=20WAIT=5FEPOCH=20escape,=20de?= =?UTF-8?q?ad-node=20PCM=20cleanup?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fail-stop reconfig could wedge a settled cluster forever on three stacked liveness holes: a BLOCKED HW remaster worker was never relaunched within the same episode (BGW_NEVER_RESTART + launched-latch), WAIT_CLUSTER had no watchdog while its cluster/HW gates were held, and WAIT_EPOCH's strict progress gate wedged permanently when an IC-piggybacked epoch bump landed before the local reconfig event (baseline re-captured as the post-bump value, cur == old forever). - HW remaster: per-dead-node result/attempts/next_attempt shmem state; the FSM relaunches BLOCKED workers with exponential backoff (cap 60s) up to cluster.hw_remaster_retry_max_attempts (SIGHUP-raisable); abnormal worker exit marks BLOCKED via before_shmem_exit; the three silent BLOCKED returns now LOG their cause. wal_threads_dir-unset is detected up front as BLOCKED_STRUCTURAL: never retried, WARNING-once with the config hint, and (level 2) a multi-node shared_catalog=on boot without it now fails fast at startup. - WAIT_CLUSTER: observational watchdog on the rebuild-timeout cadence — WARNING with the missing-survivor list, gate states and per-dead-node retry state, plus a counter; it never unfreezes anything. - WAIT_EPOCH: three proof-carrying layers — abort-to-idle on a new event id, durable observed-epoch adoption, and an event-scoped coordinator witness (REDECLARE_DONE carries the event id; stale-event DONEs are dropped at accounting; the equal-epoch escape additionally requires the coordinator's DONE to have been accounted after the accept snapshot). No timeout-only advance exists; without proof the shards stay frozen. - Dead-node PCM cleanup: the GRD dead-sweep clears the dead node's x_holder/s_holders/pi_holders/master_holder and pending-X residue (LOG summary + pcm/dead_cleanup_entries counter), and a local-master S state with no local residency can now upgrade S->X through the standard invalidate/ACK path instead of failing closed forever — post-kill DDL recovers instead of timing out on cluster locks. - Serve-gate scope alignment: the merged-materialization check in gcs_block phase_for_tag now uses the same thread-recovery scope policy as the unfreeze gate, removing the permanent-RECOVERING mismatch outside the gate's applicable scope (in-scope behavior unchanged). - Observability: hw retry/exhausted + grd cluster_gate_timeout / wait_epoch_escape + pcm dead_cleanup_entries dump keys; S4-reject and HW fail-closed diagnostics. Tests: unit relaunch-decision table (8 rows) + event-scoped DONE rejection/ witness advance + PCM dead-cleanup forms incl. pending-X; t/293 gains the same-episode self-heal and retry-exhaustion legs (unfreeze-extend positive + watchdog WARNING greps); t/337 gains the level-2 fail-fast negative leg; new t/362 runs a 4-node shared_catalog formation, kill -9, and asserts remaster convergence, post-kill DDL, cleanup counters and the 0-wedge invariant end to end. Local gates (cassert): unit 158 binaries, t/293 + t/337 + t/362 + smoke subset, cluster_regress 13/13, PG regress 219/219. Spec: spec-4.6a-grd-recovery-liveness.md --- src/backend/cluster/cluster_debug.c | 33 ++ src/backend/cluster/cluster_gcs_block.c | 41 +- src/backend/cluster/cluster_ges.c | 3 +- src/backend/cluster/cluster_grd.c | 503 ++++++++++++++++-- src/backend/cluster/cluster_guc.c | 17 + src/backend/cluster/cluster_hw_remaster.c | 186 ++++++- src/backend/cluster/cluster_hw_shmem.c | 108 +++- src/backend/cluster/cluster_lms.c | 3 + src/backend/cluster/cluster_pcm_lock.c | 125 +++++ src/backend/cluster/cluster_wal_thread.c | 19 +- src/include/cluster/cluster_grd.h | 24 +- src/include/cluster/cluster_guc.h | 2 + src/include/cluster/cluster_hw.h | 28 +- src/include/cluster/cluster_hw_remaster.h | 102 +++- src/include/cluster/cluster_pcm_lock.h | 6 + src/include/cluster/cluster_thread_recovery.h | 10 + src/test/cluster_tap/t/024_pcm_lock.pl | 7 +- .../cluster_tap/t/293_hw_online_remaster.pl | 245 ++++++++- .../t/337_shared_catalog_ddl_2node.pl | 42 +- .../362_shared_catalog_4node_kill_selfheal.pl | 310 +++++++++++ src/test/cluster_unit/Makefile | 1 + src/test/cluster_unit/test_cluster_debug.c | 95 +++- .../cluster_unit/test_cluster_gcs_block.c | 14 +- src/test/cluster_unit/test_cluster_ges.c | 3 +- src/test/cluster_unit/test_cluster_grd.c | 147 ++++- .../test_cluster_grd_starvation.c | 37 ++ .../test_cluster_hw_remaster_retry.c | 156 ++++++ .../cluster_unit/test_cluster_lock_acquire.c | 41 ++ src/test/cluster_unit/test_cluster_pcm_lock.c | 125 ++++- src/test/perl/PostgreSQL/Test/ClusterQuad.pm | 108 +++- 30 files changed, 2375 insertions(+), 166 deletions(-) create mode 100644 src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl create mode 100644 src/test/cluster_unit/test_cluster_hw_remaster_retry.c diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index 7d966746e6..e19b3e98e8 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1195,8 +1195,30 @@ static void dump_grd_recovery(ReturnSetInfo *rsinfo) { ClusterGrdRecoveryCounters c; + uint32 state; cluster_grd_recovery_counters_snapshot(&c); + state = cluster_grd_recovery_state_value(); + emit_row(rsinfo, "grd_recovery", "state", cluster_grd_recovery_state_name(state)); + emit_row(rsinfo, "grd_recovery", "state_enum_value", fmt_int32((int32)state)); + emit_row(rsinfo, "grd_recovery", "last_event_id", + fmt_int64((int64)cluster_grd_recovery_last_event_id())); + emit_row(rsinfo, "grd_recovery", "event_old_epoch", + fmt_int64((int64)cluster_grd_recovery_event_old_epoch())); + emit_row(rsinfo, "grd_recovery", "episode_epoch", + fmt_int64((int64)cluster_grd_recovery_episode_epoch_value())); + emit_row(rsinfo, "grd_recovery", "event_coordinator", + fmt_int32((int32)cluster_grd_recovery_event_coordinator())); + emit_row(rsinfo, "grd_recovery", "done_self_epoch", + fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_event_id", + fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", + fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", + fmt_int64((int64)cluster_grd_recovery_block_redeclare_epoch())); + emit_row(rsinfo, "grd_recovery", "block_redeclare_done", + fmt_bool(cluster_grd_recovery_block_redeclare_done())); emit_row(rsinfo, "grd_recovery", "remaster_started", fmt_int64((int64)c.remaster_started)); emit_row(rsinfo, "grd_recovery", "remaster_done", fmt_int64((int64)c.remaster_done)); emit_row(rsinfo, "grd_recovery", "remaster_failed", fmt_int64((int64)c.remaster_failed)); @@ -1212,6 +1234,11 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) emit_row(rsinfo, "grd_recovery", "unaffected_holder_survived", fmt_int64((int64)c.unaffected_holder_survived)); emit_row(rsinfo, "grd_recovery", "stale_holder_swept", fmt_int64((int64)c.stale_holder_swept)); + emit_row(rsinfo, "grd_recovery", "cluster_gate_timeout", + fmt_int64((int64)c.cluster_gate_timeout)); + emit_row(rsinfo, "grd_recovery", "wait_epoch_escape", fmt_int64((int64)c.wait_epoch_escape)); + /* spec-4.6a D12 — dead-node PCM residue cleanup (categorized under pcm). */ + emit_row(rsinfo, "pcm", "dead_cleanup_entries", fmt_int64((int64)c.pcm_dead_cleanup_entries)); /* spec-5.16 D5 — join-direction remaster counters (same grd_recovery * category; no new dump category, §8 Q6-A). */ emit_row(rsinfo, "grd_recovery", "join_remaster_started", @@ -2810,6 +2837,12 @@ dump_hw(ReturnSetInfo *rsinfo) fmt_int64((int64)cluster_hw_remaster_done_count())); emit_row(rsinfo, "hw", "remaster_blocked_count", fmt_int64((int64)cluster_hw_remaster_blocked_count())); + emit_row(rsinfo, "hw", "remaster_retry_count", + fmt_int64((int64)cluster_hw_remaster_retry_count())); + emit_row(rsinfo, "hw", "remaster_retry_exhausted_count", + fmt_int64((int64)cluster_hw_remaster_retry_exhausted_count())); + emit_row(rsinfo, "hw", "remaster_recoverable", + fmt_int64((int64)(cluster_hw_remaster_recoverable() ? 1 : 0))); /* * spec-6.12d D-obs: space-lease counters. bloat_ratio itself is a diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index fd3f3b7817..d0219deaf9 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -51,11 +51,12 @@ #include "cluster/cluster_gcs_block_dedup.h" /* spec-2.34 D1 — counter forward */ #include "cluster/cluster_grd.h" /* spec-4.6 D4 — block_path_failclosed counter */ #include "cluster/cluster_grd_outbound.h" -#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ -#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ -#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ -#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ -#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ +#include "cluster/cluster_membership.h" /* spec-5.16 D3b — is_member master-side gate */ +#include "cluster/cluster_qvotec.h" /* spec-5.16 D3b — in_quorum master-side gate */ +#include "cluster/cluster_recovery_merge.h" /* spec-4.7 D5 — recovered_through redo gate */ +#include "cluster/cluster_thread_recovery.h" /* spec-4.11 scope gate for online replay */ +#include "cluster/cluster_xnode_profile.h" /* spec-5.59 D2/D3/D4 profiling buckets */ +#include "cluster/cluster_xnode_lever.h" /* spec-6.12a — downgrade counters */ #include "cluster/cluster_guc.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_itl.h" /* spec-5.2 D11 — active-ITL writer-transfer guard */ @@ -65,6 +66,7 @@ #include "cluster/cluster_lmon.h" #include "cluster/cluster_pcm_lock.h" #include "cluster/cluster_shmem.h" +#include "cluster/storage/cluster_shared_fs.h" #include "cluster/cluster_sf_dep.h" #include "cluster/cluster_touched_peers.h" /* spec-5.14 D2 class 2 */ #include "common/hashfn.h" @@ -1221,12 +1223,13 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The survivor may SERVE only - * after the dead origin's merged WAL recovery passes the redo-before- - * unfreeze gate; before that the shared-storage / re-declared version may - * be stale → fail-closed RECOVERING (never a stale page). + * live survivor (recovery-aware routing). The redo-before-serve gate must + * engage on EXACTLY the same scope where online thread recovery can actually + * run. Otherwise a >2-node or GUC-off deployment waits forever on a + * materialization authority that the thread-recovery launcher intentionally + * treats as not applicable. * - * TWO conditions, both required (Q5): + * In applicable 2-node scope, two conditions are both required (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1236,13 +1239,19 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * survivor DID observe (rebuilt pi_watermark_lsn > 0), the dead * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably - * reached us → lost-write → fail-closed. This is the LSN comparison - * (NOT a bool), live in the serve path; required_lsn == 0 (cold) is - * trivially covered and (a) carries the safety. - * - * Once both hold → NORMAL → the re-routed survivor serves (rebuilt-from- - * redeclare for held blocks, lazy minimal view for cold blocks). + * reached us → lost-write → fail-closed. */ + { + bool shared_fs; + int survivors; + + shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); + survivors = cluster_conf_node_count() - 1; + if (!cluster_thread_recovery_materialization_gate_enabled( + cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) + return GCS_BLOCK_NORMAL; + } + if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index a81f80ee2b..547f0e5aac 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -552,7 +552,8 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) * record and return (no work queue, no reply, no dedup; the sender * re-announces each tick, the receiver write is a monotonic max). */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { - cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch); + cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, + ges_request_holder_request_id(req)); return; } diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..8977739433 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -46,7 +46,8 @@ #include "cluster/cluster_pcm_lock.h" /* spec-2.36 HC124 pending_x node-dead cleanup */ #include "cluster/cluster_gcs.h" /* spec-4.7 D2 — cluster_gcs_lookup_master */ #include "cluster/cluster_membership.h" /* spec-5.16 D1 — cluster_membership_is_member */ -#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ +#include "cluster/cluster_native_lock_probe.h" +#include "cluster/cluster_gcs_block.h" /* spec-4.7 D2 — block re-declare scan + send */ #include "cluster/cluster_signal.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_cssd.h" /* spec-2.16 D8 newly-dead bitmap diff */ @@ -731,9 +732,13 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->recovery_event_old_epoch, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_redeclare_generation, 0); pg_atomic_init_u64(&cluster_grd_state->recovery_barrier_deadline, 0); + pg_atomic_init_u32(&cluster_grd_state->recovery_event_coordinator, 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ - for (i = 0; i < CLUSTER_MAX_NODES; i++) + for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + } pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -747,6 +752,9 @@ cluster_grd_shmem_init(void) pg_atomic_init_u64(&cluster_grd_state->block_path_failclosed_count, 0); pg_atomic_init_u64(&cluster_grd_state->unaffected_holder_survived_count, 0); pg_atomic_init_u64(&cluster_grd_state->stale_holder_swept_count, 0); + pg_atomic_init_u64(&cluster_grd_state->cluster_gate_timeout_count, 0); + pg_atomic_init_u64(&cluster_grd_state->wait_epoch_escape_count, 0); + pg_atomic_init_u64(&cluster_grd_state->pcm_dead_cleanup_entries, 0); /* spec-5.16 D2/D3b/D5 — online-join remaster fence + counters. */ pg_atomic_init_u64(&cluster_grd_state->join_pcm_fence_epoch, 0); @@ -1630,6 +1638,78 @@ cluster_grd_redeclare_episode_epoch(void) * been re-declared to its recovery-aware master — serving it would risk an * 8.A double-grant. Reaching IDLE implies all survivor scans completed. */ +uint32 +cluster_grd_recovery_state_value(void) +{ + if (cluster_grd_state == NULL) + return (uint32)GRD_RECOVERY_IDLE; + return pg_atomic_read_u32(&cluster_grd_state->recovery_state); +} + +const char * +cluster_grd_recovery_state_name(uint32 state) +{ + switch ((ClusterGrdRecoveryState)state) { + case GRD_RECOVERY_IDLE: + return "idle"; + case GRD_RECOVERY_WAIT_EPOCH: + return "wait_epoch"; + case GRD_RECOVERY_WAIT_BARRIER: + return "wait_barrier"; + case GRD_RECOVERY_WAIT_CLUSTER: + return "wait_cluster"; + } + return "unknown"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id) + : 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch) + : 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch) + : 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return cluster_grd_state != NULL + ? pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator) + : 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node) +{ + if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); +} + bool cluster_grd_recovery_in_progress(void) { @@ -1686,6 +1766,10 @@ cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) out->unaffected_holder_survived = pg_atomic_read_u64(&cluster_grd_state->unaffected_holder_survived_count); out->stale_holder_swept = pg_atomic_read_u64(&cluster_grd_state->stale_holder_swept_count); + out->cluster_gate_timeout = pg_atomic_read_u64(&cluster_grd_state->cluster_gate_timeout_count); + out->wait_epoch_escape = pg_atomic_read_u64(&cluster_grd_state->wait_epoch_escape_count); + out->pcm_dead_cleanup_entries + = pg_atomic_read_u64(&cluster_grd_state->pcm_dead_cleanup_entries); /* spec-5.16 D5 — join-direction remaster counters. */ out->join_remaster_started = pg_atomic_read_u64(&cluster_grd_state->join_remaster_started_count); @@ -1924,6 +2008,39 @@ grd_recovery_barrier_complete(uint64 gen, uint64 episode_epoch) return true; } +static void +grd_recovery_format_waiting_backend(uint64 gen, uint64 episode_epoch, char *buf, Size buflen) +{ + int beid; + pid_t self_pid = MyProcPid; + + if (buflen == 0) + return; + buf[0] = '\0'; + for (beid = 1; beid <= MaxBackends; beid++) { + PGPROC *proc = BackendIdGetProc((BackendId)beid); + uint32 registered; + uint64 acked; + uint64 acked_epoch; + + if (proc == NULL || proc->pid == 0 || proc->pid == self_pid) + continue; + registered = pg_atomic_read_u32(&proc->cluster_grd_registered_count); + if (registered == 0) + continue; + acked = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked); + acked_epoch = pg_atomic_read_u64(&proc->cluster_grd_redeclare_acked_epoch); + if (acked < gen || acked_epoch != episode_epoch) { + snprintf(buf, buflen, + "beid=%d pid=%d registered=%u acked=" UINT64_FORMAT "/" UINT64_FORMAT + " target=" UINT64_FORMAT "/" UINT64_FORMAT, + beid, (int)proc->pid, registered, acked, acked_epoch, gen, episode_epoch); + return; + } + } + snprintf(buf, buflen, "none"); +} + /* * spec-4.6 P0#3 cluster gate — announce "my local rebind barrier is * complete for `epoch`" to every declared peer (fire-and-forget; the @@ -1934,6 +2051,7 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; + uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -1943,6 +2061,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); + req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); + req.holder_request_id_hi = (uint32)(event_id >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -1961,19 +2081,33 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) { - uint64 prev; + uint64 current_event_id; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; - /* Monotonic max: late/duplicate announcements never regress. */ - prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); - while (epoch > prev) { - if (pg_atomic_compare_exchange_u64(&cluster_grd_state->recovery_done_epoch[node], &prev, - epoch)) - break; + + /* + * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE + * from a previous episode can carry the same epoch when cur==old, so epoch + * monotonicity alone is not enough. Drop messages for any event other than + * the one this LMON has accepted; senders re-announce every tick. + */ + current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + return; + + /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic + * per-event epochs the incoming value always wins, but never regress the + * published value if a delayed lower-epoch frame ever slips through. */ + { + uint64 prev = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[node]); + + if (epoch > prev) + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); } /* @@ -2000,6 +2134,144 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +static void +grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) +{ + va_list ap; + int n; + Size avail; + + if (buflen == 0 || *off >= (int)buflen - 1) + return; + avail = buflen - (Size)*off; + va_start(ap, fmt); + n = vsnprintf(buf + *off, avail, fmt, ap); + va_end(ap); + if (n < 0) + return; + if ((Size)n >= avail) + *off = (int)buflen - 1; + else + *off += n; +} + +static void +grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, bool is_join, + char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 done_epoch; + + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + if (is_join && join_fence_is_recipient_for(i, episode_epoch)) + continue; + done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); + if (done_epoch >= episode_epoch + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + continue; + grd_recovery_appendf(buf, buflen, &off, + "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", + i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static void +grd_recovery_format_hw_dead(const uint64 *dead, char *buf, Size buflen) +{ + int off = 0; + int i; + bool any = false; + + buf[0] = '\0'; + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + ClusterHwRemasterResult result; + + if (!grd_dead_bitmap_test(dead, i) || i == cluster_node_id) + continue; + result = cluster_hw_remaster_result(i); + grd_recovery_appendf(buf, buflen, &off, "%s%d:%s/%u", any ? "," : "", i, + cluster_hw_remaster_result_name(result), + cluster_hw_remaster_attempts(i)); + any = true; + } + if (!any) + grd_recovery_appendf(buf, buflen, &off, "none"); +} + +static bool +grd_recovery_adopt_observed_epoch(const uint64 *dead, uint64 cur_epoch) +{ + uint64 max_epoch = cur_epoch; + int i; + + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + uint64 peer_epoch; + + if (i == cluster_node_id || cluster_conf_lookup_node(i) == NULL) + continue; + if (grd_dead_bitmap_test(dead, i)) + continue; + peer_epoch = cluster_reconfig_get_observed_epoch(i); + if (peer_epoch > max_epoch) + max_epoch = peer_epoch; + } + if (max_epoch <= cur_epoch) + return false; + cluster_epoch_adopt_admitted(max_epoch); + return true; +} + +static void +grd_recovery_wait_cluster_watchdog(const uint64 *dead, uint64 episode_epoch) +{ + TimestampTz now; + TimestampTz deadline; + TimestampTz next_deadline; + char missing[512]; + char hw_dead[512]; + bool thread_gate; + bool hw_gate; + bool is_join; + + deadline = (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline); + now = GetCurrentTimestamp(); + if (deadline == 0 || now <= deadline) + return; + + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + grd_recovery_format_missing_survivors(dead, episode_epoch, is_join, missing, sizeof(missing)); + thread_gate = cluster_thread_recovery_gate_unfreeze(dead, (CLUSTER_MAX_NODES + 63) / 64); + hw_gate = cluster_hw_remaster_gate_unfreeze(); + grd_recovery_format_hw_dead(dead, hw_dead, sizeof(hw_dead)); + + pg_atomic_fetch_add_u64(&cluster_grd_state->cluster_gate_timeout_count, 1); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen"), + errdetail("episode_epoch=" UINT64_FORMAT ", missing_survivors=%s, " + "thread_gate=%s, hw_gate=%s, hw_dead=%s", + episode_epoch, missing, thread_gate ? "held" : "open", hw_gate ? "held" : "open", + hw_dead), + errhint("This watchdog is observational only; it never unfreezes a shard."))); + next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); +} + /* * spec-4.7 D2 — survivor block re-declare scan (Q6-A' worker-centric). @@ -2078,6 +2350,24 @@ grd_block_redeclare_scan_complete(uint64 episode_epoch) return grd_block_redeclare_epoch == episode_epoch && grd_block_redeclare_done; } +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return grd_block_redeclare_cursor; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return grd_block_redeclare_epoch; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return grd_block_redeclare_done; +} + void cluster_grd_recovery_lmon_tick(void) { @@ -2119,6 +2409,18 @@ cluster_grd_recovery_lmon_tick(void) * epoch (the genuine pre-reconfig baseline) — do NOT overwrite it * with evt.old_epoch here. */ pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, ev_id); + pg_atomic_write_u32(&cluster_grd_state->recovery_event_coordinator, + (uint32)evt.coordinator_node_id); + if (evt.coordinator_node_id >= 0 && evt.coordinator_node_id < CLUSTER_MAX_NODES) + pg_atomic_write_u64( + &cluster_grd_state->recovery_done_epoch_at_accept, + pg_atomic_read_u64( + &cluster_grd_state->recovery_done_epoch[evt.coordinator_node_id])); + else + pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch_at_accept, 0); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms)); for (b = 0; b < (CLUSTER_MAX_NODES + 63) / 64; b++) { uint64 word = 0; int j; @@ -2174,35 +2476,116 @@ cluster_grd_recovery_lmon_tick(void) int i; int signaled; + for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) + dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + /* - * Gate on the ACCEPTED epoch having advanced past the episode's - * old epoch: the coordinator bumped it earlier this tick; a - * non-coordinator survivor observes it via IC envelope piggyback - * a tick or two later. Running the rebind before the local - * epoch advances would mint holders the new master rejects. - * - * spec-5.16 (P0, Rule 8.A) — direction-aware. A FAIL survivor captures - * old_epoch BEFORE the coordinator bumps, so it must wait for a STRICT - * advance (cur > old) to prove it adopted the new master view. A JOIN - * observer survivor instead adopts the coordinator's JOIN epoch bump (via the - * joiner_self_tick max-peer-epoch adoption, same LMON tick) BEFORE it readmits - * the rejoiner and publishes its own observer JOIN_COMMITTED event, so by the - * time its FSM accepts that event old_epoch == cur_epoch == the join epoch. - * Requiring a strict advance there wedges the survivor in WAIT_EPOCH forever: - * it never runs the re-declare barrier, never broadcasts REDECLARE_DONE, and - * the coordinator's all-members JOIN barrier (Hardening v1.1) hangs → - * join_remaster_done never advances. For JOIN the survivor already holds the - * epoch it rebinds under (that IS the post-bump master view), so equality is - * sufficient; cur_epoch is monotonic so cur < old never occurs for JOIN. + * Gate on the ACCEPTED epoch having advanced past the episode's old epoch. + * FAIL requires proof of a post-bump epoch; JOIN keeps its existing equality + * allowance because the observer already accepted the coordinator's JOIN epoch. + * For FAIL, deadline expiry is not an escape hatch by itself: we either adopt + * a durable observed epoch or require a coordinator REDECLARE_DONE witness for + * cur==old. Without either proof, stay fail-closed in WAIT_EPOCH. */ - if (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) - == (uint32)GRD_REMASTER_DIR_JOIN - ? cur_epoch < old_epoch - : cur_epoch <= old_epoch) - return; + { + ReconfigEvent latest; + bool is_join; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id != 0 + && latest.event_id + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + grd_recovery_abort_to_idle(); + return; + } - for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) - dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + is_join = (pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_JOIN); + if (is_join) { + if (cur_epoch < old_epoch) + return; + } else if (cur_epoch <= old_epoch) { + TimestampTz now = GetCurrentTimestamp(); + TimestampTz wait_deadline; + TimestampTz next_deadline; + uint32 coordinator; + uint64 coord_done; + uint64 coord_done_event_id; + uint64 accepted_event_id; + uint64 coord_done_at_accept; + + wait_deadline = (TimestampTz)pg_atomic_read_u64( + &cluster_grd_state->recovery_barrier_deadline); + if (wait_deadline == 0 || now <= wait_deadline) + return; + + if (grd_recovery_adopt_observed_epoch(dead, cur_epoch)) { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH adopted a durable observed epoch; " + "affected shards stay frozen until recovery completes"))); + return; + } + + coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); + accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + coord_done + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) + : 0; + coord_done_event_id + = coordinator < CLUSTER_MAX_NODES + ? pg_atomic_read_u64( + &cluster_grd_state->recovery_done_event_id[coordinator]) + : 0; + coord_done_at_accept + = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); + /* + * spec-4.6a §2.3 witness (all three conjuncts of the frozen + * text): the coordinator's DONE must (a) be for THIS event, + * (b) name exactly cur as the episode epoch, and (c) have been + * accounted after our P0 accept snapshot. Under strictly + * monotonic per-event epochs (c) is implied by (a)+(b) — any + * pre-accept residue carries a lower epoch — but it stays as a + * belt-and-suspenders guard against accounting bugs. + */ + if (cur_epoch == old_epoch && coord_done == cur_epoch + && coord_done_event_id == accepted_event_id + && coord_done > coord_done_at_accept) { + pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " + "equal-epoch progress"), + errdetail("coordinator=%u, epoch=" UINT64_FORMAT + ", event_id=" UINT64_FORMAT, + coordinator, cur_epoch, accepted_event_id))); + } else { + next_deadline + = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, + (uint64)next_deadline); + ereport( + WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_event_id=" UINT64_FORMAT + ", accepted_event_id=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_event_id, accepted_event_id, coord_done_at_accept))); + return; + } + } + } /* * P1 freeze affected + P3 scoped sweep + P4 remaster. Direction- @@ -2306,6 +2689,7 @@ cluster_grd_recovery_lmon_tick(void) if (state == (uint32)GRD_RECOVERY_WAIT_BARRIER) { uint64 gen = pg_atomic_read_u64(&cluster_grd_state->recovery_redeclare_generation); uint64 episode_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_episode_epoch); + TimestampTz deadline; /* P0-1 epoch-coherence guard: a SECOND epoch bump landed * mid-episode (e.g. a third node's heartbeat flap re-fired @@ -2344,7 +2728,12 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); grd_recovery_broadcast_done(episode_epoch); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), + cluster_grd_rebuild_timeout_ms); + pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)deadline); pg_atomic_write_u32(&cluster_grd_state->recovery_state, (uint32)GRD_RECOVERY_WAIT_CLUSTER); ereport(DEBUG1, @@ -2358,6 +2747,12 @@ cluster_grd_recovery_lmon_tick(void) if (GetCurrentTimestamp() > (TimestampTz)pg_atomic_read_u64(&cluster_grd_state->recovery_barrier_deadline)) { TimestampTz deadline; + char waiting_backend[256]; + bool ges_barrier_complete = grd_recovery_barrier_complete(gen, episode_epoch); + bool block_scan_complete = grd_block_redeclare_scan_complete(episode_epoch); + + grd_recovery_format_waiting_backend(gen, episode_epoch, waiting_backend, + sizeof(waiting_backend)); /* * Barrier deadline expired: fail-closed. Affected shards @@ -2376,6 +2771,12 @@ cluster_grd_recovery_lmon_tick(void) (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), errmsg("cluster GRD holder-rebuild barrier timed out; affected shards " "stay frozen"), + errdetail("gen=" UINT64_FORMAT ", episode_epoch=" UINT64_FORMAT + ", ges_barrier=%s, block_scan=%s, block_cursor=%d" + ", block_epoch=" UINT64_FORMAT ", waiting_backend=%s", + gen, episode_epoch, ges_barrier_complete ? "done" : "waiting", + block_scan_complete ? "done" : "waiting", grd_block_redeclare_cursor, + grd_block_redeclare_epoch, waiting_backend), errhint("A backend has not acked the cooperative rebind within " "cluster.grd_rebuild_timeout_ms; re-broadcasting."))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), @@ -2409,6 +2810,8 @@ cluster_grd_recovery_lmon_tick(void) for (i = 0; i < (CLUSTER_MAX_NODES + 63) / 64; i++) dead[i] = pg_atomic_read_u64(&cluster_grd_state->recovery_dead_bitmap[i]); + grd_recovery_wait_cluster_watchdog(dead, episode_epoch); + /* * spec-4.11 D1 (3b-4b Part 3) — launch one per-episode online thread- * recovery worker for each in-scope dead origin (idempotent per tick; @@ -2461,8 +2864,9 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; - if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) - < episode_epoch) { + if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { all_done = false; break; } @@ -5344,13 +5748,28 @@ cluster_grd_cleanup_on_node_dead(int32 dead_node_id) int swept = 0; int nresids; uint64 pending_x_cleared = 0; + uint64 pcm_holders_cleaned = 0; int i; pending_x_cleared = cluster_pcm_lock_clear_pending_x_for_node(dead_node_id); - if (pending_x_cleared > 0) - ereport(DEBUG1, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " - "cleared " UINT64_FORMAT " PCM pending_x entries", - dead_node_id, pending_x_cleared))); + pcm_holders_cleaned = cluster_pcm_lock_cleanup_on_node_dead(dead_node_id); + + /* + * spec-4.6a D12: dead-node PCM residue is part of the reconfig + * convergence chain -- a leftover holder/pending-X record from the dead + * node blocks post-kill DDL with lock-acquire timeouts long after the + * remaster itself finished. Surface the cleanup as a LOG summary plus an + * assertable counter (pcm/dead_cleanup_entries). + */ + if (pending_x_cleared > 0 || pcm_holders_cleaned > 0) { + if (cluster_grd_state != NULL) + pg_atomic_fetch_add_u64(&cluster_grd_state->pcm_dead_cleanup_entries, + pending_x_cleared + pcm_holders_cleaned); + ereport(LOG, (errmsg("cluster GRD dead-node cleanup for node %d: cleared " UINT64_FORMAT + " PCM pending-X waiter(s) and " UINT64_FORMAT + " PCM holder record(s) left by the dead node", + dead_node_id, pending_x_cleared, pcm_holders_cleaned))); + } if (cluster_grd_entry_htab == NULL) { ereport(DEBUG2, (errmsg_internal("cluster_grd_cleanup_on_node_dead(%d): " diff --git a/src/backend/cluster/cluster_guc.c b/src/backend/cluster/cluster_guc.c index 3334422711..b505360e11 100644 --- a/src/backend/cluster/cluster_guc.c +++ b/src/backend/cluster/cluster_guc.c @@ -226,6 +226,8 @@ int cluster_tm_convert_mode = CLUSTER_TM_CONVERT_MODE_CONVERT; /* spec-4.6 D4/D1 — failure-driven remaster tunables. */ int cluster_grd_remaster_wait_ms = 200; /* frozen-shard short wait before 53R9I */ int cluster_grd_rebuild_timeout_ms = 5000; /* holder-rebuild barrier deadline */ +int cluster_hw_remaster_retry_backoff_ms = 1000; +int cluster_hw_remaster_retry_max_attempts = 16; /* spec-5.4 D8 — SQ sequence lock tunables. */ int cluster_sequence_default_cache = 100; /* CREATE-time CACHE injection default */ @@ -2374,6 +2376,21 @@ cluster_init_guc(void) "request with a fresh deadline."), &cluster_grd_rebuild_timeout_ms, 5000, 100, 600000, PGC_SIGHUP, GUC_UNIT_MS, NULL, NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_backoff_ms", + gettext_noop("Initial backoff before retrying a BLOCKED HW remaster worker (ms)."), + gettext_noop("Range [100, 60000]. Default 1000. Same-episode HW remaster " + "retries use exponential backoff capped at 60 seconds; the adopted " + "shards stay frozen while retrying."), + &cluster_hw_remaster_retry_backoff_ms, 1000, 100, 60000, PGC_SIGHUP, GUC_UNIT_MS, NULL, + NULL, NULL); + DefineCustomIntVariable( + "cluster.hw_remaster_retry_max_attempts", + gettext_noop("Maximum same-episode retries for a BLOCKED HW remaster worker."), + gettext_noop("Range [0, 1000]. Default 16. Zero disables same-episode retry. " + "Raising the value with SIGHUP lets an exhausted episode resume retrying " + "without waiting for a new reconfig episode."), + &cluster_hw_remaster_retry_max_attempts, 16, 0, 1000, PGC_SIGHUP, 0, NULL, NULL, NULL); /* spec-2.23 D11 NEW: coordinator REPORT collect deadline. */ DefineCustomIntVariable("cluster.lmd_probe_collect_timeout_ms", diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index d1ade2a760..c585268a42 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -61,6 +61,7 @@ #include "miscadmin.h" #include "postmaster/bgworker.h" #include "storage/fd.h" +#include "storage/ipc.h" #include "utils/wait_event.h" #include "cluster/cluster_conf.h" /* CLUSTER_MAX_NODES */ @@ -74,6 +75,65 @@ #include "cluster/cluster_wal_thread.h" /* node id -> thread id */ #include "cluster/storage/cluster_undo_xlog.h" /* xl_hw_reserve, XLOG_HW_RESERVE */ +/* One worker process owns exactly one dead origin / episode. */ +static int hw_worker_dead_node = -1; +static uint64 hw_worker_episode = 0; +static bool hw_worker_armed = false; + +static uint64 +hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) +{ + uint32 backoff_ms; + TimestampTz deadline; + + backoff_ms = cluster_hw_remaster_compute_backoff_ms(cluster_hw_remaster_retry_backoff_ms, + completed_retry_attempts); + deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), (int)backoff_ms); + return (uint64)deadline; +} + +static void +hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) +{ + if (res == CLUSTER_HW_REMASTER_DONE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_done(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { + uint32 attempts = cluster_hw_remaster_attempts(dead_node); + + cluster_hw_remaster_set_next_attempt_at(dead_node, + hw_remaster_next_attempt_deadline(attempts)); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + cluster_hw_remaster_set_result(dead_node, res); + cluster_hw_bump_remaster_blocked(); + } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + cluster_hw_remaster_set_result(dead_node, res); + } +} + +static void +hw_remaster_mark_blocked_on_exit(int code pg_attribute_unused(), Datum arg) +{ + int dead_node = DatumGetInt32(arg); + + if (!hw_worker_armed) + return; + if (dead_node != hw_worker_dead_node) + return; + if (cluster_hw_remaster_launched_episode(dead_node) != hw_worker_episode) + return; + if (cluster_hw_remaster_result(dead_node) != CLUSTER_HW_REMASTER_RUNNING) + return; + + cluster_hw_bump_failclosed(); + hw_remaster_record_terminal(dead_node, CLUSTER_HW_REMASTER_BLOCKED); +} + /* ============================================================ * Minimal per-thread WAL reader over a dead origin's WAL stream. @@ -332,10 +392,24 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; if (dead_node_id < 0 || dead_node_id == cluster_node_id) return CLUSTER_HW_REMASTER_NOT_APPLICABLE; + if (!cluster_hw_remaster_recoverable()) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: cluster.wal_threads_dir is not configured; " + "adopted shards stay fail-closed"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "restart, or restart the dead node so a JOIN rebuild can replace " + "the failure-driven remaster."))); + return CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL; + } dead_tid = cluster_wal_thread_id_for(true, dead_node_id); - if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) + if (dead_tid < XLP_THREAD_ID_FIRST_REAL || dead_tid > CLUSTER_WAL_THREAD_MAX) { + cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d maps to invalid WAL thread id %u; " + "adopted shards stay fail-closed", + dead_node_id, (unsigned)dead_tid))); return CLUSTER_HW_REMASTER_BLOCKED; + } entries = (ClusterHwSnapshotEntry *)palloc(sizeof(ClusterHwSnapshotEntry) * CLUSTER_HW_AUTHORITY_MAX); @@ -384,6 +458,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_wal_state_read_slot(dead_tid, &slot) != CLUSTER_WAL_SLOT_OK || slot.highest_lsn == 0) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d WAL state slot is unusable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } validated_min = (XLogRecPtr)slot.highest_lsn; @@ -391,6 +468,9 @@ cluster_hw_remaster_rebuild_origin(int dead_node_id, uint64 episode_epoch) if (cluster_thread_recovery_validated_end(dead_tid, lower, validated_min, &upper) != CLUSTER_THREADREC_DONE) { cluster_hw_bump_failclosed(); + ereport(LOG, (errmsg("cluster HW remaster: dead node %d validated WAL end is unavailable; " + "adopted shards stay fail-closed", + dead_node_id))); return CLUSTER_HW_REMASTER_BLOCKED; } @@ -466,29 +546,26 @@ cluster_hw_remaster_worker_main(Datum main_arg) /* The bgworker framework starts the entry point with signals blocked; unblock * before any I/O so a SIGTERM during a stuck shared-storage read / shutdown is - * delivered (the default die handler FATALs, a clean fail-closed: unmarked). */ + * delivered (the default die handler FATALs into the BLOCKED exit callback). */ BackgroundWorkerUnblockSignals(); /* Capture the live reconfig episode this rebuild is for; rebuild_origin uses * it as the staleness fence before it marks any shard rebuilt. */ episode = cluster_grd_redeclare_episode_epoch(); - res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_worker_dead_node = dead_node; + hw_worker_episode = episode; + hw_worker_armed = true; + before_shmem_exit(hw_remaster_mark_blocked_on_exit, Int32GetDatum(dead_node)); - /* Observability (S5/S7): a DONE rebuilt + adopted + marked the shards; a - * BLOCKED kept them frozen (fail-closed). NOT_APPLICABLE (HW inactive) is - * neither. */ - if (res == CLUSTER_HW_REMASTER_DONE) - cluster_hw_bump_remaster_done(); - else if (res == CLUSTER_HW_REMASTER_BLOCKED) - cluster_hw_bump_remaster_blocked(); + res = cluster_hw_remaster_rebuild_origin(dead_node, episode); + hw_remaster_record_terminal(dead_node, res); + hw_worker_armed = false; ereport(LOG, (errmsg("cluster HW remaster worker: dead node %d -> %s", dead_node, - res == CLUSTER_HW_REMASTER_DONE ? "done" - : res == CLUSTER_HW_REMASTER_BLOCKED ? "blocked (shards kept frozen)" - : "not applicable"))); + cluster_hw_remaster_result_name(res)))); /* Returning is a clean exit(0). On BLOCKED / abnormal exit the adopted shards - * stay unmarked, so the serve gate keeps them fail-closed (8.A) and a later - * episode relaunches the rebuild -- no abnormal-exit handler is needed. */ + * stay unmarked, so the serve gate keeps them fail-closed (8.A); the terminal + * result lets the LMON FSM retry within the same episode when appropriate. */ } /* @@ -522,6 +599,7 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod { int node; int max_node; + uint64 now; if (dead == NULL || nwords <= 0) return; @@ -538,31 +616,93 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod max_node = nwords * 64; if (max_node > CLUSTER_MAX_NODES) max_node = CLUSTER_MAX_NODES; + now = (uint64)GetCurrentTimestamp(); for (node = 0; node < max_node; node++) { + ClusterHwRemasterResult result; + ClusterHwRemasterRelaunchDecision d; + uint64 launched; + uint64 next_attempt_at; + uint32 attempts; + if ((dead[node / 64] & (UINT64CONST(1) << (node % 64))) == 0) continue; if (node == cluster_node_id) continue; /* self is never its own dead origin */ - /* Idempotent: launch once per episode per dead origin. */ - if (cluster_hw_remaster_launched_episode(node) == episode_epoch) + + launched = cluster_hw_remaster_launched_episode(node); + if (!cluster_hw_remaster_recoverable() && launched != episode_epoch) { + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, 0); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL); + launched = episode_epoch; + } + + result = cluster_hw_remaster_result(node); + attempts = cluster_hw_remaster_attempts(node); + next_attempt_at = cluster_hw_remaster_next_attempt_at(node); + d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, + next_attempt_at, now, + cluster_hw_remaster_retry_max_attempts); + + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster is structurally blocked for dead node %d; " + "adopted shards stay fail-closed", + node), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root " + "and restart, or restart the dead node so a JOIN rebuild can " + "replace the failure-driven remaster."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED) { + cluster_hw_remaster_set_next_attempt_at(node, d.next_attempt_at); + cluster_hw_bump_remaster_retry_exhausted(); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster HW remaster retries exhausted for dead node %d after %u " + "attempt(s); adopted shards stay fail-closed", + node, attempts), + errhint("Fix the shared HW snapshot/WAL source, then raise " + "cluster.hw_remaster_retry_max_attempts with SIGHUP to resume " + "same-episode retrying."))); continue; - /* - * Claim BEFORE registering so a same-episode re-entry does not double- - * launch; revert on a registration failure so a later tick retries (the - * adopted shards stay fail-closed meanwhile, via the P7 gate). - */ + } + if (d.action != CLUSTER_HW_REMASTER_LAUNCH_INITIAL + && d.action != CLUSTER_HW_REMASTER_LAUNCH_RETRY) + continue; + cluster_hw_remaster_set_launched(node, episode_epoch); + cluster_hw_remaster_set_attempts(node, d.next_attempts); + cluster_hw_remaster_set_next_attempt_at(node, 0); + cluster_hw_remaster_set_result(node, CLUSTER_HW_REMASTER_RUNNING); if (!register_one_worker(node)) { - cluster_hw_remaster_set_launched(node, 0); + cluster_hw_remaster_set_launched( + node, d.action == CLUSTER_HW_REMASTER_LAUNCH_INITIAL ? 0 : launched); + cluster_hw_remaster_set_attempts(node, attempts); + cluster_hw_remaster_set_next_attempt_at(node, next_attempt_at); + cluster_hw_remaster_set_result(node, result); ereport(WARNING, (errmsg("could not register HW remaster worker for dead node %d", node), errhint("Background worker slots are exhausted (max_worker_processes); the " "adopted shards stay fail-closed until the rebuild can run."))); + continue; + } + if (d.action == CLUSTER_HW_REMASTER_LAUNCH_RETRY) { + cluster_hw_bump_remaster_retry(); + ereport(LOG, + (errmsg("cluster HW remaster: retrying dead node %d in episode " UINT64_FORMAT + " (attempt %u/%d)", + node, episode_epoch, d.next_attempts, + cluster_hw_remaster_retry_max_attempts))); } } } + bool cluster_hw_remaster_gate_unfreeze(void) { diff --git a/src/backend/cluster/cluster_hw_shmem.c b/src/backend/cluster/cluster_hw_shmem.c index 14bf3f7586..860241e8e9 100644 --- a/src/backend/cluster/cluster_hw_shmem.c +++ b/src/backend/cluster/cluster_hw_shmem.c @@ -100,6 +100,8 @@ typedef struct ClusterHwShared { pg_atomic_uint64 not_ready_count; /* 53RA6 serve-gate (shard adopted, unrebuilt) */ pg_atomic_uint64 remaster_done_count; /* online-remaster HW rebuild DONE (S5/S7) */ pg_atomic_uint64 remaster_blocked_count; /* online-remaster HW rebuild fail-closed (S5/S7) */ + pg_atomic_uint64 remaster_retry_count; /* same-episode retry launches */ + pg_atomic_uint64 remaster_retry_exhausted_count; /* same-episode retry cap reached */ pg_atomic_uint32 hw_rebuilt_generation[PGRAC_GRD_SHARD_COUNT]; /* §3.1b R4/R9 gate */ /* * remaster_launched_episode[node] -- the reconfig episode for which the GRD @@ -111,6 +113,9 @@ typedef struct ClusterHwShared { * race on this field. */ pg_atomic_uint64 remaster_launched_episode[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_result[CLUSTER_MAX_NODES]; + pg_atomic_uint32 remaster_attempts[CLUSTER_MAX_NODES]; + pg_atomic_uint64 remaster_next_attempt_at[CLUSTER_MAX_NODES]; } ClusterHwShared; /* @@ -196,12 +201,18 @@ cluster_hw_shmem_init(void) pg_atomic_init_u64(&hw_state->not_ready_count, 0); pg_atomic_init_u64(&hw_state->remaster_done_count, 0); pg_atomic_init_u64(&hw_state->remaster_blocked_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_count, 0); + pg_atomic_init_u64(&hw_state->remaster_retry_exhausted_count, 0); /* No shard rebuilt yet; 0 matches a never-remastered shard's GRD * master_generation (0), so boot / steady state serves. */ for (s = 0; s < PGRAC_GRD_SHARD_COUNT; s++) pg_atomic_init_u32(&hw_state->hw_rebuilt_generation[s], 0); - for (s = 0; s < CLUSTER_MAX_NODES; s++) + for (s = 0; s < CLUSTER_MAX_NODES; s++) { pg_atomic_init_u64(&hw_state->remaster_launched_episode[s], 0); + pg_atomic_init_u32(&hw_state->remaster_result[s], (uint32)CLUSTER_HW_REMASTER_NONE); + pg_atomic_init_u32(&hw_state->remaster_attempts[s], 0); + pg_atomic_init_u64(&hw_state->remaster_next_attempt_at[s], 0); + } } memset(&hctl, 0, sizeof(hctl)); @@ -408,6 +419,81 @@ cluster_hw_remaster_set_launched(int node_id, uint64 episode) pg_atomic_write_u64(&hw_state->remaster_launched_episode[node_id], episode); } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return CLUSTER_HW_REMASTER_NONE; + return (ClusterHwRemasterResult)pg_atomic_read_u32(&hw_state->remaster_result[node_id]); +} + +void +cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_result[node_id], (uint32)result); +} + +uint32 +cluster_hw_remaster_attempts(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u32(&hw_state->remaster_attempts[node_id]); +} + +void +cluster_hw_remaster_set_attempts(int node_id, uint32 attempts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u32(&hw_state->remaster_attempts[node_id], attempts); +} + +uint64 +cluster_hw_remaster_next_attempt_at(int node_id) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return 0; + return pg_atomic_read_u64(&hw_state->remaster_next_attempt_at[node_id]); +} + +void +cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts) +{ + if (hw_state == NULL || node_id < 0 || node_id >= CLUSTER_MAX_NODES) + return; + pg_atomic_write_u64(&hw_state->remaster_next_attempt_at[node_id], ts); +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result) +{ + switch (result) { + case CLUSTER_HW_REMASTER_NONE: + return "none"; + case CLUSTER_HW_REMASTER_RUNNING: + return "running"; + case CLUSTER_HW_REMASTER_DONE: + return "done"; + case CLUSTER_HW_REMASTER_BLOCKED: + return "blocked"; + case CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL: + return "blocked_structural"; + case CLUSTER_HW_REMASTER_NOT_APPLICABLE: + return "not_applicable"; + } + return "unknown"; +} + +bool +cluster_hw_remaster_recoverable(void) +{ + return !cluster_hw_authority_active() + || (cluster_wal_threads_dir != NULL && cluster_wal_threads_dir[0] != '\0'); +} + /* ============================================================ * Checkpoint write + recovery load (§3.1b R1/R2; PG-core hooked). @@ -747,6 +833,16 @@ cluster_hw_bump_remaster_blocked(void) { HW_BUMP(remaster_blocked_count); } +void +cluster_hw_bump_remaster_retry(void) +{ + HW_BUMP(remaster_retry_count); +} +void +cluster_hw_bump_remaster_retry_exhausted(void) +{ + HW_BUMP(remaster_retry_exhausted_count); +} uint64 cluster_hw_alloc_count(void) @@ -788,3 +884,13 @@ cluster_hw_remaster_blocked_count(void) { return HW_READ(remaster_blocked_count); } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return HW_READ(remaster_retry_count); +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return HW_READ(remaster_retry_exhausted_count); +} diff --git a/src/backend/cluster/cluster_lms.c b/src/backend/cluster/cluster_lms.c index c3b9a39c24..8bec6ddd71 100644 --- a/src/backend/cluster/cluster_lms.c +++ b/src/backend/cluster/cluster_lms.c @@ -59,6 +59,7 @@ #include "cluster/cluster_cr_server.h" /* spec-6.12b CR work slots */ #include "cluster/cluster_conf.h" +#include "cluster/cluster_cssd.h" #include "cluster/cluster_epoch.h" /* cluster_epoch_get_current */ #include "cluster/cluster_ges.h" #include "cluster/cluster_ges_dedup.h" @@ -1044,6 +1045,8 @@ cluster_lms_native_probe_dispatch(uint32 slot_idx) conf_node = cluster_conf_lookup_node(candidate); if (conf_node == NULL) continue; + if (cluster_cssd_get_peer_state(candidate) == CLUSTER_CSSD_PEER_DEAD) + continue; if (!native_probe_node_bit(candidate, &peer_bit)) { pg_atomic_fetch_add_u64(&cluster_lms_state->native_probe_timeout_count, 1); slot->final_status = CLUSTER_NATIVE_PROBE_FINAL_TIMEOUT; diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index 545ac33480..b9de7f6401 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -724,6 +724,96 @@ cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node) return cleared; } +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node) +{ + HASH_SEQ_STATUS scan; + struct GrdEntry *entry; + uint64 cleaned = 0; + uint32 dead_bit; + + if (cluster_pcm_htab == NULL || dead_node < 0 || dead_node >= 32) + return 0; + + dead_bit = (uint32)1u << (uint32)dead_node; + + LWLockAcquire(&ClusterPcm->htab_lock.lock, LW_SHARED); + hash_seq_init(&scan, cluster_pcm_htab); + while ((entry = (struct GrdEntry *)hash_seq_search(&scan)) != NULL) { + bool changed = false; + bool broadcast_needed = false; + bool master_holder_was_dead; + PcmState before_state; + PcmState after_state; + uint32 s_bitmap; + uint32 pi_bitmap; + + /* Cheap unlocked filter; rechecked under entry_lock below. */ + if (entry->x_holder_node != dead_node + && (pg_atomic_read_u32(&entry->s_holders_bitmap) & dead_bit) == 0 + && (pg_atomic_read_u32(&entry->pi_holders_bitmap) & dead_bit) == 0 + && (!pcm_master_holder_is_valid(entry) + || (int32)entry->master_holder.node_id != dead_node)) + continue; + + LWLockAcquire(&entry->entry_lock.lock, LW_EXCLUSIVE); + before_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + master_holder_was_dead + = pcm_master_holder_is_valid(entry) && (int32)entry->master_holder.node_id == dead_node; + + if (entry->x_holder_node == dead_node) { + entry->x_holder_node = -1; + changed = true; + } + s_bitmap = pg_atomic_read_u32(&entry->s_holders_bitmap); + if ((s_bitmap & dead_bit) != 0) { + s_bitmap &= ~dead_bit; + pg_atomic_write_u32(&entry->s_holders_bitmap, s_bitmap); + changed = true; + } + pi_bitmap = pg_atomic_read_u32(&entry->pi_holders_bitmap); + if ((pi_bitmap & dead_bit) != 0) { + pg_atomic_write_u32(&entry->pi_holders_bitmap, pi_bitmap & ~dead_bit); + changed = true; + } + + if (entry->x_holder_node >= 0) { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_X); + if (master_holder_was_dead) + pcm_master_holder_set_node(entry, entry->x_holder_node); + } else if (s_bitmap != 0) { + int32 next_holder = pcm_lowest_set_bit_node(s_bitmap); + + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_S); + if (master_holder_was_dead) { + if (next_holder >= 0) + pcm_master_holder_set_node(entry, next_holder); + else + pcm_master_holder_clear(entry); + } + } else { + pg_atomic_write_u32(&entry->master_state, (uint32)PCM_STATE_N); + if (pcm_master_holder_is_valid(entry)) + pcm_master_holder_clear(entry); + } + if (master_holder_was_dead) + changed = true; + after_state = (PcmState)pg_atomic_read_u32(&entry->master_state); + if (changed && after_state != before_state) + broadcast_needed = true; + + LWLockRelease(&entry->entry_lock.lock); + + if (broadcast_needed) + ConditionVariableBroadcast(&entry->wait_cv); + if (changed) + cleaned++; + } + LWLockRelease(&ClusterPcm->htab_lock.lock); + + return cleaned; +} + /* ======================================================================== * PGRAC MODIFICATIONS by SqlRush — spec-5.13 D5 (clean-leave PCM release). @@ -2199,6 +2289,41 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if (master_state == PCM_LOCK_MODE_X && holder >= 0 && holder != cluster_node_id) return cluster_gcs_local_master_x_transfer_and_wait(buf, holder, clean_eligible); + + /* + * PGRAC: spec-4.6a BUG-C2 follow-through for shared_catalog DDL after + * fail-stop. Local-master state=S with other live S holders but no local + * S bit used to fall through to the tag-only X acquire, which fail-closed + * as "no local S residency". This entry point is buffer-aware: the + * caller has already read or initialized the BufferDesc before asking for + * X, so first register a local S residency, then reuse the existing + * local S->X invalidate/upgrade path. If the invalidate cannot be proven, + * drop the temporary S claim and rethrow the same fail-closed error. + */ + if (master_state == PCM_LOCK_MODE_S && cluster_node_id >= 0 && cluster_node_id < 32) { + uint32 self_bit = (uint32)1u << (uint32)cluster_node_id; + + if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { + struct GrdEntry *entry; + + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + if (!cluster_gcs_block_local_x_upgrade(tag)) { + cluster_pcm_lock_release(tag); + ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), + errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), + errhint("Remote S holders did not all acknowledge in time; " + "retry the statement."))); + } + + entry = pcm_find_entry(tag); + if (entry != NULL) { + pcm_entry_lock_exclusive(entry); + entry->s_holder_refcount_local = 0; + LWLockRelease(&entry->entry_lock.lock); + } + return true; + } + } } cluster_pcm_lock_acquire(tag, mode); diff --git a/src/backend/cluster/cluster_wal_thread.c b/src/backend/cluster/cluster_wal_thread.c index 17edd98586..b5f7cab947 100644 --- a/src/backend/cluster/cluster_wal_thread.c +++ b/src/backend/cluster/cluster_wal_thread.c @@ -45,7 +45,9 @@ #include #include "access/xlog_internal.h" /* XLOGDIR */ +#include "cluster/cluster_conf.h" #include "cluster/cluster_guc.h" +#include "cluster/cluster_hw.h" #include "cluster/cluster_inject.h" #include "cluster/cluster_shmem.h" #include "cluster/cluster_wal_state.h" /* spec-4.2 ensure() */ @@ -385,8 +387,23 @@ cluster_wal_thread_init(void) cluster_wal_thread_shmem->dir_configured = dir_set ? 1 : 0; } - if (!dir_set) + if (!dir_set) { + if (cluster_shared_catalog && cluster_conf_has_peers()) + ereport(FATAL, + (errcode(ERRCODE_INVALID_PARAMETER_VALUE), + errmsg("cluster.shared_catalog requires cluster.wal_threads_dir in a " + "multi-node cluster"), + errhint("Set cluster.wal_threads_dir to the shared per-thread WAL root and " + "initialise each node with pgrac-init --wal-threads-dir."))); + if (!cluster_hw_remaster_recoverable()) + ereport(WARNING, + (errmsg("cluster HW remaster is not recoverable because " + "cluster.wal_threads_dir is unset"), + errhint("A dead node's adopted HW authority cannot be rebuilt from per-thread " + "WAL until cluster.wal_threads_dir is configured and the node is " + "restarted."))); return; /* flat layout: identity stamping only (Q3-A) */ + } /* * Configuration coherence, fail-closed (L58: enumerate the diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..a098d53ed1 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -275,6 +275,8 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 recovery_event_old_epoch; pg_atomic_uint64 recovery_redeclare_generation; pg_atomic_uint64 recovery_barrier_deadline; + pg_atomic_uint32 recovery_event_coordinator; + pg_atomic_uint64 recovery_done_epoch_at_accept; /* * spec-4.6 P0-1 (Fable review) — the epoch this episode is LOCKED to. @@ -294,6 +296,7 @@ typedef struct ClusterGrdShared { * last announced "local rebind barrier complete" (REDECLARE_DONE). * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -311,6 +314,10 @@ typedef struct ClusterGrdShared { pg_atomic_uint64 block_path_failclosed_count; /* D4 GCS/PCM 53R9K (L12) */ pg_atomic_uint64 unaffected_holder_survived_count; /* L13 sweep-scope guard */ pg_atomic_uint64 stale_holder_swept_count; /* P6 post-barrier sweep (L15) */ + pg_atomic_uint64 cluster_gate_timeout_count; /* WAIT_CLUSTER watchdog only */ + pg_atomic_uint64 wait_epoch_escape_count; /* proof-carrying equal-epoch advance */ + pg_atomic_uint64 + pcm_dead_cleanup_entries; /* spec-4.6a D12: dead-node PCM holder/pending-X records cleaned */ /* spec-5.1b D9 — convert state-machine observability counters. * convert_queue_full reuses the existing converts_full_count above; @@ -609,6 +616,18 @@ typedef enum ClusterGrdRecoveryState { } ClusterGrdRecoveryState; extern void cluster_grd_recovery_lmon_tick(void); +/* spec-4.6a D5/D6 — recovery state observability for pg_cluster_state. */ +extern uint32 cluster_grd_recovery_state_value(void); +extern const char *cluster_grd_recovery_state_name(uint32 state); +extern uint64 cluster_grd_recovery_last_event_id(void); +extern uint64 cluster_grd_recovery_event_old_epoch(void); +extern uint64 cluster_grd_recovery_episode_epoch_value(void); +extern uint32 cluster_grd_recovery_event_coordinator(void); +extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); +extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern int cluster_grd_recovery_block_redeclare_cursor(void); +extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); +extern bool cluster_grd_recovery_block_redeclare_done(void); extern uint64 cluster_grd_redeclare_generation(void); /* spec-4.6 P0-1 — the epoch the current episode is locked to (0 = none). */ extern uint64 cluster_grd_redeclare_episode_epoch(void); @@ -626,7 +645,7 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch); +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); @@ -650,6 +669,9 @@ typedef struct ClusterGrdRecoveryCounters { uint64 block_path_failclosed; uint64 unaffected_holder_survived; uint64 stale_holder_swept; + uint64 cluster_gate_timeout; + uint64 wait_epoch_escape; + uint64 pcm_dead_cleanup_entries; /* spec-5.16 D5 — join-direction remaster counters (distinct from the * failure-driven remaster_* above; §8 Q6-A). */ uint64 join_remaster_started; diff --git a/src/include/cluster/cluster_guc.h b/src/include/cluster/cluster_guc.h index e4565731eb..d091b2be3f 100644 --- a/src/include/cluster/cluster_guc.h +++ b/src/include/cluster/cluster_guc.h @@ -399,6 +399,8 @@ extern int cluster_ges_reply_wait_max_entries; * 5000ms; SIGHUP). */ extern int cluster_grd_remaster_wait_ms; extern int cluster_grd_rebuild_timeout_ms; +extern int cluster_hw_remaster_retry_backoff_ms; +extern int cluster_hw_remaster_retry_max_attempts; /* spec-5.4 D8: SQ sequence lock tunables. */ extern int cluster_sequence_default_cache; diff --git a/src/include/cluster/cluster_hw.h b/src/include/cluster/cluster_hw.h index 89656a384f..e021c1c445 100644 --- a/src/include/cluster/cluster_hw.h +++ b/src/include/cluster/cluster_hw.h @@ -312,12 +312,30 @@ extern uint32 cluster_hw_shard_rebuilt_generation(uint32 shard_id); extern void cluster_hw_mark_shard_rebuilt(uint32 shard_id, uint32 generation); /* - * Per-dead-origin online-remaster launch idempotency (S5d). The GRD FSM records - * the episode it last launched a rebuild worker for a dead origin under, and - * skips relaunching while the value equals the current episode. + * Per-dead-origin online-remaster launch state (S5d / spec-4.6a). The GRD FSM + * records the episode it last launched a rebuild worker for a dead origin under, + * and the worker records a terminal result. BLOCKED is retryable within the same + * episode; BLOCKED_STRUCTURAL is not. */ +typedef enum ClusterHwRemasterResult { + CLUSTER_HW_REMASTER_NONE = 0, + CLUSTER_HW_REMASTER_RUNNING, + CLUSTER_HW_REMASTER_DONE, + CLUSTER_HW_REMASTER_BLOCKED, + CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, + CLUSTER_HW_REMASTER_NOT_APPLICABLE, +} ClusterHwRemasterResult; + extern uint64 cluster_hw_remaster_launched_episode(int node_id); extern void cluster_hw_remaster_set_launched(int node_id, uint64 episode); +extern ClusterHwRemasterResult cluster_hw_remaster_result(int node_id); +extern void cluster_hw_remaster_set_result(int node_id, ClusterHwRemasterResult result); +extern uint32 cluster_hw_remaster_attempts(int node_id); +extern void cluster_hw_remaster_set_attempts(int node_id, uint32 attempts); +extern uint64 cluster_hw_remaster_next_attempt_at(int node_id); +extern void cluster_hw_remaster_set_next_attempt_at(int node_id, uint64 ts); +extern const char *cluster_hw_remaster_result_name(ClusterHwRemasterResult result); +extern bool cluster_hw_remaster_recoverable(void); /* * cluster_hw_apply_hwm -- redo / remaster rebuild: create-if-absent and raise @@ -383,6 +401,8 @@ extern uint64 cluster_hw_failclosed_count(void); extern uint64 cluster_hw_not_ready_count(void); extern uint64 cluster_hw_remaster_done_count(void); extern uint64 cluster_hw_remaster_blocked_count(void); +extern uint64 cluster_hw_remaster_retry_count(void); +extern uint64 cluster_hw_remaster_retry_exhausted_count(void); extern void cluster_hw_bump_alloc(void); extern void cluster_hw_bump_authority_create(void); extern void cluster_hw_bump_reserve_wal(void); @@ -391,6 +411,8 @@ extern void cluster_hw_bump_failclosed(void); extern void cluster_hw_bump_not_ready(void); extern void cluster_hw_bump_remaster_done(void); extern void cluster_hw_bump_remaster_blocked(void); +extern void cluster_hw_bump_remaster_retry(void); +extern void cluster_hw_bump_remaster_retry_exhausted(void); #endif /* !FRONTEND */ diff --git a/src/include/cluster/cluster_hw_remaster.h b/src/include/cluster/cluster_hw_remaster.h index e1bf3ba529..e2fcc481f8 100644 --- a/src/include/cluster/cluster_hw_remaster.h +++ b/src/include/cluster/cluster_hw_remaster.h @@ -47,17 +47,91 @@ #ifndef CLUSTER_HW_REMASTER_H #define CLUSTER_HW_REMASTER_H -/* - * Result of an online-remaster HW authority rebuild for one dead origin. Only - * DONE marks the adopted shards rebuilt (opens the serve gate); BLOCKED and - * NOT_APPLICABLE never do, so a shard whose HWM is not provably rebuilt stays - * fail-closed (8.A). - */ -typedef enum ClusterHwRemasterResult { - CLUSTER_HW_REMASTER_DONE = 0, /* rebuilt + adoption snapshot durable + shards marked */ - CLUSTER_HW_REMASTER_BLOCKED, /* fail-closed: keep shards frozen (53RA6) */ - CLUSTER_HW_REMASTER_NOT_APPLICABLE, /* HW authority inactive / no adopted shard / bad input */ -} ClusterHwRemasterResult; +#include "cluster/cluster_hw.h" /* ClusterHwRemasterResult */ + +#define CLUSTER_HW_REMASTER_NO_DEADLINE PG_UINT64_MAX +#define CLUSTER_HW_REMASTER_BACKOFF_CAP_MS 60000 + +typedef enum ClusterHwRemasterLaunchAction { + CLUSTER_HW_REMASTER_LAUNCH_SKIP = 0, + CLUSTER_HW_REMASTER_LAUNCH_INITIAL, + CLUSTER_HW_REMASTER_LAUNCH_RETRY, + CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED, + CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL, +} ClusterHwRemasterLaunchAction; + +typedef struct ClusterHwRemasterRelaunchDecision { + ClusterHwRemasterLaunchAction action; + uint32 next_attempts; + uint64 next_attempt_at; + ClusterHwRemasterResult next_result; +} ClusterHwRemasterRelaunchDecision; + +static inline uint32 +cluster_hw_remaster_compute_backoff_ms(int base_ms, uint32 completed_retry_attempts) +{ + uint64 ms; + uint32 shift; + + if (base_ms < 1) + base_ms = 1; + ms = (uint64)base_ms; + shift = completed_retry_attempts > 0 ? completed_retry_attempts - 1 : 0; + while (shift-- > 0 && ms < CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms <<= 1; + if (ms > CLUSTER_HW_REMASTER_BACKOFF_CAP_MS) + ms = CLUSTER_HW_REMASTER_BACKOFF_CAP_MS; + return (uint32)ms; +} + +static inline ClusterHwRemasterRelaunchDecision +cluster_hw_remaster_relaunch_decide(uint64 launched_episode, uint64 current_episode, + ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int retry_max_attempts) +{ + ClusterHwRemasterRelaunchDecision d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_SKIP; + d.next_attempts = attempts; + d.next_attempt_at = next_attempt_at; + d.next_result = result; + + if (current_episode == 0) + return d; + if (launched_episode != current_episode || result == CLUSTER_HW_REMASTER_NONE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_INITIAL; + d.next_attempts = 0; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; + } + + if (result == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (result != CLUSTER_HW_REMASTER_BLOCKED) + return d; + + if (retry_max_attempts <= 0 || attempts >= (uint32)retry_max_attempts) { + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE) { + d.action = CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED; + d.next_attempt_at = CLUSTER_HW_REMASTER_NO_DEADLINE; + } + return d; + } + if (next_attempt_at != CLUSTER_HW_REMASTER_NO_DEADLINE && now < next_attempt_at) + return d; + + d.action = CLUSTER_HW_REMASTER_LAUNCH_RETRY; + d.next_attempts = attempts + 1; + d.next_attempt_at = 0; + d.next_result = CLUSTER_HW_REMASTER_RUNNING; + return d; +} #ifndef FRONTEND @@ -81,9 +155,9 @@ extern ClusterHwRemasterResult cluster_hw_remaster_rebuild_origin(int dead_node_ /* * cluster_hw_remaster_worker_main -- dynamic-bgworker entry point (main_arg = the * dead origin node id). Captures the live reconfig episode and drives - * cluster_hw_remaster_rebuild_origin off the LMON tick. An abnormal exit simply - * leaves the shards unmarked -> the serve gate keeps them fail-closed (8.A), so - * no abnormal-exit handler is needed. + * cluster_hw_remaster_rebuild_origin off the LMON tick. A before_shmem_exit + * callback records an abnormal exit as retryable BLOCKED so the same episode can + * self-heal instead of staying pinned on the launch idempotency bit. */ extern void cluster_hw_remaster_worker_main(Datum main_arg); diff --git a/src/include/cluster/cluster_pcm_lock.h b/src/include/cluster/cluster_pcm_lock.h index d88d82dc8f..3fb909557f 100644 --- a/src/include/cluster/cluster_pcm_lock.h +++ b/src/include/cluster/cluster_pcm_lock.h @@ -415,6 +415,12 @@ extern void cluster_pcm_lock_clear_pending_x(BufferTag tag); extern int32 cluster_pcm_lock_query_pending_x_requester(BufferTag tag); extern uint64 cluster_pcm_lock_clear_pending_x_for_node(int32 dead_node); +/* PGRAC: spec-4.6a BUG-C2 — failure-path PCM holder cleanup. Removes a + * DEAD node from X/S/PI holder records in this master's PCM directory and + * demotes entries to N when no live holder remains. Idempotent under repeated + * dead-sweep ticks. */ +extern uint64 cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node); + /* PGRAC: spec-5.13 D5 (clean-leave PCM release) — a leaving node clears its * OWN holder records (X / S / PI) from the local PCM directory after the GCS * flush seam has persisted all dirty X blocks (CL-I5). Demotes an entry's diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c455b77436..c7be25e6b0 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,6 +376,16 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } +static inline bool +cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, + bool shared_fs_backend, + int live_survivor_count) +{ + return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, + live_survivor_count) + == CLUSTER_THREADREC_SCOPE_APPLICABLE; +} + /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/024_pcm_lock.pl b/src/test/cluster_tap/t/024_pcm_lock.pl index ba829df534..f2565c8fe1 100644 --- a/src/test/cluster_tap/t/024_pcm_lock.pl +++ b/src/test/cluster_tap/t/024_pcm_lock.pl @@ -55,14 +55,15 @@ # ---------- -# L1: pg_cluster_state.pcm category has 22 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5) +# L1: pg_cluster_state.pcm category has 23 keys (spec-2.30 + spec-6.14a D5 + spec-6.14 D5 +# + spec-4.6a D12 dead_cleanup_entries) # activates the state-machine diagnostics. # ---------- is($node->safe_psql( 'postgres', q{SELECT count(*) FROM pg_cluster_state WHERE category='pcm'}), - '22', - 'L1 pg_cluster_state.pcm category has 22 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5)'); + '23', + 'L1 pg_cluster_state.pcm category has 23 keys (spec-2.30 surface + spec-6.14a D5 + spec-6.14 D5 + spec-4.6a D12)'); # ---------- diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 9c156e1e53..509c2f50ff 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -30,13 +30,18 @@ # retryable transient, NOT an HW fault -- so this leg retries and SKIPs # (never fails) when only that orthogonal recovery is outstanding. # -# Negative (corrupt the dead master's snapshot -> fail-closed): +# Negative/self-heal extensions (spec-4.6a): # L6 a fresh pair; corrupt node0's HW snapshot on shared storage, then # kill node0. The survivor's rebuild must FAIL CLOSED # (hw.remaster_blocked_count advances) -- it never trusts a corrupt # snapshot, never auto-creates at 0, never reads FileSize as the -# authority. The affected shards stay frozen (P7 gate), so no extend -# can silently re-hand a block. +# authority. +# L7 hide node0's snapshot, kill node0, observe BLOCKED, then restore +# the same file. The same reconfig episode must retry and converge +# to DONE without waiting for a later episode. +# L8 keep the snapshot corrupt with retry_max_attempts=2. The survivor +# must report retry_exhausted once and keep fail-closed; DONE must not +# advance. # # Harness: ClusterPair shared_data + wal_threads_root + 3 voting disks. # Mirrors the kill/reconfig pattern of t/249 / t/274. @@ -181,17 +186,22 @@ sub retry_sql { my ($node, $sql) = @_; my $last = ''; + my $last_retryable = 0; for my $attempt (1 .. 30) { my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); - return (1, $out) if defined $rc && $rc == 0; - $last = $err // ''; + my $combined = join("\n", grep { defined $_ && $_ ne '' } ($out, $err)); + return (1, $out, 0) + if defined $rc && $rc == 0 && $combined !~ /ERROR:/; + + $last = $combined; + $last_retryable = + $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; # retry only the orthogonal post-reconfig settling transients - last - unless $last =~ /being rebuilt after reconfiguration|status unknown|not yet propagated|could not obtain X transfer|did not ship a current image/; + last unless $last_retryable; usleep(500_000); } - return (0, $last); + return (0, $last, $last_retryable); } # L5a (the robust HW proof): the rebuilt authority is SERVEABLE -- the HW serve @@ -210,22 +220,213 @@ sub retry_sql # that is NOT an HW fault. Retry, and SKIP (never fail) when only that orthogonal # recovery is outstanding; a 53RA6 from the HW gate would NOT be retried and would # fail the test. -my ($ext_ok, $ext_res) = +my ($ext_ok, $ext_res, $ext_retryable) = retry_sql($n1, 'INSERT INTO r1 SELECT g,g,g,g FROM generate_series(4001,4500) g'); SKIP: { - skip - "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", - 1 - unless $ext_ok; - my ($cnt_ok, $cnt) = retry_sql($n1, 'SELECT count(*) FROM r1'); - is($cnt, '4500', - 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') - or diag("count result: $cnt"); + if (!$ext_ok) + { + skip + "post-remaster extend gated by orthogonal dead-block recovery (spec-4.7/4.11), not HW: $ext_res", + 1 + if $ext_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("extend result: $ext_res"); + } + else + { + my ($cnt_ok, $cnt, $cnt_retryable) = retry_sql($n1, 'SELECT count(*) FROM r1'); + if (!$cnt_ok) + { + skip + "post-remaster count gated by orthogonal visibility recovery (not HW): $cnt", + 1 + if $cnt_retryable; + fail('L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite'); + diag("count result: $cnt"); + } + else + { + is($cnt, '4500', + 'L5b: node1 extended a node0-written relation after remaster -- rows landed past node0 blocks, no dup/overwrite') + or diag("count result: $cnt"); + } + } } $pair->stop_pair; +# ====================================================================== +# SELF-HEAL: first attempt BLOCKED, same episode retries after repair. +# ====================================================================== + +my $heal = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_selfheal', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +$heal->start_pair; +my $h0 = $heal->node0; +my $h1 = $heal->node1; + +ok(poll_until($h1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L7: self-heal pair in quorum'); +poll_until( + $h1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($h0, $h1, "s$_", 4000) for (1 .. 6); +$h0->safe_psql('postgres', 'CHECKPOINT'); +$h1->safe_psql('postgres', 'CHECKPOINT'); + +my $hsnap = $heal->shared_data_root . "/global/pg_hw_snapshot.0"; +my $hhold = "$hsnap.hold"; +ok(-f $hsnap, 'L7: node0 HW snapshot exists before self-heal injection'); +rename($hsnap, $hhold) or die "rename $hsnap -> $hhold: $!"; + +my $h_done_before = hw_counter($h1, 'remaster_done_count'); +my $h_blocked_before = hw_counter($h1, 'remaster_blocked_count'); +my $h_retry_before = hw_counter($h1, 'remaster_retry_count'); +$heal->kill_node9(0); +ok( poll_until( + $h1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30), + 'L7: self-heal pair declared node0 dead'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_blocked_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_blocked_count'", + 'true', 40), + 'L7: first HW rebuild attempt failed closed while the snapshot was hidden'); + +rename($hhold, $hsnap) or die "rename $hhold -> $hsnap: $!"; +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_retry_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_count'", + 'true', 40), + 'L7: same reconfig episode launched a HW remaster retry'); +ok( poll_until( + $h1, + "SELECT (value::bigint > $h_done_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_done_count'", + 'true', 60), + 'L7: restored snapshot let the same-episode retry converge to DONE'); +is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, + 'L7: self-heal path did not exhaust the retry budget'); + +# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor +# extends a table that lived on shards the dead master owned; a frozen shard +# would fail this INSERT with the remastering error. +{ + my ($ext_ok, $ext_res) = (0, ''); + for my $try (1 .. 30) + { + $ext_res = eval { + $h1->safe_psql('postgres', + 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 1; + } ? 'ok' : ($@ // 'error'); + if ($ext_res eq 'ok') { $ext_ok = 1; last; } + sleep 1; + } + ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') + or diag($ext_res); +} + +$heal->stop_pair; + +# ====================================================================== +# NEGATIVE: corrupt snapshot persists -> retries exhaust and stay closed. +# ====================================================================== + +my $exh = PostgreSQL::Test::ClusterPair->new_pair( + 'hw_remaster_exhausted', + quorum_voting_disks => 3, + shared_data => 1, + wal_threads_root => 1, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', + ]); +$exh->start_pair; +my $e0 = $exh->node0; +my $e1 = $exh->node1; + +ok(poll_until($e1, 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 20), + 'L8: exhausted pair in quorum'); +poll_until( + $e1, + q{SELECT (state='alive' AND heartbeat_recv_count>0)::text + FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 20); + +make_and_fill($e0, $e1, "x$_", 4000) for (1 .. 6); +$e0->safe_psql('postgres', 'CHECKPOINT'); +$e1->safe_psql('postgres', 'CHECKPOINT'); + +my $esnap = $exh->shared_data_root . "/global/pg_hw_snapshot.0"; +ok(-f $esnap, 'L8: node0 HW snapshot exists before exhaustion injection'); +open(my $efh, '>', $esnap) or die "open $esnap: $!"; +binmode $efh; +print $efh ("\xDE\xAD\xBE\xEF" x 64); +close $efh; + +my $e_done_before = hw_counter($e1, 'remaster_done_count'); +my $e_exh_before = hw_counter($e1, 'remaster_retry_exhausted_count'); +$exh->kill_node9(0); +poll_until( + $e1, + q{SELECT (state='dead')::text FROM pg_cluster_cssd_peers WHERE node_id=0}, + 'true', 30); +ok( poll_until( + $e1, + "SELECT (value::bigint > $e_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L8: persistent corrupt snapshot exhausted the same-episode retry budget'); +is(hw_counter($e1, 'remaster_done_count'), $e_done_before, + 'L8: DONE did not advance after retry exhaustion on corrupt snapshot'); + +# spec-4.6a section 4 (D8/L408): after exhaustion the WAIT_CLUSTER watchdog +# keeps sounding (counter + WARNING line) while shards stay frozen. +ok( poll_until( + $e1, + q{SELECT (value::bigint > 0)::text FROM pg_cluster_state } + . q{WHERE category='grd_recovery' AND key='cluster_gate_timeout'}, + 'true', 30), + 'L8: WAIT_CLUSTER watchdog counter advances while exhausted episode stays frozen'); +{ + my $log = PostgreSQL::Test::Utils::slurp_file($e1->logfile); + like( + $log, + qr/cluster GRD recovery WAIT_CLUSTER watchdog fired; affected shards stay frozen/, + 'L8: watchdog WARNING appears in the survivor log'); +} + +$exh->stop_pair; + # ====================================================================== # NEGATIVE: corrupt the dead master's snapshot -> rebuild fails closed. # ====================================================================== @@ -242,6 +443,9 @@ sub retry_sql 'cluster.ges_retransmit_max_attempts = 0', 'cluster.cssd_heartbeat_interval_ms = 1000', 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 2', ]); $neg->start_pair; my $m0 = $neg->node0; @@ -277,6 +481,7 @@ sub retry_sql } my $blocked_before = hw_counter($m1, 'remaster_blocked_count'); +my $neg_exh_before = hw_counter($m1, 'remaster_retry_exhausted_count'); $neg->kill_node9(0); poll_until( $m1, @@ -291,6 +496,12 @@ sub retry_sql . "WHERE category='hw' AND key='remaster_blocked_count'", 'true', 40), 'L6: HW rebuild FAILED CLOSED on the corrupt snapshot (remaster_blocked advanced; no auto-create-0)'); +ok( poll_until( + $m1, + "SELECT (value::bigint > $neg_exh_before)::text FROM pg_cluster_state " + . "WHERE category='hw' AND key='remaster_retry_exhausted_count'", + 'true', 70), + 'L6: corrupt snapshot eventually exhausts bounded same-episode retries'); $neg->stop_pair; diff --git a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl index a0477b88b7..86970cb280 100644 --- a/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl +++ b/src/test/cluster_tap/t/337_shared_catalog_ddl_2node.pl @@ -94,6 +94,8 @@ my $shared_root = PostgreSQL::Test::Utils::tempdir(); mkdir "$shared_root/global" or die "mkdir shared global: $!"; +my $wal_root = PostgreSQL::Test::Utils::tempdir(); + my $disk_dir = PostgreSQL::Test::Utils::tempdir(); my @disks; for my $i (0 .. 2) @@ -114,7 +116,7 @@ # Step 0: node0 init -> backup -> node1 init_from_backup (one shared sysid). # ---------- my $node0 = PostgreSQL::Test::Cluster->new('sc_ddl_node0'); -$node0->init(allows_streaming => 1); +$node0->init(allows_streaming => 1, extra => [ '-X', "$wal_root/thread_1" ]); $node0->start; $node0->backup('scb'); $node0->stop; @@ -122,6 +124,24 @@ my $node1 = PostgreSQL::Test::Cluster->new('sc_ddl_node1'); $node1->init_from_backup($node0, 'scb'); +# Relocate node1's backup-copied WAL into its shared thread dir. shared_catalog +# multi-node formation is fail-fast without cluster.wal_threads_dir, and the WAL +# thread validator requires pg_wal to resolve to the configured thread_N dir. +{ + my $pgwal = $node1->data_dir . '/pg_wal'; + my $wal2 = "$wal_root/thread_2"; + mkdir $wal2 or die "mkdir $wal2: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal2/$e") or die "rename $pgwal/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal2, $pgwal) or die "symlink $pgwal -> $wal2: $!"; +} + # Config common to the shared-catalog feature (both nodes, both phases). my $sc_common = <append_conf('postgresql.conf', $cluster_conf); @@ -819,4 +840,23 @@ sub oid_authority_hw qr/cluster\.shared_catalog is off but the shared tree holds/, 'L7: the refusal is the designed off-flip vet FATAL (never a stale serve)'); +# ---------- +# L8 (spec-4.6a D11 level-2 fail-fast): a multi-node shared_catalog=on boot +# without cluster.wal_threads_dir is refused at startup -- that shape cannot +# rebuild a dead node's HW authority from per-thread WAL, so a node failure +# would be permanently unrecoverable (the silent-BLOCKED wedge of +# spec-4.6a section 0). Flip shared_catalog back on and blank the WAL +# threads root; the boot must fail on the D11 vet, not come up degraded. +# ---------- +$node1->append_conf('postgresql.conf', "cluster.shared_catalog = on +"); +$node1->append_conf('postgresql.conf', "cluster.wal_threads_dir = '' +"); +my $nowalret = $node1->start(fail_ok => 1); +is($nowalret, 0, 'L8: multi-node shared_catalog boot without wal_threads_dir fails'); +my $nowallog = PostgreSQL::Test::Utils::slurp_file($node1->logfile); +like($nowallog, + qr/cluster\.shared_catalog requires cluster\.wal_threads_dir in a multi-node cluster/, + 'L8: the refusal is the spec-4.6a D11 startup FATAL with the config errhint'); + done_testing(); diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl new file mode 100644 index 0000000000..883860c0fb --- /dev/null +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -0,0 +1,310 @@ +#------------------------------------------------------------------------- +# +# 362_shared_catalog_4node_kill_selfheal.pl +# spec-4.6a D9 -- 4-node shared_catalog fail-stop self-heal. +# +# Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) +# so a killed node's HW authority is recoverable from its per-thread WAL. +# After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means +# the test harness is misconfigured, and lack of convergence is a real FAIL. +# +# Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group +# Portions Copyright (c) 1994, Regents of the University of California +# Portions Copyright (c) 2026, pgrac contributors +# +# Author: SqlRush +# +# IDENTIFICATION +# src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +# Portions Copyright (c) 2026, pgrac contributors +# +#------------------------------------------------------------------------- + +use strict; +use warnings; + +use FindBin; +use lib "$FindBin::RealBin/../../perl"; + +use PostgreSQL::Test::ClusterQuad; +use PostgreSQL::Test::Utils; +use Test::More; +use Time::HiRes qw(usleep); + +if ($ENV{with_pgrac_cluster} && $ENV{with_pgrac_cluster} eq 'no') +{ + plan skip_all => 'shared_catalog 4-node kill self-heal requires --enable-cluster'; +} + +sub poll_until +{ + my ($node, $query, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + $last = eval { $node->safe_psql('postgres', $query) }; + return 1 if defined $last && $last eq $want; + usleep(300_000); + } + diag("poll_until timed out: '$query' last='" . ($last // '(undef)') + . "' want='$want'"); + return 0; +} + +sub retry_sql +{ + my ($node, $sql, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($rc, $out, $err) = $node->psql('postgres', $sql, timeout => 60); + return (1, $out // '') if defined $rc && $rc == 0; + $last = $err // ''; + usleep(500_000); + } + return (0, $last); +} + +sub state_counter +{ + my ($node, $cat, $key) = @_; + my $v = eval { + $node->safe_psql('postgres', + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'") + }; + return defined $v && $v ne '' ? $v + 0 : 0; +} + +sub sum_hw +{ + my ($quad, $key, @idx) = @_; + my $sum = 0; + for my $i (@idx) + { + $sum += state_counter($quad->node($i), 'hw', $key); + } + return $sum; +} + +sub logs_contain +{ + my ($quad, $pattern, @idx) = @_; + for my $i (@idx) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + } + return 0; +} + +sub log_until +{ + my ($quad, $idx, $pattern, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + while (time() < $deadline) + { + my $log = eval { PostgreSQL::Test::Utils::slurp_file($quad->node($idx)->logfile) } // ''; + return 1 if $log =~ /$pattern/; + usleep(300_000); + } + diag("log_until timed out on node$idx waiting for /$pattern/"); + return 0; +} + +sub startup_env_blocker +{ + my ($quad) = @_; + my $log = ''; + if ($quad) + { + for my $i (0 .. 3) + { + $log .= eval { PostgreSQL::Test::Utils::slurp_file($quad->node($i)->logfile) } // ''; + } + } + return $log =~ /could not create shared memory segment|No space left on device|No space left|SHMMNI|semget/i; +} + +my $quad = eval { + PostgreSQL::Test::ClusterQuad->new_quad( + 'sc4_selfheal', + shared_catalog => 1, + quorum_voting_disks => 3, + extra_conf => [ + 'autovacuum = off', + 'cluster.grd_max_entries = 4096', + 'cluster.lms_enabled = on', + 'cluster.ges_request_timeout_ms = 3000', + 'cluster.ges_retransmit_max_attempts = 0', + 'cluster.cssd_heartbeat_interval_ms = 1000', + 'cluster.cssd_dead_deadband_factor = 6', + 'cluster.grd_rebuild_timeout_ms = 1000', + 'cluster.hw_remaster_retry_backoff_ms = 100', + 'cluster.hw_remaster_retry_max_attempts = 8', + ]); +}; +if (!$quad) +{ + my $err = $@ || 'unknown constructor failure'; + plan skip_all => "L1 shared_catalog ClusterQuad formation hit local substrate blocker: $err" + if $err =~ /could not create shared memory|No space left|SHMMNI|semget/i; + BAIL_OUT("ClusterQuad(shared_catalog) constructor failed: $err"); +} + +my $started = 1; +for my $i (1 .. 3) +{ + PostgreSQL::Test::Utils::system_log( + 'pg_ctl', '-W', '-D', $quad->node($i)->data_dir, + '-l', $quad->node($i)->logfile, + '-o', "--cluster-name=sc4_selfheal_node$i", 'start'); +} +unless ($quad->node(0)->start(fail_ok => 1)) +{ + $started = 0; +} +for my $i (1 .. 3) +{ + $quad->node($i)->_update_pid(-1); +} +if (!$started) +{ + if (startup_env_blocker($quad)) + { + eval { $quad->stop_quad; }; + plan skip_all => 'L1 shared_catalog ClusterQuad formation hit local shared-memory/resource limit'; + } + my $fatal = logs_contain($quad, qr/cluster\.shared_catalog requires cluster\.wal_threads_dir|structurally blocked|BLOCKED_STRUCTURAL/i, 0 .. 3); + eval { $quad->stop_quad; }; + ok(!$fatal, 'L1: shared_catalog quad must not hit wal_threads_dir/BLOCKED_STRUCTURAL configuration failure'); + BAIL_OUT('ClusterQuad(shared_catalog) failed to start for a non-environment reason'); +} + +# L1: all four members formed with quorum and per-thread WAL configured. +for my $i (0 .. 3) +{ + ok(poll_until($quad->node($i), 'SELECT in_quorum FROM pg_cluster_quorum_state', 't', 60), + "L1: node$i reached quorum"); + is(state_counter($quad->node($i), 'hw', 'remaster_recoverable'), 1, + "L1: node$i reports HW remaster recoverable"); +} +for my $to (1 .. 3) +{ + ok($quad->wait_for_peer_state(0, $to, 'connected', 60), + "L1: node0 sees node$to connected"); +} + +my $n0 = $quad->node0; +my $dead = 3; +my @survivors = (0, 1, 2); + +# Seed shared-catalog DDL and force node3 to own real HW state before death. +for my $t (1 .. 2) +{ + my ($ddl_ok, $ddl_res) = retry_sql($n0, + "CREATE TABLE sc4_hw_$t (id int, pad int)", 60); + ok($ddl_ok, "L1: coordinator created shared_catalog table sc4_hw_$t") + or diag($ddl_res); + my ($ok, $res) = retry_sql($quad->node($dead), + "INSERT INTO sc4_hw_$t SELECT g, g FROM generate_series(1,5000) g", 60); + ok($ok, "L1: node$dead extended shared_catalog table sc4_hw_$t before kill") + or diag($res); +} +for my $i (0 .. 3) +{ + retry_sql($quad->node($i), 'CHECKPOINT', 30); +} + +my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); +my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); + +# L2: kill node3 and require every survivor to see the real DEAD edge. +# Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, +# ordinary catalog reads may legitimately return 53R9I until the barrier opens. +$quad->kill_node9($dead); +for my $i (@survivors) +{ + ok(log_until($quad, $i, qr/cssd: peer $dead transitioned .* DEAD/, 45), + "L2: survivor node$i declared node$dead dead"); +} + +# L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +for my $i (@survivors) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + "L3: survivor node$i remastered all GRD shards off node$dead"); +} +my $done_converged = 0; +my $deadline = time() + 90; +while (time() < $deadline) +{ + if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + { + $done_converged = 1; + last; + } + usleep(500_000); +} +ok($done_converged, 'L3: HW remaster DONE advanced on at least one survivor after kill'); +ok(!logs_contain($quad, qr/blocked_structural|structurally blocked|BLOCKED_STRUCTURAL/i, @survivors), + 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); + +# L6: if a transient BLOCKED occurred, a same-episode retry must also have run. +my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +if ($blocked_after > $blocked_before) +{ + ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); +} +else +{ + pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); +} + +# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +for my $i (@survivors) +{ + my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); + ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); +} +my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); +ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); +for my $i (1, 2) +{ + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); +} + +# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is +# exposed and non-negative on every survivor; when the dead node left any +# holder/pending-X records behind, at least one survivor accounts a cleanup +# (delta assertable; zero stays legal when the dead node held nothing). +{ + my $cleanup_total = 0; + for my $i (@survivors) + { + my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); + ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); + $cleanup_total += $c; + } + diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); +} + +# L5: watchdog counters are allowed to be zero, but must be readable. +for my $i (@survivors) +{ + my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); + my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); + ok($cluster_gate >= 0 && $epoch_escape >= 0, + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); +} + +$quad->stop_quad; +done_testing(); diff --git a/src/test/cluster_unit/Makefile b/src/test/cluster_unit/Makefile index 11c235416b..521855f18a 100644 --- a/src/test/cluster_unit/Makefile +++ b/src/test/cluster_unit/Makefile @@ -65,6 +65,7 @@ TESTS = test_cluster_basic test_cluster_version test_cluster_backend_types \ test_cluster_oid_lease \ test_cluster_relmap_authority \ test_cluster_hw \ + test_cluster_hw_remaster_retry \ test_cluster_dl \ test_cluster_extend_gate \ test_cluster_ir \ diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index be82b60482..8b64c01755 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -45,6 +45,7 @@ #include "cluster/cluster_catalog_stats.h" /* spec-6.14 D10b catalog counter stubs */ #include "cluster/cluster_debug.h" +#include "cluster/cluster_grd.h" /* ClusterGrdRecoveryCounters */ #include "cluster/cluster_hang.h" /* spec-5.11: ClusterHangDumpData for dump_hang stubs */ #include "cluster/cluster_hang_resolve.h" /* spec-5.12: ClusterHangResolveCounters for dump stubs */ #include "cluster/cluster_reconfig.h" /* spec-5.14 D6 touched getter stubs */ @@ -564,6 +565,9 @@ uint64 cluster_hw_failclosed_count(void); uint64 cluster_hw_not_ready_count(void); uint64 cluster_hw_remaster_done_count(void); uint64 cluster_hw_remaster_blocked_count(void); +uint64 cluster_hw_remaster_retry_count(void); +uint64 cluster_hw_remaster_retry_exhausted_count(void); +bool cluster_hw_remaster_recoverable(void); uint64 cluster_hw_alloc_count(void) { @@ -604,6 +608,21 @@ cluster_hw_remaster_blocked_count(void) { return 0; } +uint64 +cluster_hw_remaster_retry_count(void) +{ + return 0; +} +uint64 +cluster_hw_remaster_retry_exhausted_count(void) +{ + return 0; +} +bool +cluster_hw_remaster_recoverable(void) +{ + return true; +} /* spec-5.7 D4 dump_dl stubs (cluster_dl.c not linked in this binary). */ uint64 cluster_dl_lease_count(void); @@ -3288,15 +3307,77 @@ cluster_grd_deadlock_chunk_oo_buffer_overflow_count(void) return 0; } -/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter - * snapshot. Zero-fill; layout = 13 × uint64 (must track the - * ClusterGrdRecoveryCounters struct in cluster_grd.h). */ -struct ClusterGrdRecoveryCounters; -void cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out); +uint32 +cluster_grd_recovery_state_value(void) +{ + return (uint32)GRD_RECOVERY_IDLE; +} + +const char * +cluster_grd_recovery_state_name(uint32 state pg_attribute_unused()) +{ + return "idle"; +} + +uint64 +cluster_grd_recovery_last_event_id(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_episode_epoch_value(void) +{ + return 0; +} + +uint32 +cluster_grd_recovery_event_coordinator(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +uint64 +cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +{ + return 0; +} + +int +cluster_grd_recovery_block_redeclare_cursor(void) +{ + return 0; +} + +uint64 +cluster_grd_recovery_block_redeclare_epoch(void) +{ + return 0; +} + +bool +cluster_grd_recovery_block_redeclare_done(void) +{ + return true; +} + +/* spec-4.6 D5 stub: dump_grd_recovery consumes the bulk counter snapshot. */ void -cluster_grd_recovery_counters_snapshot(struct ClusterGrdRecoveryCounters *out) +cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out) { - memset(out, 0, 13 * sizeof(uint64)); + memset(out, 0, sizeof(*out)); } uint32 diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 444accf8f0..67fcc29dc2 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -70,6 +70,7 @@ #include "cluster/cluster_gcs.h" #include "cluster/cluster_gcs_block.h" #include "cluster/cluster_ic_envelope.h" +#include "cluster/cluster_thread_recovery.h" #include "common/hashfn.h" #include "storage/buf_internals.h" #include "storage/lwlock.h" @@ -287,6 +288,16 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } +UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) +{ + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); + UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); + UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); +} + + /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -514,7 +525,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(21); + UT_PLAN(22); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -530,6 +541,7 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); + UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); diff --git a/src/test/cluster_unit/test_cluster_ges.c b/src/test/cluster_unit/test_cluster_ges.c index 1df0a6b7fc..dd2a47185a 100644 --- a/src/test/cluster_unit/test_cluster_ges.c +++ b/src/test/cluster_unit/test_cluster_ges.c @@ -296,7 +296,8 @@ cluster_cancel_token_consume(void) * completion. Standalone fixture has no recovery shmem; no-op. */ void cluster_grd_recovery_mark_peer_done(int32 node pg_attribute_unused(), - uint64 epoch pg_attribute_unused()) + uint64 epoch pg_attribute_unused(), + uint64 event_id pg_attribute_unused()) {} ClusterGrdEntryResult diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..c7f805459d 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -55,6 +55,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -85,6 +86,15 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; the GRD fixture has no PCM shmem — no-op (the cleanup + * logic itself is unit-tested in test_cluster_pcm_lock.c). */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -316,6 +326,13 @@ cluster_epoch_get_current(void) return ut_mock_epoch; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch) +{ + if (admitted_epoch > ut_mock_epoch) + ut_mock_epoch = admitted_epoch; +} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -377,10 +394,20 @@ cluster_bufmgr_redeclare_scan_chunk(int start_buf, int max_scan, return end; } +/* spec-4.6a r2-P1-1: controllable last-event mock so the recovery FSM's P0 + * accept + WAIT_EPOCH event-scoped witness can be unit-driven. Default + * zeroed = pre-existing inert behavior. */ +static ReconfigEvent ut_mock_last_event; void cluster_reconfig_get_last_event(ReconfigEvent *out) { - memset(out, 0, sizeof(*out)); + *out = ut_mock_last_event; +} + +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; } /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the @@ -419,6 +446,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { @@ -432,10 +477,12 @@ SendProcSignal(int pid pg_attribute_unused(), int reason pg_attribute_unused(), return 0; } +/* spec-4.6a r2-P1-1: controllable clock (default 0 = pre-existing value). */ +static int64 ut_mock_now = 0; int64 GetCurrentTimestamp(void) { - return 0; + return ut_mock_now; } void @@ -3903,15 +3950,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10); + cluster_grd_recovery_mark_peer_done(1, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3934,15 +3981,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5); - cluster_grd_recovery_mark_peer_done(1, 5); - cluster_grd_recovery_mark_peer_done(2, 5); + cluster_grd_recovery_mark_peer_done(0, 5, 1); + cluster_grd_recovery_mark_peer_done(1, 5, 1); + cluster_grd_recovery_mark_peer_done(2, 5, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3967,9 +4014,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4020,9 +4067,9 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10); - cluster_grd_recovery_mark_peer_done(1, 10); - cluster_grd_recovery_mark_peer_done(2, 10); + cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4035,15 +4082,74 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20); + cluster_grd_recovery_mark_peer_done(0, 20, 1); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20); + cluster_grd_recovery_mark_peer_done(1, 20, 1); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } +/* ============================================================ + * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * + * A stale DONE (previous event) must be dropped at the accounting + * layer and must never satisfy the WAIT_EPOCH coordinator witness; + * a current-event DONE at exactly cur, accounted after the P0 + * accept snapshot, is the only equal-epoch escape (proof-carrying, + * never timeout-only). + * ============================================================ */ + +UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +{ + ClusterGrdRecoveryCounters before; + ClusterGrdRecoveryCounters after; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline. ut_mock_epoch already + * holds the post-bump value, reproducing the mis-captured-baseline wedge + * shape (cur == old) from spec-4.6a section 0. */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + ut_mock_last_event.event_id = 77; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); + + /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, 76); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Current-event DONE is accounted (epoch + event id). */ + cluster_grd_recovery_mark_peer_done(0, 10, 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); + UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + + /* Witness advance: deadline expired + cur==old + coordinator DONE for + * THIS event at exactly cur, accounted after the accept snapshot. The + * FSM takes the event-scoped equal-epoch escape exactly once. */ + cluster_grd_recovery_counters_snapshot(&before); + ut_mock_now = (int64)60 * 1000000; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&after); + UT_ASSERT_EQ(after.wait_epoch_escape, before.wait_epoch_escape + 1); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4161,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4264,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..d40ee60d43 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -61,6 +61,7 @@ #include "cluster/cluster_ges_mode.h" /* spec-5.1b — frozen matrix + convert classification */ #include "access/transam.h" /* spec-5.8 D1c — InvalidTransactionId */ #include "cluster/cluster_grd.h" +#include "cluster/cluster_hw.h" /* spec-4.6a HW remaster watchdog stubs */ #include "cluster/cluster_lmd.h" /* spec-5.8 D1b — WFG vertex + submit/cancel edge */ #include "cluster/cluster_reconfig.h" /* spec-4.6 D1 — ReconfigEvent stub type */ #include "cluster/cluster_thread_recovery.h" /* spec-4.11 D3 (L238) — gate_unfreeze proto */ @@ -91,6 +92,14 @@ bool IsUnderPostmaster = false; +/* spec-4.6a D12 stub: cluster_grd_cleanup_on_node_dead chains into the PCM + * dead-holder cleanup; this fixture has no PCM shmem — no-op. */ +uint64 +cluster_pcm_lock_cleanup_on_node_dead(int32 dead_node pg_attribute_unused()) +{ + return 0; +} + void ExceptionalCondition(const char *conditionName pg_attribute_unused(), const char *fileName pg_attribute_unused(), @@ -298,6 +307,10 @@ cluster_epoch_get_current(void) return 0; } +void +cluster_epoch_adopt_admitted(uint64 admitted_epoch pg_attribute_unused()) +{} + /* * spec-4.7 D2 (L238) — cluster_grd.o's reconfig tick now references the block * re-declare scan/send (grd_block_redeclare_step / _cb). This test exercises @@ -355,6 +368,12 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +uint64 +cluster_reconfig_get_observed_epoch(int32 node_id pg_attribute_unused()) +{ + return 0; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -391,6 +410,24 @@ cluster_hw_remaster_gate_unfreeze(void) return false; } +ClusterHwRemasterResult +cluster_hw_remaster_result(int node_id pg_attribute_unused()) +{ + return CLUSTER_HW_REMASTER_NONE; +} + +uint32 +cluster_hw_remaster_attempts(int node_id pg_attribute_unused()) +{ + return 0; +} + +const char * +cluster_hw_remaster_result_name(ClusterHwRemasterResult result pg_attribute_unused()) +{ + return "none"; +} + struct PGPROC * BackendIdGetProc(int backendID pg_attribute_unused()) { diff --git a/src/test/cluster_unit/test_cluster_hw_remaster_retry.c b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c new file mode 100644 index 0000000000..456c5979f8 --- /dev/null +++ b/src/test/cluster_unit/test_cluster_hw_remaster_retry.c @@ -0,0 +1,156 @@ +/*------------------------------------------------------------------------- + * + * test_cluster_hw_remaster_retry.c + * Unit tests for spec-4.6a same-episode HW remaster retry decisions. + * + * Portions Copyright (c) 1996-2024, PostgreSQL Global Development Group + * Portions Copyright (c) 1994, Regents of the University of California + * Portions Copyright (c) 2026, pgrac contributors + * + * Author: SqlRush + * + * IDENTIFICATION + * src/test/cluster_unit/test_cluster_hw_remaster_retry.c + * + *------------------------------------------------------------------------- + */ +#include "postgres.h" + +#include "cluster/cluster_hw_remaster.h" + +#undef printf +#undef fprintf +#undef snprintf + +#include "unit_test.h" + +UT_DEFINE_GLOBALS(); + +void +ExceptionalCondition(const char *conditionName pg_attribute_unused(), + const char *fileName pg_attribute_unused(), + int lineNumber pg_attribute_unused()) +{ + abort(); +} + +static ClusterHwRemasterRelaunchDecision +decide(uint64 launched, uint64 episode, ClusterHwRemasterResult result, uint32 attempts, + uint64 next_attempt_at, uint64 now, int max_attempts) +{ + return cluster_hw_remaster_relaunch_decide(launched, episode, result, attempts, next_attempt_at, + now, max_attempts); +} + +UT_TEST(test_new_episode_initial_launch_resets_state) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(10, 11, CLUSTER_HW_REMASTER_BLOCKED, 7, 1234, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_running_done_and_not_applicable_do_not_relaunch) +{ + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_RUNNING, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_DONE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); + UT_ASSERT_EQ(decide(11, 11, CLUSTER_HW_REMASTER_NOT_APPLICABLE, 0, 0, 2000, 16).action, + CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_structural_blocked_warns_once_and_never_retries) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, 0, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_STRUCTURAL); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL, 0, CLUSTER_HW_REMASTER_NO_DEADLINE, + 3000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); +} + +UT_TEST(test_blocked_waits_until_backoff_deadline) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 4999, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 5000, 5000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 1); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_cap_exhausts_once_and_sighup_raise_recovers) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, 0, 5000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); + UT_ASSERT_EQ(d.next_attempt_at, CLUSTER_HW_REMASTER_NO_DEADLINE); + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 6000, 2); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_SKIP); + + /* Operator raised cluster.hw_remaster_retry_max_attempts via SIGHUP. */ + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 2, CLUSTER_HW_REMASTER_NO_DEADLINE, 7000, 3); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_RETRY); + UT_ASSERT_EQ(d.next_attempts, 3); +} + +UT_TEST(test_zero_max_attempts_disables_retry) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_BLOCKED, 0, 0, 5000, 0); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_MARK_EXHAUSTED); +} + +/* spec-4.6a section 2.1 truth-table row 8: latched == episode but the result + * slot reads NONE (registration failed and was reverted) -> take the + * first-launch row again; no attempt is charged. */ +UT_TEST(test_none_result_registration_failed_falls_back_to_initial) +{ + ClusterHwRemasterRelaunchDecision d; + + d = decide(11, 11, CLUSTER_HW_REMASTER_NONE, 3, 999, 2000, 16); + UT_ASSERT_EQ(d.action, CLUSTER_HW_REMASTER_LAUNCH_INITIAL); + UT_ASSERT_EQ(d.next_attempts, 0); + UT_ASSERT_EQ(d.next_attempt_at, 0); + UT_ASSERT_EQ(d.next_result, CLUSTER_HW_REMASTER_RUNNING); +} + +UT_TEST(test_backoff_exponential_cap) +{ + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 0), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 1), 1000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 2), 2000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(1000, 7), 60000); + UT_ASSERT_EQ(cluster_hw_remaster_compute_backoff_ms(100, 20), 60000); +} + + +int +main(void) +{ + UT_PLAN(8); + UT_RUN(test_new_episode_initial_launch_resets_state); + UT_RUN(test_running_done_and_not_applicable_do_not_relaunch); + UT_RUN(test_structural_blocked_warns_once_and_never_retries); + UT_RUN(test_blocked_waits_until_backoff_deadline); + UT_RUN(test_cap_exhausts_once_and_sighup_raise_recovers); + UT_RUN(test_zero_max_attempts_disables_retry); + UT_RUN(test_none_result_registration_failed_falls_back_to_initial); + UT_RUN(test_backoff_exponential_cap); + UT_DONE(); + + return ut_failed_count == 0 ? 0 : 1; +} diff --git a/src/test/cluster_unit/test_cluster_lock_acquire.c b/src/test/cluster_unit/test_cluster_lock_acquire.c index 5636b8fff8..a5ca5a719c 100644 --- a/src/test/cluster_unit/test_cluster_lock_acquire.c +++ b/src/test/cluster_unit/test_cluster_lock_acquire.c @@ -365,6 +365,47 @@ cluster_grd_lookup_master(const ClusterResId *resid pg_attribute_unused()) return -1; } +/* spec-4.6a: the S4-reject diagnostic references the CSSD peer-state view; + * fixture has no CSSD shmem — stub DEAD-free defaults. */ +#include "cluster/cluster_cssd.h" +ClusterCssdPeerState +cluster_cssd_get_peer_state(int32 peer_id pg_attribute_unused()) +{ + return CLUSTER_CSSD_PEER_ALIVE; +} +const char * +cluster_cssd_peer_state_to_string(ClusterCssdPeerState s pg_attribute_unused()) +{ + return "alive"; +} + +/* spec-4.6a: the same diagnostic is the first ereport in this .o — swallow + * elog machinery (mirror test_cluster_grd.c pattern). */ +bool +errstart(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +bool +errstart_cold(int e pg_attribute_unused(), const char *d pg_attribute_unused()) +{ + return false; +} +void +errfinish(const char *f pg_attribute_unused(), int l pg_attribute_unused(), + const char *fn pg_attribute_unused()) +{} +int +errmsg_internal(const char *fmt pg_attribute_unused(), ...) +{ + return 0; +} +int +errcode(int s pg_attribute_unused()) +{ + return 0; +} + /* spec-4.6 P0 regression harness — test-controlled S3/S4 outcomes. * stub_reserve_result default NOT_READY keeps the legacy tests on the * pre-reservation-fail path; the S4 default-deny test flips it to OK diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c0ff3b6710..c02894306f 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -100,6 +100,9 @@ static Size fake_pcm_keysize = 0; static Size fake_pcm_entrysize = 0; static LWLock *fake_lwlock_held = NULL; static LWLockMode fake_lwlock_mode = LW_EXCLUSIVE; +static LWLock *fake_lwlock_stack[16]; +static LWLockMode fake_lwlock_mode_stack[16]; +static int fake_lwlock_depth = 0; static uint32 fake_init_wait_event_seen = 0; static uint32 fake_lwlock_wait_event_seen = 0; @@ -121,6 +124,7 @@ static struct { static sigjmp_buf ut_ereport_jump; static bool ut_ereport_jump_armed = false; static int ut_ereport_fired_count = 0; +static bool fake_local_x_upgrade_result = false; void ExceptionalCondition(const char *conditionName pg_attribute_unused(), @@ -155,6 +159,10 @@ reset_fake_pcm_runtime(int max_entries) fake_pcm_keysize = 0; fake_pcm_entrysize = 0; fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + fake_lwlock_depth = 0; + memset(fake_lwlock_stack, 0, sizeof(fake_lwlock_stack)); + memset(fake_lwlock_mode_stack, 0, sizeof(fake_lwlock_mode_stack)); fake_init_wait_event_seen = 0; fake_lwlock_wait_event_seen = 0; ut_wait_event_info_storage = 0; @@ -167,6 +175,7 @@ reset_fake_pcm_runtime(int max_entries) fake_cv_broadcast_count = 0; fake_cv_sleep_wait_event = 0; fake_cv_wake_release.armed = false; + fake_local_x_upgrade_result = false; cluster_node_id = 0; NBuffers = max_entries; cluster_pcm_grd_max_entries = max_entries; @@ -191,9 +200,20 @@ bool cluster_read_scache = false; bool cluster_gcs_block_local_x_upgrade(BufferTag tag); bool -cluster_gcs_block_local_x_upgrade(BufferTag tag pg_attribute_unused()) +cluster_gcs_block_local_x_upgrade(BufferTag tag) { - return false; + uint32 holders_bm; + int n; + + if (!fake_local_x_upgrade_result) + return false; + holders_bm = cluster_pcm_lock_query_s_holders_bitmap(tag); + if (cluster_node_id >= 0 && cluster_node_id < 32) + holders_bm &= ~((uint32)1u << (uint32)cluster_node_id); + for (n = 0; n < 32; n++) + if ((holders_bm & ((uint32)1u << n)) != 0) + (void)cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_N_INVALIDATE, n); + return cluster_pcm_lock_apply_gcs_transition(tag, PCM_TRANS_S_TO_X_UPGRADE, cluster_node_id); } /* spec-4.7a D4 — stub CSSD peer liveness for the other-live-holder gate. @@ -348,6 +368,10 @@ LWLockInitialize(LWLock *lock pg_attribute_unused(), int tranche_id pg_attribute bool LWLockAcquire(LWLock *lock, LWLockMode mode) { + Assert(fake_lwlock_depth < (int)lengthof(fake_lwlock_stack)); + fake_lwlock_stack[fake_lwlock_depth] = lock; + fake_lwlock_mode_stack[fake_lwlock_depth] = mode; + fake_lwlock_depth++; fake_lwlock_held = lock; fake_lwlock_mode = mode; fake_lwlock_wait_event_seen = ut_wait_event_info_storage; @@ -357,14 +381,28 @@ LWLockAcquire(LWLock *lock, LWLockMode mode) void LWLockRelease(LWLock *lock) { - Assert(fake_lwlock_held == lock); - fake_lwlock_held = NULL; + Assert(fake_lwlock_depth > 0); + Assert(fake_lwlock_stack[fake_lwlock_depth - 1] == lock); + fake_lwlock_depth--; + fake_lwlock_stack[fake_lwlock_depth] = NULL; + if (fake_lwlock_depth > 0) { + fake_lwlock_held = fake_lwlock_stack[fake_lwlock_depth - 1]; + fake_lwlock_mode = fake_lwlock_mode_stack[fake_lwlock_depth - 1]; + } else { + fake_lwlock_held = NULL; + fake_lwlock_mode = LW_EXCLUSIVE; + } } bool LWLockHeldByMeInMode(LWLock *lock, LWLockMode mode) { - return fake_lwlock_held == lock && fake_lwlock_mode == mode; + int i; + + for (i = fake_lwlock_depth - 1; i >= 0; i--) + if (fake_lwlock_stack[i] == lock && fake_lwlock_mode_stack[i] == mode) + return true; + return false; } /* ---------- @@ -1280,6 +1318,79 @@ UT_TEST(test_pcm_d3_not_double_x) } } +UT_TEST(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades) +{ + BufferTag tag = make_tag(96); + BufferDesc buf; + bool save = cluster_gcs_block_local_cache; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + cluster_gcs_block_local_cache = true; + fake_local_x_upgrade_result = true; + + cluster_node_id = 2; + cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_S); + + memset(&buf, 0, sizeof(buf)); + buf.tag = tag; + buf.pcm_state = (uint8)PCM_STATE_N; + cluster_node_id = 0; + UT_ASSERT(cluster_pcm_lock_acquire_buffer(&buf, PCM_LOCK_MODE_X)); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(tag), (int)PCM_LOCK_MODE_X); + UT_ASSERT(!cluster_pcm_master_other_live_holder_exists(tag, 0)); + UT_ASSERT(cluster_pcm_master_requester_is_holder(tag, 0, PCM_TRANS_N_TO_X)); + + cluster_gcs_block_local_cache = save; +} + +UT_TEST(test_pcm_dead_node_cleanup_drops_holder_records) +{ + BufferTag stag = make_tag(94); + BufferTag xtag = make_tag(95); + uint32 s_bitmap; + + reset_fake_pcm_runtime(4); + fake_cssd_dead_node = -1; + + /* Dead node 2 was the first S holder, so master_holder points at it. */ + cluster_node_id = 2; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + cluster_node_id = 1; + cluster_pcm_lock_acquire(stag, PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 2); + UT_ASSERT(!cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + + cluster_node_id = 2; + cluster_pcm_lock_acquire(xtag, PCM_LOCK_MODE_X); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_X); + + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)2); + + s_bitmap = cluster_pcm_lock_query_s_holders_bitmap(stag); + UT_ASSERT_EQ((int)(s_bitmap & (1u << 2)), 0); + UT_ASSERT((s_bitmap & (1u << 1)) != 0); + UT_ASSERT_EQ((int)cluster_pcm_lock_query(stag), (int)PCM_LOCK_MODE_S); + UT_ASSERT_EQ(cluster_pcm_master_holder_node_by_tag(stag), 1); + + UT_ASSERT_EQ((int)cluster_pcm_lock_query(xtag), (int)PCM_LOCK_MODE_N); + UT_ASSERT(cluster_pcm_lock_clean_leave_verify_no_leftover(2)); + UT_ASSERT((fake_cv_broadcast_count) >= (1)); + + /* Idempotent: a repeated dead-sweep pass has nothing left to clean. */ + UT_ASSERT_EQ((uint64)cluster_pcm_lock_cleanup_on_node_dead(2), (uint64)0); + + /* spec-4.6a D12 (r2-P1-3) pending-X form: a dead requester's parked X + * intent is cleared by the companion HC124 sweep the same dead-sweep hook + * drives, and the sweep is idempotent too. */ + cluster_pcm_lock_set_pending_x(stag, 2, 1234); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), 2); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)1); + UT_ASSERT_EQ(cluster_pcm_lock_query_pending_x_requester(stag), -1); + UT_ASSERT_EQ((uint64)cluster_pcm_lock_clear_pending_x_for_node(2), (uint64)0); +} + /* spec-5.2a D2 (U1): clean-page X-transfer arm is one-shot. arm(true) sets * the backend-local flag; consume() reads-and-clears it (the acquire path * calls consume() once so the eligibility can never leak into a SUBSEQUENT @@ -1313,7 +1424,7 @@ UT_TEST(test_clean_page_xfer_arm_is_one_shot) int main(void) { - UT_PLAN(39); + UT_PLAN(41); UT_RUN(test_pcm_lock_mode_constant_aliases_match_pcm_state); UT_RUN(test_pcm_lock_transition_count_is_9); UT_RUN(test_pcm_lock_transition_enum_values_are_1_to_9); @@ -1352,6 +1463,8 @@ main(void) UT_RUN(test_pcm_d1_recovering_gate_fail_closed); UT_RUN(test_pcm_d2_rebuild_from_redeclare); UT_RUN(test_pcm_d3_not_double_x); + UT_RUN(test_pcm_acquire_buffer_local_s_nonholder_registers_s_then_upgrades); + UT_RUN(test_pcm_dead_node_cleanup_drops_holder_records); UT_RUN(test_clean_page_xfer_arm_is_one_shot); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm index 3b3a2bb29f..873852fd76 100644 --- a/src/test/perl/PostgreSQL/Test/ClusterQuad.pm +++ b/src/test/perl/PostgreSQL/Test/ClusterQuad.pm @@ -55,6 +55,29 @@ use PostgreSQL::Test::Utils; our $NODES = 4; + +# Relocate an init_from_backup node's copied pg_wal into its shared WAL thread +# directory, matching the layout initdb -X produces for the seed node. +sub _relocate_backup_pg_wal +{ + my ($node, $wal_threads_root, $thread_id) = @_; + my $pgwal = $node->data_dir . '/pg_wal'; + my $wal_thread = "$wal_threads_root/thread_$thread_id"; + + mkdir $wal_thread or die "mkdir $wal_thread: $!"; + opendir(my $dh, $pgwal) or die "opendir $pgwal: $!"; + for my $e (readdir $dh) + { + next if $e eq '.' || $e eq '..'; + rename("$pgwal/$e", "$wal_thread/$e") + or die "rename $pgwal/$e -> $wal_thread/$e: $!"; + } + closedir $dh; + rmdir $pgwal or die "rmdir $pgwal: $!"; + symlink($wal_thread, $pgwal) or die "symlink $pgwal -> $wal_thread: $!"; + return; +} + #----------------------------------------------------------------------- # new_quad($class, $cluster_name, %opts) # @@ -66,6 +89,8 @@ our $NODES = 4; # voting-disk files (QVOTEC reaches quorum OK) # wal_threads_root : 1 -> shared per-thread WAL root # shared_data : 1 -> shared data root (cluster_fs backend) +# shared_catalog : 1 -> t/337-style shared-catalog formation; +# implies shared_data + wal_threads_root #----------------------------------------------------------------------- sub new_quad { @@ -91,6 +116,12 @@ sub new_quad "${cluster_name}_node$i", port => $pg_ports[$i]); } + if ($opts{shared_catalog}) + { + $opts{shared_data} = 1; + $opts{wal_threads_root} = 1; + } + # spec-4.6: strict-mode opt-in (mirror ClusterTriple) -- pre-allocate N # shared voting-disk files so QVOTEC reaches quorum_state=OK and the GES # inbound validation (in_quorum, check 4) accepts cross-node traffic. @@ -127,23 +158,75 @@ sub new_quad if ($opts{shared_data}) { $shared_data_root = PostgreSQL::Test::Utils::tempdir(); + if ($opts{shared_catalog}) + { + mkdir "$shared_data_root/global" + or die "mkdir $shared_data_root/global: $!"; + } + } + + if ($opts{shared_catalog}) + { + my $sc_common = <init(allows_streaming => 1, + extra => [ '-X', "$wal_threads_root/thread_1" ]); + $nodes[0]->start; + $nodes[0]->backup('clusterquad_scb'); + $nodes[0]->stop; + + for my $i (1 .. $NODES - 1) + { + $nodes[$i]->init_from_backup($nodes[0], 'clusterquad_scb'); + _relocate_backup_pg_wal($nodes[$i], $wal_threads_root, $i + 1); + } + + # Seed the shared catalog/controlfile/OID authorities in a single-node era, + # then append the real cluster config below. Last GUC value wins. + $nodes[0]->append_conf('postgresql.conf', $sc_common); + $nodes[0]->append_conf('postgresql.conf', <start; + die "shared_catalog seed did not create catalog authority" + unless -e "$shared_data_root/global/pgrac_catalog_authority"; + $nodes[0]->stop; + } + else + { + my $wal_node_index = 0; + for my $node (@nodes) + { + if (defined $wal_threads_root) + { + my $thread_id = $wal_node_index + 1; + $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); + } + else + { + $node->init; + } + $wal_node_index++; + } } - my $wal_node_index = 0; for my $node (@nodes) { if (defined $wal_threads_root) { - my $thread_id = $wal_node_index + 1; - $node->init(extra => [ '-X', "$wal_threads_root/thread_$thread_id" ]); $node->append_conf('postgresql.conf', "cluster.wal_threads_dir = '$wal_threads_root'\n"); } - else - { - $node->init; - } - $wal_node_index++; if (defined $shared_data_root) { @@ -153,6 +236,15 @@ sub new_quad "cluster.shared_data_dir = '$shared_data_root'\n"); $node->append_conf('postgresql.conf', "cluster.smgr_user_relations = on\n"); + if ($opts{shared_catalog}) + { + $node->append_conf('postgresql.conf', + "cluster.controlfile_shared_authority = on\n"); + $node->append_conf('postgresql.conf', + "cluster.shared_catalog = on\n"); + $node->append_conf('postgresql.conf', + "cluster.merged_recovery = on\n"); + } } # Enable cluster + tier1, same baseline as ClusterTriple (spec-2.2). From e7ca433d8c8b2c5d5d3490b9276a5efa569debc8 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:17:40 +0800 Subject: [PATCH 11/27] ci(nightly): own shard for the t/362 4-node kill self-heal TAP t/358-361 are reserved by parallel lanes (spec-7.2 / S-xid), t/363 by the S-dead lane; occupancy verified against origin branches. The 4-node formation + kill -9 + convergence run gets its own shard for wall-clock isolation. Spec: spec-4.6a-grd-recovery-liveness.md --- .github/workflows/nightly.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5cfa8409b6..ae5dcdf16c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -182,6 +182,11 @@ jobs: # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are + # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead; + # L464 occupancy verified against origin branches 2026-07-08). Own + # shard: 4-node formation + kill + convergence needs the wall clock. + - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } steps: - name: Checkout uses: actions/checkout@v4 From 8853bc7d7e6214ee0b9f791c4fff5c80179f6e12 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:25:22 +0800 Subject: [PATCH 12/27] test(cluster): suppress pre-existing cppcheck constParameter in pi_shadow unit Same inline suppression + reason as the other cluster_unit main() signatures (test_cluster_grd.c precedent); a newer local cppcheck flags it, the CI baseline version does not. --- src/test/cluster_unit/test_cluster_pi_shadow.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/test/cluster_unit/test_cluster_pi_shadow.c b/src/test/cluster_unit/test_cluster_pi_shadow.c index b9e82a4c02..7d075852dd 100644 --- a/src/test/cluster_unit/test_cluster_pi_shadow.c +++ b/src/test/cluster_unit/test_cluster_pi_shadow.c @@ -207,6 +207,9 @@ UT_TEST(test_gate_raw_compare_traps) } int +/* cppcheck-suppress constParameter + * Reason: main() keeps the standard test harness signature used by the + * other cluster_unit binaries; argv is intentionally unused. */ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) { /* stand-in for cluster_pi_shadow_shmem_init over malloc-backed memory */ From 0204d22459dd1736fc4cc3fc201d4ca77d50382a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 18:52:51 +0800 Subject: [PATCH 13/27] =?UTF-8?q?fix(cluster):=20close=20two=20spec-2.29a?= =?UTF-8?q?=20nightly=20regressions=20=E2=80=94=20dump-category=20baseline?= =?UTF-8?q?=20+=20async=20pre-bump=20WAIT=5FEPOCH=20wedge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The ship-gate nightly on the marker-async head surfaced two lane-introduced regressions that the fast-gate matrix does not cover: 1. Dump-category baseline miss: reconfig telemetry added a pg_cluster_state category (55 -> 56), but three cross-node TAPs that assert the total (t/204/205/206) were not updated — only the locally-run L405 subset was. Bump them to 56 with the spec-2.29a note. 2. Async pre-bump WAIT_EPOCH wedge (behavior regression): the async fence / join-Phase-1 / node-remove marker staging bumps the membership epoch at stage-entry but only publishes the reconfig event once the voting-disk marker ACKs — now several LMON ticks later instead of the pre-async single-tick spin. In that window the coordinator's OWN GRD recovery IDLE tick re-captured the already-bumped epoch as its WAIT_EPOCH baseline, so the P0 accept that followed the publish read old == cur and froze the affected shards forever — the spec-4.6a section-0 shape, here triggered by the coordinator on itself with no IC piggyback (reproduced deterministically as t/293 fail-stop remaster + t/326 node-leave shard reopen). Fix: while any pre-bump stage is live, the GRD IDLE tick holds its last stable pre-reconfig baseline instead of re-capturing (cluster_reconfig_has_pending_prebump_stage guard). This restores the pre-async bump->publish atomicity as seen by GRD without touching the epoch bump timing or any user-visible reconfig contract. Adds the cluster_grd_recovery_event_old_epoch accessor + a baseline-hold unit test. Local gates (cassert): unit 158 binaries (incl. baseline-hold test), t/293 + t/325 + t/326 family all green (were red), t/204/205/206 + observability TAPs, PG regress 219/219, clang-format/headers/scn-cmp clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_grd.c | 28 ++++++++++- src/backend/cluster/cluster_reconfig.c | 24 +++++++++ src/include/cluster/cluster_grd.h | 1 + src/include/cluster/cluster_reconfig.h | 6 +++ .../t/204_cluster_visibility_fork_2node.pl | 4 +- .../t/205_cluster_visibility_scn_2node.pl | 4 +- .../t/206_cluster_itl_writable_2node.pl | 2 +- src/test/cluster_unit/test_cluster_grd.c | 50 ++++++++++++++++++- .../test_cluster_grd_starvation.c | 7 +++ 9 files changed, 118 insertions(+), 8 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 33c5b2b7c9..d85321d2b3 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1638,6 +1638,20 @@ cluster_grd_recovery_in_progress(void) return pg_atomic_read_u32(&cluster_grd_state->recovery_state) != (uint32)GRD_RECOVERY_IDLE; } +/* + * spec-2.29a: the pre-reconfig baseline epoch the WAIT_EPOCH gate compares + * against. Exposed for diagnostics + the IDLE baseline-hold unit test (the + * async pre-bump staging window must not let this re-capture a post-bump + * value; see the IDLE branch of cluster_grd_recovery_lmon_tick). + */ +uint64 +cluster_grd_recovery_event_old_epoch(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_old_epoch); +} + /* spec-4.6 D4/D5 — recovery counter bump helpers for out-of-module * call sites (S4 stale mapping in cluster_lock_acquire.c; GCS block * fail-closed guard in cluster_gcs_block.c). */ @@ -2108,9 +2122,19 @@ cluster_grd_recovery_lmon_tick(void) * fires, making old_epoch already == the post-bump epoch and * wedging WAIT_EPOCH forever (cur <= old). The last stable * idle epoch is the reliable "before reconfig" value. + * + * spec-2.29a nightly-regression fix: the coordinator's own + * async pre-bump stages (fail-stop / node-remove / join + * Phase-1) bump the epoch several ticks before publishing the + * event. During that window this IDLE tick must NOT re-capture + * the already-bumped epoch as the baseline, or the P0 accept + * that follows the publish reads old == cur and wedges + * WAIT_EPOCH — the coordinator wedging itself, no IC piggyback + * needed. Hold the last stable pre-reconfig baseline instead. */ - pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, - cluster_epoch_get_current()); + if (!cluster_reconfig_has_pending_prebump_stage()) + pg_atomic_write_u64(&cluster_grd_state->recovery_event_old_epoch, + cluster_epoch_get_current()); return; } diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index cf03d3e8b7..7edc315c52 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -1253,6 +1253,30 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) stage->removal_event_id = 0; } +/* + * spec-2.29a nightly-regression fix — pre-bump staging window guard. + * + * The three pre-bump coordinator paths (fail-stop fence, node-remove, + * join Phase-1) advance the membership epoch at stage-entry but only + * publish the reconfig event once the voting-disk marker ACKs, which now + * spans several LMON ticks instead of the pre-async single-tick spin. + * Between the bump and the publish the GRD recovery IDLE tick would + * re-capture recovery_event_old_epoch as the post-bump value, so its P0 + * accept later reads old == cur and wedges WAIT_EPOCH forever (the + * spec-4.6a section 0 shape, here triggered by the coordinator on ITSELF + * — even in a 2-node cluster with no IC piggyback). While any pre-bump + * stage is live the GRD IDLE tick must hold its last stable (genuine + * pre-reconfig) baseline instead of re-capturing. join Phase-2 COMMITTED + * does not pre-bump, so it is intentionally excluded. + */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return failstop_fence_stage.async.has_staged_event + || node_removed_fence_stage.async.has_staged_event + || join_prepare_stage.async.has_staged_event; +} + static bool cluster_reconfig_submit_fence_stage(ClusterReconfigFenceStage *stage, ClusterMarkerAsyncKind kind, int32 target_node, TimestampTz now) diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index 7015600fec..08d475a9d6 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -660,6 +660,7 @@ typedef struct ClusterGrdRecoveryCounters { } ClusterGrdRecoveryCounters; extern void cluster_grd_recovery_counters_snapshot(ClusterGrdRecoveryCounters *out); +extern uint64 cluster_grd_recovery_event_old_epoch(void); /* spec-2.29a WAIT_EPOCH baseline */ /* spec-4.6 P0#2 — pre-remaster stale-epoch sweep SCOPED to the affected * (dead-master) shards; affected_shards is a PGRAC_GRD_SHARD_COUNT-bit diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index 4ce28e13fd..df42f84d77 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -409,6 +409,12 @@ extern void cluster_reconfig_shmem_register(void); */ extern void cluster_reconfig_get_last_event(ReconfigEvent *out); +/* spec-2.29a: true while a pre-bump coordinator stage (fail-stop fence / + * node-remove / join Phase-1) has bumped the epoch but not yet published its + * reconfig event -- the GRD recovery IDLE tick must hold its baseline instead + * of re-capturing the post-bump epoch (else WAIT_EPOCH wedges). */ +extern bool cluster_reconfig_has_pending_prebump_stage(void); + /* ============================================================ * Coordinator path APIs (Step 2 D2 wiring). diff --git a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl index f12abe0f14..9bc0e1fccf 100644 --- a/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl +++ b/src/test/cluster_tap/t/204_cluster_visibility_fork_2node.pl @@ -325,8 +325,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L12a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L12a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); my $tt_categories = $pair->node0->safe_psql('postgres', q{ SELECT string_agg(c, ',' ORDER BY c) diff --git a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl index 4e65595eff..56d8cd35f9 100644 --- a/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl +++ b/src/test/cluster_tap/t/205_cluster_visibility_scn_2node.pl @@ -296,8 +296,8 @@ sub wait_hint_delta # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', - 'L14a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', + 'L14a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl index 80447a5161..8f4f1c5ce7 100644 --- a/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl +++ b/src/test/cluster_tap/t/206_cluster_itl_writable_2node.pl @@ -287,7 +287,7 @@ # ============================================================ is($pair->node0->safe_psql('postgres', q{SELECT count(DISTINCT category) FROM pg_cluster_state}), - '55', 'L13a pg_cluster_state has 55 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog)'); + '56', 'L13a pg_cluster_state has 56 categories (spec-6.15 adds xid_stripe; spec-6.2 adds smart_fusion; spec-6.12 adds xnode_lever; spec-6.14 adds catalog; spec-2.29a adds reconfig)'); is($pair->node0->safe_psql('postgres', q{SELECT count(*) FROM pg_cluster_state diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2a0b568524..77e7862d6f 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -383,6 +383,16 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a nightly-regression fix: controllable pre-bump-stage flag so the + * GRD IDLE baseline-hold can be unit-driven (default false = pre-fix + * re-capture behavior; existing tests are unaffected). */ +static bool ut_mock_pending_prebump = false; +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return ut_mock_pending_prebump; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op @@ -4044,6 +4054,43 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) } +/* ============================================================ + * spec-2.29a nightly-regression fix — GRD IDLE baseline hold while a + * pre-bump coordinator stage is in flight. + * + * The async fence/join/node-remove marker staging bumps the epoch + * several ticks before publishing. While that window is open the GRD + * IDLE tick must NOT re-capture the post-bump epoch as its WAIT_EPOCH + * baseline, or the P0 accept that follows reads old == cur and wedges. + * ============================================================ */ + +UT_TEST(test_recovery_idle_holds_baseline_during_prebump_stage) +{ + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_pending_prebump = false; + + /* Idle tick with epoch 5 captures baseline 5 (no staging; get_last_event + * returns event_id 0 so the tick stays IDLE and re-captures). */ + ut_mock_epoch = 5; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* A pre-bump stage goes live and the coordinator bumps the epoch to 6 + * before publishing. The IDLE tick must HOLD baseline 5. */ + ut_mock_pending_prebump = true; + ut_mock_epoch = 6; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 5); + + /* Marker ACKs, stage clears; a subsequent idle tick resumes tracking. */ + ut_mock_pending_prebump = false; + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_event_old_epoch(), 6); + + cluster_enabled = false; +} + int /* cppcheck-suppress constParameter * Reason: main() keeps the standard test harness signature used by the @@ -4055,7 +4102,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(80); + UT_PLAN(81); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4158,6 +4205,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); + UT_RUN(test_recovery_idle_holds_baseline_during_prebump_stage); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; diff --git a/src/test/cluster_unit/test_cluster_grd_starvation.c b/src/test/cluster_unit/test_cluster_grd_starvation.c index 44b24f4099..af5f119123 100644 --- a/src/test/cluster_unit/test_cluster_grd_starvation.c +++ b/src/test/cluster_unit/test_cluster_grd_starvation.c @@ -355,6 +355,13 @@ cluster_reconfig_get_last_event(ReconfigEvent *out) memset(out, 0, sizeof(*out)); } +/* spec-2.29a: no pre-bump staging in the starvation fixture (no-op false). */ +bool +cluster_reconfig_has_pending_prebump_stage(void) +{ + return false; +} + /* spec-4.11 D3 stub: cluster_grd.c's WAIT_CLUSTER->IDLE transition consults the * thread-recovery unfreeze gate before P7. These tests drive the GES/GRD * remaster FSM, not online thread recovery, so the gate is out of scope -> no-op From 87f793084b2e78b4c27ccc27e0bf269d2fb14173 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:06:25 +0800 Subject: [PATCH 14/27] =?UTF-8?q?fix(cluster):=20clean-leave=20coherence?= =?UTF-8?q?=20excludes=20the=20leaving=20node=20(regression=20=E2=91=A1b,?= =?UTF-8?q?=20others-dead)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The spec-2.29a marker-async change turned the clean-leave COMMITTING marker into an async, multi-tick wait. In a slow environment (nightly Linux) the leaving node finishes its drain and stops heart-beating inside that wait window, so CSSD marks it SUSPECTED->DEAD and bumps the global dead_generation — and the coherence gate, which compared that SCALAR dead_generation, wrongly read it as a third-party death intruding mid-drain and escalated an otherwise healthy clean leave (t/310 dormant-member legs, t/331 clean_leave x idle). apply_clean_leave_as_coordinator bumps + publishes atomically, so this is NOT the baseline-hold wedge (②a); it is the coherence predicate counting the leaving node's OWN expected DEAD. Fix: compare an others-dead bitmap (the dead set EXCLUDING the leaving node) instead of the scalar dead_generation, at all four coherence sites (CL_COHERENT macro, drive_drain commit-point, staged-ACK re-check, non-staged pre-check). cl_state snapshots the others-dead set at bind; the pure predicate does a bitmap compare (fail-closed on a missing view) and is unit-tested against the reflexive-case matrix. Every third-party incoherence still escalates (epoch move OR a non-leaving node entering the others-dead set); only the leaving node's own expected transition is tolerated. 8.A / CL-I3 protection is unchanged — argued in spec-2.29a §②b with a case matrix. Local gates (cassert): policy unit reflexive-case matrix, cluster_unit 158 binaries, PG regress 219/219, clang-format/headers/scn-cmp clean. NOTE: t/310/331 are timing-sensitive — the false escalation only reproduces in a slow environment, so the fix is verified by nightly, not locally (a local run passes the assertions). A separate pre-existing cassert teardown SIGABRT flake in t/310 (isolated: reproduces without this change) is unrelated. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 122 +++++++++++++----- .../cluster/cluster_clean_leave_policy.c | 32 ++++- src/include/cluster/cluster_clean_leave.h | 22 +++- .../cluster_unit/test_cluster_clean_leave.c | 30 +++-- 4 files changed, 152 insertions(+), 54 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 56b4b7fc62..6ea7873643 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -382,6 +382,11 @@ cluster_clean_leave_check_pending_in_proc_interrupts(void) * disabled survivor MUST reply LEAVE_DRAIN_NAK rather than go silent, else the * leaving node waits for an ACK that never comes and false-times-out. */ + +/* spec-2.29a ②b: forward decls — the bind sites (below) snapshot the + * others-dead set that the coherence gate (defined near drive_drain) compares. */ +static void cl_others_dead_snapshot(int32 leaving, uint8 *out); + static void cl_announce_handler(const ClusterICEnvelope *env, const void *payload) { @@ -510,6 +515,9 @@ cl_announce_handler(const ClusterICEnvelope *env, const void *payload) * fails the commit closed instead of committing on a stale membership view. */ cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes the leaving node) + * the coherence gate compares against. */ + cl_others_dead_snapshot(leaving, cl_state->leave_baseline_others_dead); pg_atomic_write_u32(&cl_state->survivor_acked, 0); pg_atomic_write_u32(&cl_state->commit_ready_received, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); @@ -1430,6 +1438,8 @@ cl_request_body(void) cl_state->leaving_node_id = cluster_node_id; cl_state->leave_epoch = baseline_epoch; cl_state->leave_baseline_dead_gen = cluster_cssd_get_dead_generation(); + /* spec-2.29a ②b: snapshot the others-dead set (excludes self, the leaver). */ + cl_others_dead_snapshot(cluster_node_id, cl_state->leave_baseline_others_dead); cl_state->barrier_deadline_us = (uint64)GetCurrentTimestamp() + (uint64)cluster_clean_leave_drain_timeout_ms * 1000ULL; /* @@ -1563,6 +1573,49 @@ cluster_clean_leave_request(void) * re-checks version coherence (CL-I3: an external epoch bump = a real death * intruded -> escalate) and the mixed-mode NAK (clean abort). */ + +/* + * spec-2.29a ②b: snapshot the CSSD dead set EXCLUDING the leaving node. The + * coherence gate compares this "others-dead" set rather than the scalar global + * dead_generation, so the leaving node's own expected alive->DEAD transition + * (it stops heart-beating once its drain finishes) never falsely escalates the + * leave. A third-party death still changes this set and escalates (CL-I3). + */ +static void +cl_others_dead_snapshot(int32 leaving, uint8 *out /* CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES */) +{ + int i; + + memset(out, 0, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); + for (i = 0; i < CLUSTER_MAX_NODES; i++) { + if (i == leaving) + continue; /* the leaving node's own DEAD is expected, not incoherence */ + if (i >= CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES * 8) + break; + if (cluster_conf_lookup_node(i) == NULL) + continue; + if (cluster_cssd_get_peer_state(i) == CLUSTER_CSSD_PEER_DEAD) + out[i / 8] |= (uint8)(1u << (i % 8)); + } +} + +/* + * spec-2.29a ②b: the coherence gate used by every clean-leave step. Coherent + * iff the epoch has not moved AND no THIRD-PARTY death changed the others-dead + * set since the leave was bound. cl_state->leaving_node_id names the node + * whose own DEAD is excluded. + */ +static bool +cl_coherent(uint64 baseline_epoch) +{ + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + return cluster_clean_leave_version_coherent( + baseline_epoch, cluster_epoch_get_current(), cl_state->leave_baseline_others_dead, + now_others_dead, CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES); +} + void cluster_clean_leave_drive_drain(void) { @@ -1623,16 +1676,14 @@ cluster_clean_leave_drive_drain(void) /* * CL-I3 coherence gate, re-checked before every drain step. Uses the * dead_gen-aware helper, not an epoch-only compare: CSSD increments the - * dead_generation the moment it declares a peer dead, which is STRICTLY - * BEFORE the reconfig coordinator bumps the membership epoch. Checking - * dead_gen too lets us escalate in that earlier window instead of draining - * into a death that has not yet reached the epoch. At commit the guarded - * CAS in apply_clean_leave_as_coordinator is the final authority. + * others-dead set the moment it declares a THIRD-PARTY peer dead, which is + * STRICTLY BEFORE the reconfig coordinator bumps the membership epoch. + * Checking that set too lets us escalate in that earlier window instead of + * draining into a death that has not yet reached the epoch. The leaving + * node's OWN DEAD is excluded (spec-2.29a ②b). At commit the guarded CAS + * in apply_clean_leave_as_coordinator is the final authority. */ -#define CL_COHERENT() \ - cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), \ - cl_state->leave_baseline_dead_gen, \ - cluster_cssd_get_dead_generation()) +#define CL_COHERENT() cl_coherent(baseline_epoch) /* REQUESTED -> QUIESCING: abort local writable backends (53R62). */ if (!CL_COHERENT()) { @@ -1726,14 +1777,22 @@ cl_leaving_barrier_tick(void) * leave is active (cluster_clean_leave_in_progress gates drive_joins + * commit_member), and the leave does not start while a join is pending * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * dead_gen-unchanged bump during a leave can only be the leave's own commit. + * bump with an unchanged others-dead set during a leave can only be the + * leave's own commit (spec-2.29a ②b: the leaving node's own DEAD is excluded + * so its expected heartbeat stop no longer looks like a third-party death). */ if (cluster_epoch_get_current() > baseline_epoch) { - if (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen) { + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + if (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0) { /* - * Commit point observed (our clean-leave epoch was published; no real - * death intruded). The leave can no longer be un-committed, so from - * here the barrier deadline must NOT escalate (Hardening v1.0.1 P1-1). + * Commit point observed (our clean-leave epoch was published; no + * third-party death intruded). The leave can no longer be + * un-committed, so from here the barrier deadline must NOT escalate + * (Hardening v1.0.1 P1-1). */ pg_atomic_write_u32(&cl_state->commit_point_observed, 1); LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); @@ -1817,11 +1876,9 @@ cl_coordinator_commit(int32 leaving) * applies. On failure the leave does not commit and the leaving * node escalates to fail-stop (identical to the pre-check path). */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { + if (!cl_coherent(baseline_epoch)) { ereport(LOG, - (errmsg("cluster clean-leave: version moved (epoch or dead_generation) " + (errmsg("cluster clean-leave: version moved (epoch or third-party death) " "across the COMMITTING marker wait for node %d; not committing " "(escalate to fail-stop, CL-I3)", leaving))); @@ -1862,22 +1919,21 @@ cl_coordinator_commit(int32 leaving) * guarded CAS inside apply_clean_leave_as_coordinator below, which closes the * check-then-bump TOCTOU at >=3 nodes. */ /* - * CL-I3 commit-handoff coherence (epoch AND dead_gen): refuse to commit if a - * real death intruded since this survivor started tracking the leave — either - * the death's fail-stop already bumped the epoch, OR (the >=3-node window, - * Hardening v1.0.1 P1-2) CSSD bumped its dead_generation but the fail-stop - * epoch has NOT yet advanced, which an epoch-only check (and the guarded CAS - * below) would miss and commit on a stale membership view. Uses the same - * unit-tested version_coherent helper as drive_drain so the dead_gen-aware - * coherence holds through the commit handoff; the leaving node then observes - * the eventual foreign event and escalates. + * CL-I3 commit-handoff coherence (epoch AND others-dead): refuse to commit if + * a THIRD-PARTY death intruded since this survivor started tracking the leave + * — either the death's fail-stop already bumped the epoch, OR (the >=3-node + * window, Hardening v1.0.1 P1-2) CSSD added it to the others-dead set but the + * fail-stop epoch has NOT yet advanced, which an epoch-only check (and the + * guarded CAS below) would miss and commit on a stale membership view. Uses + * the same others-dead coherence helper as drive_drain (spec-2.29a ②b: the + * leaving node's own expected DEAD is excluded so it never falsely escalates); + * the leaving node then observes the eventual foreign event and escalates. */ - if (!cluster_clean_leave_version_coherent(baseline_epoch, cluster_epoch_get_current(), - cl_state->leave_baseline_dead_gen, - cluster_cssd_get_dead_generation())) { - ereport(LOG, (errmsg("cluster clean-leave: version moved (epoch or dead_generation) before " - "committing node %d; not committing (escalate to fail-stop, CL-I3)", - leaving))); + if (!cl_coherent(baseline_epoch)) { + ereport(LOG, + (errmsg("cluster clean-leave: version moved (epoch or third-party death) before " + "committing node %d; not committing (escalate to fail-stop, CL-I3)", + leaving))); return; } diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 0a4eeebd04..095f4b082b 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -136,17 +136,35 @@ cluster_clean_leave_should_abort_writable(bool in_transaction, bool has_top_xid) /* * cluster_clean_leave_version_coherent -- the bound leave is still coherent - * only if neither the cluster epoch nor the CSSD dead_generation moved since - * the leave was bound. Any external bump (a real death intruding mid-drain) - * makes it incoherent → caller must ABORTED_ESCALATE (never complete a clean - * leave on a stale version, which would let a destructive sweep run on a newer - * version and double-grant). + * only if neither the cluster epoch nor the OTHERS-dead set (every dead node + * except the leaving node itself) moved since the leave was bound. Any + * external membership change — a third-party death intruding mid-drain, or a + * third-party fail-stop that already bumped the epoch — makes it incoherent → + * caller must ABORTED_ESCALATE (never complete a clean leave on a stale + * membership view, which would let a destructive sweep run on a newer view + * and double-grant, CL-I3). + * + * spec-2.29a ②b fix: the pre-fix predicate compared the SCALAR global CSSD + * dead_generation, which also counts the leaving node's OWN alive→DEAD + * transition (it stops heart-beating once its drain finishes — the EXPECTED + * terminal state of a clean leave). Under the async COMMITTING marker that + * spans several LMON ticks, a slow environment lets that transition land + * inside the wait window and the scalar check wrongly escalated an otherwise + * healthy leave. Comparing the others-dead bitmap (which excludes the + * leaving node) removes exactly that false positive while keeping every + * third-party incoherence escalation (see the reflexive-case matrix in + * spec-2.29a §②b 8.A argument). */ bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen) + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes) { - return bound_epoch == current_epoch && bound_dead_gen == current_dead_gen; + if (bound_epoch != current_epoch) + return false; + if (bound_others_dead == NULL || current_others_dead == NULL) + return false; /* fail-closed: cannot prove coherence without both views */ + return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 1330a40490..3d3a3801e4 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -151,11 +151,17 @@ typedef struct ClusterLeaveState { int32 leaving_node_id; /* -1 if none */ uint32 _pad0; uint64 leave_epoch; /* epoch this leave is bound to (CL-I3 immutable) */ - uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound; - * an unchanged value at the epoch bump means OUR - * clean-leave committed (no real death intruded) */ + uint64 leave_baseline_dead_gen; /* CSSD dead_generation when the leave was bound + * (retained for observability; the coherence gate + * now uses leave_baseline_others_dead, spec-2.29a + * ②b — the scalar counted the leaving node's own + * expected DEAD and falsely escalated) */ uint64 barrier_deadline_us; /* fail-closed deadline (drain_timeout_ms) */ uint8 ack_bitmap[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; /* survivor acks */ + /* spec-2.29a ②b: CSSD dead set (EXCLUDING the leaving node) snapshotted when + * the leave was bound; the coherence gate escalates only if a THIRD-PARTY + * death changed this set mid-drain. */ + uint8 leave_baseline_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; pg_atomic_uint64 ges_drained_count; /* shards/grants drained (observability) */ pg_atomic_uint64 gcs_flushed_count; /* dirty/X pages flushed */ pg_atomic_uint64 shards_remastered; /* shards moved off leaving node */ @@ -337,10 +343,14 @@ extern bool cluster_clean_leave_should_abort_writable(bool in_transaction, bool /* version-coherent leave (U3 / CL-I3 / L235): the leave is still coherent only * if neither the cluster epoch nor the CSSD dead_generation moved since the - * leave was bound; any external bump (a real death intruding) => not coherent - * => the caller must ABORTED_ESCALATE. */ + * leave was bound; any external membership change (a third-party death, or a + * third-party fail-stop that already bumped the epoch) => not coherent => the + * caller must ABORTED_ESCALATE. spec-2.29a ②b: the dead set compared here + * EXCLUDES the leaving node itself (others-dead bitmap), so the leaving node's + * own expected alive→DEAD transition never falsely escalates the leave. */ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, - uint64 bound_dead_gen, uint64 current_dead_gen); + const uint8 *bound_others_dead, + const uint8 *current_others_dead, int nbytes); /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index b703798b38..24a7ce43a4 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -159,14 +159,28 @@ UT_TEST(test_phase_fsm) UT_TEST(test_version_coherent) { - /* nothing moved -> coherent */ - UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, 3, 3)); - /* epoch bumped by an external death -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 3)); - /* dead_generation moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, 3, 4)); - /* both moved -> incoherent */ - UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, 3, 4)); + /* spec-2.29a ②b: coherence = epoch unchanged AND the others-dead bitmap + * (dead set EXCLUDING the leaving node) unchanged. The reflexive-case + * matrix from the spec §②b 8.A argument, exercised on the pure predicate. */ + uint8 od_none[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + uint8 od_third[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES] = { 0 }; + const int n = CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES; + + od_third[0] = 0x04; /* a THIRD-PARTY node (e.g. node 2) became DEAD */ + + /* (i) nothing moved (leaving node's own DEAD is already excluded upstream, + * so its transition does not appear here) -> coherent */ + UT_ASSERT(cluster_clean_leave_version_coherent(7, 7, od_none, od_none, n)); + /* (iii) epoch bumped by an external fail-stop -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_none, n)); + /* (ii) a third-party death entered the others-dead set, epoch not yet + * bumped (the P1-b window) -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, od_third, n)); + /* (iv) both moved -> incoherent */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 8, od_none, od_third, n)); + /* fail-closed on a missing view */ + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, NULL, od_none, n)); + UT_ASSERT(!cluster_clean_leave_version_coherent(7, 7, od_none, NULL, n)); } From 55dd3f34057ddeec164f1fcff5ee47c9e133b909 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:37 +0800 Subject: [PATCH 15/27] fix(cluster): converge REDECLARE_DONE on (episode_epoch, dead_bitmap_hash) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The P6 all-done gate and the WAIT_EPOCH coordinator witness keyed the cross-node DONE on event_id, but event_id folds the sender-local cssd_dead_generation, which drifts with each survivor's private flap observation history and never converges across nodes: any flap asymmetry made the survivors compute different ids for the same episode, drop each other's DONEs, and wedge P6 forever — nondeterministically reintroducing the permanent-freeze shape this branch exists to remove. The quorum-accepted dead SET is what actually converges, so the DONE payload now carries a hash over the dead bitmap alone (same kernel as the event_id hash, riding the same request-id field pair — no envelope change), stamped per episode at P0 accept. Accounting, the P6 gate and the witness compare against the stamp; event_id stays a purely local accept-dedup scope. A late DONE from a previous episode still cannot back a new one: a coordinator re-election changes the dead set (hash mismatch) and a same-node re-death rides a higher epoch through the interposed JOIN bump (epoch conjunct). Pre-accept frames now mismatch the previous stamp and are dropped; senders re-announce every tick, so accounting always lands after the accept snapshot — closing the first-event pre-accept window as a side effect. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R2/R4) --- src/backend/cluster/cluster_debug.c | 4 +- src/backend/cluster/cluster_ges.c | 4 +- src/backend/cluster/cluster_grd.c | 146 ++++++++++++++------- src/include/cluster/cluster_grd.h | 28 +++- src/test/cluster_unit/test_cluster_debug.c | 2 +- src/test/cluster_unit/test_cluster_grd.c | 124 ++++++++++++----- 6 files changed, 216 insertions(+), 92 deletions(-) diff --git a/src/backend/cluster/cluster_debug.c b/src/backend/cluster/cluster_debug.c index e19b3e98e8..1e50f2b3a9 100644 --- a/src/backend/cluster/cluster_debug.c +++ b/src/backend/cluster/cluster_debug.c @@ -1211,8 +1211,8 @@ dump_grd_recovery(ReturnSetInfo *rsinfo) fmt_int32((int32)cluster_grd_recovery_event_coordinator())); emit_row(rsinfo, "grd_recovery", "done_self_epoch", fmt_int64((int64)cluster_grd_recovery_done_epoch_for(cluster_node_id))); - emit_row(rsinfo, "grd_recovery", "done_self_event_id", - fmt_int64((int64)cluster_grd_recovery_done_event_id_for(cluster_node_id))); + emit_row(rsinfo, "grd_recovery", "done_self_bitmap_hash", + fmt_int64((int64)cluster_grd_recovery_done_bitmap_hash_for(cluster_node_id))); emit_row(rsinfo, "grd_recovery", "block_redeclare_cursor", fmt_int32((int32)cluster_grd_recovery_block_redeclare_cursor())); emit_row(rsinfo, "grd_recovery", "block_redeclare_epoch", diff --git a/src/backend/cluster/cluster_ges.c b/src/backend/cluster/cluster_ges.c index 547f0e5aac..f68e407239 100644 --- a/src/backend/cluster/cluster_ges.c +++ b/src/backend/cluster/cluster_ges.c @@ -550,7 +550,9 @@ cluster_ges_request_handler(const ClusterICEnvelope *env, const void *payload) /* spec-4.6 P0#3 cluster gate — fire-and-forget barrier announcement: * record and return (no work queue, no reply, no dedup; the sender - * re-announces each tick, the receiver write is a monotonic max). */ + * re-announces each tick, the receiver write is a monotonic max). + * Amendment v1.2 (R2): the request-id field pair carries the sender's + * dead_bitmap hash (composite convergence key), not an event_id. */ if (req->opcode == GES_REQ_OPCODE_REDECLARE_DONE) { cluster_grd_recovery_mark_peer_done((int32)env->source_node_id, holder_epoch, ges_request_holder_request_id(req)); diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index 8977739433..ae87b62dfa 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -737,8 +737,9 @@ cluster_grd_shmem_init(void) /* spec-4.6 P0#3 cluster gate — per-node barrier-done epochs. */ for (i = 0; i < CLUSTER_MAX_NODES; i++) { pg_atomic_init_u64(&cluster_grd_state->recovery_done_epoch[i], 0); - pg_atomic_init_u64(&cluster_grd_state->recovery_done_event_id[i], 0); + pg_atomic_init_u64(&cluster_grd_state->recovery_done_bitmap_hash[i], 0); } + pg_atomic_init_u64(&cluster_grd_state->recovery_event_bitmap_hash, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_started_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_done_count, 0); pg_atomic_init_u64(&cluster_grd_state->remaster_failed_count, 0); @@ -1703,11 +1704,36 @@ cluster_grd_recovery_done_epoch_for(int32 node) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node) +cluster_grd_recovery_done_bitmap_hash_for(int32 node) { if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return 0; - return pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[node]); + return pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[node]); +} + +uint64 +cluster_grd_recovery_event_bitmap_hash_value(void) +{ + if (cluster_grd_state == NULL) + return 0; + return pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); +} + +/* + * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). + * + * The cross-node half of the REDECLARE_DONE convergence key: a hash over + * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — + * the generation is per-instance observation history and does NOT converge + * across survivors, which is exactly why event_id cannot key the P6 gate. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, + * so the composite key degrades to the epoch-only gate there. + */ +uint64 +cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) +{ + return hash_bytes_extended(dead_bitmap, CLUSTER_RECONFIG_DEAD_BITMAP_BYTES, 0); } bool @@ -2051,7 +2077,11 @@ static void grd_recovery_broadcast_done(uint64 epoch) { GesRequestPayload req; - uint64 event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + /* Amendment v1.2 (R2): the DONE payload carries the sender's accepted + * dead-bitmap hash (cross-node convergence key), riding the request-id + * field pair — NOT the sender-local event_id (its dead_generation fold + * diverges across survivors' flap observation histories). */ + uint64 bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); uint64 master_gen = cluster_lms_get_shard_master_generation(); int i; @@ -2061,8 +2091,8 @@ grd_recovery_broadcast_done(uint64 epoch) req.holder_procno = 0; req.holder_cluster_epoch_lo = (uint32)(epoch & 0xffffffffu); req.holder_cluster_epoch_hi = (uint32)(epoch >> 32); - req.holder_request_id_lo = (uint32)(event_id & 0xffffffffu); - req.holder_request_id_hi = (uint32)(event_id >> 32); + req.holder_request_id_lo = (uint32)(bitmap_hash & 0xffffffffu); + req.holder_request_id_hi = (uint32)(bitmap_hash >> 32); req.shard_master_generation_lo = (uint32)(master_gen & 0xffffffffu); req.shard_master_generation_hi = (uint32)(master_gen >> 32); @@ -2081,21 +2111,29 @@ grd_recovery_broadcast_done(uint64 epoch) /* REDECLARE_DONE receiver (cluster_ges.c inbound handler). */ void -cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) +cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash) { - uint64 current_event_id; + uint64 episode_bitmap_hash; if (cluster_grd_state == NULL || node < 0 || node >= CLUSTER_MAX_NODES) return; /* - * spec-4.6a D4-v2: REDECLARE_DONE is an event-scoped proof. A stale DONE - * from a previous episode can carry the same epoch when cur==old, so epoch - * monotonicity alone is not enough. Drop messages for any event other than - * the one this LMON has accepted; senders re-announce every tick. + * spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE converges on the COMPOSITE + * key (episode_epoch, dead_bitmap_hash). A stale DONE from a previous + * episode can carry the same epoch when cur==old, so epoch monotonicity + * alone is not enough — but the previous episode's dead SET necessarily + * differs (a coordinator re-election adds the old coordinator to it; a + * re-death of the same node rides a higher epoch via the interposed JOIN + * bump), so the bitmap hash is the ABA guard. Flap-history drift changes + * only the per-instance dead_generation, never the quorum dead SET, so + * DONEs for the same episode always match here (the R2 wedge fix). + * Pre-accept frames mismatch the previous episode's stamp and are dropped; + * senders re-announce every tick, so accounting lands after our P0 accept + * (which also closes the R4 pre-accept window by construction). */ - current_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); - if (event_id == 0 || (current_event_id != 0 && event_id != current_event_id)) + episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); + if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) return; /* spec-4.6a §2.3: monotonic-max accounting. Under strictly-monotonic @@ -2107,7 +2145,7 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id) if (epoch > prev) pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[node], epoch); } - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[node], event_id); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[node], dead_bitmap_hash); } /* @@ -2175,13 +2213,13 @@ grd_recovery_format_missing_survivors(const uint64 *dead, uint64 episode_epoch, continue; done_epoch = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]); if (done_epoch >= episode_epoch - && pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + && pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) continue; grd_recovery_appendf(buf, buflen, &off, - "%s%d(done=" UINT64_FORMAT "/event=" UINT64_FORMAT ")", any ? "," : "", - i, done_epoch, - pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i])); + "%s%d(done=" UINT64_FORMAT "/bitmap_hash=" UINT64_FORMAT ")", + any ? "," : "", i, done_epoch, + pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i])); any = true; } if (!any) @@ -2433,6 +2471,12 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } + /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node + * half. Hash over the accepted dead bitmap alone — every survivor that + * accepted the same quorum dead SET computes the same value regardless + * of its private dead_generation observation history. */ + pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, + cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2510,8 +2554,8 @@ cluster_grd_recovery_lmon_tick(void) TimestampTz next_deadline; uint32 coordinator; uint64 coord_done; - uint64 coord_done_event_id; - uint64 accepted_event_id; + uint64 coord_done_bitmap_hash; + uint64 episode_bitmap_hash; uint64 coord_done_at_accept; wait_deadline = (TimestampTz)pg_atomic_read_u64( @@ -2533,29 +2577,33 @@ cluster_grd_recovery_lmon_tick(void) } coordinator = pg_atomic_read_u32(&cluster_grd_state->recovery_event_coordinator); - accepted_event_id = pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id); + episode_bitmap_hash + = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); coord_done = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[coordinator]) : 0; - coord_done_event_id + coord_done_bitmap_hash = coordinator < CLUSTER_MAX_NODES ? pg_atomic_read_u64( - &cluster_grd_state->recovery_done_event_id[coordinator]) + &cluster_grd_state->recovery_done_bitmap_hash[coordinator]) : 0; coord_done_at_accept = pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch_at_accept); /* - * spec-4.6a §2.3 witness (all three conjuncts of the frozen - * text): the coordinator's DONE must (a) be for THIS event, - * (b) name exactly cur as the episode epoch, and (c) have been + * spec-4.6a §2.3 witness, composite-key form (Amendment v1.2 + * R2): the coordinator's DONE must (a) name this episode's + * quorum dead SET (bitmap hash — the cross-node-consistent + * key; the sender-local event_id folds the per-instance + * dead_generation and diverges under flap asymmetry), (b) + * name exactly cur as the episode epoch, and (c) have been * accounted after our P0 accept snapshot. Under strictly * monotonic per-event epochs (c) is implied by (a)+(b) — any * pre-accept residue carries a lower epoch — but it stays as a * belt-and-suspenders guard against accounting bugs. */ if (cur_epoch == old_epoch && coord_done == cur_epoch - && coord_done_event_id == accepted_event_id + && coord_done_bitmap_hash == episode_bitmap_hash && coord_done > coord_done_at_accept) { pg_atomic_fetch_add_u64(&cluster_grd_state->wait_epoch_escape_count, 1); ereport(WARNING, @@ -2563,25 +2611,25 @@ cluster_grd_recovery_lmon_tick(void) errmsg("cluster GRD recovery WAIT_EPOCH used coordinator witness for " "equal-epoch progress"), errdetail("coordinator=%u, epoch=" UINT64_FORMAT - ", event_id=" UINT64_FORMAT, - coordinator, cur_epoch, accepted_event_id))); + ", dead_bitmap_hash=" UINT64_FORMAT, + coordinator, cur_epoch, episode_bitmap_hash))); } else { next_deadline = TimestampTzPlusMilliseconds(now, cluster_grd_rebuild_timeout_ms); pg_atomic_write_u64(&cluster_grd_state->recovery_barrier_deadline, (uint64)next_deadline); - ereport( - WARNING, - (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), - errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " - "affected shards stay frozen"), - errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT - ", coordinator=%u, coordinator_done=" UINT64_FORMAT - ", coordinator_done_event_id=" UINT64_FORMAT - ", accepted_event_id=" UINT64_FORMAT - ", coordinator_done_at_accept=" UINT64_FORMAT, - cur_epoch, old_epoch, coordinator, coord_done, - coord_done_event_id, accepted_event_id, coord_done_at_accept))); + ereport(WARNING, + (errcode(ERRCODE_CLUSTER_GRD_SHARD_REMASTERING), + errmsg("cluster GRD recovery WAIT_EPOCH has no post-bump proof; " + "affected shards stay frozen"), + errdetail("cur_epoch=" UINT64_FORMAT ", old_epoch=" UINT64_FORMAT + ", coordinator=%u, coordinator_done=" UINT64_FORMAT + ", coordinator_done_bitmap_hash=" UINT64_FORMAT + ", episode_bitmap_hash=" UINT64_FORMAT + ", coordinator_done_at_accept=" UINT64_FORMAT, + cur_epoch, old_epoch, coordinator, coord_done, + coord_done_bitmap_hash, episode_bitmap_hash, + coord_done_at_accept))); return; } } @@ -2728,8 +2776,8 @@ cluster_grd_recovery_lmon_tick(void) */ pg_atomic_write_u64(&cluster_grd_state->recovery_done_epoch[cluster_node_id], episode_epoch); - pg_atomic_write_u64(&cluster_grd_state->recovery_done_event_id[cluster_node_id], - pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)); + pg_atomic_write_u64(&cluster_grd_state->recovery_done_bitmap_hash[cluster_node_id], + pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)); grd_recovery_broadcast_done(episode_epoch); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), cluster_grd_rebuild_timeout_ms); @@ -2864,9 +2912,13 @@ cluster_grd_recovery_lmon_tick(void) */ if (is_join && join_fence_is_recipient_for(i, episode_epoch)) continue; + /* Amendment v1.2 (R2): composite key — the peer's DONE must name + * this episode's epoch AND the same quorum dead SET. Keying on + * event_id here wedged the gate whenever survivors' private + * dead_generation histories diverged (flap asymmetry). */ if (pg_atomic_read_u64(&cluster_grd_state->recovery_done_epoch[i]) < episode_epoch - || pg_atomic_read_u64(&cluster_grd_state->recovery_done_event_id[i]) - != pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) { + || pg_atomic_read_u64(&cluster_grd_state->recovery_done_bitmap_hash[i]) + != pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { all_done = false; break; } diff --git a/src/include/cluster/cluster_grd.h b/src/include/cluster/cluster_grd.h index a098d53ed1..c6fa42ecaf 100644 --- a/src/include/cluster/cluster_grd.h +++ b/src/include/cluster/cluster_grd.h @@ -294,9 +294,21 @@ typedef struct ClusterGrdShared { /* spec-4.6 P0#3 cluster gate — per-node epoch for which that node * last announced "local rebind barrier complete" (REDECLARE_DONE). - * P6 requires done_epoch[s] >= current epoch for EVERY survivor. */ + * P6 requires done_epoch[s] >= current epoch for EVERY survivor. + * + * spec-4.6a Amendment v1.2 (R2): the cross-node convergence key is the + * COMPOSITE (episode_epoch, dead_bitmap_hash). event_id is NOT globally + * consistent (it folds the per-instance cssd_dead_generation, which + * drifts with each node's own flap observation history), so keying the + * P6 gate on it wedges the cluster whenever two survivors observed a + * different flap history. The quorum-accepted dead SET is what actually + * converges; its hash rides the DONE payload instead. event_id stays a + * purely LOCAL ABA scope (P0 accept dedup). */ pg_atomic_uint64 recovery_done_epoch[CLUSTER_MAX_NODES]; - pg_atomic_uint64 recovery_done_event_id[CLUSTER_MAX_NODES]; + pg_atomic_uint64 recovery_done_bitmap_hash[CLUSTER_MAX_NODES]; + /* hash of THIS episode's accepted dead_bitmap (LMON writes at P0 accept; + * empty-bitmap hash for JOIN episodes, identical on every node). */ + pg_atomic_uint64 recovery_event_bitmap_hash; /* spec-4.6 D5 — 13 grd_recovery counters (dump category * 'grd_recovery'; each has a t/249 leg). Incremented along @@ -624,7 +636,11 @@ extern uint64 cluster_grd_recovery_event_old_epoch(void); extern uint64 cluster_grd_recovery_episode_epoch_value(void); extern uint32 cluster_grd_recovery_event_coordinator(void); extern uint64 cluster_grd_recovery_done_epoch_for(int32 node); -extern uint64 cluster_grd_recovery_done_event_id_for(int32 node); +extern uint64 cluster_grd_recovery_done_bitmap_hash_for(int32 node); +extern uint64 cluster_grd_recovery_event_bitmap_hash_value(void); +/* Amendment v1.2 (R2): the cross-node DONE key — hash over the dead bitmap + * ALONE (no dead_generation fold; same kernel as the event_id hash). */ +extern uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap); extern int cluster_grd_recovery_block_redeclare_cursor(void); extern uint64 cluster_grd_recovery_block_redeclare_epoch(void); extern bool cluster_grd_recovery_block_redeclare_done(void); @@ -644,8 +660,10 @@ extern bool grd_block_redeclare_scan_complete(uint64 episode_epoch); /* spec-4.6 P0#3 cluster gate — REDECLARE_DONE receiver (cluster_ges.c * inbound handler): record that `node` completed its local rebind - * barrier for `epoch`. */ -extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 event_id); + * barrier for `epoch`. Amendment v1.2 (R2): the third argument is the + * sender's dead_bitmap hash (the cross-node half of the composite + * convergence key), NOT the sender-local event_id. */ +extern void cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap_hash); /* spec-4.6 D4/D5 — recovery counter bumps for out-of-module call sites. */ extern void cluster_grd_inc_stale_request_drop(void); diff --git a/src/test/cluster_unit/test_cluster_debug.c b/src/test/cluster_unit/test_cluster_debug.c index 8b64c01755..8b741f7c42 100644 --- a/src/test/cluster_unit/test_cluster_debug.c +++ b/src/test/cluster_unit/test_cluster_debug.c @@ -3350,7 +3350,7 @@ cluster_grd_recovery_done_epoch_for(int32 node pg_attribute_unused()) } uint64 -cluster_grd_recovery_done_event_id_for(int32 node pg_attribute_unused()) +cluster_grd_recovery_done_bitmap_hash_for(int32 node pg_attribute_unused()) { return 0; } diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index c7f805459d..2e4b813e17 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -3748,6 +3748,30 @@ ut_jr_setup_3node(void) cluster_grd_master_map_init(); } +/* spec-4.6a Amendment v1.2 (R2): REDECLARE_DONE accounting converges on the + * composite key (episode_epoch, dead_bitmap_hash), so a test that feeds DONEs + * must first drive a P0 accept to stamp the episode's bitmap hash. Accept an + * empty-dead-bitmap FAIL event (hash identical to a JOIN episode's, no fence + * side effects; the FSM parks in WAIT_EPOCH awaiting a strict bump, which the + * fence assertions never touch). Returns the stamped hash for the callers' + * mark_peer_done payloads. */ +static uint64 +ut_jr_stamp_episode_bitmap_hash(uint64 event_id) +{ + bool saved_enabled = cluster_enabled; + + cluster_enabled = true; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle tick: baseline capture */ + ut_mock_last_event.event_id = event_id; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + cluster_grd_recovery_lmon_tick(); /* P0 accept stamps the bitmap hash */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_enabled = saved_enabled; + return cluster_grd_recovery_event_bitmap_hash_value(); +} + /* U1 — recompute moves the joiner's home shards back from the survivor. */ UT_TEST(test_jr_u1_recompute_moves_joiner_home) { @@ -3937,9 +3961,12 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(901); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* the rejoiner's home block */ @@ -3950,15 +3977,15 @@ UT_TEST(test_jr_u13_view_rebuilt_all_members_barrier) /* HARDENING v1.1 — the joiner (node 1) announcing its OWN trivial barrier * must NOT lift the fence: survivors 0 and 2 have not re-declared yet. */ - cluster_grd_recovery_mark_peer_done(1, 10, 1); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* One survivor done is still not enough. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* All members done → view rebuilt → fence lifts. */ - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -3969,10 +3996,13 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) int n1[1] = { 1 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(902); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); @@ -3981,15 +4011,15 @@ UT_TEST(test_jr_u14_fence_arm_monotonic_and_scope) ut_mock_epoch = 5; cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; - cluster_grd_recovery_mark_peer_done(0, 5, 1); - cluster_grd_recovery_mark_peer_done(1, 5, 1); - cluster_grd_recovery_mark_peer_done(2, 5, 1); + cluster_grd_recovery_mark_peer_done(0, 5, done_hash); + cluster_grd_recovery_mark_peer_done(1, 5, done_hash); + cluster_grd_recovery_mark_peer_done(2, 5, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* 5 < fence epoch 10 */ /* Done at the real fence epoch lifts it. */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } @@ -4002,10 +4032,12 @@ UT_TEST(test_jr_u15_master_side_gate_decision) int n1[1] = { 1 }; BufferTag tag; bool deny; + uint64 done_hash; memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(903); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; @@ -4014,9 +4046,9 @@ UT_TEST(test_jr_u15_master_side_gate_decision) deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(deny); - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); deny = cluster_grd_join_remaster_active_for_shard(tag) && !cluster_grd_block_view_rebuilt(tag); UT_ASSERT(!deny); } @@ -4059,17 +4091,20 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) int n2[1] = { 2 }; BufferTag tag; + uint64 done_hash; + memset(&tag, 0, sizeof(tag)); ut_jr_setup_3node(); /* declared {0,1,2}, all members */ /* --- Episode 1: node 1 rejoins and completes (all survivors re-declare). --- */ ut_mock_epoch = 10; + done_hash = ut_jr_stamp_episode_bitmap_hash(904); /* v1.2 R2 composite key */ ut_jr_build_bitmap(join1, n1, 1); cluster_grd_arm_join_pcm_fence(join1); ut_mock_static_master = 1; /* a node-1-home block */ - cluster_grd_recovery_mark_peer_done(0, 10, 1); - cluster_grd_recovery_mark_peer_done(1, 10, 1); - cluster_grd_recovery_mark_peer_done(2, 10, 1); + cluster_grd_recovery_mark_peer_done(0, 10, done_hash); + cluster_grd_recovery_mark_peer_done(1, 10, done_hash); + cluster_grd_recovery_mark_peer_done(2, 10, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); /* episode 1 converged */ /* --- Episode 2: node 2 rejoins. node 1 is now a steady survivor whose held @@ -4082,29 +4117,35 @@ UT_TEST(test_jr_u17_stale_recipient_not_excluded_next_episode) /* Only the OTHER survivor (node 0) has re-declared for episode 2; node 1 has * NOT. node 1's stale episode-1 fence bit must NOT exclude it from the * barrier — it must still gate the fence lift. */ - cluster_grd_recovery_mark_peer_done(0, 20, 1); + cluster_grd_recovery_mark_peer_done(0, 20, done_hash); UT_ASSERT(!cluster_grd_block_view_rebuilt(tag)); /* must still wait for node 1 */ /* Once node 1 re-declares for episode 2 the barrier converges and lifts. */ - cluster_grd_recovery_mark_peer_done(1, 20, 1); + cluster_grd_recovery_mark_peer_done(1, 20, done_hash); UT_ASSERT(cluster_grd_block_view_rebuilt(tag)); } /* ============================================================ - * spec-4.6a r2-P1-1 — event-scoped REDECLARE_DONE proof. + * spec-4.6a Amendment v1.2 (R2) — composite-key REDECLARE_DONE proof. * - * A stale DONE (previous event) must be dropped at the accounting - * layer and must never satisfy the WAIT_EPOCH coordinator witness; - * a current-event DONE at exactly cur, accounted after the P0 - * accept snapshot, is the only equal-epoch escape (proof-carrying, - * never timeout-only). + * The cross-node convergence key is (episode_epoch, dead_bitmap_hash). + * A DONE naming a DIFFERENT dead set (the old-event ABA shape) must be + * dropped at the accounting layer and never satisfy the WAIT_EPOCH + * coordinator witness. A DONE naming the SAME dead set must be + * accounted even though the sender's local event_id differs (the + * dead_generation-drift shape that wedged the event_id key: R2), and + * then satisfies the witness — the only equal-epoch escape + * (proof-carrying, never timeout-only). * ============================================================ */ -UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) +UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) { ClusterGrdRecoveryCounters before; ClusterGrdRecoveryCounters after; + uint8 old_dead[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 episode_hash; + uint64 old_event_hash; ut_jr_setup_3node(); cluster_enabled = true; @@ -4117,27 +4158,38 @@ UT_TEST(test_recovery_stale_event_done_rejected_and_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); cluster_grd_recovery_lmon_tick(); - /* Stage the fail-stop event: dead node 2, coordinator 0, event 77. */ + /* Stage the fail-stop event: dead node 2, coordinator 0. The local + * event_id (77) folds this node's own dead_generation; a peer that saw a + * different flap history computes a DIFFERENT id for the same episode. */ ut_mock_last_event.event_id = 77; ut_mock_last_event.coordinator_node_id = 0; ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; ut_mock_last_event.dead_bitmap[0] = 0x04; cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH, no proof yet */ UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 77); - - /* Stale-event DONE (event 76) is dropped: nothing is accounted. */ - cluster_grd_recovery_mark_peer_done(0, 10, 76); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 0); + episode_hash = cluster_grd_recovery_event_bitmap_hash_value(); + UT_ASSERT_EQ(episode_hash, cluster_grd_dead_bitmap_hash(ut_mock_last_event.dead_bitmap)); + + /* Old-event ABA shape: a late DONE naming a DIFFERENT dead set (node 3 + * instead of node 2 — e.g. the previous episode before a coordinator + * re-election) is dropped: nothing is accounted. */ + memset(old_dead, 0, sizeof(old_dead)); + old_dead[0] = 0x08; + old_event_hash = cluster_grd_dead_bitmap_hash(old_dead); + cluster_grd_recovery_mark_peer_done(0, 10, old_event_hash); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); - /* Current-event DONE is accounted (epoch + event id). */ - cluster_grd_recovery_mark_peer_done(0, 10, 77); + /* Generation-drift shape (the R2 wedge): the peer's local event_id + * differs, but its DONE names the SAME quorum dead set -> the composite + * key matches and the DONE is accounted. */ + cluster_grd_recovery_mark_peer_done(0, 10, episode_hash); UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 10); - UT_ASSERT_EQ(cluster_grd_recovery_done_event_id_for(0), 77); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), episode_hash); - /* Witness advance: deadline expired + cur==old + coordinator DONE for - * THIS event at exactly cur, accounted after the accept snapshot. The - * FSM takes the event-scoped equal-epoch escape exactly once. */ + /* Witness advance: deadline expired + cur==old + coordinator DONE naming + * THIS dead set at exactly cur, accounted after the accept snapshot. The + * FSM takes the composite-key equal-epoch escape exactly once. */ cluster_grd_recovery_counters_snapshot(&before); ut_mock_now = (int64)60 * 1000000; cluster_grd_recovery_lmon_tick(); @@ -4264,7 +4316,7 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u15_master_side_gate_decision); UT_RUN(test_jr_u16_pre_member_guard); UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); - UT_RUN(test_recovery_stale_event_done_rejected_and_witness_advance); + UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From ddefc0003592ae537598ac18601d7633d3fc10ae Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 16/27] fix(cluster): dead-master block serve proof is unconditional again MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Revert the scope short-circuit that returned NORMAL for a dead static master's blocks wherever online thread recovery cannot run (the default configuration and every >2-node deployment): it skipped both the is_materialized cold-block door and the redo-coverage lost-write door, so a committed write only the dead node saw could be silently read stale from shared storage. No other guard sits on that read path — the GRD freeze ends with the episode, the HW gate covers only extend high-water marks, and the page-SCN checks ride the ship path. Out of scope the posture is now an explicit bounded retryable error (53R9L, hint updated to name the way out: restart the failed node, or enable online thread recovery in a supported scope) for every tag whose static master is dead — including never-written blocks, which the cold-block door cannot distinguish from not-yet-replayed ones — and the door reopens the moment the failed node stops being DEAD. In-scope behavior is byte-identical. The now-orphaned scope helper is removed. t/362 asserts the honest two-outcome contract on fresh connections (success or explicit SQLSTATE — 53R9L or the TT-unknown visibility door — never a hang), rides pre-warmed sessions for convergence observability, and pins the D12 cleanup counter to a positive delta; t/293's post-kill extend leg follows the same posture. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R1/R3) --- src/backend/cluster/cluster_gcs_block.c | 31 ++- src/include/cluster/cluster_thread_recovery.h | 10 - .../cluster_tap/t/293_hw_online_remaster.pl | 30 ++- .../362_shared_catalog_4node_kill_selfheal.pl | 218 +++++++++++++++--- .../cluster_unit/test_cluster_gcs_block.c | 13 +- 5 files changed, 223 insertions(+), 79 deletions(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index d0219deaf9..ee200ffafe 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1223,13 +1223,8 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) /* * spec-4.7 D7 + D5 — static master is DEAD; the block is remastered to a - * live survivor (recovery-aware routing). The redo-before-serve gate must - * engage on EXACTLY the same scope where online thread recovery can actually - * run. Otherwise a >2-node or GUC-off deployment waits forever on a - * materialization authority that the thread-recovery launcher intentionally - * treats as not applicable. - * - * In applicable 2-node scope, two conditions are both required (Q5): + * live survivor (recovery-aware routing). Two conditions are both + * required before the on-disk version may be served (Q5): * (a) is_materialized(origin): the dead origin's merged replay completed * (publish is atomic at end-of-replay with the max EndRecPtr). This * is the cold-block safety door — a block NO survivor observed has no @@ -1240,18 +1235,18 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * origin's recovered_lsn must reach that observed page_lsn — else the * dead node wrote a version a survivor saw but whose WAL never durably * reached us → lost-write → fail-closed. + * + * spec-4.6a Amendment v1.2 (R1): this proof is UNCONDITIONAL. Where + * online thread recovery cannot run (GUC off — the default — or any + * >2-node deployment) the materialization authority is never published, + * so a dead master's blocks stay RECOVERING until the failed node + * restarts and runs its own instance recovery: a bounded, retryable + * ERROR on the request path (53R9L), never an unproven serve. A scope + * predicate must never gate a correctness proof — a committed write on + * a cold block that only the dead node saw has NO other guard on this + * read path (GRD freeze ends with the episode; the HW gate covers only + * extend high-water marks; pd_block_scn checks ride the ship path). */ - { - bool shared_fs; - int survivors; - - shared_fs = (cluster_shared_storage_backend == CLUSTER_SHARED_FS_BACKEND_CLUSTER_FS); - survivors = cluster_conf_node_count() - 1; - if (!cluster_thread_recovery_materialization_gate_enabled( - cluster_online_thread_recovery, cluster_conf_has_peers(), shared_fs, survivors)) - return GCS_BLOCK_NORMAL; - } - if (!cluster_merged_instance_is_materialized(static_master)) { if (ClusterGcsBlock != NULL) pg_atomic_fetch_add_u64(&ClusterGcsBlock->recovery_block_resources_recovering, 1); diff --git a/src/include/cluster/cluster_thread_recovery.h b/src/include/cluster/cluster_thread_recovery.h index c7be25e6b0..c455b77436 100644 --- a/src/include/cluster/cluster_thread_recovery.h +++ b/src/include/cluster/cluster_thread_recovery.h @@ -376,16 +376,6 @@ cluster_thread_recovery_gate_decide(ClusterThreadRecScope scope, const uint64 *d return false; /* every dead origin materialized -> ready to unfreeze */ } -static inline bool -cluster_thread_recovery_materialization_gate_enabled(bool guc_on, bool has_peers, - bool shared_fs_backend, - int live_survivor_count) -{ - return cluster_thread_recovery_decide_scope(guc_on, has_peers, shared_fs_backend, - live_survivor_count) - == CLUSTER_THREADREC_SCOPE_APPLICABLE; -} - /* * cluster_thread_recovery_replay_epoch_aborts -- the L235 episode-epoch * staleness guard (spec-4.11 3b-4b). The per-thread replay slot stamps the GRD diff --git a/src/test/cluster_tap/t/293_hw_online_remaster.pl b/src/test/cluster_tap/t/293_hw_online_remaster.pl index 509c2f50ff..2447312952 100644 --- a/src/test/cluster_tap/t/293_hw_online_remaster.pl +++ b/src/test/cluster_tap/t/293_hw_online_remaster.pl @@ -329,23 +329,39 @@ sub retry_sql is(hw_counter($h1, 'remaster_retry_exhausted_count'), 0, 'L7: self-heal path did not exhaust the retry budget'); -# spec-4.6a section 4 (D8): DONE must actually unfreeze (P7) -- the survivor -# extends a table that lived on shards the dead master owned; a frozen shard -# would fail this INSERT with the remastering error. +# spec-4.6a section 4 (D8, Amendment v1.2 R1): after the same-episode DONE the +# episode has converged (P7 ran: DONE counter + no exhaustion asserted above), +# but in this out-of-scope deployment (online_thread_recovery off) EVERY tag +# whose static master is the dead node stays fail-closed until that node +# returns — including brand-new blocks, which have no materialization proof +# either (the cold-block door cannot distinguish never-existed from +# not-yet-replayed; main's t/293 L5b documents the same posture). The +# survivor write is therefore the honest two-outcome contract: success (all +# touched tags survivor-mastered) or the explicit bounded 53R9L — never a +# hang and never an unbounded wedge. { my ($ext_ok, $ext_res) = (0, ''); - for my $try (1 .. 30) + for my $try (1 .. 10) { $ext_res = eval { $h1->safe_psql('postgres', - 'INSERT INTO s1 SELECT g, g, g, g FROM generate_series(1, 4096) g'); + 'CREATE TABLE IF NOT EXISTS s1_heal (a int, b int, c int, d int); ' + . 'INSERT INTO s1_heal SELECT g, g, g, g FROM generate_series(1, 4096) g'); 1; } ? 'ok' : ($@ // 'error'); if ($ext_res eq 'ok') { $ext_ok = 1; last; } sleep 1; } - ok($ext_ok, 'L7: survivor extend succeeds after same-episode DONE (P7 unfreeze)') - or diag($ext_res); + if ($ext_ok) + { + ok(1, 'L7: survivor new-table extend succeeded after same-episode DONE (P7 unfreeze)'); + } + else + { + like($ext_res, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + 'L7: survivor extend fail-closed with an explicit SQLSTATE (dead-master tag, out-of-scope posture)'); + } } $heal->stop_pair; diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index 883860c0fb..a5f6719eea 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -5,6 +5,13 @@ # # Formation uses ClusterQuad(shared_catalog + wal_threads_root + shared_data) # so a killed node's HW authority is recoverable from its per-thread WAL. +# Amendment v1.2 (R1): in this out-of-scope deployment (4 nodes, online +# thread recovery off) the dead master's PRE-EXISTING writes stay +# fail-closed until the failed node restarts — its unproven blocks (bounded +# 53R9L) and equally its transaction verdicts (TT status unknown for its +# xids, the visibility door). Reads and DDL are asserted on the honest +# two-outcome contract (success or explicit SQLSTATE, never a hang); +# new-object DDL/writes prove P7 unfreeze. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -88,6 +95,52 @@ sub sum_hw return $sum; } +# Amendment v1.2 (R1): after the kill, a NEW backend's parse may cold-read a +# dead-master catalog page and honestly fail closed (53R9L) until the failed +# node returns. The convergence observability therefore rides PRE-WARMED +# long-lived psql sessions (catcache hot; pg_cluster_* reads touch shmem, not +# dead-master blocks), while fresh-connection legs assert the two-outcome +# contract. bg_query returns (1, value) or (0, error-ish) without dying. +sub bg_query +{ + my ($bg, $sql) = @_; + my $out = eval { $bg->query($sql) }; + return (0, $@ // 'query failed') if $@; + return (1, $out // ''); +} + +sub bg_counter +{ + my ($bg, $cat, $key) = @_; + my ($ok, $v) = bg_query($bg, + "SELECT value FROM pg_cluster_state WHERE category='$cat' AND key='$key'"); + return ($ok && defined $v && $v ne '') ? $v + 0 : 0; +} + +sub bg_sum +{ + my ($bgs, $cat, $key, @idx) = @_; + my $sum = 0; + $sum += bg_counter($bgs->{$_}, $cat, $key) for @idx; + return $sum; +} + +sub bg_poll_until +{ + my ($bg, $sql, $want, $timeout_s) = @_; + my $deadline = time() + $timeout_s; + my $last = ''; + while (time() < $deadline) + { + my ($ok, $out) = bg_query($bg, $sql); + $last = $out; + return 1 if $ok && defined $out && $out eq $want; + usleep(500_000); + } + diag("bg_poll_until timed out: '$sql' last='$last' want='$want'"); + return 0; +} + sub logs_contain { my ($quad, $pattern, @idx) = @_; @@ -217,9 +270,20 @@ sub startup_env_blocker retry_sql($quad->node($i), 'CHECKPOINT', 30); } -my $done_before = sum_hw($quad, 'remaster_done_count', @survivors); -my $blocked_before = sum_hw($quad, 'remaster_blocked_count', @survivors); -my $retry_before = sum_hw($quad, 'remaster_retry_count', @survivors); +# Amendment v1.2 (R1): pre-warmed observability sessions (see helper note). +my %bg; +for my $i (@survivors) +{ + $bg{$i} = $quad->node($i)->background_psql('postgres', on_error_stop => 0); + bg_query($bg{$i}, "SELECT value FROM pg_cluster_state WHERE category='hw' AND key='remaster_done_count'"); + bg_query($bg{$i}, "SELECT count(*) FROM pg_cluster_grd_shards WHERE master_node_id=$dead"); + bg_query($bg{$i}, 'SELECT txid_current()'); +} + +my $done_before = bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors); +my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); +my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); +my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -232,18 +296,26 @@ sub startup_env_blocker } # L3: GRD leaves no shard mastered by the dead node, and HW remaster converges. +# The map read is hard-asserted on the warm COORDINATOR session only: the view +# scan on a cold survivor deterministically cold-reads a dead-master catalog +# page out of the small test buffer pool and honestly fails closed (53R9L) +# until node3 returns — the map itself is a deterministic recompute barriered +# by P6, and each cold survivor is instead hard-asserted on its own HW +# remaster worker DONE log line below. +ok(bg_poll_until($bg{0}, + "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", + 'true', 60), + 'L3: coordinator remastered all GRD shards off node3 (warm session)'); for my $i (@survivors) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=0)::text FROM pg_cluster_grd_shards WHERE master_node_id=$dead", - 'true', 60), - "L3: survivor node$i remastered all GRD shards off node$dead"); + ok(log_until($quad, $i, qr/HW remaster worker: dead node $dead -> done/, 90), + "L3: survivor node$i HW remaster worker reported done"); } my $done_converged = 0; my $deadline = time() + 90; while (time() < $deadline) { - if (sum_hw($quad, 'remaster_done_count', @survivors) > $done_before) + if (bg_sum(\%bg, 'hw', 'remaster_done_count', @survivors) > $done_before) { $done_converged = 1; last; @@ -255,10 +327,10 @@ sub startup_env_blocker 'L3: no survivor reported BLOCKED_STRUCTURAL after kill'); # L6: if a transient BLOCKED occurred, a same-episode retry must also have run. -my $blocked_after = sum_hw($quad, 'remaster_blocked_count', @survivors); +my $blocked_after = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); if ($blocked_after > $blocked_before) { - ok(sum_hw($quad, 'remaster_retry_count', @survivors) > $retry_before, + ok(bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors) > $retry_before, 'L6: transient BLOCKED was followed by same-episode HW remaster retry'); } else @@ -266,45 +338,127 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } -# L4: survivor SQL and shared-catalog DDL recover after fail-stop. +# L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an +# out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a +# BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a +# hang. sc4_hw_* spans ~20+ pages, so some page's static master is node3 with +# overwhelming probability; the redo-coverage proof cannot be published without +# online thread recovery, so the scan hits 53R9L (resource recovering). If by +# hash luck no scanned page is node3-mastered the scan succeeds — both are the +# honest two-outcome contract; only a hang/timeout or an unexpected error fails. +for my $t (1 .. 2) +{ + my $t0 = time(); + my ($rc, $out, $err) = $quad->node(1)->psql('postgres', + "SELECT count(*) FROM sc4_hw_$t", timeout => 60); + my $elapsed = time() - $t0; + if (defined $rc && $rc == 0) + { + ok(1, "L4a: dead-master table sc4_hw_$t read succeeded (no node$dead-mastered page scanned)"); + diag("L4a: sc4_hw_$t count=$out"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|cluster TT status unknown/i, + "L4a: dead-master table sc4_hw_$t read fail-closed with an explicit SQLSTATE"); + } + cmp_ok($elapsed, '<', 60, "L4a: sc4_hw_$t read returned bounded (no hang)"); +} + +# L4: survivor SQL recovers after fail-stop — hard-asserted on the pre-warmed +# sessions; a FRESH connection is the honest two-outcome contract (v1.2 R1): +# success, or an explicit fail-closed SQLSTATE on a dead-master catalog page — +# never a hang. for my $i (@survivors) { - my ($ok, $res) = retry_sql($quad->node($i), 'SELECT txid_current()', 60); - ok($ok, "L4: survivor node$i accepts txid_current after reconfig") or diag($res); + my $got = 0; + my $dl = time() + 60; + while (time() < $dl) + { + my ($ok, $out) = bg_query($bg{$i}, 'SELECT txid_current()'); + if ($ok && defined $out && $out =~ /^\d+$/) { $got = 1; last; } + usleep(500_000); + } + ok($got, "L4: survivor node$i accepts txid_current after reconfig (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($i)->psql('postgres', 'SELECT txid_current()', + timeout => 60); + if (defined $rc && $rc == 0) + { + ok(1, "L4: fresh connection txid_current succeeded on node$i"); + } + else + { + like($err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); + } + cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); +} +my ($ddl_rc, $ddl_out, $ddl_err) = $n0->psql('postgres', + 'CREATE TABLE sc4_after_kill (id int)', timeout => 60); +my $ddl_ok = defined $ddl_rc && $ddl_rc == 0; +if ($ddl_ok) +{ + ok(1, 'L4: coordinator survivor DDL succeeded after node kill'); +} +else +{ + like($ddl_err // '', + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); + diag("L4: DDL fail-closed honestly: $ddl_err"); } -my ($ddl_ok, $ddl_res) = retry_sql($n0, 'CREATE TABLE sc4_after_kill (id int)', 60); -ok($ddl_ok, 'L4: coordinator survivor can run DDL after node kill') or diag($ddl_res); for my $i (1, 2) { - ok(poll_until($quad->node($i), - "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", - 'true', 60), - "L4: survivor node$i sees post-kill shared-catalog DDL"); + if ($ddl_ok) + { + ok(poll_until($quad->node($i), + "SELECT (count(*)=1)::text FROM pg_class WHERE relname='sc4_after_kill'", + 'true', 60), + "L4: survivor node$i sees post-kill shared-catalog DDL"); + } + else + { + ok(1, "L4: survivor node$i visibility leg not applicable (DDL fail-closed honestly)"); + } } -# L4 (spec-4.6a D12, r2-P1-3): the dead-node PCM residue cleanup counter is -# exposed and non-negative on every survivor; when the dead node left any -# holder/pending-X records behind, at least one survivor accounts a cleanup -# (delta assertable; zero stays legal when the dead node held nothing). +# L4 (spec-4.6a D12, Amendment v1.2 R3): node3 held real PCM/HW state at kill +# time (it extended sc4_hw_1/2 with 5000 rows each), so the dead-node residue +# cleanup MUST account a positive delta across the survivors — a >= 0 +# assertion would stay green with D12 entirely disabled. { my $cleanup_total = 0; for my $i (@survivors) { - my $c = state_counter($quad->node($i), 'pcm', 'dead_cleanup_entries'); - ok($c >= 0, "L4: survivor node$i exposes pcm/dead_cleanup_entries"); - $cleanup_total += $c; + my ($ok, $n) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='pcm' AND key='dead_cleanup_entries'"); + ok($ok && defined $n && $n eq '1', + "L4: survivor node$i exposes the pcm/dead_cleanup_entries key"); + $cleanup_total += bg_counter($bg{$i}, 'pcm', 'dead_cleanup_entries'); } - diag("L4: pcm/dead_cleanup_entries total across survivors = $cleanup_total"); + cmp_ok($cleanup_total - $cleanup_before, '>', 0, + 'L4: dead-node PCM residue cleanup accounted a positive delta after kill'); + diag("L4: pcm/dead_cleanup_entries delta across survivors = " + . ($cleanup_total - $cleanup_before)); } -# L5: watchdog counters are allowed to be zero, but must be readable. +# L5: watchdog counters are allowed to be zero, but the KEYS must exist +# (state_counter reads a missing key as 0, so presence is the real assertion). for my $i (@survivors) { - my $cluster_gate = state_counter($quad->node($i), 'grd_recovery', 'cluster_gate_timeout'); - my $epoch_escape = state_counter($quad->node($i), 'grd_recovery', 'wait_epoch_escape'); - ok($cluster_gate >= 0 && $epoch_escape >= 0, - "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counters"); + my ($ok1, $n1) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='cluster_gate_timeout'"); + my ($ok2, $n2) = bg_query($bg{$i}, + "SELECT count(*) FROM pg_cluster_state WHERE category='grd_recovery' AND key='wait_epoch_escape'"); + ok($ok1 && $ok2 && defined $n1 && $n1 eq '1' && defined $n2 && $n2 eq '1', + "L5: survivor node$i exposes WAIT_CLUSTER/WAIT_EPOCH watchdog counter keys"); } +$bg{$_}->quit for @survivors; + $quad->stop_quad; done_testing(); diff --git a/src/test/cluster_unit/test_cluster_gcs_block.c b/src/test/cluster_unit/test_cluster_gcs_block.c index 67fcc29dc2..8729fbd744 100644 --- a/src/test/cluster_unit/test_cluster_gcs_block.c +++ b/src/test/cluster_unit/test_cluster_gcs_block.c @@ -288,16 +288,6 @@ UT_TEST(test_gcs_block_tag_is_standard_buffer_tag_20b) } -UT_TEST(test_dead_static_master_materialization_gate_scope_matches_thread_recovery) -{ - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(false, true, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, false, true, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, false, 1)); - UT_ASSERT(!cluster_thread_recovery_materialization_gate_enabled(true, true, true, 2)); - UT_ASSERT(cluster_thread_recovery_materialization_gate_enabled(true, true, true, 1)); -} - - /* spec-5.2 D2 (U3): pure master-side decision for an X-held N→S read. * node0 = holder/master in DIRECT, node1 = requester. */ UT_TEST(test_xheld_read_ship_decision_truth_table) @@ -525,7 +515,7 @@ UT_TEST(test_clean_xfer_stale_break_predicate) int main(void) { - UT_PLAN(22); + UT_PLAN(21); UT_RUN(test_gcs_block_msg_type_enum_values_no_collision); UT_RUN(test_gcs_block_payload_sizes_locked); UT_RUN(test_gcs_block_request_field_offsets); @@ -541,7 +531,6 @@ main(void) UT_RUN(test_gcs_block_data_size_equals_blcksz); UT_RUN(test_gcs_block_msg_type_enum_extends_without_gap); UT_RUN(test_gcs_block_tag_is_standard_buffer_tag_20b); - UT_RUN(test_dead_static_master_materialization_gate_scope_matches_thread_recovery); UT_RUN(test_xheld_read_ship_decision_truth_table); UT_RUN(test_forward_payload_read_image_flag_roundtrip); UT_RUN(test_clean_page_xfer_eligible_flag_roundtrip_and_orthogonal); From 0d57fa90bd6500743988b9653735330a5660bbcc Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 20:26:58 +0800 Subject: [PATCH 17/27] fix(cluster): remaster retry ordering, runtime-structural warning, upgrade cleanup - Order the worker's terminal-result store after its backoff-deadline store (pg_memory_barrier) so the relaunch decider never pairs a fresh BLOCKED with a stale deadline and skips one backoff wait. - A structural cause discovered only by the WORKER (SIGHUP race) now leaves the retry deadline unset so the FSM's next tick takes the MARK_STRUCTURAL branch and emits the once-per-episode operator WARNING with the configuration hint. - Wrap the local S->X upgrade in PG_TRY and release the temporary S claim when the ACK wait throws (cancel via CHECK_FOR_INTERRUPTS), not only on the false return; document why the completed upgrade hard-resets the local S refcount (the X grant subsumes all local S declarations). Linker-only exception stubs for the units that link cluster_pcm_lock.o. Spec: spec-4.6a-grd-recovery-liveness.md (Amendment v1.2 R5/R6/R7/R9) --- src/backend/cluster/cluster_hw_remaster.c | 19 ++++++++++++- src/backend/cluster/cluster_pcm_lock.c | 27 +++++++++++++++++-- .../test_cluster_bufmgr_pcm_hook.c | 11 ++++++++ src/test/cluster_unit/test_cluster_pcm_lock.c | 13 +++++++++ 4 files changed, 67 insertions(+), 3 deletions(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index c585268a42..03ee6751a3 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -95,8 +95,16 @@ hw_remaster_next_attempt_deadline(uint32 completed_retry_attempts) static void hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) { + /* Amendment v1.2 (R6): the worker stores next_attempt_at BEFORE the + * terminal result, and the LMON relaunch decision reads them in the + * opposite order (result first). Order the two stores so a decider + * that observes the terminal result also observes the matching backoff + * deadline — without the fence a weakly-ordered CPU could let it pair a + * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait + * (bounded self-correcting, but cheap to close outright). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_done(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED) { @@ -104,14 +112,23 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) cluster_hw_remaster_set_next_attempt_at(dead_node, hw_remaster_next_attempt_deadline(attempts)); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_BLOCKED_STRUCTURAL) { - cluster_hw_remaster_set_next_attempt_at(dead_node, CLUSTER_HW_REMASTER_NO_DEADLINE); + /* Amendment v1.2 (R7): leave next_attempt_at at 0 (not NO_DEADLINE) + * so the FSM's next tick takes the MARK_STRUCTURAL decision branch + * and emits the episode-once operator WARNING + errhint — covering + * the SIGHUP race where only the WORKER (not the launch-side + * precheck) discovers the structural cause. MARK_STRUCTURAL then + * stamps NO_DEADLINE, making the WARNING once-per-episode. */ + cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); cluster_hw_bump_remaster_blocked(); } else if (res == CLUSTER_HW_REMASTER_NOT_APPLICABLE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); + pg_memory_barrier(); cluster_hw_remaster_set_result(dead_node, res); } } diff --git a/src/backend/cluster/cluster_pcm_lock.c b/src/backend/cluster/cluster_pcm_lock.c index b9de7f6401..88ddaad4cf 100644 --- a/src/backend/cluster/cluster_pcm_lock.c +++ b/src/backend/cluster/cluster_pcm_lock.c @@ -2221,7 +2221,10 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) errmsg("block-level cache protocol state is being rebuilt " "after reconfiguration"), errhint("The block resource is recovering (survivor re-declare / " - "master rebuild after node failure); retry the transaction."))); + "master rebuild after node failure); retry the transaction. " + "If the failed node stays down, restart it to run its " + "instance recovery, or enable online thread recovery in a " + "supported scope (cluster.online_thread_recovery)."))); CHECK_FOR_INTERRUPTS(); pgstat_report_wait_start(WAIT_EVENT_GCS_BLOCK_RECOVERING); @@ -2305,9 +2308,24 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) if ((cluster_pcm_lock_query_s_holders_bitmap(tag) & self_bit) == 0) { struct GrdEntry *entry; + bool upgraded = false; cluster_pcm_lock_acquire(tag, PCM_LOCK_MODE_S); - if (!cluster_gcs_block_local_x_upgrade(tag)) { + /* Amendment v1.2 (R5): the upgrade waits on remote ACKs with + * CHECK_FOR_INTERRUPTS in the loop, so a cancel can THROW out + * of it — release the temporary S claim on that path too, not + * only on the false return. */ + PG_TRY(); + { + upgraded = cluster_gcs_block_local_x_upgrade(tag); + } + PG_CATCH(); + { + cluster_pcm_lock_release(tag); + PG_RE_THROW(); + } + PG_END_TRY(); + if (!upgraded) { cluster_pcm_lock_release(tag); ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("cluster_pcm: S->X upgrade invalidate did not complete"), @@ -2318,6 +2336,11 @@ cluster_pcm_lock_acquire_buffer(BufferDesc *buf, PcmLockMode mode) entry = pcm_find_entry(tag); if (entry != NULL) { pcm_entry_lock_exclusive(entry); + /* Amendment v1.2 (R9): the completed S->X upgrade replaces + * ALL of this node's S declarations for the tag (the X + * grant subsumes them), so the local S refcount is + * intentionally hard-reset rather than decremented — a + * later release of the X does not owe any S releases. */ entry->s_holder_refcount_local = 0; LWLockRelease(&entry->entry_lock.lock); } diff --git a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c index 540cff5b7d..90151fc3cb 100644 --- a/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c +++ b/src/test/cluster_unit/test_cluster_bufmgr_pcm_hook.c @@ -181,6 +181,17 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): linker-only exception-stack stubs (the + * linked cluster_pcm_lock.o now wraps the S->X upgrade in PG_TRY). */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { diff --git a/src/test/cluster_unit/test_cluster_pcm_lock.c b/src/test/cluster_unit/test_cluster_pcm_lock.c index c02894306f..08ba568886 100644 --- a/src/test/cluster_unit/test_cluster_pcm_lock.c +++ b/src/test/cluster_unit/test_cluster_pcm_lock.c @@ -134,6 +134,19 @@ ExceptionalCondition(const char *conditionName pg_attribute_unused(), abort(); } +/* spec-4.6a Amendment v1.2 (R5): the S->X upgrade is now wrapped in + * PG_TRY/PG_CATCH, which references the exception stack + re-throw. The + * unit's errfinish mock longjmps to its own buffer (never through + * PG_exception_stack), so these exist for the linker only. */ +sigjmp_buf *PG_exception_stack = NULL; +ErrorContextCallback *error_context_stack = NULL; + +void +pg_re_throw(void) +{ + abort(); +} + static BufferTag make_tag(uint32 blockno) { From 1be30fed70488c2c367c6366142373adadb076c1 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Wed, 8 Jul 2026 21:10:46 +0800 Subject: [PATCH 18/27] fix(cluster): r2 P2-1 barrier-tick scalar conjunct + t/274 backend pre-bump shmem bit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review-r2 items on top of the ②b others-dead coherence fix: P2-1 — the leaving node's barrier-tick own-commit latch (cl_leaving_barrier_tick) inferred 'my leave committed' from epoch>baseline AND the others-dead bitmap unchanged. The bitmap is not monotone: a third-party node that false-fail- stopped (bumping the epoch) then recovered leaves CSSD hysteresis DEAD->ALIVE, so the bitmap rebounds to its bound value while the scalar dead_generation only advances. If the leaver's first epoch>baseline observation lands after that rebound it would mis-latch a leave the survivor coordinator actually refused, suppress the barrier-deadline escalation, and hang forever in BARRIER_WAIT. Restore the scalar conjunct (epoch>baseline AND others_dead==bound AND dead_gen==baseline) via the pure, unit-tested cluster_clean_leave_own_commit_ latched predicate. This does not reintroduce the ②b false positive: the leaver's OWN alive->DEAD never bumps ITS OWN dead_generation (a node does not observe itself dead), so on the leaver side the scalar stays at baseline through its drain. Survivor-side coherence keeps the bitmap-only check. Adds the rebound negative-leg unit (others_dead unchanged + dead_gen advanced -> escalate). t/274 — a backend-context coordinator (the fail-stop inject test) advances the epoch then SYNCHRONOUSLY waits for the fence marker before publishing (MyBackendType != B_LMON path). During that seconds-long wait the epoch is bumped but unpublished, and a concurrent LMON GRD IDLE tick re-captures the post-bump epoch as its WAIT_EPOCH baseline -> old==cur -> the thread replay slot never reaches DONE -> HG#1b/HG#5 done-poll timeout. The ②a baseline-hold guard only reads LMON-process-local staged flags, invisible to a backend's pre-bump window. Publish it in shmem: ReconfigShmem.prebump_sync_active set before the bump, cleared at every return; the guard reads the shmem bit OR the local stage. 8.A (durable-marker-before-publish) unchanged. Reproduced the wedge locally at 87f793084b (HG#1b slot-DONE poll times out forever), then verified the fix: t/274 goes 18/18 green (exit 2 -> 0). This also fixes the long-standing main flake (the wedge is deterministic once LMON is freed from the sync-marker park; it was merely probabilistic before) — the t/274 known-flake / L66 registration should be revised. Local gates (cassert): cluster_unit 158 binaries (incl. own-commit-latched + version-coherent matrices), t/274 18/18, PG regress 219/219, clang-format/ headers/scn-cmp clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 27 ++++++++++++---- .../cluster/cluster_clean_leave_policy.c | 30 +++++++++++++++++ src/backend/cluster/cluster_reconfig.c | 23 +++++++++++++ src/include/cluster/cluster_clean_leave.h | 9 ++++++ src/include/cluster/cluster_reconfig.h | 10 ++++++ .../cluster_unit/test_cluster_clean_leave.c | 32 ++++++++++++++++++- 6 files changed, 124 insertions(+), 7 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 6ea7873643..31e25bc8a4 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -1777,17 +1777,32 @@ cl_leaving_barrier_tick(void) * leave is active (cluster_clean_leave_in_progress gates drive_joins + * commit_member), and the leave does not start while a join is pending * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * bump with an unchanged others-dead set during a leave can only be the - * leave's own commit (spec-2.29a ②b: the leaving node's own DEAD is excluded - * so its expected heartbeat stop no longer looks like a third-party death). + * bump with an unchanged version during a leave can only be the leave's own + * commit. + * + * spec-2.29a ②b + r2 P2-1: "unchanged version" here means the others-dead + * bitmap AND the scalar dead_generation both unchanged. The bitmap excludes + * the leaving node's own expected DEAD (②b: else its heartbeat stop would + * falsely escalate), but the bitmap is not monotone — a third-party + * false-DEAD→ALIVE rebound restores it while the scalar dead_generation only + * advances (r2 P2-1: else the leaver could mis-latch a refused leave and + * hang). The leaver's own DEAD never bumps its OWN dead_generation, so the + * scalar conjunct is safe on this side (it would NOT be safe on the survivor + * side, which keeps the bitmap-only coherence check). */ if (cluster_epoch_get_current() > baseline_epoch) { uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool others_dead_unchanged; + bool dead_gen_unchanged; cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); - if (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, - CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) - == 0) { + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged + = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(true, others_dead_unchanged, + dead_gen_unchanged)) { /* * Commit point observed (our clean-leave epoch was published; no * third-party death intruded). The leave can no longer be diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index 095f4b082b..ece1d0813f 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -167,6 +167,36 @@ cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, return memcmp(bound_others_dead, current_others_dead, (size_t)nbytes) == 0; } +/* + * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick + * infers "my clean-leave committed" from the membership epoch advancing past + * its bound baseline (the coordinator publishes the CLEAN_LEAVE event into ITS + * own reconfig state, but the epoch piggybacks to the leaver). Latching that + * inference suppresses the barrier-deadline escalation, so a FALSE latch hangs + * the leaver in BARRIER_WAIT forever. + * + * spec-2.29a r2 P2-1: the leaver's own-commit inference needs BOTH a bitmap + * check AND the scalar dead_generation. The others-dead bitmap alone is not + * monotone: a third-party node that false-fail-stopped (bumping the epoch) + * then recovered leaves the CSSD hysteresis DEAD→ALIVE, so the others-dead + * bitmap returns to its bound value while the scalar dead_generation (which + * only advances) does not. If the leaver's first epoch>baseline observation + * lands after that rebound, a bitmap-only check would mis-latch a leave the + * survivor coordinator actually refused. The scalar conjunct closes it — and + * it does NOT reintroduce the ②b false positive, because the leaving node's + * OWN alive→DEAD transition never bumps ITS OWN dead_generation (a node does + * not observe itself dead), so on the leaver side dead_gen stays at baseline + * through its own drain. (The survivor-side coherence sites keep the + * others-dead bitmap: a survivor DOES observe the leaver's DEAD and would be + * falsely escalated by the scalar there.) + */ +bool +cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, + bool dead_gen_unchanged) +{ + return epoch_advanced && others_dead_unchanged && dead_gen_unchanged; +} + /* ============================================================ * leave-intent marker structural validation (D2 / §2.5) diff --git a/src/backend/cluster/cluster_reconfig.c b/src/backend/cluster/cluster_reconfig.c index 7edc315c52..ccaf5991c1 100644 --- a/src/backend/cluster/cluster_reconfig.c +++ b/src/backend/cluster/cluster_reconfig.c @@ -177,6 +177,7 @@ cluster_reconfig_shmem_init(void) pg_atomic_init_u64(&ReconfigShmem->apply_counter, 0); pg_atomic_init_u64(&ReconfigShmem->dedup_skip_counter, 0); pg_atomic_init_u64(&ReconfigShmem->procsig_broadcast_count, 0); + pg_atomic_init_u32(&ReconfigShmem->prebump_sync_active, 0); /* spec-2.29a r2 t/274 */ /* spec-5.14 D6 — touched_peers observability counters. */ pg_atomic_init_u64(&ReconfigShmem->touched_abort_count, 0); pg_atomic_init_u64(&ReconfigShmem->touched_stamp_count, 0); @@ -1272,6 +1273,11 @@ cluster_reconfig_release_fence_stage(ClusterReconfigFenceStage *stage) bool cluster_reconfig_has_pending_prebump_stage(void) { + /* spec-2.29a r2 t/274: the shmem bit covers a BACKEND-context coordinator's + * bump→publish window (LMON cannot see that process's local stage); the + * three LMON-local staged flags cover the LMON-driven paths. */ + if (ReconfigShmem != NULL && pg_atomic_read_u32(&ReconfigShmem->prebump_sync_active) != 0) + return true; return failstop_fence_stage.async.has_staged_event || node_removed_fence_stage.async.has_staged_event || join_prepare_stage.async.has_staged_event; @@ -2533,6 +2539,16 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( CLUSTER_INJECTION_POINT("cluster-reconfig-epoch-bump-pre"); + /* + * spec-2.29a r2 t/274: mark the pre-bump→publish window BEFORE bumping the + * epoch, so a concurrent LMON GRD IDLE tick holds its WAIT_EPOCH baseline + * while this (possibly backend-context, synchronous) path bumps but has not + * yet published. Cleared at every return below. Harmless on the LMON / + * fence-off paths (they either hand off to the local staged flag or publish + * within this same call with no intervening tick). + */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 1); + /* D18: atomic CAS-loop increment. Returns pre/post snapshots. */ cluster_epoch_advance_for_reconfig(&old_epoch, &new_epoch); @@ -2650,11 +2666,13 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( "majority for epoch %llu; not publishing reconfig event " "(write-fenced, will retry)", (unsigned long long)new_epoch))); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; } cluster_reconfig_publish_event(&evt); cluster_reconfig_log_failstop_epoch_bump(&evt); + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; } @@ -2665,6 +2683,8 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( (void)cluster_reconfig_submit_fence_stage(&failstop_fence_stage, CLUSTER_MARKER_KIND_FENCE_FAILSTOP, coordinator_node_id, GetCurrentTimestamp()); + /* LMON path: the local staged flag now covers the pending window. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); return; /* fail-closed until the staged marker is majority-durable */ } @@ -2680,6 +2700,9 @@ cluster_reconfig_apply_epoch_bump_as_coordinator( * tick free of allocator traffic. */ cluster_reconfig_log_failstop_epoch_bump(&evt); + + /* spec-2.29a r2 t/274: fence-off path published within this call. */ + pg_atomic_write_u32(&ReconfigShmem->prebump_sync_active, 0); } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 3d3a3801e4..14975b9e7e 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -352,6 +352,15 @@ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 curr const uint8 *bound_others_dead, const uint8 *current_others_dead, int nbytes); +/* spec-2.29a r2 P2-1: the leaving node's barrier-tick own-commit latch. Needs + * BOTH the others-dead bitmap unchanged AND the scalar dead_generation + * unchanged — the bitmap alone is not monotone under a third-party + * false-DEAD→ALIVE rebound and could mis-latch a refused leave (hang). The + * leaver's own DEAD never bumps its own dead_generation, so the scalar + * conjunct does not reintroduce the ②b false positive. */ +extern bool cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, + bool dead_gen_unchanged); + /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the * leaving node is the expected declared peer, and the dead_bitmap names only the diff --git a/src/include/cluster/cluster_reconfig.h b/src/include/cluster/cluster_reconfig.h index df42f84d77..1aa106f690 100644 --- a/src/include/cluster/cluster_reconfig.h +++ b/src/include/cluster/cluster_reconfig.h @@ -226,6 +226,16 @@ typedef struct ClusterReconfigState { pg_atomic_uint64 apply_counter; /* total events observed */ pg_atomic_uint64 dedup_skip_counter; /* duplicate event_id skipped */ pg_atomic_uint64 procsig_broadcast_count; /* PROCSIG broadcast tally */ + /* spec-2.29a r2 t/274: a backend-context coordinator (e.g. the fail-stop + * inject test) advances the epoch then SYNCHRONOUSLY waits for the fence + * marker before publishing. During that (seconds-long) wait the epoch is + * bumped but the reconfig event is not yet published, so a concurrently + * running LMON GRD IDLE tick would re-capture the post-bump epoch as its + * WAIT_EPOCH baseline and wedge (old==cur). The LMON-process-local staged + * flags cannot see a backend's pre-bump window, so publish it in shmem: + * set before the bump, cleared after publish / on failure. The GRD IDLE + * baseline-hold guard reads this OR the local stage. */ + pg_atomic_uint32 prebump_sync_active; /* spec-5.14 D6 — touched_peers fail-stop observability (Q8 A': these * live in this existing region; the region count is unchanged). */ diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 24a7ce43a4..53dfe32045 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -184,6 +184,35 @@ UT_TEST(test_version_coherent) } +/* ============================================================ + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r2 P2-1) + * ============================================================ */ + +UT_TEST(test_own_commit_latched) +{ + /* own commit latches only when the epoch advanced AND both the others-dead + * bitmap and the scalar dead_generation are unchanged. */ + + /* clean commit: epoch advanced, nothing else moved -> latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); + + /* epoch has not advanced yet -> not our commit (keep waiting) */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + + /* a third-party death is currently in the others-dead set -> escalate */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, true)); + + /* r2 P2-1 rebound case: the others-dead bitmap has REBOUND to its bound + * value (a third-party false-DEAD then recovered) but the scalar + * dead_generation ADVANCED — the non-monotone bitmap must NOT be trusted; + * the scalar conjunct forces escalate instead of a false latch/hang. */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, true, false)); + + /* both diverged -> escalate */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, false)); +} + + /* ============================================================ * U5 — writable-only quiesce gate * ============================================================ */ @@ -396,10 +425,11 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(8); + UT_PLAN(9); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); + UT_RUN(test_own_commit_latched); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); From ee2c09fd34ead1a615034923d80b54a2d021df8a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 07:59:22 +0800 Subject: [PATCH 19/27] test(cluster): pin evidence-over-inference own-commit latch (known-red) The leaving node's barrier-tick own-commit latch must be backed by direct evidence (the durable COMMITTED marker confirmation for THIS leave attempt) and must be immune to third-party transient false-DEAD flap noise: the scalar dead_generation is monotone, so a flap during the leave window advances it forever and the r2 P2-1 three-conjunct inference then refuses a healthy committed leave until the barrier deadline escalates it (nightly t/331 C1/C4 false-escalation, run 28948167577). The reworked U3b matrix pins: (a) evidence, no noise -> latch (d) evidence + flap noise -> still latch (RED on current code) (c) no evidence, any noise state -> never latch (deadline escalation stays armed; the r2 P2-1 refused-leave mis-latch wedge stays shut) Known-red TDD commit: leg (d) fails on the current three-conjunct predicate; the follow-up commit flips the latch to marker evidence. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- .../cluster_unit/test_cluster_clean_leave.c | 47 ++++++++++++------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 53dfe32045..5da4135ddc 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -185,31 +185,44 @@ UT_TEST(test_version_coherent) /* ============================================================ - * U3b — leaver barrier-tick own-commit latch (spec-2.29a r2 P2-1) + * U3b — leaver barrier-tick own-commit latch (spec-2.29a r3, evidence + * over inference) * ============================================================ */ UT_TEST(test_own_commit_latched) { - /* own commit latches only when the epoch advanced AND both the others-dead - * bitmap and the scalar dead_generation are unchanged. */ - - /* clean commit: epoch advanced, nothing else moved -> latch */ + /* The latch verdict is EVIDENCE-only: the first argument is "the durable + * COMMITTED marker for THIS leave attempt was confirmed" (the coordinator's + * nonce-bound LEAVE_COMMITTED attestation). The two coherence observations + * (others-dead bitmap / scalar dead_generation) stay in the signature as + * contract inputs the verdict must IGNORE — a third-party transient + * false-DEAD flap on the leaver's local CSSD view advances the (monotone) + * dead_generation and can transiently disturb the bitmap, and neither may + * refuse a latch that marker evidence backs (the t/331 C1/C4 false- + * escalation), nor may any combination latch without evidence (the r2 P2-1 + * refused-leave mis-latch hang). */ + + /* (a) evidence present, no flap noise -> latch */ UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, true)); - /* epoch has not advanced yet -> not our commit (keep waiting) */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); - - /* a third-party death is currently in the others-dead set -> escalate */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, true)); + /* (d) PINNING LEG (t/331 C1/C4 regression): a third-party flap advanced the + * scalar dead_generation (it never rebounds) while the bitmap rebounded to + * its bound value — WITH marker evidence the leave still latches; the flap + * must not false-escalate a committed leave. */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, true, false)); - /* r2 P2-1 rebound case: the others-dead bitmap has REBOUND to its bound - * value (a third-party false-DEAD then recovered) but the scalar - * dead_generation ADVANCED — the non-monotone bitmap must NOT be trusted; - * the scalar conjunct forces escalate instead of a false latch/hang. */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, true, false)); + /* (d) flap currently visible in the others-dead bitmap too -> still latch */ + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, true)); + UT_ASSERT(cluster_clean_leave_own_commit_latched(true, false, false)); - /* both diverged -> escalate */ - UT_ASSERT(!cluster_clean_leave_own_commit_latched(true, false, false)); + /* (c) no evidence -> never latch, whatever the coherence observations say + * (a refused leave never produces a COMMITTED marker, so the barrier + * deadline escalation stays armed and bounds the wait — the r2 P2-1 + * mis-latch wedge stays closed). */ + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, true, false)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, true)); + UT_ASSERT(!cluster_clean_leave_own_commit_latched(false, false, false)); } From 1bd762925d6088c39d3793ed4c570a14789c6f7e Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 08:05:30 +0800 Subject: [PATCH 20/27] fix(cluster): repair torn dual-copy xid-authority header on restart A crash between write_header_both's two durable renames leaves the transitioned flag (unseal / CLUSTER_ERA stamp) in the primary copy only, with a stale pre-transition .bak behind it. Nothing on the restart path rewrote the pair -- the bootstrap gates skipped the transition call when the primary already showed the new flag, and the transition functions early-returned on the same observation -- so the stale .bak survived the whole native run. Any later transient primary read failure then fell back to it: a stale SEALED .bak hands joiners the previous pass's high-water (false-invisible + xid reissue), and a stale pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Make the two flag transitions idempotent re-asserts: the bootstrap calls them unconditionally on every boot of their respective arm, and they rewrite BOTH copies until both validate with the post-transition flags (a missing or invalid copy counts as unsettled and is restored too). Once both copies are settled the call is a no-write no-op, so the steady-state boot cost is two header reads. All fail-closed semantics (53RB5 refusals, CLUSTER_ERA re-entry FATAL, corrupt-authority PANIC) are unchanged. Red-first unit coverage: torn-window tests for both transitions (complete the transition, re-install the old image as .bak, re-run the boot path, assert the .bak is repaired and that a primary read failure falls back to the repaired image, not the stale one). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 27 +++- src/backend/cluster/cluster_xid_authority.c | 68 +++++++++- src/include/cluster/cluster_xid_authority.h | 10 +- .../cluster_unit/test_cluster_xid_authority.c | 127 +++++++++++++++++- 4 files changed, 216 insertions(+), 16 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index f4c1718f6a..69591fd2e6 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -183,12 +183,19 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, * fail-closed (unsealed) instead of adopting the previous pass's * stale high-water (spec-6.15b §3.1 multi-pass arm, rule 8.A). The * clean shutdown of this run re-publishes and re-seals. + * + * Called unconditionally (not gated on the primary's SEALED flag): + * the transition re-asserts BOTH on-disk copies, so a crash between + * a previous unseal's two renames -- a stale SEALED .bak behind an + * already-unsealed primary, which a later primary read failure + * would resurrect -- is repaired here instead of surviving the run + * (review r3-X1). Once both copies are open this is a no-write + * no-op. */ - if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) { - cluster_xid_authority_begin_native_run(); + cluster_xid_authority_begin_native_run(); + if (auth.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) elog(LOG, "cluster shared_catalog: re-opened native seed era " "(XID authority unsealed for this run)"); - } return; } @@ -268,13 +275,21 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, (unsigned long long)own_next, (unsigned long long)auth.native_hw_full); } - if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) { - cluster_xid_authority_mark_cluster_era(); + /* + * One-way CLUSTER_ERA stamp, re-run unconditionally on every cluster + * boot: the transition re-asserts BOTH on-disk copies, so a crash + * between a previous stamp's two renames -- a stale pre-CLUSTER_ERA + * .bak behind an already-stamped primary, which a later primary read + * failure would hand to an enabled=off boot as a re-entry bypass -- is + * repaired here instead of surviving the run (review r3-X1). Once + * both copies are stamped this is a no-write no-op. + */ + cluster_xid_authority_mark_cluster_era(); + if ((auth.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) == 0) elog(LOG, "cluster shared_catalog: marked XID authority cluster era started at native " "high-water %llu", (unsigned long long)auth.native_hw_full); - } } void diff --git a/src/backend/cluster/cluster_xid_authority.c b/src/backend/cluster/cluster_xid_authority.c index 08904b7445..6a16671c57 100644 --- a/src/backend/cluster/cluster_xid_authority.c +++ b/src/backend/cluster/cluster_xid_authority.c @@ -171,6 +171,46 @@ read_image(const char *path, char *image) return cluster_xid_authority_classify(image, CLUSTER_XID_AUTHORITY_FILE_SIZE); } +/* + * copy_flags_settled -- does the on-disk copy at relpath validate AND carry + * all `must_set` flag bits with all `must_clear` bits clear? + */ +static bool +copy_flags_settled(const char *relpath, uint32 must_set, uint32 must_clear) +{ + ClusterXidAuthorityHeader hdr; + char path[MAXPGPATH]; + char image[CLUSTER_XID_AUTHORITY_FILE_SIZE]; + + if (!build_path(path, sizeof(path), relpath)) + return false; + if (read_image(path, image) != CLUSTER_XID_AUTHORITY_VALID) + return false; + memcpy(&hdr, image, sizeof(hdr)); + return (hdr.flags & must_set) == must_set && (hdr.flags & must_clear) == 0; +} + +/* + * both_copies_flags_settled -- is a one-way flag transition complete in BOTH + * on-disk copies? Flag transitions install the same image as primary and + * .bak (primary first), so a crash between the two durable renames leaves a + * pre-transition .bak behind an already-transitioned primary. Nothing else + * on the restart path rewrites the pair, so without a re-assert the stale + * copy survives the whole run, and any later transient primary read failure + * resurrects the pre-transition flags through the .bak fallback -- a stale + * SEALED .bak hands joiners the previous pass's high-water and a stale + * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard (review + * r3-X1, 8.A). The transition entry points therefore skip their write only + * when this returns true; a missing or invalid copy is "not settled" too, + * which restores the damaged copy for free. + */ +static bool +both_copies_flags_settled(uint32 must_set, uint32 must_clear) +{ + return copy_flags_settled(CLUSTER_XID_AUTHORITY_REL_PATH, must_set, must_clear) + && copy_flags_settled(CLUSTER_XID_AUTHORITY_BAK_REL_PATH, must_set, must_clear); +} + /* * cluster_xid_authority_read -- fail-closed read of the shared authority. * Tries primary then .bak; returns false when neither is trustworthy. Never @@ -284,8 +324,10 @@ write_header(ClusterXidAuthorityHeader *hdr) * pre-CLUSTER_ERA .bak re-opens the native-era re-entry guard. Crash * points: both copies old (transition not yet taken: state still * consistent), primary new + .bak old (read prefers the valid primary), - * both new. A later single-copy corruption therefore never resurrects the - * pre-transition flags. + * both new. The primary-new/.bak-old window does not self-repair on its + * own: the transition entry points are re-run by every boot and re-assert + * both copies until both_copies_flags_settled holds (review r3-X1), so a + * later single-copy corruption never resurrects the pre-transition flags. */ static void write_header_both(ClusterXidAuthorityHeader *hdr) @@ -378,7 +420,11 @@ cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, b /* * cluster_xid_authority_mark_cluster_era -- one-way CLUSTER_ERA stamp (first - * cluster.enabled=on boot; spec §3.1). No-op when already stamped. A + * cluster.enabled=on boot; spec §3.1). Idempotent re-assert: re-run by + * every cluster boot, it rewrites BOTH copies until both carry the flag, so + * a crash between write_header_both's two renames (stamped primary, stale + * .bak) is repaired on the next boot instead of surviving the run (review + * r3-X1); once both copies are settled it is a no-write no-op. A * missing/corrupt authority PANICs: the catalog bootstrap seeds it before * any caller can reach this point, so absence here is real damage. */ @@ -391,8 +437,9 @@ cluster_xid_authority_mark_cluster_era(void) ereport(PANIC, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID authority is missing or corrupt at cluster-era stamp"))); - if (hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) - return; + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA) != 0 + && both_copies_flags_settled(CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, 0)) + return; /* transition complete in both copies */ hdr.flags |= CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA; write_header_both(&hdr); /* review F1 variant: one-way flag in BOTH copies */ @@ -417,6 +464,12 @@ cluster_xid_authority_mark_cluster_era(void) * the stamp is re-applied by the next cluster boot (mark is re-issued * whenever unset) and this authority is left unsealed -- which only ever * blocks adoption, never yields a wrong answer. + * + * Idempotent re-assert (review r3-X1): re-run by every native-era boot, + * it rewrites BOTH copies until neither retains SEALED, so a crash + * between write_header_both's two renames (unsealed primary, stale + * SEALED .bak) is repaired on the next boot instead of surviving the + * run; once both copies are settled it is a no-write no-op. */ void cluster_xid_authority_begin_native_run(void) @@ -432,8 +485,9 @@ cluster_xid_authority_begin_native_run(void) (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("native seed era cannot be re-entered on a formed shared catalog tree"))); - if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0) - return; /* already open (first run, or a prior pass crashed) */ + if ((hdr.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED) == 0 + && both_copies_flags_settled(0, CLUSTER_XID_AUTHORITY_FLAG_SEALED)) + return; /* open, and no copy retains SEALED */ hdr.flags &= ~CLUSTER_XID_AUTHORITY_FLAG_SEALED; write_header_both(&hdr); /* review F1: no copy may retain SEALED */ diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index c9c51f7ab2..3287f77c22 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -194,14 +194,20 @@ extern bool cluster_xid_authority_seed_if_absent(uint64 initial_native_hw); extern void cluster_xid_authority_publish_native(uint64 native_hw_full, uint64 next_multi, bool seal); -/* One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). */ +/* + * One-way: stamp CLUSTER_ERA (first cluster.enabled=on boot). Idempotent + * re-assert: rewrites BOTH copies until both carry the flag (repairing a + * torn previous stamp); no-write no-op once both are stamped. + */ extern void cluster_xid_authority_mark_cluster_era(void); /* * A follow-up native-era (cluster.enabled=off) boot on a sealed authority * re-opens the era: clears SEALED so a crash of this run never exposes the * previous pass's stale high-water to joiners. Caller vets CLUSTER_ERA - * first (re-entry is FATAL); no-op when already unsealed. + * first (re-entry is FATAL). Idempotent re-assert: rewrites BOTH copies + * until neither retains SEALED (repairing a torn previous unseal); + * no-write no-op once both are open. */ extern void cluster_xid_authority_begin_native_run(void); diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index 4a646270e1..cb745fd2a0 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -544,6 +544,129 @@ UT_TEST(test_mark_cluster_era_survives_primary_corruption_via_bak) CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); } +/* + * write_raw_image / read_raw_image -- install / fetch one fixed-size + * authority image at `path` (raw open/write, no CRC games: the images + * moved around here are byte copies of real, valid on-disk images). + */ +static void +write_raw_image(const char *path, const char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDWR | O_CREAT | O_TRUNC, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(write(fd, image, len), (long long)len); + close(fd); +} + +static void +read_raw_image(const char *path, char *image, size_t len) +{ + int fd; + + fd = open(path, O_RDONLY, 0); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(read(fd, image, len), (long long)len); + close(fd); +} + +UT_TEST(test_unseal_torn_crash_repairs_stale_sealed_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1: a crash BETWEEN write_header_both's two renames leaves + * primary=UNSEALED(new) / .bak=SEALED(old high-water). Every native-era + * boot re-runs the transition, which must re-assert BOTH copies; before + * the fix it early-returned on the already-unsealed primary and the + * stale SEALED .bak survived the whole run -- any later primary read + * failure resurrected the previous pass's high-water (8.A). */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + /* capture the pre-transition (SEALED) primary image */ + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + /* complete the transition, then re-install the old SEALED image as + * .bak, simulating the crash window between the two durable renames */ + cluster_xid_authority_begin_native_run(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next native-era boot re-runs the transition */ + cluster_xid_authority_begin_native_run(); + + /* the .bak copy itself must have been repaired to an unsealed image */ + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); + + /* a later primary read failure must fall back to the REPAIRED image, + * not resurrect the previous pass's sealed high-water */ + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.native_hw_full, 816); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_SEALED, 0); +} + +UT_TEST(test_mark_cluster_era_torn_crash_repairs_stale_bak) +{ + ClusterXidAuthorityHeader got; + char primary_path[MAXPGPATH]; + char bak_path[MAXPGPATH]; + char old_image[sizeof(ClusterXidAuthorityHeader)]; + char bak_image[sizeof(ClusterXidAuthorityHeader)]; + int fd; + + /* review r3-X1 variant: the same torn-crash window across the + * CLUSTER_ERA stamp leaves a .bak without the one-way flag; a .bak + * fallback would then let an enabled=off boot re-enter the native era + * on a formed tree. The stamp is re-run by every cluster boot and + * must repair the lagging copy. */ + unlink_files(); + UT_ASSERT_EQ(cluster_xid_authority_seed_if_absent(791), true); + cluster_xid_authority_publish_native(816, 1, true); + + snprintf(primary_path, sizeof(primary_path), "%s/%s", test_root, + CLUSTER_XID_AUTHORITY_REL_PATH); + snprintf(bak_path, sizeof(bak_path), "%s/%s", test_root, CLUSTER_XID_AUTHORITY_BAK_REL_PATH); + read_raw_image(primary_path, old_image, sizeof(old_image)); + + cluster_xid_authority_mark_cluster_era(); + write_raw_image(bak_path, old_image, sizeof(old_image)); + + /* restart repair path: the next cluster boot re-runs the stamp */ + cluster_xid_authority_mark_cluster_era(); + + read_raw_image(bak_path, bak_image, sizeof(bak_image)); + UT_ASSERT_EQ(cluster_xid_authority_classify(bak_image, sizeof(bak_image)), + CLUSTER_XID_AUTHORITY_VALID); + memcpy(&got, bak_image, sizeof(got)); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); + + fd = open(primary_path, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + (void)!write(fd, "garbage!", 8); + close(fd); + UT_ASSERT_EQ(cluster_xid_authority_read(&got), true); + UT_ASSERT_EQ(got.flags & CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA, + CLUSTER_XID_AUTHORITY_FLAG_CLUSTER_ERA); +} + UT_TEST(test_prefix_check_divergence_truth_table) { const uint64 xids_per_page = (uint64)BLCKSZ * 4; @@ -682,7 +805,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(15); + UT_PLAN(17); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -692,6 +815,8 @@ main(void) UT_RUN(test_begin_native_run_unseals_before_cluster_era); UT_RUN(test_unseal_survives_primary_corruption_via_bak); UT_RUN(test_mark_cluster_era_survives_primary_corruption_via_bak); + UT_RUN(test_unseal_torn_crash_repairs_stale_sealed_bak); + UT_RUN(test_mark_cluster_era_torn_crash_repairs_stale_bak); UT_RUN(test_primary_corrupt_falls_back_to_bak); UT_RUN(test_present_distinguishes_corrupt_from_absent); UT_RUN(test_prehistory_publish_adopt_round_trip); From d8b1d6d414b249d755c3b8a93ae62277d80bea0d Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:04:50 +0800 Subject: [PATCH 21/27] fix(cluster): close prefix-check front-truncation blind spot (fail-closed) The divergent-lineage prefix check broke out of its comparison loop and returned CONSISTENT on the FIRST missing local pg_xact page. That is only sound for a limit that ends where the local tree ends; pg_xact front truncation (SimpleLruTruncate removes whole low segments once every xid they cover is frozen past oldestXid) means a joiner can have segment 0000 legitimately absent while later segments are present AND divergent -- the first-page ENOENT then declared the clone CONSISTENT without ever comparing the surviving pages, and the adopt arm overwrote the joiner's own live-xid outcome pages with the seed's bits (false-visible / false-invisible). Pass the caller's recovery-anchor oldestXid into the check and start the comparison at oldestXid's own 2-bit slot: bits below it are frozen truth that CLOG never consults again and whose on-disk survival is an accident of truncation timing, so they are exempt whether the pages survive or not (neither truncation nor tuple freezing rewrites surviving CLOG bytes, so this can never hide live evidence). Within the comparable range [oldestXid, min(own_next, native_hw)) a missing local page is an anomaly -- pg_xact always covers [oldestXid, nextXid) on a well-formed node -- and now returns UNAVAILABLE (fail-closed 53RB5 at the caller), never CONSISTENT. Sub-byte boundaries at oldestXid are masked at 2-bit precision. Red-first unit coverage: front-truncated divergent clone must DIVERGE (returned CONSISTENT pre-fix), divergence strictly below oldestXid stays CONSISTENT (frozen exemption, mid-byte lead mask), oldestXid at the divergent slot DIVERGES, and a hole inside the comparable range is UNAVAILABLE (the old truth-table leg expecting CONSISTENT for a missing segment encoded exactly the blind spot and now expects UNAVAILABLE). Spec: spec-6.15b-xid-authority-native-era.md --- .../cluster/cluster_catalog_bootstrap.c | 20 ++- src/backend/cluster/cluster_xid_prehistory.c | 123 ++++++++++++++---- src/include/cluster/cluster_xid_authority.h | 23 ++-- .../cluster_unit/test_cluster_xid_authority.c | 86 ++++++++++-- 4 files changed, 203 insertions(+), 49 deletions(-) diff --git a/src/backend/cluster/cluster_catalog_bootstrap.c b/src/backend/cluster/cluster_catalog_bootstrap.c index 69591fd2e6..413af44ebb 100644 --- a/src/backend/cluster/cluster_catalog_bootstrap.c +++ b/src/backend/cluster/cluster_catalog_bootstrap.c @@ -234,16 +234,21 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, * native range, so skipping would trust divergent local truth and * adopting would overwrite the node's own outcomes; both are * silently wrong (8.A). Byte-compare the comparable prefix - * [0, min(own_next, native_hw)) against the sealed blob and fail - * closed on any contradiction. Bypass only when the local + * [oldestXid, min(own_next, native_hw)) against the sealed blob and + * fail closed on any contradiction. Bypass only when the local * oldestXid has advanced past the native range (frozen+truncated: - * those bits are never consulted again). + * those bits are never consulted again). Below that bypass the + * check starts at oldestXid itself (review r3-X2): segments frozen + * and truncated away are no alibi for the surviving range, and a + * missing local page inside the comparable range fails closed + * (UNAVAILABLE) instead of passing as a shorter clone. */ if ((uint64)ra.checkPointCopy.oldestXid <= auth.native_hw_full) { ClusterXidPrefixVerdict pv; pv = cluster_xid_prehistory_prefix_check(DataDir, auth.native_hw_full, - Min(own_next, auth.native_hw_full)); + Min(own_next, auth.native_hw_full), + (uint64)ra.checkPointCopy.oldestXid); if (pv == CLUSTER_XID_PREFIX_DIVERGED) ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), @@ -257,8 +262,11 @@ cluster_catalog_prepare_xid_authority(const ControlFileData *cf, ereport(FATAL, (errcode(ERRCODE_CLUSTER_XID_AUTHORITY_UNAVAILABLE), errmsg("shared XID prehistory is unavailable for the lineage check"), - errhint("Restore \"%s\" (or its .bak) from the shared tree's " - "backup.", + errdetail("Either no sealed prehistory blob passes validation, or the " + "local pg_xact has a hole inside the comparable native " + "range."), + errhint("Restore \"%s\" (or its .bak) from the shared tree's backup, " + "or re-provision this node if its local pg_xact is damaged.", CLUSTER_XID_PREHISTORY_REL_PATH))); } diff --git a/src/backend/cluster/cluster_xid_prehistory.c b/src/backend/cluster/cluster_xid_prehistory.c index a61550e009..490804ccac 100644 --- a/src/backend/cluster/cluster_xid_prehistory.c +++ b/src/backend/cluster/cluster_xid_prehistory.c @@ -458,9 +458,11 @@ cluster_xid_prehistory_was_adopted(void) /* * read_local_clog_page_optional -- read local pg_xact page `pageno` into - * buf. Returns false when the segment file or the page does not exist - * (short clone: the comparable prefix ends there); PANICs on a real read - * error so an I/O fault is never mistaken for a short prefix. + * buf. Returns false when the segment file or the page does not exist; + * PANICs on a real read error so an I/O fault is never mistaken for a + * missing page. The caller decides what a missing page means: below + * page(oldestXid) it is a legitimately truncated frozen segment, at or + * above it is an anomaly (review r3-X2). */ static bool read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf) @@ -492,14 +494,85 @@ read_local_clog_page_optional(const char *local_pgdata, uint32 pageno, char *buf return r == BLCKSZ; } +/* + * clog_slot_mask -- byte mask covering the 2-bit CLOG status slots + * [lo, hi) within one byte (0 <= lo < hi <= 4). Slot n of a byte holds + * xid bits at (n * CLOG_BITS_PER_XACT), low-order first (clog.c + * TransactionIdToBIndex). + */ +static inline unsigned char +clog_slot_mask(uint32 lo, uint32 hi) +{ + unsigned int m; + + m = ((hi >= 4) ? 0xFFu : ((1u << (hi * 2)) - 1u)) & ~((1u << (lo * 2)) - 1u); + return (unsigned char)m; +} + +/* + * clog_page_range_equal -- compare blob vs local CLOG page content for the + * page-local 2-bit slots [from_xact, to_xact) only. Sub-byte boundaries + * are masked so bits outside the range never influence the verdict. + */ +static bool +clog_page_range_equal(const char *blob_page, const char *local_page, uint32 from_xact, + uint32 to_xact) +{ + uint32 first_byte = from_xact / 4; + uint32 last_byte = (to_xact - 1) / 4; + + Assert(from_xact < to_xact && to_xact <= PREHISTORY_XACTS_PER_PAGE); + + if (first_byte == last_byte) + return ((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, to_xact - first_byte * 4)) + == 0; + + if (from_xact % 4 != 0) { + if (((unsigned char)(blob_page[first_byte] ^ local_page[first_byte]) + & clog_slot_mask(from_xact % 4, 4)) + != 0) + return false; + first_byte++; + } + if (to_xact % 4 != 0) { + if (((unsigned char)(blob_page[last_byte] ^ local_page[last_byte]) + & clog_slot_mask(0, to_xact % 4)) + != 0) + return false; + last_byte--; + } + return first_byte > last_byte + || memcmp(blob_page + first_byte, local_page + first_byte, last_byte - first_byte + 1) + == 0; +} + /* * cluster_xid_prehistory_prefix_check -- see header. Streams the sealed * blob (primary, then .bak) and byte-compares the local pg_xact prefix - * covering xids [0, limit_xid_full) at per-xact (2-bit) precision. + * covering xids [oldest_xid_full, min(limit_xid_full, native_hw_full)) at + * per-xact (2-bit) precision. + * + * Frozen-prefix exemption (review r3-X2): SimpleLruTruncate removes whole + * pg_xact SEGMENTS once every page they cover precedes page(oldestXid) + * (slru.c SlruMayDeleteSegment), so a joiner that froze past >=1 segment + * legitimately has no file where the blob still has pages -- the old + * break-on-first-missing-local-page treated exactly that shape as a + * tail-short clone and returned CONSISTENT without ever comparing the + * surviving pages. Neither truncation nor tuple freezing rewrites the + * CLOG bytes that survive, but bits below oldestXid are dead content + * anyway (every tuple with such an xmin/xmax is frozen, so CLOG is never + * consulted for them again) and whether they still exist is an accident + * of truncation timing; lineage evidence therefore starts at oldestXid's + * own 2-bit slot, and everything below it is skipped whether the page + * survives or not. Within the comparable range [oldestXid, limit) a + * MISSING local page is an anomaly -- pg_xact always covers + * [page(oldestXid), page(nextXid)] on a well-formed node -- and returns + * UNAVAILABLE (fail-closed), never CONSISTENT. */ ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, - uint64 limit_xid_full) + uint64 limit_xid_full, uint64 oldest_xid_full) { ClusterXidPrehistoryHeader hdr; char primary[MAXPGPATH]; @@ -508,8 +581,10 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f char local_page[BLCKSZ]; const char *src_path; uint32 pages = 0; + uint32 start_page; uint32 p; uint64 limit; + uint64 oldest; int src; if (!build_path(primary, sizeof(primary), CLUSTER_XID_PREHISTORY_REL_PATH) @@ -527,6 +602,11 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f if (limit == 0) return CLUSTER_XID_PREFIX_CONSISTENT; + oldest = Min(oldest_xid_full, limit); + if (oldest >= limit) + return CLUSTER_XID_PREFIX_CONSISTENT; /* everything comparable is frozen */ + start_page = (uint32)(oldest / PREHISTORY_XACTS_PER_PAGE); + src = OpenTransientFile(src_path, O_RDONLY | PG_BINARY); if (src < 0 || read(src, &hdr, sizeof(hdr)) != sizeof(hdr)) { if (src >= 0) @@ -536,9 +616,8 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f for (p = 0; p < pages; p++) { uint64 page_first_xid = (uint64)p * PREHISTORY_XACTS_PER_PAGE; - uint64 in_scope; - uint32 full_bytes; - uint32 partial_bits; + uint64 cmp_from; + uint64 cmp_to; if (read(src, blob_page, BLCKSZ) != BLCKSZ) { CloseTransientFile(src); @@ -546,26 +625,24 @@ cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_f } if (page_first_xid >= limit) break; /* comparison scope ended on a page boundary */ + if (p < start_page) + continue; /* wholly frozen page: skipped, present or not */ - if (!read_local_clog_page_optional(local_pgdata, p, local_page)) - break; /* short clone: no local bits left to contradict */ + cmp_from = Max(page_first_xid, oldest); + cmp_to = Min(page_first_xid + PREHISTORY_XACTS_PER_PAGE, limit); + /* oldest < limit and p >= start_page guarantee a non-empty range */ + Assert(cmp_from < cmp_to); - in_scope = Min((uint64)PREHISTORY_XACTS_PER_PAGE, limit - page_first_xid); - full_bytes = (uint32)(in_scope / 4); - partial_bits = (uint32)(in_scope % 4) * 2; - - if (full_bytes > 0 && memcmp(blob_page, local_page, full_bytes) != 0) { + if (!read_local_clog_page_optional(local_pgdata, p, local_page)) { + /* hole inside the comparable range: fail closed (r3-X2) */ CloseTransientFile(src); - return CLUSTER_XID_PREFIX_DIVERGED; + return CLUSTER_XID_PREFIX_UNAVAILABLE; } - if (partial_bits > 0) { - unsigned char mask = (unsigned char)((1 << partial_bits) - 1); - if (((unsigned char)blob_page[full_bytes] & mask) - != ((unsigned char)local_page[full_bytes] & mask)) { - CloseTransientFile(src); - return CLUSTER_XID_PREFIX_DIVERGED; - } + if (!clog_page_range_equal(blob_page, local_page, (uint32)(cmp_from - page_first_xid), + (uint32)(cmp_to - page_first_xid))) { + CloseTransientFile(src); + return CLUSTER_XID_PREFIX_DIVERGED; } } CloseTransientFile(src); diff --git a/src/include/cluster/cluster_xid_authority.h b/src/include/cluster/cluster_xid_authority.h index 3287f77c22..0e73859d46 100644 --- a/src/include/cluster/cluster_xid_authority.h +++ b/src/include/cluster/cluster_xid_authority.h @@ -244,16 +244,23 @@ typedef enum ClusterXidPrefixVerdict { } ClusterXidPrefixVerdict; /* - * Compare the local pg_xact bytes covering xids [0, limit_xid_full) with the - * sealed prehistory blob at 2-bit (per-xact) precision. A local segment or - * page that does not exist ends the comparable prefix (a shorter clone has - * no bits to contradict). Callers FATAL on DIVERGED/UNAVAILABLE: a joiner - * whose own native-era history contradicts the seed's is not a pre-seed - * lineage, and neither skipping (trusting local bits) nor adopting - * (overwriting the joiner's own outcomes) is sound for it. + * Compare the local pg_xact bytes covering xids [oldest_xid_full, + * limit_xid_full) with the sealed prehistory blob at 2-bit (per-xact) + * precision. Bits below oldest_xid_full are frozen truth that CLOG never + * consults again and that SimpleLruTruncate may already have removed + * (whole segments); they are exempt whether the pages survive or not. A + * local page missing INSIDE the comparable range is fail-closed + * UNAVAILABLE, never CONSISTENT: pg_xact covers [oldestXid, nextXid) on a + * well-formed node, so a hole is an anomaly, and a front-truncated + * divergent clone must not pass as a "shorter clone" (review r3-X2). + * Callers FATAL on DIVERGED/UNAVAILABLE: a joiner whose own native-era + * history contradicts the seed's is not a pre-seed lineage, and neither + * skipping (trusting local bits) nor adopting (overwriting the joiner's + * own outcomes) is sound for it. */ extern ClusterXidPrefixVerdict cluster_xid_prehistory_prefix_check(const char *local_pgdata, uint64 native_hw_full, - uint64 limit_xid_full); + uint64 limit_xid_full, + uint64 oldest_xid_full); #endif /* CLUSTER_XID_AUTHORITY_H */ diff --git a/src/test/cluster_unit/test_cluster_xid_authority.c b/src/test/cluster_unit/test_cluster_xid_authority.c index cb745fd2a0..19b98855fa 100644 --- a/src/test/cluster_unit/test_cluster_xid_authority.c +++ b/src/test/cluster_unit/test_cluster_xid_authority.c @@ -680,7 +680,7 @@ UT_TEST(test_prefix_check_divergence_truth_table) make_fake_pgdata(pgdata_seed, sizeof(pgdata_seed), "f2seed", 1, 0xAA); make_fake_pgdata(pgdata_join, sizeof(pgdata_join), "f2join", 1, 0xAA); cluster_xid_prehistory_publish(pgdata_seed, 816); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_CONSISTENT); /* flip one status byte INSIDE the limit -> DIVERGED */ @@ -690,27 +690,29 @@ UT_TEST(test_prefix_check_divergence_truth_table) UT_ASSERT_EQ(lseek(fd, 100, SEEK_SET), 100); /* byte 100 = xids 400..403 */ (void)!write(fd, "X", 1); close(fd); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_DIVERGED); /* the same flipped byte OUTSIDE the limit -> CONSISTENT (adopt arm: * bytes at/after own_next belong to the seed alone) */ - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 400, 0), CLUSTER_XID_PREFIX_CONSISTENT); /* partial-byte boundary: xid 400's 2-bit slot enters scope at limit 401 */ - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 401, 0), CLUSTER_XID_PREFIX_DIVERGED); - /* missing local pg_xact segment -> comparable prefix is empty -> - * CONSISTENT (a shorter clone has no bits to contradict) */ + /* missing local page inside the comparable range -> fail-closed + * UNAVAILABLE, never CONSISTENT: with oldestXid=0 nothing below the + * hole is frozen, so "shorter clone" is not a possible innocent + * explanation (review r3-X2) */ unlink(seg); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), - CLUSTER_XID_PREFIX_CONSISTENT); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), + CLUSTER_XID_PREFIX_UNAVAILABLE); /* no trustworthy blob -> UNAVAILABLE */ unlink_files(); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, 816, 816, 0), CLUSTER_XID_PREFIX_UNAVAILABLE); /* multi-segment divergence: 33-page trees, flip a byte in segment 0001 */ @@ -722,18 +724,77 @@ UT_TEST(test_prefix_check_divergence_truth_table) make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "f2seedms", n_pages); make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "f2joinms", n_pages); cluster_xid_prehistory_publish(pgdata_seed, hw); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), CLUSTER_XID_PREFIX_CONSISTENT); snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); fd = open(seg, O_RDWR, 0600); UT_ASSERT(fd >= 0); (void)!write(fd, "Z", 1); close(fd); - UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw), + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, 0), CLUSTER_XID_PREFIX_DIVERGED); } } +UT_TEST(test_prefix_check_front_truncation) +{ + const uint64 xids_per_page = (uint64)BLCKSZ * 4; + const int n_pages = 33; + const uint64 hw = (uint64)n_pages * xids_per_page; + const uint64 page32_first = 32 * xids_per_page; + char pgdata_seed[MAXPGPATH]; + char pgdata_join[MAXPGPATH]; + char seg[MAXPGPATH]; + unsigned char b; + int fd; + + /* review r3-X2 (a): front truncation is no alibi. SimpleLruTruncate + * removed the joiner's segment 0000 (oldestXid frozen past it), but the + * surviving page 32 diverges inside the comparable range -> DIVERGED. + * Pre-fix, break-on-first-missing-local-page returned CONSISTENT and + * the adopt arm then overwrote the joiner's own live-xid outcomes. */ + unlink_files(); + make_fake_pgdata_multiseg(pgdata_seed, sizeof(pgdata_seed), "ftseed", n_pages); + make_fake_pgdata_multiseg(pgdata_join, sizeof(pgdata_join), "ftjoin", n_pages); + cluster_xid_prehistory_publish(pgdata_seed, hw); + snprintf(seg, sizeof(seg), "%s/pg_xact/0000", pgdata_join); + UT_ASSERT_EQ(unlink(seg), 0); + snprintf(seg, sizeof(seg), "%s/pg_xact/0001", pgdata_join); + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); /* byte 600 = slots 2400..2403 */ + (void)!write(fd, "X", 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first), + CLUSTER_XID_PREFIX_DIVERGED); + + /* (b) frozen exemption: restore byte 600 (page fill 0x20), then flip + * ONLY slot 2400's two low bits (0x20 -> 0x21). With oldestXid at slot + * 2401 the divergence sits strictly below the boundary, mid-byte: the + * lead mask must exclude it -> CONSISTENT (missing segment 0000 below + * page(oldestXid) is equally exempt). */ + fd = open(seg, O_RDWR, 0600); + UT_ASSERT(fd >= 0); + UT_ASSERT_EQ(lseek(fd, 600, SEEK_SET), 600); + b = 0x21; + (void)!write(fd, &b, 1); + close(fd); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2401), + CLUSTER_XID_PREFIX_CONSISTENT); + + /* same bytes with oldestXid AT the divergent slot -> DIVERGED (the + * boundary slot itself is comparable) */ + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_DIVERGED); + + /* review r3-X2 (c): a hole AT/ABOVE page(oldestXid) inside the + * comparable range is an anomaly -> UNAVAILABLE (fail-closed), never + * CONSISTENT */ + UT_ASSERT_EQ(unlink(seg), 0); + UT_ASSERT_EQ(cluster_xid_prehistory_prefix_check(pgdata_join, hw, hw, page32_first + 2400), + CLUSTER_XID_PREFIX_UNAVAILABLE); +} + UT_TEST(test_prehistory_classify_corrupt) { char buf[64]; @@ -805,7 +866,7 @@ main(void) { setup_shared_dir(); - UT_PLAN(17); + UT_PLAN(18); UT_RUN(test_layout_offsets_locked); UT_RUN(test_classify_short_magic_crc_valid); UT_RUN(test_payload_bytes_boundaries); @@ -822,6 +883,7 @@ main(void) UT_RUN(test_prehistory_publish_adopt_round_trip); UT_RUN(test_prehistory_multi_segment_round_trip); UT_RUN(test_prefix_check_divergence_truth_table); + UT_RUN(test_prefix_check_front_truncation); UT_RUN(test_prehistory_classify_corrupt); UT_DONE(); return ut_failed_count == 0 ? 0 : 1; From ea2cdc99ee8f5ff5b855f8a222365a51e7e03f38 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:21:29 +0800 Subject: [PATCH 22/27] fix(cluster): clean-leave own-commit latch consumes marker evidence, not inference Nightly t/331 C1 (clean_leave x idle) / C4 (leave_remove x idle) regressed at the r2 P2-1 head (run 28948167577; parent green): the leaver's barrier-tick own-commit latch inferred its commit from epoch>baseline + others-dead bitmap unchanged + scalar dead_generation unchanged. The scalar is monotone, so a third-party transient SUSPECTED->DEAD->ALIVE flap on the leaver's local CSSD view during the leave window advances it forever; the latch then refuses a leave the coordinator ACTUALLY committed and the immediate-escalate arm (or the barrier deadline) aborts it: phase ends ABORTED_ESCALATE -> IDLE, never 'committed', breaking the C1/C4 drains+commits assertions. Reverting to bitmap-only would re-open the r2 P2-1 wedge (rebound mis-latch of a REFUSED leave suppresses escalation forever), so neither inference is usable. Replace inference with direct evidence: latch <=> the durable COMMITTED marker for THIS leave attempt is confirmed. The evidence already reaches the leaver with zero extra IO: the coordinator sends the nonce-bound LEAVE_COMMITTED only after its qvotec ACKs the COMMITTED marker majority-durable (there is no runtime voting-disk read path on the leaver, and none is needed - the LMON tick budget this spec protects is untouched). cl_committed_handler now routes through a pure identity gate (self-addressed + currently leaving + per-attempt nonce + committed epoch past the bound baseline; fail-closed on any mismatch, so a stale confirmation - and through it a stale COMMITTED marker from a previous leave of the same node - can never false-latch) and records the attested committed epoch E before publishing the evidence flag; the barrier tick consumes E instead of re-reading its possibly-stale local epoch view. The latch and the P1-V0.7 exit gate collapse into one step (the evidence IS the durable-truth-source confirmation). Escalation semantics are unchanged: no evidence by the barrier deadline -> the existing cl_escalate path, which is exactly what bounds a refused leave (it never gets a COMMITTED marker). Third-party flaps no longer matter: the coherence observations are now contract inputs the predicate must ignore (pinned by the U3b unit matrix, red-first in the parent commit) and feed only a flap-noise LOG at the latch. Survivor-side coherence sites (drive_drain / staged-ACK / pre-check) are untouched. Unit: U3b matrix flipped green; new U3c pins the evidence identity gate (match / stale-nonce / misrouted / not-leaving / epoch-not-advanced). Local gates: t/331 x4 all green (6/6), t/310 24/24, t/363 18/18, t/274 18/18 (prebump shmem-bit stays green), cluster_unit 158 binaries, cluster_regress 13/13, PG core 219/219, clang-format-18 + comment-headers clean. Spec: spec-2.29a-reconfig-marker-async-lmon-liveness.md --- src/backend/cluster/cluster_clean_leave.c | 176 +++++++++--------- .../cluster/cluster_clean_leave_policy.c | 89 ++++++--- src/include/cluster/cluster_clean_leave.h | 32 +++- .../cluster_unit/test_cluster_clean_leave.c | 37 +++- 4 files changed, 218 insertions(+), 116 deletions(-) diff --git a/src/backend/cluster/cluster_clean_leave.c b/src/backend/cluster/cluster_clean_leave.c index 31e25bc8a4..24be2345d8 100644 --- a/src/backend/cluster/cluster_clean_leave.c +++ b/src/backend/cluster/cluster_clean_leave.c @@ -122,6 +122,8 @@ cluster_clean_leave_shmem_init(void) pg_atomic_init_u32(&cl_state->commit_point_observed, 0); pg_atomic_init_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_init_u32(&cl_state->committed_marker_durable, 0); + /* spec-2.29a r3 (evidence latch). */ + cl_state->committed_confirmed_epoch = 0; /* Hardening v1.0.2 (P1 preflight / P2 nonce). */ pg_atomic_init_u64(&cl_state->leave_attempt_nonce, 0); pg_atomic_init_u32(&cl_state->preflight_pending, 0); @@ -555,11 +557,15 @@ cl_commit_ready_handler(const ClusterICEnvelope *env, const void *payload) } /* - * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate): - * the survivor coordinator has made the COMMITTED marker majority-durable and - * signals that the durable truth-source now exists, so this leaving node may - * proceed to COMMITTED and exit. Only acted on if it is about OUR leave; idem- - * potent (the coordinator re-sends each tick until we are gone). + * cl_committed_handler -- leaving node side (Hardening v1.0.1, P1-V0.7 exit gate; + * spec-2.29a r3 evidence latch): the survivor coordinator has made the COMMITTED + * marker majority-durable and attests that the durable truth-source now exists. + * This confirmation is the leaver's own-commit MARKER EVIDENCE: the barrier tick + * latches on it (and only on it) and proceeds to COMMITTED/exit. Identity is + * checked by the pure evidence gate (self-addressed + currently leaving + + * per-attempt nonce + committed epoch past the bound baseline) — any mismatch is + * dropped fail-closed. Idempotent (the coordinator re-sends each tick until we + * are gone). */ static void cl_committed_handler(const ClusterICEnvelope *env, const void *payload) @@ -571,17 +577,19 @@ cl_committed_handler(const ClusterICEnvelope *env, const void *payload) return; if (!cluster_clean_leave_announce_payload_valid(p)) return; - if (p->leaving_node_id != cluster_node_id) - return; /* only the leaving node consumes its own COMMITTED confirmation */ - /* Hardening v1.0.2 (P2): bind to THIS attempt — a stale LEAVE_COMMITTED from a - * prior same-epoch attempt must not prematurely confirm the current one. Only - * meaningful once we are the leaver actively awaiting confirmation. */ - if (p->leave_nonce != pg_atomic_read_u64(&cl_state->leave_attempt_nonce)) + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + if (!cluster_clean_leave_committed_evidence_matches( + p->leaving_node_id, p->leave_nonce, p->leave_epoch, cluster_node_id, + cl_state->leaving_node_id, pg_atomic_read_u64(&cl_state->leave_attempt_nonce), + cl_state->leave_epoch)) { + LWLockRelease(&cl_state->lock); return; - if (cl_state->leaving_node_id != cluster_node_id) - return; /* we are not currently leaving */ - + } + /* the committed epoch E is recorded BEFORE the evidence flag is published, + * inside the same critical section the barrier tick reads it back under. */ + cl_state->committed_confirmed_epoch = p->leave_epoch; pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 1); + LWLockRelease(&cl_state->lock); } /* @@ -1467,6 +1475,7 @@ cl_request_body(void) pg_atomic_write_u32(&cl_state->commit_point_observed, 0); pg_atomic_write_u32(&cl_state->committed_durable_confirmed, 0); pg_atomic_write_u32(&cl_state->committed_marker_durable, 0); + cl_state->committed_confirmed_epoch = 0; /* spec-2.29a r3: per-attempt evidence */ LWLockRelease(&cl_state->lock); cl_set_phase(CLUSTER_LEAVE_REQUESTED); @@ -1755,83 +1764,72 @@ cl_leaving_barrier_tick(void) { uint64 baseline_epoch = cl_state->leave_epoch; int32 coordinator; + uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; + bool committed_evidence; + bool others_dead_unchanged; + bool dead_gen_unchanged; /* - * 1. observe the commit. The survivor coordinator runs the actual two-phase - * commit and publishes the CLEAN_LEAVE event into ITS OWN reconfig state — - * the leaving node's last_applied never carries it. What DOES propagate to - * the leaving node is the membership epoch (every IC envelope piggybacks it). - * So the leaving node detects its own commit by the epoch advancing past the - * bound baseline. Discriminate from a real death intruding (CL-I3) by the - * CSSD dead_generation: a cooperative leave does NOT mark anyone CSSD-dead, so - * an unchanged dead_generation at the bump means OUR clean-leave committed; a - * changed one means a real death (escalate). + * 1. observe the commit — EVIDENCE over inference (spec-2.29a r3). The + * survivor coordinator runs the actual two-phase commit, publishes the + * CLEAN_LEAVE event into ITS OWN reconfig state, drives the COMMITTED marker + * to voting-disk majority-durability, and only then sends the nonce-bound + * LEAVE_COMMITTED (re-sent each tick while we are alive). That confirmation + * — validated by the pure identity gate in cl_committed_handler — is the ONLY + * basis on which this node latches its own commit: latch <=> a valid + * COMMITTED marker for THIS leave attempt exists. * - * Hardening v1.0.4 (P2): this "epoch bump + dead_gen unchanged == OUR commit" - * inference is sound ONLY because there is no OTHER dead_gen-unchanged reconfig - * in flight. A spec-5.15 online JOIN also bumps the epoch with dead_gen - * unchanged and would be mis-observed here as the leave's commit (then the node - * stops escalating but never gets the real LEAVE_COMMITTED -> BARRIER_WAIT - * hang). That collision is removed STRUCTURALLY by the one-membership-reconfig- - * at-a-time serialization: the join driver does not bump the epoch while a clean - * leave is active (cluster_clean_leave_in_progress gates drive_joins + - * commit_member), and the leave does not start while a join is pending - * (cluster_reconfig_join_in_progress gates the request). Under that invariant a - * bump with an unchanged version during a leave can only be the leave's own - * commit. + * Two inference generations preceded this and each failed one way (see the + * predicate comment in cluster_clean_leave_policy.c): "epoch advanced + + * others-dead bitmap unchanged" could mis-latch a REFUSED leave after a + * third-party false-DEAD rebound (r2 P2-1 wedge); adding the monotone scalar + * dead_generation conjunct then false-escalated a healthy committed leave + * whenever a transient third-party flap advanced the leaver's local + * dead_generation during the leave window (nightly t/331 C1/C4). Marker + * evidence is immune to both: flaps cannot erase a durable marker, and a + * refused leave never produces one — no latch, so the barrier deadline below + * still bounds the wait (fail-closed escalation, unchanged semantics). * - * spec-2.29a ②b + r2 P2-1: "unchanged version" here means the others-dead - * bitmap AND the scalar dead_generation both unchanged. The bitmap excludes - * the leaving node's own expected DEAD (②b: else its heartbeat stop would - * falsely escalate), but the bitmap is not monotone — a third-party - * false-DEAD→ALIVE rebound restores it while the scalar dead_generation only - * advances (r2 P2-1: else the leaver could mis-latch a refused leave and - * hang). The leaver's own DEAD never bumps its OWN dead_generation, so the - * scalar conjunct is safe on this side (it would NOT be safe on the survivor - * side, which keeps the bitmap-only coherence check). + * The coherence observations are still taken, but ONLY for the flap-noise + * LOG at the latch (the predicate contract pins that they never affect the + * verdict). A real third-party death intruding mid-leave is refused on the + * survivor side (cl_coherent pre-check + guarded CAS, CL-I3), so no evidence + * arrives here and the deadline escalates this leaver — same outcome as the + * old immediate escalate arm, now bounded by the barrier deadline instead. + * + * Latching collapses the old two-step gate: the evidence IS the P1-V0.7 + * durable-truth-source confirmation, so the leave can no longer be + * un-committed (deadline must not escalate past here, Hardening v1.0.1 + * P1-1) AND the node may exit (COMMITTED) in the same tick. */ - if (cluster_epoch_get_current() > baseline_epoch) { - uint8 now_others_dead[CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES]; - bool others_dead_unchanged; - bool dead_gen_unchanged; - - cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); - others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, - CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) - == 0); - dead_gen_unchanged - = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); - if (cluster_clean_leave_own_commit_latched(true, others_dead_unchanged, - dead_gen_unchanged)) { - /* - * Commit point observed (our clean-leave epoch was published; no - * third-party death intruded). The leave can no longer be - * un-committed, so from here the barrier deadline must NOT escalate - * (Hardening v1.0.1 P1-1). - */ - pg_atomic_write_u32(&cl_state->commit_point_observed, 1); - LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); - cl_state->leave_epoch = cluster_epoch_get_current(); /* the committed epoch E */ - LWLockRelease(&cl_state->lock); + committed_evidence = (pg_atomic_read_u32(&cl_state->committed_durable_confirmed) != 0); + cl_others_dead_snapshot(cl_state->leaving_node_id, now_others_dead); + others_dead_unchanged = (memcmp(now_others_dead, cl_state->leave_baseline_others_dead, + CLUSTER_CLEAN_LEAVE_ACK_BITMAP_BYTES) + == 0); + dead_gen_unchanged = (cluster_cssd_get_dead_generation() == cl_state->leave_baseline_dead_gen); + if (cluster_clean_leave_own_commit_latched(committed_evidence, others_dead_unchanged, + dead_gen_unchanged)) { + uint64 committed_epoch; + + pg_atomic_write_u32(&cl_state->commit_point_observed, 1); + LWLockAcquire(&cl_state->lock, LW_EXCLUSIVE); + committed_epoch = cl_state->committed_confirmed_epoch; /* the committed epoch E */ + cl_state->leave_epoch = committed_epoch; + LWLockRelease(&cl_state->lock); - /* - * P1-V0.7 exit gate: reach COMMITTED ("may exit") ONLY after the - * coordinator confirms the COMMITTED marker is majority-durable - * (LEAVE_COMMITTED). Until then stay in BARRIER_WAIT and re-tick — the - * durable truth-source must exist before this node departs, else a - * survivor restart could not rebuild the clean-departed fact. - */ - if (pg_atomic_read_u32(&cl_state->committed_durable_confirmed)) { - cl_set_phase(CLUSTER_LEAVE_COMMITTED); - CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); - ereport(LOG, - (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " - "majority-durable; this node has drained and may exit", - (unsigned long long)cl_state->leave_epoch))); - } - } else { - cl_escalate(); /* a real death changed the version mid-leave (CL-I3) */ - } + if (!others_dead_unchanged || !dead_gen_unchanged) + ereport(LOG, (errmsg("cluster clean-leave: third-party liveness flap observed at the " + "commit latch (others-dead %s, dead_generation %s); durable " + "COMMITTED marker evidence overrides it", + others_dead_unchanged ? "unchanged" : "changed", + dead_gen_unchanged ? "unchanged" : "advanced"))); + + cl_set_phase(CLUSTER_LEAVE_COMMITTED); + CLUSTER_INJECTION_POINT("cluster-clean-leave-barrier-complete"); + ereport(LOG, (errmsg("cluster clean-leave: committed at epoch %llu + COMMITTED marker " + "majority-durable; this node has drained and may exit", + (unsigned long long)committed_epoch))); return; } @@ -1842,8 +1840,12 @@ cl_leaving_barrier_tick(void) } /* 3. fail-closed deadline — ONLY before the commit point (P1-1: a committed - * leave is never un-committed; post-commit we wait for the durable marker, - * bounded by disk health, and never escalate). */ + * leave is never un-committed). With the r3 evidence latch this arm is what + * bounds EVERY no-evidence outcome: a refused leave, a foreign death that + * moved the version (the coordinator then refuses, CL-I3), or a lost/never- + * sent confirmation all end here instead of hanging in BARRIER_WAIT. The + * commit_point_observed guard is belt-and-braces: the latch above transitions + * out of BARRIER_WAIT in the same tick it sets the flag. */ if (!pg_atomic_read_u32(&cl_state->commit_point_observed) && (uint64)GetCurrentTimestamp() > cl_state->barrier_deadline_us) { cl_escalate(); diff --git a/src/backend/cluster/cluster_clean_leave_policy.c b/src/backend/cluster/cluster_clean_leave_policy.c index ece1d0813f..91a2c4d768 100644 --- a/src/backend/cluster/cluster_clean_leave_policy.c +++ b/src/backend/cluster/cluster_clean_leave_policy.c @@ -169,32 +169,79 @@ cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 current_epoch, /* * cluster_clean_leave_own_commit_latched -- the LEAVING node's barrier tick - * infers "my clean-leave committed" from the membership epoch advancing past - * its bound baseline (the coordinator publishes the CLEAN_LEAVE event into ITS - * own reconfig state, but the epoch piggybacks to the leaver). Latching that - * inference suppresses the barrier-deadline escalation, so a FALSE latch hangs - * the leaver in BARRIER_WAIT forever. + * latches "my clean-leave committed" on direct EVIDENCE, never on inference + * (spec-2.29a r3). The evidence is committed_marker_evidence: the survivor + * coordinator made the COMMITTED marker for THIS leave attempt majority- + * durable on the voting disk and attested it with a nonce-bound + * LEAVE_COMMITTED (validated by cluster_clean_leave_committed_evidence_ + * matches before the flag this argument mirrors is ever set). Latching + * suppresses the barrier-deadline escalation and lets the leaver exit, so + * the verdict must be exactly: latch <=> evidence. * - * spec-2.29a r2 P2-1: the leaver's own-commit inference needs BOTH a bitmap - * check AND the scalar dead_generation. The others-dead bitmap alone is not - * monotone: a third-party node that false-fail-stopped (bumping the epoch) - * then recovered leaves the CSSD hysteresis DEAD→ALIVE, so the others-dead - * bitmap returns to its bound value while the scalar dead_generation (which - * only advances) does not. If the leaver's first epoch>baseline observation - * lands after that rebound, a bitmap-only check would mis-latch a leave the - * survivor coordinator actually refused. The scalar conjunct closes it — and - * it does NOT reintroduce the ②b false positive, because the leaving node's - * OWN alive→DEAD transition never bumps ITS OWN dead_generation (a node does - * not observe itself dead), so on the leaver side dead_gen stays at baseline - * through its own drain. (The survivor-side coherence sites keep the - * others-dead bitmap: a survivor DOES observe the leaver's DEAD and would be - * falsely escalated by the scalar there.) + * History of the two inference failure modes this replaces: + * - r2 P2-1 (mis-latch wedge): the pre-r2 "epoch advanced + others-dead + * bitmap unchanged" inference could mis-latch a REFUSED leave after a + * third-party false-DEAD -> ALIVE rebound restored the (non-monotone) + * bitmap, suppressing the deadline escalation forever. + * - r3 (t/331 C1/C4 false-escalation): the r2 scalar dead_generation + * conjunct is monotone the other way — a third-party transient flap on + * the leaver's local CSSD view advances it forever, so a healthy + * committed leave was refused until the deadline escalated it. + * Marker evidence is immune to both: a refused leave never gets a COMMITTED + * marker (no latch -> bounded deadline escalation), and a flap cannot make + * durable evidence disappear (latch -> no false escalation). + * + * others_dead_unchanged / dead_gen_unchanged are the leaver's live coherence + * observations. They are deliberately kept in the signature as contract + * inputs that MUST NOT affect the verdict — the U3b unit matrix pins the + * flap-immunity on them — and the runtime uses them only for the flap-noise + * LOG at the latch point (observability, never control flow). */ bool -cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, +cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, bool others_dead_unchanged, bool dead_gen_unchanged) { - return epoch_advanced && others_dead_unchanged && dead_gen_unchanged; + (void)others_dead_unchanged; /* observability-only input (see above) */ + (void)dead_gen_unchanged; /* observability-only input (see above) */ + return committed_marker_evidence; +} + +/* + * cluster_clean_leave_committed_evidence_matches -- may the leaving node + * accept a LEAVE_COMMITTED confirmation as marker evidence for THIS leave + * attempt? (spec-2.29a r3; the payload's magic/version/CRC were already + * checked by cluster_clean_leave_announce_payload_valid.) + * + * Identity is bound three ways, all fail-closed: + * - payload_leaving_node == self_node: the confirmation is addressed to + * this node's own leave (LEAVE_COMMITTED is point-to-point, but a + * misrouted frame must still not latch). + * - current_leaving_node == self_node: this node IS currently the leaver + * (not idle, not a survivor of someone else's leave). + * - payload_nonce == current_attempt_nonce: the per-attempt nonce + * (Hardening v1.0.2) pins the confirmation to THIS attempt, so a stale + * LEAVE_COMMITTED — and through it a stale COMMITTED marker — from a + * PREVIOUS leave of the same node can never false-latch a new attempt. + * - payload_epoch > bound_leave_epoch: the committed epoch E the + * coordinator attests must lie past the baseline this leave bound + * (sanity; the commit is a guarded CAS off that baseline). + */ +bool +cluster_clean_leave_committed_evidence_matches(int32 payload_leaving_node, uint64 payload_nonce, + uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, + uint64 current_attempt_nonce, + uint64 bound_leave_epoch) +{ + if (payload_leaving_node != self_node) + return false; + if (current_leaving_node != self_node) + return false; + if (payload_nonce != current_attempt_nonce) + return false; + if (payload_epoch <= bound_leave_epoch) + return false; + return true; } diff --git a/src/include/cluster/cluster_clean_leave.h b/src/include/cluster/cluster_clean_leave.h index 14975b9e7e..6a4808f793 100644 --- a/src/include/cluster/cluster_clean_leave.h +++ b/src/include/cluster/cluster_clean_leave.h @@ -206,6 +206,14 @@ typedef struct ClusterLeaveState { pg_atomic_uint32 commit_point_observed; pg_atomic_uint32 committed_durable_confirmed; pg_atomic_uint32 committed_marker_durable; + /* spec-2.29a r3 (evidence latch): the committed epoch E the coordinator's + * LEAVE_COMMITTED attested (the epoch stamped into the majority-durable + * COMMITTED marker). Written by the leaving node's confirmation handler + * under `lock` BEFORE committed_durable_confirmed is set; consumed once by + * the barrier tick when the evidence latches (so the leaver records the + * exact committed epoch instead of inferring it from its possibly-stale + * local epoch view). Guarded by `lock`. */ + uint64 committed_confirmed_epoch; /* * Hardening v1.0.2 fields. @@ -352,15 +360,25 @@ extern bool cluster_clean_leave_version_coherent(uint64 bound_epoch, uint64 curr const uint8 *bound_others_dead, const uint8 *current_others_dead, int nbytes); -/* spec-2.29a r2 P2-1: the leaving node's barrier-tick own-commit latch. Needs - * BOTH the others-dead bitmap unchanged AND the scalar dead_generation - * unchanged — the bitmap alone is not monotone under a third-party - * false-DEAD→ALIVE rebound and could mis-latch a refused leave (hang). The - * leaver's own DEAD never bumps its own dead_generation, so the scalar - * conjunct does not reintroduce the ②b false positive. */ -extern bool cluster_clean_leave_own_commit_latched(bool epoch_advanced, bool others_dead_unchanged, +/* spec-2.29a r3: the leaving node's barrier-tick own-commit latch — evidence + * over inference. Latches iff the durable COMMITTED marker for THIS leave + * attempt was confirmed (nonce-bound LEAVE_COMMITTED attestation); the two + * coherence observations are contract inputs the verdict must ignore (a + * third-party transient flap must neither refuse an evidenced latch — the + * t/331 C1/C4 false-escalation — nor latch anything without evidence — the + * r2 P2-1 refused-leave mis-latch wedge). */ +extern bool cluster_clean_leave_own_commit_latched(bool committed_marker_evidence, + bool others_dead_unchanged, bool dead_gen_unchanged); +/* spec-2.29a r3: identity gate for a LEAVE_COMMITTED confirmation — accept it + * as marker evidence only for THIS node's CURRENT leave attempt (self- + * addressed + currently leaving + per-attempt nonce match + committed epoch + * past the bound baseline); fail-closed on any mismatch. */ +extern bool cluster_clean_leave_committed_evidence_matches( + int32 payload_leaving_node, uint64 payload_nonce, uint64 payload_epoch, int32 self_node, + int32 current_leaving_node, uint64 current_attempt_nonce, uint64 bound_leave_epoch); + /* leave-intent marker structural validation (magic/version/CRC/identity). Pure: * computes CRC32C over [magic..phase] and checks magic, version, that the * leaving node is the expected declared peer, and the dead_bitmap names only the diff --git a/src/test/cluster_unit/test_cluster_clean_leave.c b/src/test/cluster_unit/test_cluster_clean_leave.c index 5da4135ddc..04363fd736 100644 --- a/src/test/cluster_unit/test_cluster_clean_leave.c +++ b/src/test/cluster_unit/test_cluster_clean_leave.c @@ -226,6 +226,40 @@ UT_TEST(test_own_commit_latched) } +/* ============================================================ + * U3c — LEAVE_COMMITTED evidence identity gate (spec-2.29a r3) + * ============================================================ */ + +UT_TEST(test_committed_evidence_matches) +{ + /* args: payload_leaving_node, payload_nonce, payload_epoch, + * self_node, current_leaving_node, current_attempt_nonce, + * bound_leave_epoch */ + + /* (a) confirmation for THIS node's CURRENT attempt, committed epoch past + * the bound baseline -> evidence accepted */ + UT_ASSERT(cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 1, 42, 7)); + + /* (b) stale identity: a LEAVE_COMMITTED (and through it a COMMITTED + * marker) from a PREVIOUS leave attempt of the same node — nonce differs + * -> fail-closed, no latch */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 41, 8, 1, 1, 42, 7)); + + /* (b) misrouted: confirmation addressed to a different leaving node */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(2, 42, 8, 1, 1, 42, 7)); + + /* (b) this node is not currently leaving (idle: leaving_node_id == -1, + * or tracking someone else's leave) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, -1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 8, 1, 2, 42, 7)); + + /* (b) committed epoch not past the bound baseline (attested epoch must be + * the guarded-CAS successor of the baseline this leave bound) */ + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 7, 1, 1, 42, 7)); + UT_ASSERT(!cluster_clean_leave_committed_evidence_matches(1, 42, 0, 1, 1, 42, 7)); +} + + /* ============================================================ * U5 — writable-only quiesce gate * ============================================================ */ @@ -438,11 +472,12 @@ UT_TEST(test_ic_payload_validation) int main(void) { - UT_PLAN(9); + UT_PLAN(10); UT_RUN(test_struct_layout); UT_RUN(test_phase_fsm); UT_RUN(test_version_coherent); UT_RUN(test_own_commit_latched); + UT_RUN(test_committed_evidence_matches); UT_RUN(test_writable_only_gate); UT_RUN(test_marker_validation); UT_RUN(test_should_invalidate); From 0155eb94975bae86fc2f385eb119d085b885884f Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:17 +0800 Subject: [PATCH 23/27] fix(cluster): re-consume same-epoch dead-set growth mid recovery episode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WAIT_BARRIER / WAIT_CLUSTER carried only epoch-coherence guards, so a survivor whose CSSD detected a coordinator-folded multi-death late kept announcing REDECLARE_DONE under its stale dead-bitmap hash: peers that stamped the full set dropped those frames on the composite key and P6 — which has no timeout by design — wedged permanently (survivor-behind detection skew). Add a mid-episode guard that aborts to IDLE on a newer local event whose dead-bitmap hash differs: the next tick re-consumes the event, re-stamps recovery_event_bitmap_hash, re-runs recovery against the fuller dead set and re-announces DONE under the new composite key. Same-set dead_generation drift is absorbed without episode churn (the event_id keeps its local ABA-scoping role). The coordinator-behind direction needs no handling: the coordinator's own later detection bumps the epoch again and the epoch guards abort. Correct the stamp-site / accounting / hash-helper comments that overstated the bitmap as quorum-ratified (each node stamps its OWN accepted event's bitmap; convergence is eventual via the CSSD deadband plus this guard), and assert a stamped episode hash is never 0 — 0 is reserved as unstamped in the mark_peer_done drop test, and a JOIN episode's all-zero bitmap hashes to a fixed nonzero constant. Unit: survivor-behind growth leg (H({1}) stamped at the folded epoch, grown {1,2} event arrives -> abort, re-stamp, full-set remaster, DONE accounting converges) plus a same-set drift no-churn leg; both go red with the guard disabled. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-2, r3-P3a) --- src/backend/cluster/cluster_grd.c | 118 ++++++++++++++++++-- src/test/cluster_unit/test_cluster_grd.c | 134 ++++++++++++++++++++++- 2 files changed, 241 insertions(+), 11 deletions(-) diff --git a/src/backend/cluster/cluster_grd.c b/src/backend/cluster/cluster_grd.c index ae87b62dfa..3df1166222 100644 --- a/src/backend/cluster/cluster_grd.c +++ b/src/backend/cluster/cluster_grd.c @@ -1723,12 +1723,16 @@ cluster_grd_recovery_event_bitmap_hash_value(void) * cluster_grd_dead_bitmap_hash — spec-4.6a Amendment v1.2 (R2). * * The cross-node half of the REDECLARE_DONE convergence key: a hash over - * the quorum-accepted dead bitmap ALONE. Same kernel as the event_id hash + * the sender's ACCEPTED dead bitmap ALONE (each node's own local event — + * there is no cross-node ratification of the bitmap; see the P0 stamp-site + * note and the r3-P2-2 mid-episode re-consume guard for how detection skew + * converges). Same kernel as the event_id hash * (cluster_reconfig_compute_event_id) minus the cssd_dead_generation fold — * the generation is per-instance observation history and does NOT converge * across survivors, which is exactly why event_id cannot key the P6 gate. - * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere, - * so the composite key degrades to the epoch-only gate there. + * A JOIN episode's dead bitmap is all-zero, hashing identically everywhere + * (a fixed NONZERO constant — 0 stays reserved for "unstamped"), so the + * composite key degrades to the epoch-only gate there. */ uint64 cluster_grd_dead_bitmap_hash(const uint8 *dead_bitmap) @@ -2126,11 +2130,23 @@ cluster_grd_recovery_mark_peer_done(int32 node, uint64 epoch, uint64 dead_bitmap * differs (a coordinator re-election adds the old coordinator to it; a * re-death of the same node rides a higher epoch via the interposed JOIN * bump), so the bitmap hash is the ABA guard. Flap-history drift changes - * only the per-instance dead_generation, never the quorum dead SET, so - * DONEs for the same episode always match here (the R2 wedge fix). + * only the per-instance dead_generation, never the sender's accepted dead + * SET, so same-set DONEs match here (the R2 wedge fix). r3-P2-2: the + * bitmap is NOT quorum-ratified — under multi-death detection skew a + * behind survivor transiently stamps (and announces) a SMALLER set's hash + * at the same epoch; those frames are dropped HERE until its own CSSD + * catches up and the mid-episode re-consume guard re-stamps + re-announces + * the full set (grd_recovery_consume_new_event_mid_episode). Per-tick + * re-announce then converges the accounting. * Pre-accept frames mismatch the previous episode's stamp and are dropped; * senders re-announce every tick, so accounting lands after our P0 accept * (which also closes the R4 pre-accept window by construction). + * + * r3-P3(a) — the `== 0` arm treats 0 as "unstamped/pre-accept" (our own + * recovery_event_bitmap_hash is 0 before the first P0 accept). This + * relies on a stamped hash never being 0: JOIN episodes hash an ALL-ZERO + * bitmap and hash_bytes_extended(zero[16], 0) is a fixed nonzero + * constant (asserted at the stamp site). */ episode_bitmap_hash = pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash); if (dead_bitmap_hash == 0 || dead_bitmap_hash != episode_bitmap_hash) @@ -2172,6 +2188,61 @@ grd_recovery_abort_to_idle(void) "mid-recovery); shards stay frozen, re-running under the new epoch"))); } +/* + * grd_recovery_consume_new_event_mid_episode — spec-4.6a r3-P2-2. + * + * WAIT_BARRIER / WAIT_CLUSTER guard against a NEWER local reconfig event at + * the SAME epoch. Multi-death detection skew: when two nodes die near- + * simultaneously the coordinator can fold both into ONE epoch bump with dead + * set {A,B}, while a survivor whose CSSD saw only {B} first stamps + * recovery_event_bitmap_hash = H({B}) and sails past WAIT_EPOCH on that + * bump. Its later local detection of A publishes a new event with dead set + * {A,B} but does NOT move the epoch, so the WAIT_BARRIER / WAIT_CLUSTER + * epoch guards never fire, the survivor keeps announcing DONE under H({B}), + * every peer that stamped H({A,B}) keeps dropping it (composite-key + * mismatch), and P6 — which has no timeout by design — wedges permanently. + * + * Disposition (runs in the single-writer LMON tick, after the epoch guard): + * - new event, FAIL direction, SAME dead-bitmap hash: only the sender- + * local dead_generation fold moved (flap drift re-fired the same quorum + * dead set). Absorb the event_id — the episode semantics (epoch, dead + * set) are unchanged, so re-running P1-P7 would be pure churn. + * Absorbing also keeps the IDLE dedup from re-consuming the event after + * this episode completes. + * - new event with a DIFFERENT dead-bitmap hash (grown dead set — the + * skew above — or a JOIN/FAIL interleave): abort to IDLE. The next + * tick re-consumes the current event, re-stamps + * recovery_event_bitmap_hash, re-runs recovery against the fuller dead + * set, and re-announces DONE under the new composite key — closing the + * survivor-behind wedge. The coordinator-behind direction needs no + * handling here: the coordinator's own later detection bumps the epoch + * again and the existing epoch guards abort. + * + * Returns true when the caller must return (episode aborted to IDLE). + */ +static bool +grd_recovery_consume_new_event_mid_episode(void) +{ + ReconfigEvent latest; + + cluster_reconfig_get_last_event(&latest); + if (latest.event_id == 0 + || latest.event_id == pg_atomic_read_u64(&cluster_grd_state->recovery_last_event_id)) + return false; + + if (latest.reconfig_kind == (uint8)RECONFIG_KIND_FAIL_STOP + && pg_atomic_read_u32(&cluster_grd_state->recovery_direction) + == (uint32)GRD_REMASTER_DIR_FAIL + && cluster_grd_dead_bitmap_hash(latest.dead_bitmap) + == pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash)) { + pg_atomic_write_u64(&cluster_grd_state->recovery_last_event_id, latest.event_id); + return false; + } + + grd_recovery_abort_to_idle(); + return true; +} + static void grd_recovery_appendf(char *buf, Size buflen, int *off, const char *fmt, ...) { @@ -2471,12 +2542,28 @@ cluster_grd_recovery_lmon_tick(void) } pg_atomic_write_u64(&cluster_grd_state->recovery_dead_bitmap[b], word); } - /* Amendment v1.2 (R2): stamp the composite convergence key's cross-node - * half. Hash over the accepted dead bitmap alone — every survivor that - * accepted the same quorum dead SET computes the same value regardless - * of its private dead_generation observation history. */ + /* Amendment v1.2 (R2) + r3-P2-2: stamp the composite convergence key's + * cross-node half. The hash is over THIS node's OWN accepted event's + * dead bitmap — there is NO cross-node ratification of the bitmap (the + * envelope propagates only the epoch). Convergence is eventual, not + * instantaneous: every survivor's CSSD deadband converges on the same + * dead set, and until it does, a survivor that accepted a SMALLER set + * at the same epoch stamps a different hash. The mid-episode + * re-consume guard (grd_recovery_consume_new_event_mid_episode) closes + * that skew: the survivor's own later detection re-stamps the full + * set's hash and re-announces DONE. dead_generation drift alone never + * changes the bitmap, so same-set re-fires hash identically. */ pg_atomic_write_u64(&cluster_grd_state->recovery_event_bitmap_hash, cluster_grd_dead_bitmap_hash(evt.dead_bitmap)); + /* r3-P3(a) — 0 is reserved as "unstamped/pre-accept" in + * mark_peer_done's drop test, so a stamped episode hash must never be + * 0. JOIN episodes hash an ALL-ZERO dead bitmap: the invariant is + * that hash_bytes_extended(zero[16], 0) != 0 (a fixed nonzero + * constant). For FAIL bitmaps a zero hash is a 2^-64 collision, the + * same trust level as the event_id hash; the Assert turns the + * otherwise-undebuggable "all DONEs dropped" wedge into a loud stop + * on cassert builds. */ + Assert(pg_atomic_read_u64(&cluster_grd_state->recovery_event_bitmap_hash) != 0); /* * spec-5.16 D2 — record the remaster direction and, for JOIN, arm the * joiner-home PCM block fence on THIS node. The joiner already armed it @@ -2749,6 +2836,12 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — a newer local event at the SAME epoch (multi-death + * detection skew grew the dead set) re-consumes; same-set drift is + * absorbed without churn. */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — advance the survivor block re-declare scan one chunk * while the GES rebind barrier is still pending (worker-centric, runs * in this tick; epoch-coherent via the guard just above). */ @@ -2850,6 +2943,13 @@ cluster_grd_recovery_lmon_tick(void) return; } + /* r3-P2-2 — same-epoch dead-set growth re-consumes here too: this + * node may already have announced DONE under the stale bitmap hash; + * the re-run re-announces under the full set's hash so peers' + * composite-key accounting converges (P6 has no timeout). */ + if (grd_recovery_consume_new_event_mid_episode()) + return; + /* spec-4.7 D2 — keep advancing the block re-declare scan after the GES * rebind barrier completed, so a large pool is fully re-declared within * the recovery window (no-op once the cursor reaches NBuffers). */ diff --git a/src/test/cluster_unit/test_cluster_grd.c b/src/test/cluster_unit/test_cluster_grd.c index 2e4b813e17..981aaa967e 100644 --- a/src/test/cluster_unit/test_cluster_grd.c +++ b/src/test/cluster_unit/test_cluster_grd.c @@ -4201,6 +4201,131 @@ UT_TEST(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance) memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); } +/* r3-P2-2 — survivor-behind multi-death detection skew. The coordinator + * folded {1,2} into ONE epoch bump; this survivor accepted only {1} first and + * stamped H({1}) at that epoch. Its later local detection of node 2 publishes + * a new event with the GROWN set at the SAME epoch (no further bump), which + * pre-fix was never re-consumed past WAIT_EPOCH: DONE hashes could never match + * and P6 (no timeout) wedged permanently. The mid-episode guard must abort to + * IDLE, re-consume, re-stamp the full set's hash, re-run the remaster against + * the fuller dead set, and let full-set DONEs account. */ +UT_TEST(test_recovery_same_epoch_dead_set_growth_restamps) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint8 grown[CLUSTER_RECONFIG_DEAD_BITMAP_BYTES]; + uint64 h_small; + uint64 h_grown; + int i; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + /* Idle tick captures the pre-reconfig baseline (old = 10). */ + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); + + /* Event 1: this survivor's CSSD has seen only node 1 dead so far. */ + ut_mock_last_event.event_id = 601; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x02; /* {1} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> parks in WAIT_EPOCH (cur==old) */ + h_small = cluster_grd_recovery_event_bitmap_hash_value(); + + /* The coordinator's single folded bump arrives via piggyback: this node + * sails past WAIT_EPOCH with the SMALL set stamped. */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER, episode epoch 11 */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_small); + + /* The wedge shape: a full-set DONE from the coordinator is dropped while + * this node's stamp is behind. */ + memset(grown, 0, sizeof(grown)); + grown[0] = 0x06; /* {1,2} */ + h_grown = cluster_grd_dead_bitmap_hash(grown); + UT_ASSERT_NE(h_grown, h_small); + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), 0); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 0); + + /* Local CSSD catches up: NEW event_id, SAME epoch, GROWN dead set. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 602; + ut_mock_last_event.dead_bitmap[0] = 0x06; + cluster_grd_recovery_lmon_tick(); /* WAIT_BARRIER guard -> abort to IDLE */ + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed + 1); + + /* Re-consume: re-stamp to the FULL set's hash, re-run the remaster against + * the fuller set (nothing may stay mastered by 1 OR 2), land back in + * WAIT_BARRIER under the same epoch. */ + cluster_grd_recovery_lmon_tick(); + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_grown); + for (i = 0; i < PGRAC_GRD_SHARD_COUNT; i++) + UT_ASSERT_EQ(cluster_grd_shard_master(i), (int32)0); + + /* The coordinator's per-tick full-set DONE re-announce now accounts. */ + cluster_grd_recovery_mark_peer_done(0, 11, h_grown); + UT_ASSERT_EQ(cluster_grd_recovery_done_epoch_for(0), 11); + UT_ASSERT_EQ(cluster_grd_recovery_done_bitmap_hash_for(0), h_grown); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + +/* r3-P2-2 no-regression leg — dead_generation drift re-fires the SAME dead + * set under the same epoch (new event_id, identical bitmap). The guard must + * ABSORB it: no abort-to-IDLE churn, stamp unchanged, and the event_id is + * consumed so the post-episode IDLE dedup does not re-run the episode. */ +UT_TEST(test_recovery_same_epoch_same_set_drift_absorbed_no_churn) +{ + ClusterGrdRecoveryCounters c0; + ClusterGrdRecoveryCounters c1; + uint64 h_set; + + ut_jr_setup_3node(); + cluster_enabled = true; + ut_mock_now = 0; + ut_mock_epoch = 10; + + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); + cluster_grd_recovery_lmon_tick(); /* idle baseline (old = 10) */ + + ut_mock_last_event.event_id = 611; + ut_mock_last_event.coordinator_node_id = 0; + ut_mock_last_event.reconfig_kind = (uint8)RECONFIG_KIND_FAIL_STOP; + ut_mock_last_event.dead_bitmap[0] = 0x04; /* {2} */ + cluster_grd_recovery_lmon_tick(); /* P0 accept -> WAIT_EPOCH */ + ut_mock_epoch = 11; + cluster_grd_recovery_lmon_tick(); /* P1-P5 -> WAIT_BARRIER */ + UT_ASSERT_EQ(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_WAIT_BARRIER); + h_set = cluster_grd_recovery_event_bitmap_hash_value(); + + /* Same-set re-fire (drift): absorbed, no episode churn. The tick may + * legitimately advance the episode (WAIT_BARRIER -> WAIT_CLUSTER with the + * trivial unit barrier) — the assertion is that it never falls back to + * IDLE and the stamp/counters do not move. */ + cluster_grd_recovery_counters_snapshot(&c0); + ut_mock_last_event.event_id = 612; + cluster_grd_recovery_lmon_tick(); + cluster_grd_recovery_counters_snapshot(&c1); + UT_ASSERT_EQ(c1.remaster_failed, c0.remaster_failed); /* no abort */ + UT_ASSERT_NE(cluster_grd_recovery_state_value(), (uint32)GRD_RECOVERY_IDLE); + UT_ASSERT_EQ(cluster_grd_recovery_event_bitmap_hash_value(), h_set); + UT_ASSERT_EQ(cluster_grd_recovery_last_event_id(), 612); + + cluster_enabled = false; + ut_mock_now = 0; + memset(&ut_mock_last_event, 0, sizeof(ut_mock_last_event)); +} + int /* cppcheck-suppress constParameter @@ -4212,8 +4337,9 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) * spec-6.3a:+6 lifecycle; 5.8 D1b:+4 (U2a-d); D1c:+2 (U3a-b); * D1e:+2 (U4a-b); 5.9 Hardening:+1 (convert ABA); * spec-5.16:+12 (join-remaster U1-U5/U10-U16); - * +1 (U17 cross-episode fence Hardening). */ - UT_PLAN(81); + * +1 (U17 cross-episode fence Hardening); + * spec-4.6a r3-P2-2:+2 (same-epoch dead-set growth re-stamp + no-churn). */ + UT_PLAN(83); UT_RUN(test_grd_clusterresid_size_16); UT_RUN(test_grd_resid_encode_decode_roundtrip); @@ -4318,6 +4444,10 @@ main(int argc pg_attribute_unused(), char *argv[] pg_attribute_unused()) UT_RUN(test_jr_u17_stale_recipient_not_excluded_next_episode); UT_RUN(test_recovery_stale_bitmap_done_rejected_and_drifted_witness_advance); + /* spec-4.6a r3-P2-2 — same-epoch multi-death detection-skew closure. */ + UT_RUN(test_recovery_same_epoch_dead_set_growth_restamps); + UT_RUN(test_recovery_same_epoch_same_set_drift_absorbed_no_churn); + UT_DONE(); return ut_failed_count == 0 ? 0 : 1; } From 2a35cb2f8c2a1ebe0c0ae23e6efbdc3c6f162987 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 24/27] fix(cluster): pair the HW remaster terminal-state fence with a read barrier MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hw_remaster_record_terminal stores the backoff deadline before the terminal result with a release fence between the stores, but the LMON relaunch decider loaded result-then-deadline with no read-side pairing: on a weakly-ordered CPU it could still pair a fresh BLOCKED with a stale zero deadline and skip one backoff wait. Insert pg_read_barrier() between the result and deadline loads (attempts needs no fence — it is written only by the single-writer LMON FSM) and cross-reference the pairing at the write site. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P3b) --- src/backend/cluster/cluster_hw_remaster.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_hw_remaster.c b/src/backend/cluster/cluster_hw_remaster.c index 03ee6751a3..f2231b59b0 100644 --- a/src/backend/cluster/cluster_hw_remaster.c +++ b/src/backend/cluster/cluster_hw_remaster.c @@ -101,7 +101,9 @@ hw_remaster_record_terminal(int dead_node, ClusterHwRemasterResult res) * that observes the terminal result also observes the matching backoff * deadline — without the fence a weakly-ordered CPU could let it pair a * fresh BLOCKED with a stale (zero) deadline and skip one backoff wait - * (bounded self-correcting, but cheap to close outright). */ + * (bounded self-correcting, but cheap to close outright). r3-P3(b): + * paired with the pg_read_barrier() between the result and deadline + * loads in cluster_hw_remaster_launch_workers (acquire/release pair). */ if (res == CLUSTER_HW_REMASTER_DONE) { cluster_hw_remaster_set_next_attempt_at(dead_node, 0); pg_memory_barrier(); @@ -657,6 +659,16 @@ cluster_hw_remaster_launch_workers(const uint64 *dead, int nwords, uint64 episod } result = cluster_hw_remaster_result(node); + /* r3-P3(b) — read-side pairing for the hw_remaster_record_terminal + * write fence (deadline stored BEFORE the terminal result, with + * pg_memory_barrier between). Load the result FIRST, fence, then the + * deadline: a decider that observed a terminal result is then + * guaranteed to observe the matching backoff deadline. Without this + * the release fence is one-sided and a weakly-ordered reader could + * still pair a fresh BLOCKED with a stale (zero) deadline. attempts + * needs no fence: it is written only by this FSM (single-writer + * LMON). */ + pg_read_barrier(); attempts = cluster_hw_remaster_attempts(node); next_attempt_at = cluster_hw_remaster_next_attempt_at(node); d = cluster_hw_remaster_relaunch_decide(launched, episode_epoch, result, attempts, From bf11f6a29099d145ec9784330477e88b4c1a09a7 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:32 +0800 Subject: [PATCH 25/27] docs(cluster): ground the dead-master unseal predicate safety proof MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The serve-gate unseal (static master no longer CSSD-DEAD -> NORMAL) is heartbeat liveness, not a direct recovery-completion signal. Document the traced proof of why returning NORMAL is nevertheless safe: CSSD — the only heartbeat sender — is spawned by the phase-4 driver at the PM_RUN transition, after the startup process completed the node's crash recovery against shared storage; and even under a stale-ALIVE view a fetch cannot complete against a still-recovering node, because the master-side handler default-denies until the node is an in-quorum MEMBER and only the (equally post-PM_RUN) QVOTEC process can establish the quorum lease after a restart wiped shmem. Deny replies surface as bounded 53R9L, endpoint unavailability as bounded 53R90, and no path falls back to a silent local storage read. Replaces the previous wording that asserted recovery-completion semantics without grounding. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P2-1 prove-safe) --- src/backend/cluster/cluster_gcs_block.c | 37 ++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/backend/cluster/cluster_gcs_block.c b/src/backend/cluster/cluster_gcs_block.c index ee200ffafe..15d0643910 100644 --- a/src/backend/cluster/cluster_gcs_block.c +++ b/src/backend/cluster/cluster_gcs_block.c @@ -1201,6 +1201,39 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) return GCS_BLOCK_RECOVERING; } + /* + * r3-P2-1 unseal-safety proof — this predicate is heartbeat LIVENESS + * (CSSD hysteresis flips DEAD->ALIVE on heartbeat receipt alone, + * cluster_cssd.c deadband scan), NOT a direct "instance recovery + * complete" signal. It is nevertheless safe to return NORMAL here, + * because on a crash-restarted master the heartbeat source itself is + * recovery-gated: + * (1) CSSD — the only heartbeat sender — is spawned by the cluster + * phase-4 driver, which the postmaster reaper invokes only at the + * PM_RUN transition, i.e. after the startup process exited 0 and + * the node's crash recovery fully replayed its WAL thread to shared + * storage (the ServerLoop respawn is equally PM_RUN-gated). A + * still-recovering node sends NO heartbeats, so a survivor's DEAD + * verdict cannot flip back early. + * (2) Belt-and-suspenders: even under a stale-ALIVE view (fast restart + * inside the deadband), a fetch cannot complete against a + * still-recovering node. The master-side handler + * (cluster_gcs_handle_block_request_envelope) default-denies unless + * the node is an in-quorum MEMBER, and cluster_qvotec_in_quorum() + * demands QUORUM_OK plus a live lease — state only the QVOTEC + * process (phase-4 / post-PM_RUN as well) can establish after a + * restart wiped shmem. The deny replies map to bounded 53R9L; an + * unresponsive endpoint exhausts the retransmit budget into bounded + * 53R90. Neither path ever falls back to a silent local storage + * read (STORAGE_FALLBACK is a master REPLY status, not a local + * fallback). + * (3) For online_join rejoin, MEMBER additionally requires coordinator + * admission, which vets the joiner's post-recovery voting slot + * (the slot_generation != 0 readiness sub-gate). + * So heartbeat-ALIVE implies the returned master completed its own + * instance recovery: its committed WAL is on shared storage and no + * merged-materialization proof is needed for its blocks. + */ if (static_master == cluster_node_id || cluster_cssd_get_peer_state(static_master) != CLUSTER_CSSD_PEER_DEAD) return GCS_BLOCK_NORMAL; @@ -1240,7 +1273,9 @@ cluster_gcs_block_phase_for_tag(BufferTag tag) * online thread recovery cannot run (GUC off — the default — or any * >2-node deployment) the materialization authority is never published, * so a dead master's blocks stay RECOVERING until the failed node - * restarts and runs its own instance recovery: a bounded, retryable + * restarts and completes its own instance recovery (the unseal above is + * heartbeat liveness; the r3-P2-1 note explains why heartbeats imply + * recovery completion): a bounded, retryable * ERROR on the request path (53R9L), never an unproven serve. A scope * predicate must never gate a correctness proof — a committed write on * a cold block that only the dead node saw has NO other guard on this From 5b8e8ef040c500825fd696c9b51497124efda83a Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:32:48 +0800 Subject: [PATCH 26/27] test(cluster): lock the dead-master serve gate with a must-fail 53R9L leg MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Amendment v1.2 (R1) serve-gate fix had no test lock: the t/362 two-outcome legs stay green under a reverted gate (their success arm accepts a NORMAL serve of a dead-master page) and no unit links the real phase decision. Add L4h: after the kill, with the dead node never restarted, once the observed survivor's own recovery episode reached IDLE, a FRESH backend on a cold survivor runs a wide pg_class/pg_attribute scan that MUST fail with exactly 53R9L — no success arm, bounded return. An out-of-scope formation never publishes the dead node's materialization proof, so every dead-master page stays RECOVERING while the node is down; the scan spans dozens of catalog pages hashed ~1/4 to the dead node and cold-reads them past the 16MB test pool, and it touches no dead-node-written rows (heap INSERTs only, no DDL), so 53R9L is the only legal outcome. Verified red-first: with the gate reverted the suite fails at exactly this leg while every other leg stays green. The episode-IDLE wait polls the pre-warmed session with the exact query shape it ran pre-kill and compares in Perl: a cast or operator added to the SQL performs fresh syscache lookups that themselves cold-read a dead-master catalog page and trip 53R9L (the very mechanism under test). Also accept the 53R51 write-fence rejection in the pre-existing two-outcome write legs: a transient lease/epoch fence right after the reconfig is an explicit, retryable fail-closed rejection inside the same honest contract. Spec: spec-4.6a-grd-recovery-liveness.md (r3-P1-1 TAP hard leg) --- .../362_shared_catalog_4node_kill_selfheal.pl | 64 ++++++++++++++++++- 1 file changed, 61 insertions(+), 3 deletions(-) diff --git a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl index a5f6719eea..75747a223c 100644 --- a/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl +++ b/src/test/cluster_tap/t/362_shared_catalog_4node_kill_selfheal.pl @@ -11,7 +11,9 @@ # 53R9L) and equally its transaction verdicts (TT status unknown for its # xids, the visibility door). Reads and DDL are asserted on the honest # two-outcome contract (success or explicit SQLSTATE, never a hang); -# new-object DDL/writes prove P7 unfreeze. +# new-object DDL/writes prove P7 unfreeze. L4h (r3-P1-1) is the serve-gate +# LOCK: a fresh backend's wide catalog scan on a cold survivor MUST fail +# 53R9L (no success arm) while the dead node is down. # After the kill legs start there is no SKIP path: BLOCKED_STRUCTURAL means # the test harness is misconfigured, and lack of convergence is a real FAIL. # @@ -284,6 +286,9 @@ sub startup_env_blocker my $blocked_before = bg_sum(\%bg, 'hw', 'remaster_blocked_count', @survivors); my $retry_before = bg_sum(\%bg, 'hw', 'remaster_retry_count', @survivors); my $cleanup_before = bg_sum(\%bg, 'pcm', 'dead_cleanup_entries', @survivors); +my %grd_done_before; +$grd_done_before{$_} = bg_counter($bg{$_}, 'grd_recovery', 'remaster_done') + for @survivors; # L2: kill node3 and require every survivor to see the real DEAD edge. # Use the CSSD log instead of a SQL view: during fail-closed GRD recovery, @@ -338,6 +343,56 @@ sub startup_env_blocker pass('L6: no transient HW BLOCKED occurred on the clean 4-node path'); } +# L4h (r3-P1-1 hard leg): with the serve gate in place, an out-of-scope +# formation (4 nodes / online_thread_recovery off) NEVER publishes the dead +# node's materialization proof, so EVERY node3-mastered page stays RECOVERING +# until node3 returns — and node3 is never restarted in this test. A wide +# catalog scan from a FRESH backend on a cold survivor therefore MUST fail +# closed with 53R9L: pg_class + pg_attribute span dozens of pages hashed ~1/4 +# to node3, and a fresh backend's full scan cold-reads pages no prior session +# pulled into the 16MB test pool. Deliberately NO success arm — a NORMAL +# serve of a dead-master page here is the exact 8.A regression this leg locks +# (reverting the serve-gate fix turns this leg red while the two-outcome legs +# stay green). Gate on the survivor's OWN episode reaching IDLE first so the +# failure surface is deterministically the post-episode materialization proof +# (53R9L), never the in-episode shard freeze (53R9I). The scan touches no +# node3-written rows (node3 ran only heap INSERTs, no DDL), so the TT +# visibility door cannot fire first: 53R9L is the ONLY legal outcome. +{ + my $cold = 2; + + # Compare in Perl, not SQL: the warm session's catcache covers exactly the + # bg_counter query shape it ran pre-kill; a cast/operator added in SQL + # would itself cold-read a dead-master catalog page and trip 53R9L. + my $idle = 0; + my $idle_deadline = time() + 90; + while (time() < $idle_deadline) + { + if (bg_counter($bg{$cold}, 'grd_recovery', 'remaster_done') + > $grd_done_before{$cold}) + { + $idle = 1; + last; + } + usleep(500_000); + } + ok($idle, "L4h: survivor node$cold GRD recovery episode reached IDLE (warm session)"); + + my $t0 = time(); + my ($rc, $out, $err) = $quad->node($cold)->psql('postgres', + 'SELECT count(*) FROM pg_class c JOIN pg_attribute a ON a.attrelid = c.oid', + timeout => 60); + my $elapsed = time() - $t0; + isnt(defined $rc ? $rc : 0, 0, + "L4h: fresh wide catalog scan on survivor node$cold MUST fail while node$dead is down"); + like($err // '', + qr/block-level cache protocol state is being rebuilt after reconfiguration/, + 'L4h: the failure is the explicit 53R9L dead-master fail-closed gate'); + cmp_ok($elapsed, '<', 60, 'L4h: fail-closed scan returned bounded (no hang)'); + diag("L4h: scan failed closed as required: " . ($err // '(no stderr)')) + if defined $err && $err ne ''; +} + # L4a (Amendment v1.2 R1): reading the dead master's PRE-EXISTING blocks in an # out-of-scope deployment (online_thread_recovery off / 4 nodes) must be a # BOUNDED, EXPLICIT fail-closed error — never a silent stale read, never a @@ -392,7 +447,7 @@ sub startup_env_blocker else { like($err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, "L4: fresh connection on node$i fail-closed with an explicit SQLSTATE"); } cmp_ok(time() - $t0, '<', 60, "L4: fresh connection probe on node$i returned bounded"); @@ -406,8 +461,11 @@ sub startup_env_blocker } else { + # write-fenced (53R51, spec-4.12): a transient lease/epoch fence on the + # writer right after the reconfig — an explicit, retryable fail-closed + # rejection, squarely inside the honest two-outcome contract. like($ddl_err // '', - qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown/i, + qr/being rebuilt after reconfiguration|resource recovering|53R9L|being remastered|53R9I|cluster TT status unknown|write-fenced/i, 'L4: coordinator DDL fail-closed with an explicit SQLSTATE (dead-master catalog block)'); diag("L4: DDL fail-closed honestly: $ddl_err"); } From bfe00bc2e9c5bd727fc81baa59210c1db5837fa2 Mon Sep 17 00:00:00 2001 From: SqlRush Date: Thu, 9 Jul 2026 09:55:22 +0800 Subject: [PATCH 27/27] fix(ci): fuse nightly TAP shard matrix after 3-way merge (362 + 363 shards, 361 into crossnode-b) --- .github/workflows/nightly.yml | 29 ++++++++--------------------- 1 file changed, 8 insertions(+), 21 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6900330eec..905b89fab7 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -176,33 +176,20 @@ jobs: # the release-cut 4-node evidence is HG#2a-CI or HG#2a-EXT (external). - { name: stage5-beta-chaos-soak, ranges: "344-345", unit: false, regress: false } # spec-6.12/6.15 cross-node perf + correctness family (t/346-350: -<<<<<<< HEAD # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash), - # the renumbered t/351-356 family, t/357 multi-xmax alias floor. + # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and + # t/361 XID authority native-era adoption. # Closes the 346+ shard hole (every new t/ file must land in a shard # the same commit; see docs/code-review-checklist.md). - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357", unit: false, regress: false } -<<<<<<< HEAD - # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-361 are - # reserved by the parallel spec-7.2 / S-xid lanes, t/363 by S-dead; - # L464 occupancy verified against origin branches 2026-07-08). Own - # shard: 4-node formation + kill + convergence needs the wall clock. + - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } + # t/362 spec-4.6a 4-node shared_catalog kill self-heal (t/358-360 + # reserved by the parallel spec-7.2 lane; L464 occupancy verified + # against origin branches 2026-07-08). Own shard: 4-node formation + # + kill + convergence needs the wall clock. - { name: stage7-reconfig-liveness, ranges: "362-362", unit: false, regress: false } -======= - # t/363 spec-2.29a marker-async LMON liveness (t/358-362 reserved by - # the parallel spec-7.2 / S-xid / S-reconfig lanes; L464 occupancy - # verified against origin branches 2026-07-08). + # t/363 spec-2.29a marker-async LMON liveness. - { name: stage7-marker-async, ranges: "363-363", unit: false, regress: false } ->>>>>>> stage7-p0-dead-diagnosis -======= - # aged-seed verdict / ledger / 3-node nudge / PI rebuild + crash) and - # the renumbered t/351-356 family, t/357 multi-xmax alias floor, and t/361 XID authority native-era adoption. - # Closes the 346+ shard hole (every new t/ file must land in a shard - # the same commit; see docs/code-review-checklist.md). - - { name: stage6-crossnode-a, ranges: "346-350", unit: false, regress: false } - - { name: stage6-crossnode-b, ranges: "351-357 361", unit: false, regress: false } ->>>>>>> stage7-p0-xid-authority steps: - name: Checkout uses: actions/checkout@v4