From 81993afd60b4e6232e76b3ceaaa119e4d909b0aa Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:09:46 -0600 Subject: [PATCH 1/8] Prevent shutdown exceptions after partial startup failures --- .../votingplugin/VotingPluginMain.java | 108 ++++--- .../VotingPluginMainShutdownTest.java | 269 ++++++++++++++++++ docs/startup-failure-shutdown.md | 6 + 3 files changed, 344 insertions(+), 39 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java create mode 100644 docs/startup-failure-shutdown.md diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index cde4d636b..331a0ec0c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -317,6 +317,7 @@ public File getLoadedPluginJarFile() { private volatile VotingPluginBackgroundTask backgroundTask; private volatile AtomicBoolean basicBungeeUpdateRunning; + private final AtomicBoolean unloadCompleted = new AtomicBoolean(); private synchronized AtomicBoolean basicBungeeUpdateAdmission() { if (basicBungeeUpdateRunning == null) basicBungeeUpdateRunning = new AtomicBoolean(); @@ -1903,6 +1904,7 @@ private void migrateVoteBroadcast(Config configFile) { @Override public void onPreLoad() { plugin = this; + unloadCompleted.set(false); setupFiles(); if ("SQLITE".equalsIgnoreCase(configFile.getData().getString("DataStorage", "SQLITE"))) { @@ -1944,11 +1946,13 @@ public void onFinish() { @Override public void onDisable() { - if (offlineVoteOwnerHandoffs != null) offlineVoteOwnerHandoffs.close(); try { - shutdownVoteReminders(); + runShutdownStep("offline vote handoffs", () -> { + if (offlineVoteOwnerHandoffs != null) offlineVoteOwnerHandoffs.close(); + }); } finally { - super.onDisable(); + try { shutdownVoteReminders(); } + finally { super.onDisable(); } } } @@ -1961,48 +1965,74 @@ private void shutdownVoteReminders() { @Override public void onUnLoad() { + if (!unloadCompleted.compareAndSet(false, true)) return; placeholderPlayerPresence.clear(); - stopBackendHostedControlLifecycle(); - stopBackendControlConnectorLifecycle(); - if (getBackendProxyHandler() != null) { - try { - getBackendProxyHandler().closeForShutdown(); - } catch (Exception e) { - debug(e); + runShutdownStep("backend hosted Control", this::stopBackendHostedControlLifecycle); + runShutdownStep("backend Control connector", this::stopBackendControlConnectorLifecycle); + runShutdownStep("backend proxy", () -> { + if (getBackendProxyHandler() != null) getBackendProxyHandler().closeForShutdown(); + }); + runShutdownStep("backend ordered vote queue", () -> { + if (backendOrderedVoteOverflowQueue != null) { + backendOrderedVoteOverflowQueue.close(); + backendOrderedVoteOverflowQueue = null; } - } - if (backendOrderedVoteOverflowQueue != null) { - backendOrderedVoteOverflowQueue.close(); - backendOrderedVoteOverflowQueue = null; - } - if (webhookManager != null) { - webhookManager.shutdown(); - } - voteTimer.shutdown(); + }); + runShutdownStep("webhook manager", () -> { + if (webhookManager != null) webhookManager.shutdown(); + }); + runShutdownStep("vote timer", this::shutdownVoteTimer); + runShutdownStep("Votifier vote queue", () -> { + if (votifierVoteOverflowQueue != null) { + votifierVoteOverflowQueue.close(); + votifierVoteOverflowQueue = null; + } + }); + runShutdownStep("time queue", () -> { + if (timeQueueHandler != null) timeQueueHandler.save(); + }); + runShutdownStep("vote reminders", this::shutdownVoteReminders); + runShutdownStep("cooldown checker", () -> { + if (coolDownCheck != null) { + coolDownCheck.shutdown(); + coolDownCheck = null; + } + }); + runShutdownStep("signs", () -> { + if (signs != null) signs.storeSigns(); + }); + runShutdownStep("event handlers", () -> HandlerList.unregisterAll(this)); + if (plugin == this) plugin = null; + } + + private void runShutdownStep(String name, Runnable step) { try { - voteTimer.awaitTermination(1, TimeUnit.SECONDS); - } catch (InterruptedException e) { - e.printStackTrace(); - } - voteTimer.shutdownNow(); - if (votifierVoteOverflowQueue != null) { - votifierVoteOverflowQueue.close(); - votifierVoteOverflowQueue = null; - } - if (timeQueueHandler != null) { - timeQueueHandler.save(); + step.run(); + } catch (RuntimeException | Error failure) { + try { + getLogger().log(java.util.logging.Level.WARNING, "Failed to shut down " + name, failure); + } catch (RuntimeException | Error ignored) { + // Logging must not prevent the next independent cleanup step. + } } + } - shutdownVoteReminders(); - - if (coolDownCheck != null) { - coolDownCheck.shutdown(); - coolDownCheck = null; + private void shutdownVoteTimer() { + if (voteTimer == null) return; + try { + voteTimer.shutdown(); + if (!voteTimer.awaitTermination(1, TimeUnit.SECONDS)) voteTimer.shutdownNow(); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + voteTimer.shutdownNow(); + } catch (RuntimeException failure) { + try { + getLogger().log(java.util.logging.Level.WARNING, "Failed to shut down vote timer", failure); + } catch (RuntimeException | Error ignored) { } + voteTimer.shutdownNow(); + } finally { + voteTimer = null; } - - getSigns().storeSigns(); - HandlerList.unregisterAll(plugin); - plugin = null; } /** diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java new file mode 100644 index 000000000..9187cff1a --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java @@ -0,0 +1,269 @@ +package com.bencodez.votingplugin; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.doThrow; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.concurrent.ScheduledExecutorService; + +import org.junit.jupiter.api.Test; + +import com.bencodez.votingplugin.signs.Signs; +import com.bencodez.votingplugin.timequeue.TimeQueueHandler; +import com.bencodez.votingplugin.placeholders.PlaceholderPlayerPresence; +import com.bencodez.votingplugin.webhook.VotingPluginWebhookManager; + +class VotingPluginMainShutdownTest { + @Test + void lateVoteTimerIsOptionalDuringPartialStartup() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + invokeShutdownVoteTimer(plugin); + assertNull(field(plugin, "voteTimer")); + } + + @Test + void voteTimerCleanupIsRepeatable() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + ScheduledExecutorService timer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); + setField(plugin, "voteTimer", timer); + invokeShutdownVoteTimer(plugin); + assertDoesNotThrow(() -> invokeShutdownVoteTimer(plugin)); + assertNull(field(plugin, "voteTimer")); + } + + @Test + void publicUnloadSkipsLateOptionalResourcesAndIsIdempotent() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "plugin", plugin); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + + assertDoesNotThrow(plugin::onUnLoad); + assertDoesNotThrow(plugin::onUnLoad); + assertNull(field(plugin, "signs")); + assertNull(field(plugin, "timeQueueHandler")); + verify((java.util.concurrent.ExecutorService) field(plugin, "backendHostedControlLifecycle")).shutdownNow(); + verify((java.util.concurrent.ExecutorService) field(plugin, "backendControlConnectorLifecycle")).shutdownNow(); + } + + @Test + void publicUnloadCleansInitializedQueueAndSignResources() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "plugin", plugin); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + Signs signs = mock(Signs.class); + TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); + setField(plugin, "signs", signs); + setField(plugin, "timeQueueHandler", timeQueue); + + plugin.onUnLoad(); + + verify(signs).storeSigns(); + verify(timeQueue).save(); + plugin.onUnLoad(); + verify(signs).storeSigns(); + } + + @Test + void failedCleanupDoesNotSkipLaterResources() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "plugin", plugin); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + Signs signs = mock(Signs.class); + TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); + VotingPluginWebhookManager webhook = mock(VotingPluginWebhookManager.class); + doThrow(new IllegalStateException("webhook close failed")).when(webhook).shutdown(); + doThrow(new IllegalStateException("queue save failed")).when(timeQueue).save(); + setField(plugin, "webhookManager", webhook); + setField(plugin, "signs", signs); + setField(plugin, "timeQueueHandler", timeQueue); + + assertDoesNotThrow(plugin::onUnLoad); + + verify(webhook).shutdown(); + verify(timeQueue).save(); + verify(signs).storeSigns(); + } + + @Test + void gracefulTimerShutdownDoesNotForceStop() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + org.mockito.Mockito.when(timer.awaitTermination(1, java.util.concurrent.TimeUnit.SECONDS)).thenReturn(true); + setField(plugin, "voteTimer", timer); + + invokeShutdownVoteTimer(plugin); + + verify(timer).shutdown(); + verify(timer, never()).shutdownNow(); + } + + @Test + void timerTimeoutForcesStop() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + org.mockito.Mockito.when(timer.awaitTermination(1, java.util.concurrent.TimeUnit.SECONDS)).thenReturn(false); + setField(plugin, "voteTimer", timer); + + invokeShutdownVoteTimer(plugin); + + verify(timer).shutdownNow(); + } + + @Test + void interruptedTimerShutdownRestoresInterruptAndForcesStop() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + doThrow(new InterruptedException()).when(timer).awaitTermination(1, java.util.concurrent.TimeUnit.SECONDS); + setField(plugin, "voteTimer", timer); + + invokeShutdownVoteTimer(plugin); + + try { + org.junit.jupiter.api.Assertions.assertTrue(Thread.currentThread().isInterrupted()); + } finally { + Thread.interrupted(); + } + verify(timer).shutdownNow(); + } + + @Test + void failedForcedTimerStopStillClosesInitializedQueuesAndListenersOnce() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "plugin", plugin); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + doThrow(new IllegalStateException("controlled forced-stop failure")).when(timer).shutdownNow(); + setField(plugin, "voteTimer", timer); + var queue = mock(com.bencodez.votingplugin.listeners.VotifierVoteOverflowQueue.class); + var cooldown = mock(com.bencodez.votingplugin.cooldown.CoolDownCheck.class); + var reminders = mock(com.bencodez.votingplugin.votereminding.VoteRemindersManager.class); + setField(plugin, "votifierVoteOverflowQueue", queue); + setField(plugin, "coolDownCheck", cooldown); + setField(plugin, "voteRemindersManager", reminders); + try (var handlers = org.mockito.Mockito.mockStatic(org.bukkit.event.HandlerList.class)) { + assertDoesNotThrow(plugin::onUnLoad); + assertDoesNotThrow(plugin::onUnLoad); + verify(queue).close(); + verify(cooldown).shutdown(); + verify(reminders).shutdown(); + handlers.verify(() -> org.bukkit.event.HandlerList.unregisterAll(plugin)); + } + assertNull(field(plugin, "voteTimer")); + assertNull(field(plugin, "votifierVoteOverflowQueue")); + } + + @Test + void rejectedMysqlStartupRetainsItsCauseAndFullDisableCleansEarlyResources() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + var options = mock(com.bencodez.advancedcore.AdvancedCoreConfigOptions.class, + org.mockito.Mockito.RETURNS_DEEP_STUBS); + org.mockito.Mockito.doReturn(options).when(plugin).getOptions(); + org.mockito.Mockito.doReturn(java.util.logging.Logger.getLogger("startup-fixture")).when(plugin).getLogger(); + org.mockito.Mockito.when(options.getDebug().isDebug()).thenReturn(true); + setField(plugin, "plugin", plugin); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + var logger = mock(java.util.logging.Logger.class); + org.mockito.Mockito.doReturn(logger).when(plugin).getLogger(); + ScheduledExecutorService coreTimer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); + setField(plugin, "timer", coreTimer); + setField(plugin, "loadUserData", true); + var configuration = new org.bukkit.configuration.file.YamlConfiguration(); + configuration.set("Host", "127.0.0.1"); configuration.set("Port", 3306); + configuration.set("Username", "fixture"); configuration.set("Password", "fixture"); + configuration.set("Database", "fixture"); configuration.set("MaxConnections", 1); + configuration.set("ConnectionTimeout", 1000); configuration.set("MinimumIdle", 0); + configuration.set("Driver", RejectingDriver.class.getName()); + configuration.set("Debug", true); + try { + IllegalStateException failure = org.junit.jupiter.api.Assertions.assertThrows(IllegalStateException.class, + () -> new com.bencodez.advancedcore.api.user.userstorage.mysql.MySQL(plugin, "Users", configuration)); + Throwable original = failure; + while (original.getCause() != null) original = original.getCause(); + org.junit.jupiter.api.Assertions.assertInstanceOf(java.sql.SQLException.class, original); + org.junit.jupiter.api.Assertions.assertEquals("controlled authentication rejection", original.getMessage()); + assertNull(plugin.getMysql(), "rejected constructor must not publish storage"); + assertDoesNotThrow(plugin::onDisable); + org.junit.jupiter.api.Assertions.assertTrue(coreTimer.isTerminated()); + assertNull(field(plugin, "voteTimer")); + assertNull(field(plugin, "signs")); + assertDoesNotThrow(plugin::onDisable); + org.junit.jupiter.api.Assertions.assertSame(original, rootCause(failure)); + verify(logger, never()).warning(org.mockito.ArgumentMatchers.anyString()); + } finally { coreTimer.shutdownNow(); } + } + + private static Throwable rootCause(Throwable failure) { + while (failure.getCause() != null) failure = failure.getCause(); + return failure; + } + + public static final class RejectingDriver implements java.sql.Driver { + public java.sql.Connection connect(String url, java.util.Properties properties) throws java.sql.SQLException { + throw new java.sql.SQLException("controlled authentication rejection", "28000", 1045); + } + public boolean acceptsURL(String url) { return url != null && url.startsWith("jdbc:"); } + public java.sql.DriverPropertyInfo[] getPropertyInfo(String url, java.util.Properties properties) { + return new java.sql.DriverPropertyInfo[0]; + } + public int getMajorVersion() { return 1; } + public int getMinorVersion() { return 0; } + public boolean jdbcCompliant() { return false; } + public java.util.logging.Logger getParentLogger() { return java.util.logging.Logger.getGlobal(); } + } + + private static void initializeEarlyLifecycle(VotingPluginMain plugin) throws Exception { + var hosted = mock(java.util.concurrent.ExecutorService.class); + var connector = mock(ScheduledExecutorService.class); + org.mockito.Mockito.when(hosted.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)).thenReturn(true); + org.mockito.Mockito.when(connector.awaitTermination(10, java.util.concurrent.TimeUnit.SECONDS)).thenReturn(true); + setField(plugin, "pendingNativeUserStorageCloseLock", new Object()); + setField(plugin, "pendingNativeUserStorageCloses", new java.util.ArrayList<>()); + setField(plugin, "backendHostedControlLifecycle", hosted); + setField(plugin, "backendControlConnectorLifecycle", connector); + setField(plugin, "backendHostedControlManagers", java.util.concurrent.ConcurrentHashMap.newKeySet()); + org.mockito.Mockito.doReturn(java.util.logging.Logger.getLogger("shutdown-fixture")).when(plugin).getLogger(); + } + + private static void invokeShutdownVoteTimer(VotingPluginMain plugin) throws Exception { + Method method = VotingPluginMain.class.getDeclaredMethod("shutdownVoteTimer"); + method.setAccessible(true); + method.invoke(plugin); + } + + private static Object field(VotingPluginMain plugin, String name) throws Exception { + Field field = VotingPluginMain.class.getDeclaredField(name); + field.setAccessible(true); + return field.get(plugin); + } + + private static void setField(VotingPluginMain plugin, String name, Object value) throws Exception { + Field field = findField(name); + field.setAccessible(true); + field.set(plugin, value); + } + + private static Field findField(String name) throws Exception { + Class type = VotingPluginMain.class; + while (type != null) { + try { return type.getDeclaredField(name); } + catch (NoSuchFieldException ignored) { type = type.getSuperclass(); } + } + throw new NoSuchFieldException(name); + } +} diff --git a/docs/startup-failure-shutdown.md b/docs/startup-failure-shutdown.md new file mode 100644 index 000000000..b2a5b987b --- /dev/null +++ b/docs/startup-failure-shutdown.md @@ -0,0 +1,6 @@ +# Startup failure and shutdown + +VotingPlugin may be unloaded after a partial startup. Shutdown therefore treats each optional subsystem independently: +a failed close does not prevent the remaining queues, reminders, cooldown checks, signs, handlers, or executors from being +cleaned up. The vote timer uses a bounded termination wait, forces cancellation if it does not terminate, and restores +the interrupt flag when shutdown is interrupted. Repeated unload callbacks are safe after the first unload completes. From 4409f8fdd93740933b21f50b3e7268002c05c268 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 23:25:48 -0600 Subject: [PATCH 2/8] Assert partial-startup shutdown emits no cleanup failure warnings --- .../com/bencodez/votingplugin/VotingPluginMainShutdownTest.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java index 9187cff1a..74b3c6169 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java @@ -205,6 +205,8 @@ void rejectedMysqlStartupRetainsItsCauseAndFullDisableCleansEarlyResources() thr assertDoesNotThrow(plugin::onDisable); org.junit.jupiter.api.Assertions.assertSame(original, rootCause(failure)); verify(logger, never()).warning(org.mockito.ArgumentMatchers.anyString()); + verify(logger, never()).log(org.mockito.ArgumentMatchers.eq(java.util.logging.Level.WARNING), + org.mockito.ArgumentMatchers.anyString(), org.mockito.ArgumentMatchers.any(Throwable.class)); } finally { coreTimer.shutdownNow(); } } From a89bb5594e9e6daa16091e1cba0e86ba3fff808c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:25:34 -0600 Subject: [PATCH 3/8] Build startup regressions against corrected SQL dependencies --- .github/workflows/maven.yml | 21 ++++++++++++++++++++- docs/startup-failure-shutdown.md | 21 +++++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 8eceaf272..56a5b8627 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -22,8 +22,27 @@ jobs: java-version: '21' distribution: 'temurin' cache: maven + # Build the corrected SQL lifecycle dependencies until their published snapshots catch up. + - name: Check out SimpleAPI SQL lifecycle dependency + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: BenCodez/SimpleAPI + ref: fdbf26eec885ec3f1c1dbd937a243a0acfaa0a5e + path: .ci/simpleapi + persist-credentials: false + - name: Build and install SimpleAPI dependency + run: mvn -B -f .ci/simpleapi/SimpleAPI/pom.xml clean install + - name: Check out AdvancedCore SQL lifecycle dependency + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: BenCodez/AdvancedCore + ref: a68cc2dfe6e44c1c0854e659c614708492fcd58d + path: .ci/advancedcore + persist-credentials: false + - name: Build and install AdvancedCore dependency + run: mvn -B -nsu -f .ci/advancedcore/AdvancedCore/pom.xml clean install - name: Build with Maven - run: mvn -B -f VotingPlugin/pom.xml package + run: mvn -B -nsu -f VotingPlugin/pom.xml package dependency-submission: if: github.event_name == 'push' diff --git a/docs/startup-failure-shutdown.md b/docs/startup-failure-shutdown.md index b2a5b987b..80441898a 100644 --- a/docs/startup-failure-shutdown.md +++ b/docs/startup-failure-shutdown.md @@ -4,3 +4,24 @@ VotingPlugin may be unloaded after a partial startup. Shutdown therefore treats a failed close does not prevent the remaining queues, reminders, cooldown checks, signs, handlers, or executors from being cleaned up. The vote timer uses a bounded termination wait, forces cancellation if it does not terminate, and restores the interrupt flag when shutdown is interrupted. Repeated unload callbacks are safe after the first unload completes. + +## Building the complete SQL failure fix + +The shutdown change works independently, but the MySQL authentication-failure regression also exercises the corrected +SimpleAPI and AdvancedCore constructors. SimpleAPI PR #111 is merged; AdvancedCore PR #361 supplies the remaining SQL +initialization fix. Build these dependencies in that order before building VotingPlugin. CI pins the merged SimpleAPI +revision `fdbf26eec885ec3f1c1dbd937a243a0acfaa0a5e` and AdvancedCore revision +`a68cc2dfe6e44c1c0854e659c614708492fcd58d` so an older published snapshot cannot silently replace the tested fixes. + +With those source revisions checked out in sibling directories, run: + +```shell +mvn -B -f ../SimpleAPI/SimpleAPI/pom.xml clean install +mvn -B -nsu -f ../AdvancedCore/AdvancedCore/pom.xml clean install +mvn -B -nsu -f VotingPlugin/pom.xml clean package +``` + +`-nsu` preserves the locally installed snapshot dependencies for these builds. The CI pins can be changed deliberately +once corrected dependencies are reliably published. This is a build dependency only: the distributable VotingPlugin JAR +includes the required libraries, and server owners do not need to install separate SimpleAPI or AdvancedCore JARs. +Invalid MySQL credentials still abort startup; no storage fallback or configuration migration is introduced. From a9b4af5fc5ea6c91064dce3fb9ac7fb192dd6d39 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 11 Oct 2026 10:58:49 -0600 Subject: [PATCH 4/8] Retain timed votes across shutdown races and failed persistence --- .github/workflows/maven.yml | 13 +- VotingPlugin/pom.xml | 2 +- .../votingplugin/VotingPluginMain.java | 9 +- .../votingplugin/data/ServerData.java | 44 ++++- .../timequeue/TimeQueueHandler.java | 119 ++++++++--- .../VotingPluginMainShutdownTest.java | 24 ++- .../TimeQueueShutdownPersistenceTest.java | 187 ++++++++++++++++++ docs/startup-failure-shutdown.md | 26 ++- 8 files changed, 363 insertions(+), 61 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownPersistenceTest.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 56a5b8627..9a0894776 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -22,21 +22,12 @@ jobs: java-version: '21' distribution: 'temurin' cache: maven - # Build the corrected SQL lifecycle dependencies until their published snapshots catch up. - - name: Check out SimpleAPI SQL lifecycle dependency - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: BenCodez/SimpleAPI - ref: fdbf26eec885ec3f1c1dbd937a243a0acfaa0a5e - path: .ci/simpleapi - persist-credentials: false - - name: Build and install SimpleAPI dependency - run: mvn -B -f .ci/simpleapi/SimpleAPI/pom.xml clean install + # AdvancedCore startup fix is still under review; SimpleAPI is pinned in the POM. - name: Check out AdvancedCore SQL lifecycle dependency uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: BenCodez/AdvancedCore - ref: a68cc2dfe6e44c1c0854e659c614708492fcd58d + ref: 4ca3d3530e002bcafd0d815e752d4f61c9bd0a8f path: .ci/advancedcore persist-credentials: false - name: Build and install AdvancedCore dependency diff --git a/VotingPlugin/pom.xml b/VotingPlugin/pom.xml index ea1707452..7a628e6dc 100644 --- a/VotingPlugin/pom.xml +++ b/VotingPlugin/pom.xml @@ -519,7 +519,7 @@ com.bencodez simpleapi - 1.0.2-SNAPSHOT + 1.0.2-20261011.161601-40 compile diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index 331a0ec0c..37e32e1e6 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -1965,7 +1965,12 @@ private void shutdownVoteReminders() { @Override public void onUnLoad() { - if (!unloadCompleted.compareAndSet(false, true)) return; + if (!unloadCompleted.compareAndSet(false, true)) { + runShutdownStep("time queue retry", () -> { + if (timeQueueHandler != null) timeQueueHandler.saveForShutdown(); + }); + return; + } placeholderPlayerPresence.clear(); runShutdownStep("backend hosted Control", this::stopBackendHostedControlLifecycle); runShutdownStep("backend Control connector", this::stopBackendControlConnectorLifecycle); @@ -1989,7 +1994,7 @@ public void onUnLoad() { } }); runShutdownStep("time queue", () -> { - if (timeQueueHandler != null) timeQueueHandler.save(); + if (timeQueueHandler != null) timeQueueHandler.saveForShutdown(); }); runShutdownStep("vote reminders", this::shutdownVoteReminders); runShutdownStep("cooldown checker", () -> { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java index a255b8d58..66f703dff 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java @@ -87,10 +87,12 @@ public void addSign(Location location, String data, int position) { * @param vote the vote time queue entry */ public void addTimeVoted(int num, VoteTimeQueue vote) { - getData().set("TimedVoteCache." + num + ".Name", vote.getName()); - getData().set("TimedVoteCache." + num + ".Service", vote.getService()); - getData().set("TimedVoteCache." + num + ".Time", vote.getTime()); - saveData(); + synchronized (plugin.getServerDataFile()) { + getData().set("TimedVoteCache." + num + ".Name", vote.getName()); + getData().set("TimedVoteCache." + num + ".Service", vote.getService()); + getData().set("TimedVoteCache." + num + ".Time", vote.getTime()); + saveTimedVoteCache(); + } } /** @@ -106,8 +108,10 @@ public void addVoteShopPurchase(String ident) { * Clears the timed vote cache. */ public void clearTimedVoteCache() { - getData().set("TimedVoteCache", null); - saveData(); + synchronized (plugin.getServerDataFile()) { + getData().set("TimedVoteCache", null); + saveTimedVoteCache(); + } } /** @@ -384,11 +388,31 @@ public void removeSign(String sign) { saveData(); } - /** - * Save data. - */ + /** Retain an ambiguous event delivery without making it eligible for automatic replay. */ + public void addUncertainTimedVote(java.util.UUID id, VoteTimeQueue vote) { + synchronized (plugin.getServerDataFile()) { + String path = "UncertainTimedVotes." + id; + getData().set(path + ".Name", vote.getName()); + getData().set(path + ".Service", vote.getService()); + getData().set(path + ".Time", vote.getTime()); + saveTimedVoteCache(); + } + } + + /** Timed votes must observe write failures before their in-memory queue is retired. */ + private void saveTimedVoteCache() { + try { + plugin.getServerDataFile().getData().save(plugin.getServerDataFile().getdFile()); + } catch (java.io.IOException failure) { + throw new java.io.UncheckedIOException("Failed to save timed vote cache", failure); + } + } + + /** Saves server data through the existing general-purpose file helper. */ public synchronized void saveData() { - plugin.getServerDataFile().saveData(); + synchronized (plugin.getServerDataFile()) { + plugin.getServerDataFile().saveData(); + } } /** diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java index 3bf50342a..1755fbeae 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java @@ -3,6 +3,7 @@ import java.time.LocalDateTime; import java.time.ZoneId; import java.util.Queue; +import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; @@ -29,6 +30,12 @@ public class TimeQueueHandler implements Listener { private Queue timeChangeQueue = new ConcurrentLinkedQueue<>(); private VotingPluginMain plugin; + private final Object persistenceLock = new Object(); + private boolean stopping; + private final AtomicBoolean processing = new AtomicBoolean(); + private VoteTimeQueue dispatchingVote; + private UUID dispatchId; + private int persistedShutdownVotes; private final AtomicBoolean retryPending = new AtomicBoolean(); private final AtomicInteger retryAttempts = new AtomicInteger(); @@ -49,8 +56,13 @@ public TimeQueueHandler(VotingPluginMain plugin) { * @param voteSiteName the vote site name */ public void addVote(String voteUsername, String voteSiteName) { - timeChangeQueue.add(new VoteTimeQueue(voteUsername, voteSiteName, - LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli())); + synchronized (persistenceLock) { + timeChangeQueue.add(new VoteTimeQueue(voteUsername, voteSiteName, + LocalDateTime.now().atZone(ZoneId.systemDefault()).toInstant().toEpochMilli())); + // A vote task can outlive shutdownNow(). Persist its late admission + // through the same owner rather than appending to an already-saved queue. + if (stopping) persistPendingVotes(); + } } /** @@ -76,12 +88,8 @@ public void postTimeChange(DateChangedEvent event) { } private void scheduleQueueProcessing(long delay, TimeUnit unit) { - boolean admitted = VoteTaskAdmission.trySchedule(plugin.getVoteTimer(), () -> { - // Clear only after the bounded executor has admitted the task. If it is - // rejected, shutdown persistence can still recover the in-memory queue. - plugin.getServerData().clearTimedVoteCache(); - processQueue(); - }, delay, unit); + synchronized (persistenceLock) { if (stopping) return; } + boolean admitted = VoteTaskAdmission.trySchedule(plugin.getVoteTimer(), this::processQueue, delay, unit); if (!admitted) { plugin.getLogger().warning("Unable to schedule time-queue processing because vote processing is busy; queued votes were retained."); scheduleRetry(); @@ -110,32 +118,93 @@ private void scheduleRetry() { * Processes all votes in the queue. */ public void processQueue() { - while (getTimeChangeQueue().size() > 0) { - VoteTimeQueue vote = getTimeChangeQueue().remove(); - PlayerVoteEvent voteEvent = new PlayerVoteEvent( - plugin.getVoteSiteManager().getVoteSite(plugin.getVoteSiteManager().getVoteSiteName(true, vote.getService()), true), vote.getName(), - vote.getService(), true); - voteEvent.setTime(vote.getTime()); - plugin.getServer().getPluginManager().callEvent(voteEvent); - - if (voteEvent.isCancelled()) { - plugin.debug("Vote cancelled"); - return; + if (!processing.compareAndSet(false, true)) return; + try { + synchronized (persistenceLock) { + if (stopping || dispatchingVote != null) return; + // Every processing entry point retires the previous disk snapshot + // before handing any occurrence to event listeners. + plugin.getServerData().clearTimedVoteCache(); + persistedShutdownVotes = 0; + } + while (true) { + VoteTimeQueue vote; + synchronized (persistenceLock) { + if (stopping || dispatchingVote != null) return; + vote = timeChangeQueue.peek(); + } + if (vote == null) return; + PlayerVoteEvent voteEvent = new PlayerVoteEvent( + plugin.getVoteSiteManager().getVoteSite(plugin.getVoteSiteManager().getVoteSiteName(true, vote.getService()), true), vote.getName(), + vote.getService(), true); + voteEvent.setTime(vote.getTime()); + synchronized (persistenceLock) { + if (stopping) return; + if (timeChangeQueue.peek() != vote) continue; + dispatchingVote = vote; + dispatchId = UUID.randomUUID(); + } + try { + plugin.getServer().getPluginManager().callEvent(voteEvent); + } catch (RuntimeException | Error failure) { + synchronized (persistenceLock) { + try { + retainUncertainDispatch(); + } catch (RuntimeException | Error persistenceFailure) { + if (failure != persistenceFailure) failure.addSuppressed(persistenceFailure); + } + } + throw failure; + } + synchronized (persistenceLock) { + timeChangeQueue.remove(vote); + dispatchingVote = null; + dispatchId = null; + } + if (voteEvent.isCancelled()) { + plugin.debug("Vote cancelled"); + return; + } } + } finally { + processing.set(false); } } + /** Uncertain event delivery is retained for review, never automatically replayed. */ + private void retainUncertainDispatch() { + if (dispatchingVote == null || !timeChangeQueue.contains(dispatchingVote)) return; + plugin.getServerData().addUncertainTimedVote(dispatchId, dispatchingVote); + timeChangeQueue.remove(dispatchingVote); + plugin.getLogger().warning("Timed vote event delivery is uncertain; retained in ServerData.yml VotingPlugin.UncertainTimedVotes for manual review."); + } + /** * Saves pending votes to server data. */ public void save() { - if (!timeChangeQueue.isEmpty()) { - int num = 0; - for (VoteTimeQueue vote : timeChangeQueue) { - plugin.getServerData().addTimeVoted(num, vote); - num++; + synchronized (persistenceLock) { persistPendingVotes(); } + } + + /** Seal processing and durably retain late admissions from tasks that ignore interruption. */ + public void saveForShutdown() { + synchronized (persistenceLock) { + stopping = true; + persistPendingVotes(); + } + } + + /** Remove only a successfully saved prefix; failures retain the same next slot for retry. */ + private void persistPendingVotes() { + VoteTimeQueue vote; + while ((vote = timeChangeQueue.peek()) != null) { + if (vote == dispatchingVote) { + retainUncertainDispatch(); + } else { + plugin.getServerData().addTimeVoted(persistedShutdownVotes, vote); + persistedShutdownVotes++; + timeChangeQueue.remove(vote); } } - timeChangeQueue.clear(); } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java index 74b3c6169..1269d88e0 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/VotingPluginMainShutdownTest.java @@ -68,7 +68,7 @@ void publicUnloadCleansInitializedQueueAndSignResources() throws Exception { plugin.onUnLoad(); verify(signs).storeSigns(); - verify(timeQueue).save(); + verify(timeQueue).saveForShutdown(); plugin.onUnLoad(); verify(signs).storeSigns(); } @@ -84,7 +84,7 @@ void failedCleanupDoesNotSkipLaterResources() throws Exception { TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); VotingPluginWebhookManager webhook = mock(VotingPluginWebhookManager.class); doThrow(new IllegalStateException("webhook close failed")).when(webhook).shutdown(); - doThrow(new IllegalStateException("queue save failed")).when(timeQueue).save(); + doThrow(new IllegalStateException("queue save failed")).when(timeQueue).saveForShutdown(); setField(plugin, "webhookManager", webhook); setField(plugin, "signs", signs); setField(plugin, "timeQueueHandler", timeQueue); @@ -92,7 +92,25 @@ void failedCleanupDoesNotSkipLaterResources() throws Exception { assertDoesNotThrow(plugin::onUnLoad); verify(webhook).shutdown(); - verify(timeQueue).save(); + verify(timeQueue).saveForShutdown(); + verify(signs).storeSigns(); + } + + @Test + void repeatedUnloadRetriesOnlyFailedTimedVotePersistence() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "unloadCompleted", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); + Signs signs = mock(Signs.class); + org.mockito.Mockito.doThrow(new IllegalStateException("disk unavailable")).doNothing() + .when(timeQueue).saveForShutdown(); + setField(plugin, "timeQueueHandler", timeQueue); + setField(plugin, "signs", signs); + assertDoesNotThrow(plugin::onUnLoad); + assertDoesNotThrow(plugin::onUnLoad); + verify(timeQueue, org.mockito.Mockito.times(2)).saveForShutdown(); verify(signs).storeSigns(); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownPersistenceTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownPersistenceTest.java new file mode 100644 index 000000000..4a80ea2ba --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownPersistenceTest.java @@ -0,0 +1,187 @@ +package com.bencodez.votingplugin.tests.timequeue; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.*; +import java.util.concurrent.*; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.bukkit.configuration.file.YamlConfiguration; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.data.ServerData; +import com.bencodez.votingplugin.timequeue.TimeQueueHandler; +import com.bencodez.votingplugin.timequeue.VoteTimeQueue; + +class TimeQueueShutdownPersistenceTest { + private record Fixture(VotingPluginMain plugin, ServerData data, TimeQueueHandler handler) {} + private Fixture fixture() { + VotingPluginMain plugin = mock(VotingPluginMain.class, RETURNS_DEEP_STUBS); + ServerData data = mock(ServerData.class); + when(plugin.getServerData()).thenReturn(data); + when(data.getTimedVoteCacheKeys()).thenReturn(Set.of()); + return new Fixture(plugin, data, new TimeQueueHandler(plugin)); + } + + @Test void failureRetainsOnlyUnsavedSuffixAndRetriesAtSameIndex() { + Fixture f = fixture(); + Map saved = new TreeMap<>(); + java.util.concurrent.atomic.AtomicBoolean fail = new java.util.concurrent.atomic.AtomicBoolean(true); + doAnswer(call -> { + int index = call.getArgument(0); + VoteTimeQueue vote = call.getArgument(1); + if (index == 1 && fail.getAndSet(false)) throw new IllegalStateException("disk unavailable"); + saved.put(index, vote.getName()); return null; + }).when(f.data).addTimeVoted(anyInt(), any()); + f.handler.addVote("First", "site"); f.handler.addVote("Second", "site"); + assertThrows(IllegalStateException.class, f.handler::saveForShutdown); + assertEquals(Map.of(0, "First"), saved); + assertEquals("Second", f.handler.getTimeChangeQueue().peek().getName()); + f.handler.saveForShutdown(); f.handler.saveForShutdown(); + f.handler.addVote("Late", "site"); + assertEquals(Map.of(0, "First", 1, "Second", 2, "Late"), saved); + assertTrue(f.handler.getTimeChangeQueue().isEmpty()); + verify(f.data, times(1)).addTimeVoted(eq(0), any()); + verify(f.data, times(2)).addTimeVoted(eq(1), any()); + } + + @Test void lateAdmissionDuringSaveIsAppendedWithoutBeingClearedOrOverwriting() throws Exception { + Fixture f = fixture(); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + Map saved = new ConcurrentHashMap<>(); + doAnswer(call -> { + int index = call.getArgument(0); + if (index == 0) { entered.countDown(); assertTrue(release.await(5, TimeUnit.SECONDS)); } + saved.put(index, ((VoteTimeQueue) call.getArgument(1)).getName()); return null; + }).when(f.data).addTimeVoted(anyInt(), any()); + f.handler.addVote("First", "site"); + ExecutorService workers = Executors.newFixedThreadPool(2); + try { + Future save = workers.submit(f.handler::saveForShutdown); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + var adding = new CountDownLatch(1); + Future late = workers.submit(() -> { adding.countDown(); f.handler.addVote("Late", "site"); }); + assertTrue(adding.await(5, TimeUnit.SECONDS)); release.countDown(); + save.get(5, TimeUnit.SECONDS); late.get(5, TimeUnit.SECONDS); + assertEquals(Map.of(0, "First", 1, "Late"), saved); + assertTrue(f.handler.getTimeChangeQueue().isEmpty()); + } finally { release.countDown(); workers.shutdownNow(); assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); } + } + + @Test void admittedProcessingCallbackCannotClearShutdownSnapshot() { + Fixture f = fixture(); + var scheduled = org.mockito.ArgumentCaptor.forClass(Runnable.class); + verify(f.plugin.getVoteTimer()).schedule(scheduled.capture(), eq(120L), eq(TimeUnit.SECONDS)); + f.handler.addVote("First", "site"); f.handler.saveForShutdown(); + scheduled.getValue().run(); f.handler.processQueue(); + verify(f.data, never()).clearTimedVoteCache(); + verify(f.plugin.getServer().getPluginManager(), never()).callEvent(any()); + } + + @Test void ordinarySaveDoesNotSealFutureProcessing() { + Fixture f = fixture(); + f.handler.addVote("Saved", "site"); f.handler.save(); + f.handler.addVote("New", "site"); f.handler.processQueue(); + verify(f.plugin.getServer().getPluginManager()).callEvent(any()); + } + + @Test void shutdownRetainsInFlightEventForReviewWithoutWaitingOrReplaying() throws Exception { + Fixture f = fixture(); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + var events = f.plugin.getServer().getPluginManager(); + doAnswer(call -> { entered.countDown(); assertTrue(release.await(5, TimeUnit.SECONDS)); return null; }) + .when(events).callEvent(any()); + f.handler.addVote("InFlight", "site"); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future delivery = worker.submit(f.handler::processQueue); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + f.handler.saveForShutdown(); f.handler.saveForShutdown(); + f.handler.processQueue(); + var vote = org.mockito.ArgumentCaptor.forClass(VoteTimeQueue.class); + verify(f.data, times(1)).addUncertainTimedVote(any(UUID.class), vote.capture()); + assertEquals("InFlight", vote.getValue().getName()); + verify(f.data, never()).addTimeVoted(anyInt(), any()); + assertTrue(f.handler.getTimeChangeQueue().isEmpty()); + release.countDown(); delivery.get(5, TimeUnit.SECONDS); + verify(f.plugin.getServer().getPluginManager(), times(1)).callEvent(any()); + } finally { release.countDown(); worker.shutdownNow(); assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS)); } + } + + @Test void shutdownBeforeEventDispatchPersistsVoteForNormalRecovery() throws Exception { + Fixture f = fixture(); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + var sites = f.plugin.getVoteSiteManager(); + doAnswer(call -> { entered.countDown(); assertTrue(release.await(5, TimeUnit.SECONDS)); return null; }) + .when(sites).getVoteSite(any(), eq(true)); + f.handler.addVote("NotDispatched", "site"); + ExecutorService worker = Executors.newSingleThreadExecutor(); + try { + Future delivery = worker.submit(f.handler::processQueue); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + f.handler.saveForShutdown(); + release.countDown(); delivery.get(5, TimeUnit.SECONDS); + verify(f.data).addTimeVoted(eq(0), any()); + verify(f.data, never()).addUncertainTimedVote(any(), any()); + verify(f.plugin.getServer().getPluginManager(), never()).callEvent(any()); + } finally { release.countDown(); worker.shutdownNow(); assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS)); } + } + + @Test void failedEventAndFailedQuarantineRemainRetryableWithoutRepeatingEvent() { + Fixture f = fixture(); + RuntimeException original = new IllegalStateException("event failure"); + var events = f.plugin.getServer().getPluginManager(); + doThrow(original).when(events).callEvent(any()); + doThrow(new IllegalStateException("disk failure")).doNothing() + .when(f.data).addUncertainTimedVote(any(), any()); + f.handler.addVote("Ambiguous", "site"); + assertSame(original, assertThrows(IllegalStateException.class, f.handler::processQueue)); + assertEquals(1, original.getSuppressed().length); + assertEquals(1, f.handler.getTimeChangeQueue().size()); + f.handler.processQueue(); + f.handler.saveForShutdown(); f.handler.saveForShutdown(); + var ids = org.mockito.ArgumentCaptor.forClass(UUID.class); + verify(f.data, times(2)).addUncertainTimedVote(ids.capture(), any()); + assertEquals(ids.getAllValues().get(0), ids.getAllValues().get(1)); + verify(f.plugin.getServer().getPluginManager(), times(1)).callEvent(any()); + verify(f.data, never()).addTimeVoted(anyInt(), any()); + } + + @Test void directProcessingClearsPreviousCacheBeforeHandingOffRecoveredVote() { + Fixture f = fixture(); + f.handler.addVote("Recovered", "site"); + f.handler.processQueue(); + var calls = inOrder(f.data, f.plugin.getServer().getPluginManager()); + calls.verify(f.data).clearTimedVoteCache(); + calls.verify(f.plugin.getServer().getPluginManager()).callEvent(any()); + f.handler.addVote("New", "site"); + f.handler.saveForShutdown(); + verify(f.data).addTimeVoted(eq(0), any()); + } + + @TempDir java.nio.file.Path directory; + @Test void actualTimedCacheWriteReportsIoFailureAndSucceedsAfterRepair() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class); + var file = mock(com.bencodez.advancedcore.data.ServerData.class); + when(plugin.getServerDataFile()).thenReturn(file); + YamlConfiguration yaml = new YamlConfiguration(); + when(file.getData()).thenReturn(yaml); + java.nio.file.Path target = directory.resolve("ServerData.yml"); + java.nio.file.Files.createDirectory(target); + when(file.getdFile()).thenReturn(target.toFile()); + ServerData data = new ServerData(plugin); + VoteTimeQueue vote = new VoteTimeQueue("First", "site", 123L); + assertThrows(java.io.UncheckedIOException.class, () -> data.addTimeVoted(0, vote)); + java.nio.file.Files.delete(target); + data.addTimeVoted(0, vote); + YamlConfiguration stored = YamlConfiguration.loadConfiguration(target.toFile()); + assertEquals("First", stored.getString("VotingPlugin.TimedVoteCache.0.Name")); + assertEquals(123L, stored.getLong("VotingPlugin.TimedVoteCache.0.Time")); + UUID uncertain = UUID.randomUUID(); + data.addUncertainTimedVote(uncertain, vote); + stored = YamlConfiguration.loadConfiguration(target.toFile()); + assertEquals("First", stored.getString("VotingPlugin.UncertainTimedVotes." + uncertain + ".Name")); + assertEquals(Set.of("0"), data.getTimedVoteCacheKeys()); + } +} diff --git a/docs/startup-failure-shutdown.md b/docs/startup-failure-shutdown.md index 80441898a..171c74b2d 100644 --- a/docs/startup-failure-shutdown.md +++ b/docs/startup-failure-shutdown.md @@ -9,19 +9,27 @@ the interrupt flag when shutdown is interrupted. Repeated unload callbacks are s The shutdown change works independently, but the MySQL authentication-failure regression also exercises the corrected SimpleAPI and AdvancedCore constructors. SimpleAPI PR #111 is merged; AdvancedCore PR #361 supplies the remaining SQL -initialization fix. Build these dependencies in that order before building VotingPlugin. CI pins the merged SimpleAPI -revision `fdbf26eec885ec3f1c1dbd937a243a0acfaa0a5e` and AdvancedCore revision -`a68cc2dfe6e44c1c0854e659c614708492fcd58d` so an older published snapshot cannot silently replace the tested fixes. +initialization fix. The POM pins published SimpleAPI `1.0.2-20261011.161601-40`, so no manual SimpleAPI source installation is needed. CI builds AdvancedCore revision +`4ca3d3530e002bcafd0d815e752d4f61c9bd0a8f` while that PR remains under review. -With those source revisions checked out in sibling directories, run: +With that AdvancedCore revision checked out in a sibling directory, run: ```shell -mvn -B -f ../SimpleAPI/SimpleAPI/pom.xml clean install -mvn -B -nsu -f ../AdvancedCore/AdvancedCore/pom.xml clean install +mvn -B -f ../AdvancedCore/AdvancedCore/pom.xml clean install mvn -B -nsu -f VotingPlugin/pom.xml clean package ``` -`-nsu` preserves the locally installed snapshot dependencies for these builds. The CI pins can be changed deliberately -once corrected dependencies are reliably published. This is a build dependency only: the distributable VotingPlugin JAR -includes the required libraries, and server owners do not need to install separate SimpleAPI or AdvancedCore JARs. +`-nsu` preserves the locally installed AdvancedCore snapshot for this build. The CI source pin can be removed once that +fix is merged and published. This is a build dependency only: the distributable VotingPlugin JAR includes the required +libraries, and server owners do not need to install separate SimpleAPI or AdvancedCore JARs. Invalid MySQL credentials still abort startup; no storage fallback or configuration migration is introduced. + +Timed-vote shutdown publication shares an admission fence with queue additions and cache-clearing callbacks. A task that +outlives the executor's bounded termination wait persists its late timed vote through that same owner. Successfully +saved prefixes are retired; an unsuccessful write retains its vote and cache index for a later unload retry, without +repeating unrelated cleanup. Timed-cache writes report I/O failures rather than relying on the legacy file helper's +log-only error handling. Ordinary `TimeQueueHandler.save()` remains a checkpoint; `saveForShutdown()` seals processing. +The existing ServerData file format and persistence model are retained: this is not a new transactional reward system or +an exactly-once guarantee across arbitrary vote event handlers, chat, rewards and disk writes. + +A timed vote remains owned while its event is being prepared. If shutdown wins before event dispatch, it is saved for ordinary timed-cache recovery. Once event dispatch starts, external handlers may already have acted: shutdown or a thrown dispatch error retains that occurrence under `VotingPlugin.UncertainTimedVotes.` in `ServerData.yml` and logs a warning. These records contain the original name, service and timestamp and are **not automatically replayed**. An administrator must compare them with voting data/logs before deciding whether any recovery is needed; a completed event can still leave a conservative review record when shutdown raced it. Repeated persistence attempts use the same ID. Shutdown does not wait indefinitely for external event handlers. From cf8def4c8a77068c7d1a7bd033a4a5ede94ea6b0 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 11 Oct 2026 11:06:37 -0600 Subject: [PATCH 5/8] Validate shutdown against latest AdvancedCore initialization fix --- .github/workflows/maven.yml | 2 +- docs/startup-failure-shutdown.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 9a0894776..7101cc5b4 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -27,7 +27,7 @@ jobs: uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: repository: BenCodez/AdvancedCore - ref: 4ca3d3530e002bcafd0d815e752d4f61c9bd0a8f + ref: d09f0d891127952f0787aef879380a0b8ee0dde8 path: .ci/advancedcore persist-credentials: false - name: Build and install AdvancedCore dependency diff --git a/docs/startup-failure-shutdown.md b/docs/startup-failure-shutdown.md index 171c74b2d..75fa2d332 100644 --- a/docs/startup-failure-shutdown.md +++ b/docs/startup-failure-shutdown.md @@ -10,7 +10,7 @@ the interrupt flag when shutdown is interrupted. Repeated unload callbacks are s The shutdown change works independently, but the MySQL authentication-failure regression also exercises the corrected SimpleAPI and AdvancedCore constructors. SimpleAPI PR #111 is merged; AdvancedCore PR #361 supplies the remaining SQL initialization fix. The POM pins published SimpleAPI `1.0.2-20261011.161601-40`, so no manual SimpleAPI source installation is needed. CI builds AdvancedCore revision -`4ca3d3530e002bcafd0d815e752d4f61c9bd0a8f` while that PR remains under review. +`d09f0d891127952f0787aef879380a0b8ee0dde8` while that PR remains under review. With that AdvancedCore revision checked out in a sibling directory, run: From 3b4208c595526e57042065f05d8b8972719e9781 Mon Sep 17 00:00:00 2001 From: Ben Date: Tue, 13 Oct 2026 23:15:15 -0600 Subject: [PATCH 6/8] Bound timed-vote shutdown persistence and retain retryable recovery --- .../votingplugin/data/ServerData.java | 18 ++ .../timequeue/TimeQueueHandler.java | 85 +++++++++- .../TimeQueueRecoveryCheckpointTest.java | 68 ++++++++ .../timequeue/TimeQueueShutdownBoundTest.java | 156 ++++++++++++++++++ docs/startup-failure-shutdown.md | 8 + 5 files changed, 329 insertions(+), 6 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownBoundTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java index 6bfa8f79c..7e4f4817c 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java @@ -422,6 +422,24 @@ public void prepareTimedVoteDispatch(UUID id, VoteTimeQueue vote, } } + /** Restore an undispatched occurrence and its pending suffix for ordinary restart recovery. */ + public void abortTimedVoteDispatch(UUID id, java.util.Collection pending) { + synchronized (plugin.getServerDataFile()) { + String path = "UncertainTimedVotes." + id; + Object previousCache = getData().get("TimedVoteCache"); + Object previousRecord = getData().get(path); + try { + setTimedVoteCache(pending); + getData().set(path, null); + saveTimedVoteCache(); + } catch (RuntimeException | Error failure) { + getData().set("TimedVoteCache", previousCache); + getData().set(path, previousRecord); + throw failure; + } + } + } + /** Clear only a confirmed dispatch record; failed writes retain its live representation. */ public void completeTimedVoteDispatch(UUID id) { synchronized (plugin.getServerDataFile()) { diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java index 7e817833b..bd59d2d9d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/timequeue/TimeQueueHandler.java @@ -5,9 +5,13 @@ import java.util.Queue; import java.util.UUID; import java.util.concurrent.ConcurrentLinkedQueue; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; import org.bukkit.configuration.ConfigurationSection; import org.bukkit.event.EventHandler; @@ -31,11 +35,13 @@ public class TimeQueueHandler implements Listener { private VotingPluginMain plugin; private final Object persistenceLock = new Object(); - private boolean stopping; + private volatile boolean stopping; + private final AtomicReference> shutdownSave = new AtomicReference<>(); private final AtomicBoolean processing = new AtomicBoolean(); private VoteTimeQueue dispatchingVote; private UUID dispatchId; private boolean dispatchEnded; + private boolean dispatchStarted; private int persistedShutdownVotes; private final AtomicBoolean retryPending = new AtomicBoolean(); private final AtomicInteger retryAttempts = new AtomicInteger(); @@ -155,6 +161,9 @@ public void processQueue() { clearDispatch(); throw failure; } + // A preparation resumed after stop is still eligible for normal recovery. + if (stopping) return; + dispatchStarted = true; } try { plugin.getServer().getPluginManager().callEvent(voteEvent); @@ -192,6 +201,7 @@ private void clearDispatch() { dispatchingVote = null; dispatchId = null; dispatchEnded = false; + dispatchStarted = false; } private void checkpointPendingVotes() { @@ -212,17 +222,73 @@ private void retainUncertainDispatch() { * Saves pending votes to server data. */ public void save() { + if (stopping) { + saveForShutdown(); + return; + } synchronized (persistenceLock) { if (stopping) persistPendingVotes(); else checkpointPendingVotes(); } } - /** Seal processing and durably retain late admissions from tasks that ignore interruption. */ + /** Seal processing and retain one persistence owner without an unbounded lifecycle wait. */ public void saveForShutdown() { - synchronized (persistenceLock) { - stopping = true; - persistPendingVotes(); + stopping = true; + FutureTask save; + while (true) { + save = shutdownSave.get(); + if (save != null) { + if (!save.isDone()) break; + shutdownSave.compareAndSet(save, null); + continue; + } + FutureTask candidate = new FutureTask(() -> { + try { + synchronized (persistenceLock) { persistPendingVotes(); } + } catch (RuntimeException | Error failure) { + try { + plugin.getLogger().warning("Timed vote shutdown persistence failed; pending votes remain owned for retry. Recovery is not confirmed."); + } catch (RuntimeException | Error ignored) { } + throw failure; + } + return null; + }) { + @Override protected void done() { + // Late admission failures must be retryable after this owner completes. + shutdownSave.compareAndSet(this, null); + } + }; + FutureTask existing = shutdownSave.compareAndExchange(null, candidate); + if (existing == null) { + save = candidate; + try { + Thread worker = new Thread(candidate, "VotingPlugin-timed-vote-shutdown"); + worker.setDaemon(true); + worker.start(); + } + catch (RuntimeException | Error failure) { + candidate.cancel(false); + shutdownSave.compareAndSet(candidate, null); + throw failure; + } + break; + } + } + try { + save.get(1, TimeUnit.SECONDS); + shutdownSave.compareAndSet(save, null); + } catch (TimeoutException pending) { + plugin.getLogger().warning("Timed vote shutdown persistence is still pending; its worker retains ownership. Recovery is not yet confirmed."); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + plugin.getLogger().warning("Timed vote shutdown persistence wait was interrupted; its worker retains ownership."); + } catch (ExecutionException failure) { + shutdownSave.compareAndSet(save, null); + Throwable cause = failure.getCause(); + if (cause instanceof Error error) throw error; + if (cause instanceof RuntimeException runtime) throw runtime; + throw new IllegalStateException("Timed vote shutdown persistence failed", cause); } } @@ -231,7 +297,14 @@ private void persistPendingVotes() { VoteTimeQueue vote; while ((vote = timeChangeQueue.peek()) != null) { if (vote == dispatchingVote) { - retainUncertainDispatch(); + if (dispatchStarted) retainUncertainDispatch(); + else { + var pending = timeChangeQueue.stream().toList(); + plugin.getServerData().abortTimedVoteDispatch(dispatchId, pending); + timeChangeQueue.removeAll(pending); + persistedShutdownVotes = pending.size(); + clearDispatch(); + } } else { plugin.getServerData().addTimeVoted(persistedShutdownVotes, vote); persistedShutdownVotes++; diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueRecoveryCheckpointTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueRecoveryCheckpointTest.java index c8187377a..3fea52f21 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueRecoveryCheckpointTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueRecoveryCheckpointTest.java @@ -24,6 +24,74 @@ class TimeQueueRecoveryCheckpointTest { @TempDir Path directory; + @Test + void stoppingDuringPreparationRestoresTheUndispatchedVoteForActualRestartRecovery() throws Exception { + Fixture fixture = new Fixture(directory.resolve("aborted-preparation.yml")); + fixture.handler.addVote("First", "site"); + fixture.handler.addVote("Later", "site"); + fixture.handler.save(); + var entered = new java.util.concurrent.CountDownLatch(1); + var release = new java.util.concurrent.CountDownLatch(1); + doAnswer(call -> { + call.callRealMethod(); + entered.countDown(); + assertTrue(release.await(10, java.util.concurrent.TimeUnit.SECONDS)); + return null; + }).when(fixture.yaml).save(fixture.path.toFile()); + var callers = java.util.concurrent.Executors.newFixedThreadPool(2); + try { + var delivery = callers.submit(fixture.handler::processQueue); + assertTrue(entered.await(5, java.util.concurrent.TimeUnit.SECONDS)); + callers.submit(fixture.handler::saveForShutdown).get(2, java.util.concurrent.TimeUnit.SECONDS); + release.countDown(); + delivery.get(5, java.util.concurrent.TimeUnit.SECONDS); + fixture.handler.saveForShutdown(); + verify(fixture.plugin.getServer().getPluginManager(), never()).callEvent(any()); + assertEquals(List.of("First", "Later"), new Fixture(fixture.path).names()); + assertTrue(fixture.stored().getConfigurationSection("VotingPlugin.UncertainTimedVotes").getKeys(false).isEmpty()); + assertEquals("keep", fixture.stored().getString("Adjacent.Value")); + } finally { + release.countDown(); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)); + fixture.handler.saveForShutdown(); + } + } + + @Test + void failedLateAdmissionRemainsRetryableAfterTheEarlierShutdownSaveSucceeded() throws Exception { + Fixture fixture = new Fixture(directory.resolve("late-admission.yml")); + fixture.handler.addVote("First", "site"); + fixture.handler.saveForShutdown(); + Path saved = fixture.blockWrites(); + assertThrows(java.io.UncheckedIOException.class, () -> fixture.handler.addVote("Late", "site")); + assertEquals(List.of("Late"), fixture.names()); + fixture.repair(saved); + fixture.handler.saveForShutdown(); + assertTrue(fixture.names().isEmpty()); + assertEquals(List.of("First", "Late"), new Fixture(fixture.path).names()); + assertEquals("keep", fixture.stored().getString("Adjacent.Value")); + } + + @Test + void abortedPreparationWriteFailurePreservesBothWrappersUntilRepairAndRetry() throws Exception { + Fixture fixture = new Fixture(directory.resolve("abort-retry.yml")); + VoteTimeQueue first = new VoteTimeQueue("First", "site", 123L); + VoteTimeQueue later = new VoteTimeQueue("Later", "site", 456L); + UUID id = new UUID(1, 2); + fixture.data.prepareTimedVoteDispatch(id, first, List.of(later)); + Path saved = fixture.blockWrites(); + assertThrows(java.io.UncheckedIOException.class, + () -> fixture.data.abortTimedVoteDispatch(id, List.of(first, later))); + assertEquals("Later", fixture.yaml.getString("VotingPlugin.TimedVoteCache.0.Name")); + assertEquals("First", fixture.yaml.getString("VotingPlugin.UncertainTimedVotes." + id + ".Name")); + fixture.repair(saved); + fixture.data.abortTimedVoteDispatch(id, List.of(first, later)); + assertEquals(List.of("First", "Later"), new Fixture(fixture.path).names()); + assertTrue(fixture.stored().getConfigurationSection("VotingPlugin.UncertainTimedVotes").getKeys(false).isEmpty()); + assertEquals("keep", fixture.stored().getString("Adjacent.Value")); + } + @Test void cancelledDispatchKeepsTheUndispatchedSuffixBothLiveAndRecoverable() throws Exception { Fixture fixture = new Fixture(directory.resolve("ServerData.yml")); diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownBoundTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownBoundTest.java new file mode 100644 index 000000000..6970e3102 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/timequeue/TimeQueueShutdownBoundTest.java @@ -0,0 +1,156 @@ +package com.bencodez.votingplugin.tests.timequeue; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.util.Set; +import java.util.concurrent.*; + +import org.junit.jupiter.api.Test; + +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.data.ServerData; +import com.bencodez.votingplugin.timequeue.TimeQueueHandler; + +class TimeQueueShutdownBoundTest { + @Test void storageFailureAfterTimeoutIsReportedAndCanRetryAtTheSameIndex() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, RETURNS_DEEP_STUBS); + ServerData data = mock(ServerData.class); + when(plugin.getServerData()).thenReturn(data); + when(data.getTimedVoteCacheKeys()).thenReturn(Set.of()); + TimeQueueHandler handler = new TimeQueueHandler(plugin); + CountDownLatch entered = new CountDownLatch(1), release = new CountDownLatch(1), failed = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicBoolean fail = new java.util.concurrent.atomic.AtomicBoolean(true); + doAnswer(call -> { + if (fail.getAndSet(false)) { + entered.countDown(); + assertTrue(release.await(10, TimeUnit.SECONDS)); + failed.countDown(); + throw new IllegalStateException("synthetic storage failure"); + } + return null; + }).when(data).addTimeVoted(anyInt(), any()); + handler.addVote("First", "site"); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future shutdown = caller.submit(handler::saveForShutdown); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + shutdown.get(2, TimeUnit.SECONDS); + release.countDown(); + assertTrue(failed.await(5, TimeUnit.SECONDS)); + verify(plugin.getLogger(), timeout(5000)).warning( + "Timed vote shutdown persistence failed; pending votes remain owned for retry. Recovery is not confirmed."); + assertEquals(1, handler.getTimeChangeQueue().size()); + try { handler.saveForShutdown(); } + catch (IllegalStateException observedCompletion) { + assertEquals("synthetic storage failure", observedCompletion.getMessage()); + handler.saveForShutdown(); + } + assertTrue(handler.getTimeChangeQueue().isEmpty()); + verify(data, times(2)).addTimeVoted(eq(0), any()); + } finally { + release.countDown(); + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test void interruptedWaitPreservesTheFlagAndLeavesPersistenceOwned() throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, RETURNS_DEEP_STUBS); + ServerData data = mock(ServerData.class); + when(plugin.getServerData()).thenReturn(data); + when(data.getTimedVoteCacheKeys()).thenReturn(Set.of()); + TimeQueueHandler handler = new TimeQueueHandler(plugin); + CountDownLatch entered = new CountDownLatch(1), release = new CountDownLatch(1); + doAnswer(call -> { entered.countDown(); assertTrue(release.await(5, TimeUnit.SECONDS)); return null; }) + .when(data).addTimeVoted(anyInt(), any()); + handler.addVote("First", "site"); + try { + Thread.currentThread().interrupt(); + handler.saveForShutdown(); + assertTrue(Thread.currentThread().isInterrupted()); + Thread.interrupted(); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + assertEquals(1, handler.getTimeChangeQueue().size()); + } finally { + Thread.interrupted(); + release.countDown(); + handler.saveForShutdown(); + } + assertTrue(handler.getTimeChangeQueue().isEmpty()); + verify(data, times(1)).addTimeVoted(eq(0), any()); + } + + @Test void stalledPreparationDoesNotBlockShutdownOrAuthorizeAnEventAfterStopping() throws Exception { + exerciseStalledWrite("prepare"); + } + + @Test void stalledCompletionDoesNotBlockShutdownOrRepeatTheCompletedEvent() throws Exception { + exerciseStalledWrite("complete"); + } + + @Test void stalledShutdownWriteRetainsOneOwnerAndDoesNotBlockTheLifecycleCaller() throws Exception { + exerciseStalledWrite("shutdown"); + } + + private void exerciseStalledWrite(String phase) throws Exception { + VotingPluginMain plugin = mock(VotingPluginMain.class, RETURNS_DEEP_STUBS); + ServerData data = mock(ServerData.class); + when(plugin.getServerData()).thenReturn(data); + when(data.getTimedVoteCacheKeys()).thenReturn(Set.of()); + TimeQueueHandler handler = new TimeQueueHandler(plugin); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + org.mockito.stubbing.Answer stalled = call -> { + entered.countDown(); + boolean interrupted = false; + try { + while (true) { + try { assertTrue(release.await(10, TimeUnit.SECONDS)); break; } + catch (InterruptedException ignored) { interrupted = true; } + } + } finally { if (interrupted) Thread.currentThread().interrupt(); } + return null; + }; + if (phase.equals("prepare")) doAnswer(stalled).when(data).prepareTimedVoteDispatch(any(), any(), any()); + if (phase.equals("complete")) doAnswer(stalled).when(data).completeTimedVoteDispatch(any()); + if (phase.equals("shutdown")) doAnswer(stalled).when(data).addTimeVoted(anyInt(), any()); + handler.addVote("First", "site"); + handler.addVote("Later", "site"); + ExecutorService callers = Executors.newFixedThreadPool(3); + try { + Future processing = phase.equals("shutdown") ? null : callers.submit(handler::processQueue); + Future shutdown; + if (processing == null) { + shutdown = callers.submit(handler::saveForShutdown); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + } else { + assertTrue(entered.await(5, TimeUnit.SECONDS)); + shutdown = callers.submit(handler::saveForShutdown); + } + shutdown.get(2, TimeUnit.SECONDS); + // Repeated unload must join the same retained persistence owner. + Future repeated = callers.submit(handler::saveForShutdown); + repeated.get(2, TimeUnit.SECONDS); + assertFalse(handler.getTimeChangeQueue().isEmpty()); + release.countDown(); + if (processing != null) processing.get(5, TimeUnit.SECONDS); + handler.saveForShutdown(); + assertTrue(handler.getTimeChangeQueue().isEmpty()); + verify(plugin.getServer().getPluginManager(), times(phase.equals("complete") ? 1 : 0)).callEvent(any()); + if (phase.equals("prepare")) verify(data, times(1)).abortTimedVoteDispatch(any(), + argThat(pending -> pending.stream().map(vote -> vote.getName()).toList() + .equals(java.util.List.of("First", "Later")))); + else verify(data, times(1)).addTimeVoted(eq(phase.equals("shutdown") ? 1 : 0), + argThat(vote -> vote.getName().equals("Later"))); + if (phase.equals("shutdown")) verify(data, times(1)).addTimeVoted(eq(0), + argThat(vote -> vote.getName().equals("First"))); + } finally { + release.countDown(); + callers.shutdownNow(); + assertTrue(callers.awaitTermination(5, TimeUnit.SECONDS)); + handler.saveForShutdown(); + } + } +} diff --git a/docs/startup-failure-shutdown.md b/docs/startup-failure-shutdown.md index 3947e5bdd..c59c7cc83 100644 --- a/docs/startup-failure-shutdown.md +++ b/docs/startup-failure-shutdown.md @@ -49,3 +49,11 @@ not prove a reward ran or failed. The stopped vote timer remains visible until overflow and event-handler cleanup finishes. A missing timer is treated as rejected admission, allowing a late Votifier callback to use the existing overflow path rather than fail on a null reference. No accepted task is run inline on the event thread as a fallback. + +Timed-vote shutdown persistence has one retained daemon worker per handler. Each shutdown call waits at most one second +for that worker, including when a preparation/completion write holds the queue monitor or the shutdown write itself +stalls. Repeated calls join the same pending work; a completed failure remains retryable with the original queue/index. +An interrupted wait preserves the caller's interrupt flag and does not cancel persistence. A timeout reports pending +ownership, **not confirmed recovery**: if the process exits before storage resumes, new pending writes may not finish. +Stopping is published before waiting, so a stalled preparation cannot authorize a new event when it resumes after stop. +This bound covers the timed-queue persistence handoff; it does not promise a deadline for every other subsystem's I/O. From dc127ce3005649fc4027e6ee9f76abd1dd3c9123 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 17:18:36 -0600 Subject: [PATCH 7/8] Keep shutdown sign publication within the bounded process drain Capture sign state before asynchronous persistence, retain failed attempts for retry, and keep outstanding generation publication leases in the process-exit drain without blocking lifecycle cleanup. --- .../votingplugin/VotingPluginMain.java | 29 +- .../votingplugin/data/ServerData.java | 40 +++ .../votingplugin/signs/SignHandler.java | 6 + .../bencodez/votingplugin/signs/Signs.java | 63 +++++ .../util/ServerDataPublicationFence.java | 65 ++++- .../tests/DisplayUnloadIntegrationTest.java | 3 +- .../GenerationFencedShutdownStageTest.java | 20 ++ .../tests/ServerDataProcessExitDrainTest.java | 249 ++++++++++++++++++ .../tests/SignShutdownPublicationTest.java | 220 ++++++++++++++++ .../tests/VotingPluginMainShutdownTest.java | 162 ++++++++++-- 10 files changed, 835 insertions(+), 22 deletions(-) create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ServerDataProcessExitDrainTest.java create mode 100644 VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java index bb08900b1..24a46a79d 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/VotingPluginMain.java @@ -341,6 +341,7 @@ public String formatVoteCommandMessage(String message) { private volatile AtomicBoolean basicBungeeUpdateRunning; private final AtomicBoolean unloadInProgress = new AtomicBoolean(); private Set completedUnloadSteps; + private java.util.concurrent.CompletionStage signShutdownPublication; private synchronized AtomicBoolean basicBungeeUpdateAdmission() { if (basicBungeeUpdateRunning == null) basicBungeeUpdateRunning = new AtomicBoolean(); @@ -2063,6 +2064,15 @@ public void onEnable() { if (serverDataPublicationFence != null) getServerDataFile(); serverDataPublicationFence = com.bencodez.votingplugin.util.ServerDataPublicationFence.beginGeneration( getDataFolder().toPath().resolve("ServerData.yml")); + try { serverDataPublicationFence.retainForProcessExit(); } + catch (RuntimeException | Error registrationFailure) { + serverDataEnableAdmissionRejected = true; + try { serverDataPublicationFence.retire(); } + catch (RuntimeException | Error cleanupFailure) { + if (cleanupFailure != registrationFailure) registrationFailure.addSuppressed(cleanupFailure); + } + throw registrationFailure; + } serverDataFinalPublication = new java.util.concurrent.CompletableFuture<>(); serverDataFinalPublicationReserved = false; serverDataFinalStageAttached = false; @@ -2093,6 +2103,7 @@ public void onPreLoad() { plugin = this; unloadInProgress.set(false); completedUnloadSteps = new java.util.HashSet<>(); + signShutdownPublication = null; setupFiles(); if ("SQLITE".equalsIgnoreCase(configFile.getData().getString("DataStorage", "SQLITE"))) { @@ -2136,6 +2147,7 @@ public void onFinish() { public void onDisable() { // The rejected enable owns no SDK shutdown stage to complete a new reservation. if (serverDataEnableAdmissionRejected) return; + runShutdownStep("server-data process drain", this::beginServerDataProcessDrain); serverDataEnableCleanupStarted = true; runShutdownStep("Discord leaderboards", () -> { if (discordHandler != null) discordHandler.close(); @@ -2222,6 +2234,7 @@ private void shutdownVoteReminders() { @Override public void onUnLoad() { + runShutdownStep("server-data process drain", this::beginServerDataProcessDrain); if (!unloadInProgress.compareAndSet(false, true)) { runShutdownStep("time queue retry", () -> { if (timeQueueHandler != null) timeQueueHandler.saveForShutdown(); @@ -2270,9 +2283,7 @@ public void onUnLoad() { coolDownCheck = null; } }); - runUnloadStep("signs", () -> { - if (signs != null) signs.storeSigns(); - }); + runShutdownStep("signs", this::shutdownSigns); runUnloadStep("event handlers", () -> HandlerList.unregisterAll(this)); // Late producers must see rejection until their overflow and handler cleanup finishes. if (completedUnloadSteps.contains("vote timer")) voteTimer = null; @@ -2280,6 +2291,18 @@ public void onUnLoad() { } finally { unloadInProgress.set(false); } } + private void beginServerDataProcessDrain() { + var owner = serverDataPublicationFence; + if (owner != null) owner.beginShutdown(System.nanoTime()); + } + + private void shutdownSigns() { + if (signs == null) return; + var previous = signShutdownPublication; + if (previous != null && !previous.toCompletableFuture().isCompletedExceptionally()) return; + signShutdownPublication = java.util.Objects.requireNonNull(signs.storeSignsForShutdown()); + } + private void runUnloadStep(String name, Runnable step) { if (!completedUnloadSteps.contains(name) && runShutdownStep(name, step)) completedUnloadSteps.add(name); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java index 0261abf55..29070f28f 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java @@ -78,6 +78,46 @@ void store(ConfigurationSection data, String path) { } } + + /** Detached shutdown payload: no platform object is accessed by its publisher. */ + public static final class SignSnapshot { + private final String count; + private final SignCoordinates location; + private final SignCoordinates skull; + private final String data; + private final int position; + + private SignSnapshot(String count, Location location, Location skull, String data, int position) { + this.count = count; + this.location = SignCoordinates.capture(location); + this.skull = skull == null ? null : SignCoordinates.capture(skull); + this.data = data; + this.position = position; + } + + private void store(ConfigurationSection target) { + location.store(target, "Signs." + count); + target.set("Signs." + count + ".Data", data); + target.set("Signs." + count + ".Position", position); + if (skull != null) skull.store(target, "Signs." + count + ".Skull"); + } + } + + public static SignSnapshot captureSignSnapshot(String count, Location location, Location skull, + String data, int position) { + return new SignSnapshot(count, location, skull, data, position); + } + + /** Worker-only: publish the whole captured sign batch through the actual durable SDK owner. */ + public java.util.concurrent.CompletionStage publishShutdownSigns(List signs) { + var snapshot = List.copyOf(signs); + return getFile().mutateFinalData(candidate -> { + var target = candidate.getConfigurationSection("VotingPlugin"); + if (target == null) target = candidate.createSection("VotingPlugin"); + for (var sign : snapshot) sign.store(target); + }); + } + /** Only YAML work belongs here; platform callbacks run outside the file owner. */ private void mutateData(Runnable mutation) { var fence = getPublicationFence(); diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/SignHandler.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/SignHandler.java index b16d45124..914568845 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/SignHandler.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/SignHandler.java @@ -286,6 +286,12 @@ public void removeSign() { /** * Store sign. */ + /** Capture platform state on the caller before shutdown transfers publication ownership. */ + public com.bencodez.votingplugin.data.ServerData.SignSnapshot captureShutdownSnapshot() { + return com.bencodez.votingplugin.data.ServerData.captureSignSnapshot( + sign, location, skullLocation, data, position); + } + public void storeSign() { plugin.getServerData().setSign(sign, location, skullLocation, data, position); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/Signs.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/Signs.java index 7d540bab4..98a70e983 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/Signs.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/signs/Signs.java @@ -64,6 +64,69 @@ public void storeSigns() { } } + private final Object shutdownLock = new Object(); + private ShutdownSigns shutdownSigns; + private java.util.concurrent.CompletableFuture shutdownAttempt; + + private record ShutdownSigns(com.bencodez.votingplugin.data.ServerData data, + java.util.List snapshot, + com.bencodez.votingplugin.util.ServerDataPublicationFence.Lease lease) { } + + /** No YAML lock or disk wait on the lifecycle caller; failures retain the original payload. */ + public java.util.concurrent.CompletionStage storeSignsForShutdown() { + synchronized (shutdownLock) { + if (shutdownAttempt != null && !shutdownAttempt.isCompletedExceptionally()) + return shutdownAttempt.minimalCompletionStage(); + if (shutdownSigns == null) { + var snapshot = new java.util.ArrayList(); + for (var sign : java.util.List.copyOf(getSigns())) snapshot.add(sign.captureShutdownSnapshot()); + if (snapshot.isEmpty()) { + shutdownAttempt = java.util.concurrent.CompletableFuture.completedFuture(null); + return shutdownAttempt.minimalCompletionStage(); + } + var data = java.util.Objects.requireNonNull(plugin.getServerData()); + var fence = java.util.Objects.requireNonNull(plugin.getServerDataPublicationFence()); + shutdownSigns = new ShutdownSigns(data, java.util.List.copyOf(snapshot), fence.acquire()); + } + var publication = shutdownSigns; + var attempt = new java.util.concurrent.CompletableFuture(); + shutdownAttempt = attempt; + try { + startShutdownWorker(() -> { + try { + java.util.Objects.requireNonNull(publication.data().publishShutdownSigns(publication.snapshot())) + .whenComplete((ignored, failure) -> finishShutdownSigns(publication, attempt, failure)); + } catch (Throwable failure) { finishShutdownSigns(publication, attempt, failure); } + }); + } catch (RuntimeException | Error failure) { + finishShutdownSigns(publication, attempt, failure); + throw failure; + } + return attempt.minimalCompletionStage(); + } + } + + private void finishShutdownSigns(ShutdownSigns publication, + java.util.concurrent.CompletableFuture attempt, Throwable failure) { + if (failure == null) { + try { publication.lease().close(); attempt.complete(null); } + catch (Throwable releaseFailure) { attempt.completeExceptionally(releaseFailure); } + } else { + // Neither rejected admission nor failed I/O proves durability or writer drain. + // Keep the same detached payload and lease for an explicit repeated-unload attempt. + try { plugin.getLogger().warning("Sign shutdown persistence failed; its payload remains owned. Recovery is not confirmed."); } + catch (RuntimeException | Error ignored) { } + attempt.completeExceptionally(failure); + } + } + + /** Admission seam for deterministic rejection/cancellation tests. */ + protected void startShutdownWorker(Runnable task) { + var worker = new Thread(task, "VotingPlugin-sign-shutdown"); + worker.setDaemon(true); + worker.start(); + } + /** * Update signs. */ diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ServerDataPublicationFence.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ServerDataPublicationFence.java index 12c451b30..1e0788612 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ServerDataPublicationFence.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/util/ServerDataPublicationFence.java @@ -9,6 +9,10 @@ import java.util.Properties; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; /** Memory-only admission for participating plugin generations sharing one server-data path in a JVM. */ public final class ServerDataPublicationFence { @@ -20,6 +24,56 @@ public final class ServerDataPublicationFence { private final AtomicBoolean retirementStarted = new AtomicBoolean(); private final java.util.concurrent.CompletableFuture retirementCompleted = new java.util.concurrent.CompletableFuture<>(); + private final CompletableFuture processShutdownStarted = new CompletableFuture<>(); + private final CompletableFuture processExitCompletion = new CompletableFuture<>(); + private final AtomicBoolean processHookRemoved = new AtomicBoolean(); + private volatile Runtime processRuntime; + private volatile Thread processHook; + + /** Opt in during startup, before the JVM can begin invoking shutdown hooks. */ + public synchronized void retainForProcessExit() { + if (processHook != null) return; + var runtime = Runtime.getRuntime(); + var hook = new Thread(this::awaitProcessExit, "VotingPlugin-ServerData-Exit"); + runtime.addShutdownHook(hook); + processRuntime = runtime; + processHook = hook; + processExitCompletion.whenComplete((ignored, failure) -> removeProcessHook()); + } + + /** The first disable entry fixes the deadline; retries cannot renew it. No caller wait. */ + public void beginShutdown(long startedNanos) { + if (processShutdownStarted.complete(startedNanos) && processHook != null) { + CompletableFuture.delayedExecutor(remainingProcessWait(startedNanos), TimeUnit.NANOSECONDS) + .execute(this::removeProcessHook); + } + } + + private static long remainingProcessWait(long startedNanos) { + long elapsed = Math.max(0L, System.nanoTime() - startedNanos); + return Math.max(0L, TimeUnit.SECONDS.toNanos(15) - elapsed); + } + + private void awaitProcessExit() { + long exitStarted = System.nanoTime(); + try { + long disableStarted = processShutdownStarted.get(remainingProcessWait(exitStarted), TimeUnit.NANOSECONDS); + long remaining = Math.min(remainingProcessWait(exitStarted), remainingProcessWait(disableStarted)); + if (remaining > 0) processExitCompletion.get(remaining, TimeUnit.NANOSECONDS); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } catch (ExecutionException | TimeoutException pendingOrExpired) { + // Process exit is bounded. Timeout does not acknowledge publication or release a lease. + } + } + + private void removeProcessHook() { + var hook = processHook; + if (hook == null || !processHookRemoved.compareAndSet(false, true)) return; + try { processRuntime.removeShutdownHook(hook); } + catch (IllegalStateException stopping) { /* The JVM is already invoking its hooks. */ } + } + private ServerDataPublicationFence(Properties registry, String key, String generation) { this.registry = registry; this.key = key; @@ -63,10 +117,16 @@ public Lease acquire() { /** Existing operations/retries retain ownership; a successor requires zero outstanding leases. */ public void retire() { + boolean drained = false; synchronized (registry) { State current = state(registry.get(key)); - if (generation.equals(current.generation)) registry.setProperty(key, generation + ":" + current.users + ":true"); + if (generation.equals(current.generation)) { + registry.setProperty(key, generation + ":" + current.users + ":true"); + drained = current.users == 0; + } } + // Completion callbacks may remove a JVM hook; never run them under the registry monitor. + if (drained) processExitCompletion.complete(null); } /** Reserve before asynchronous admission; a failed stage never proves drain. */ @@ -142,13 +202,16 @@ public static final class Lease implements AutoCloseable { private final AtomicBoolean closed = new AtomicBoolean(); private Lease(ServerDataPublicationFence owner) { this.owner = owner; } @Override public void close() { + boolean drained; synchronized (owner.registry) { if (closed.get()) return; State current = owner.current(); if (current.users <= 0) throw new IllegalStateException("Server-data publication ownership is invalid"); owner.registry.setProperty(owner.key, owner.generation + ":" + (current.users - 1) + ":" + current.retiring); closed.set(true); + drained = current.retiring && current.users == 1; } + if (drained) owner.processExitCompletion.complete(null); } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/DisplayUnloadIntegrationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/DisplayUnloadIntegrationTest.java index 1dda7ef6f..8b1e05467 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/DisplayUnloadIntegrationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/DisplayUnloadIntegrationTest.java @@ -46,6 +46,7 @@ private ScheduledExecutorService stoppedExecutor() throws Exception { field(plugin, "backendControlConnectorLifecycle", stoppedExecutor()); field(plugin, "voteTimer", stoppedExecutor()); var signs = mock(Signs.class); doReturn(signs).when(plugin).getSigns(); + when(signs.storeSignsForShutdown()).thenReturn(CompletableFuture.completedFuture(null)); field(plugin, "signs", signs); var pending = new CompletableFuture(); var filter = new TopVoterDisplayFilter(row -> pending); @@ -69,7 +70,7 @@ private ScheduledExecutorService stoppedExecutor() throws Exception { assertEquals(1, generation.get()); menu.open(player); verifyNoInteractions(player); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } } } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/GenerationFencedShutdownStageTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/GenerationFencedShutdownStageTest.java index 8ef940433..37db2c063 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/GenerationFencedShutdownStageTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/GenerationFencedShutdownStageTest.java @@ -56,6 +56,26 @@ private VotingPluginMain plugin(ServerDataPublicationFence fence) throws Excepti } } + @Test void disableStartsTheProcessDrainBeforeSdkCleanupAndNeverRenewsIt() throws Exception { + var fence = ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")); + var plugin = plugin(fence); + var startedField = ServerDataPublicationFence.class.getDeclaredField("processShutdownStarted"); + startedField.setAccessible(true); + var started = (CompletableFuture) startedField.get(fence); + try (var cleanup = mockConstruction(AdvancedCoreLifecycle.class, (owner, context) -> { + doAnswer(call -> { + assertTrue(started.isDone(), "The consumer deadline must precede SDK cleanup"); + return null; + }).when(owner).shutdown(anyLong()); + })) { + plugin.onDisable(); + var original = started.get(5, TimeUnit.SECONDS); + plugin.onDisable(); + assertEquals(original, started.get()); + plugin.onShutdownComplete(CompletableFuture.completedFuture(null)); + } + } + @Test void failedFinalStageRetainsTheGenerationAfterAllCapturedExecutorsTerminate() throws Exception { checkFailure(false); } diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ServerDataProcessExitDrainTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ServerDataProcessExitDrainTest.java new file mode 100644 index 000000000..8104b8482 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/ServerDataProcessExitDrainTest.java @@ -0,0 +1,249 @@ +package com.bencodez.votingplugin.tests; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.bukkit.Location; +import org.bukkit.World; +import org.bukkit.configuration.file.YamlConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.data.GenerationFencedServerDataFile; +import com.bencodez.votingplugin.data.ServerData; +import com.bencodez.votingplugin.signs.SignHandler; +import com.bencodez.votingplugin.signs.Signs; +import com.bencodez.votingplugin.util.ServerDataPublicationFence; + +class ServerDataProcessExitDrainTest { + @TempDir Path directory; + + /** Observe entry to the real bounded wait without replacing its completion semantics. */ + private static final class ObservedCompletion extends CompletableFuture { + final CountDownLatch entered = new CountDownLatch(1); + volatile long nanos; + @Override public Void get(long timeout, TimeUnit unit) + throws InterruptedException, ExecutionException, TimeoutException { + nanos = unit.toNanos(timeout); + entered.countDown(); + return super.get(timeout, unit); + } + } + + private static Object field(Object owner, String name) throws Exception { + var f = owner.getClass().getDeclaredField(name); + f.setAccessible(true); + return f.get(owner); + } + + private static ObservedCompletion observe(ServerDataPublicationFence fence) throws Exception { + var completion = new ObservedCompletion(); + var f = ServerDataPublicationFence.class.getDeclaredField("processExitCompletion"); + f.setAccessible(true); + f.set(fence, completion); + return completion; + } + + @Test void outstandingLeaseKeepsTheActualProcessHookWaitingUntilRetirementDrains() throws Exception { + var fence = ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")); + var completion = observe(fence); + var lease = fence.acquire(); + fence.retainForProcessExit(); + var hook = (Thread) field(fence, "processHook"); + fence.retainForProcessExit(); + assertSame(hook, field(fence, "processHook"), "Repeated enrollment must not register another hook"); + fence.beginShutdown(System.nanoTime()); + fence.retire(); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + var exit = caller.submit(hook::run); + assertTrue(completion.entered.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + assertFalse(exit.isDone(), "A daemon reservation must still be represented at process exit"); + lease.close(); + exit.get(5, TimeUnit.SECONDS); + assertTrue(completion.isDone()); + ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")).retire(); + } finally { + lease.close(); + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test void repeatedDisableCannotRenewTheOriginalSharedDeadline() throws Exception { + var fence = ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")); + var completion = observe(fence); + var lease = fence.acquire(); + fence.retainForProcessExit(); + long original = System.nanoTime() - TimeUnit.SECONDS.toNanos(1); + fence.beginShutdown(original); + fence.beginShutdown(System.nanoTime()); + var started = (CompletableFuture) field(fence, "processShutdownStarted"); + assertEquals(original, started.get()); + fence.retire(); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + var hook = (Thread) field(fence, "processHook"); + var exit = caller.submit(hook::run); + assertTrue(completion.entered.await(5, TimeUnit.SECONDS)); + assertTrue(completion.nanos > 0 && completion.nanos <= TimeUnit.SECONDS.toNanos(14), + "Process exit must use the remaining disable budget, not another fifteen seconds"); + lease.close(); + exit.get(5, TimeUnit.SECONDS); + } finally { + lease.close(); + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test void expiredProcessBudgetDoesNotCertifyFailedPublicationOrLiftGenerationAdmission() throws Exception { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var completion = observe(fence); + fence.retainForProcessExit(); + fence.retainUntilSuccessfulPublication(CompletableFuture.failedFuture(new IllegalStateException("Synthetic I/O"))); + var hook = (Thread) field(fence, "processHook"); + fence.beginShutdown(System.nanoTime() - TimeUnit.SECONDS.toNanos(16)); + fence.retire(); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + caller.submit(hook::run).get(5, TimeUnit.SECONDS); + assertFalse(completion.isDone(), "Deadline expiry cannot report persistence success"); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + } finally { + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test void interruptedProcessWaitPreservesInterruptionWithoutAcknowledgingTheWriter() throws Exception { + var fence = ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")); + var completion = observe(fence); + var lease = fence.acquire(); + fence.retainForProcessExit(); + fence.beginShutdown(System.nanoTime()); + fence.retire(); + var interrupted = new java.util.concurrent.atomic.AtomicBoolean(); + var hook = (Thread) field(fence, "processHook"); + var observer = new Thread(() -> { + hook.run(); + interrupted.set(Thread.currentThread().isInterrupted()); + }); + try { + observer.start(); + assertTrue(completion.entered.await(5, TimeUnit.SECONDS)); + observer.interrupt(); + observer.join(5000); + assertFalse(observer.isAlive()); + assertTrue(interrupted.get()); + assertFalse(completion.isDone()); + } finally { + lease.close(); + observer.interrupt(); + observer.join(5000); + } + } + + @Test void retainedExecutorTerminationIsIncludedWithoutAPreStorageDependencyCycle() throws Exception { + var fence = ServerDataPublicationFence.beginGeneration(directory.resolve("ServerData.yml")); + var completion = observe(fence); + fence.retainForProcessExit(); + var writer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + writer.execute(() -> { + entered.countDown(); + try { assertTrue(release.await(5, TimeUnit.SECONDS)); } + catch (InterruptedException failure) { throw new AssertionError(failure); } + }); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + writer.shutdown(); + var retirement = fence.retainUntilTermination(new java.util.concurrent.ScheduledExecutorService[] { writer }); + fence.beginShutdown(System.nanoTime()); + fence.retire(); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + var exit = caller.submit(((Thread) field(fence, "processHook"))::run); + assertTrue(completion.entered.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone()); + release.countDown(); + retirement.toCompletableFuture().get(5, TimeUnit.SECONDS); + exit.get(5, TimeUnit.SECONDS); + } finally { + release.countDown(); + writer.shutdownNow(); + caller.shutdownNow(); + assertTrue(writer.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + + @Test void queuedSignPublicationRemainsInTheProcessDrainAfterTheNativeSdkOwnerReleases() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var nativeOwner = com.bencodez.advancedcore.lifecycle.ServerDataOwnership.acquire(target); + var fence = ServerDataPublicationFence.beginGeneration(target); + var completion = observe(fence); + fence.retainForProcessExit(); + var plugin = mock(VotingPluginMain.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getLogger()).thenReturn(mock(java.util.logging.Logger.class)); + when(plugin.getServerDataPublicationFence()).thenReturn(fence); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin); + backing.setData(new YamlConfiguration()); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = new ServerData(plugin); + when(plugin.getServerData()).thenReturn(data); + var workers = new ArrayList(); + var signs = new Signs(plugin) { + @Override protected void startShutdownWorker(Runnable worker) { workers.add(worker); } + }; + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + var sign = mock(SignHandler.class, CALLS_REAL_METHODS); + for (var entry : java.util.Map.of("sign", "0", "position", 1, + "data", "SyntheticSign").entrySet()) { + var f = SignHandler.class.getDeclaredField(entry.getKey()); + f.setAccessible(true); + f.set(sign, entry.getValue()); + } + sign.setLocation(new Location(world, 1, 2, 3)); + signs.getSigns().add(sign); + var published = signs.storeSignsForShutdown(); + long started = System.nanoTime(); + nativeOwner.beginShutdown(started); + // Exercise the actual SDK ownership release once its own aggregate has settled. + // This fixture does not simulate unrelated native cleanup actions. + nativeOwner.releaseAfter(CompletableFuture.completedFuture(null)); + assertTrue(nativeOwner.isReleased()); + fence.beginShutdown(started); + fence.retire(); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + var exit = caller.submit(((Thread) field(fence, "processHook"))::run); + assertTrue(completion.entered.await(5, TimeUnit.SECONDS)); + assertFalse(completion.isDone(), "The native release cannot discard the consumer reservation"); + assertFalse(published.toCompletableFuture().isDone()); + workers.getFirst().run(); + published.toCompletableFuture().get(5, TimeUnit.SECONDS); + exit.get(5, TimeUnit.SECONDS); + assertEquals("SyntheticSign", YamlConfiguration.loadConfiguration(target.toFile()) + .getString("VotingPlugin.Signs.0.Data")); + } finally { + if (!published.toCompletableFuture().isDone()) workers.getFirst().run(); + caller.shutdownNow(); + assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java new file mode 100644 index 000000000..c33f9b4b3 --- /dev/null +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java @@ -0,0 +1,220 @@ +package com.bencodez.votingplugin.tests; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import org.bukkit.Location; +import org.bukkit.World; +import org.bukkit.configuration.file.YamlConfiguration; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import com.bencodez.votingplugin.VotingPluginMain; +import com.bencodez.votingplugin.data.GenerationFencedServerDataFile; +import com.bencodez.votingplugin.data.ServerData; +import com.bencodez.votingplugin.signs.SignHandler; +import com.bencodez.votingplugin.signs.Signs; +import com.bencodez.votingplugin.util.ServerDataPublicationFence; + +class SignShutdownPublicationTest { + @TempDir Path directory; + + private static final class Admission extends Signs { + final List workers = new ArrayList<>(); + boolean reject; + Admission(VotingPluginMain plugin) { super(plugin); } + @Override protected void startShutdownWorker(Runnable work) { + if (reject) throw new RejectedExecutionException("Synthetic rejection"); + workers.add(work); + } + } + + private VotingPluginMain plugin(ServerDataPublicationFence fence) { + var plugin = mock(VotingPluginMain.class); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getLogger()).thenReturn(mock(java.util.logging.Logger.class)); + when(plugin.getServerDataPublicationFence()).thenReturn(fence); + return plugin; + } + + private SignHandler sign(VotingPluginMain plugin, String id, Location location, Location skull) throws Exception { + var sign = mock(SignHandler.class, CALLS_REAL_METHODS); + for (var entry : java.util.Map.of("plugin", plugin, "sign", id, + "data", "SyntheticSign" + id, "position", 2).entrySet()) { + var field = SignHandler.class.getDeclaredField(entry.getKey()); + field.setAccessible(true); + field.set(sign, entry.getValue()); + } + sign.setLocation(location); + sign.setSkullLocation(skull); + return sign; + } + + @Test void rejectedAdmissionRetainsTheOriginalBatchAndPublishesItOnceOnRetry() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin); + backing.setData(new YamlConfiguration()); + backing.getData().set("VotingPlugin.Unrelated", "preserved"); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = new ServerData(plugin); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var owner = Thread.currentThread(); + var world = mock(World.class); + when(world.getName()).thenAnswer(call -> { + assertSame(owner, Thread.currentThread(), "Platform state must be captured before handoff"); + return "SyntheticWorld"; + }); + var location = new Location(world, -1.5, 64, 2.5); + var first = sign(plugin, "0", location, new Location(world, 3, 65, 4)); + signs.getSigns().add(first); + signs.getSigns().add(sign(plugin, "1", new Location(world, 5, 66, 6), null)); + signs.reject = true; + assertThrows(RejectedExecutionException.class, signs::storeSignsForShutdown); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + // Retry must use the accepted detached snapshot, not newer platform state. + location.setX(90); + first.setData("LaterUnacceptedState"); + signs.getSigns().clear(); + signs.reject = false; + var completion = signs.storeSignsForShutdown(); + signs.storeSignsForShutdown(); + assertEquals(1, signs.workers.size()); + var worker = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + worker.submit(signs.workers.getFirst()).get(5, TimeUnit.SECONDS); + completion.toCompletableFuture().get(5, TimeUnit.SECONDS); + } finally { + worker.shutdownNow(); + assertTrue(worker.awaitTermination(5, TimeUnit.SECONDS)); + } + var disk = YamlConfiguration.loadConfiguration(target.toFile()); + assertEquals(-1, disk.getInt("VotingPlugin.Signs.0.X"), "Preserve legacy coordinate truncation"); + assertEquals(3, disk.getInt("VotingPlugin.Signs.0.Skull.X")); + assertEquals("SyntheticSign0", disk.getString("VotingPlugin.Signs.0.Data")); + assertEquals(2, disk.getInt("VotingPlugin.Signs.1.Position")); + assertEquals("preserved", disk.getString("VotingPlugin.Unrelated")); + signs.storeSignsForShutdown(); + assertEquals(1, signs.workers.size(), "Successful publication is idempotent"); + ServerDataPublicationFence.beginGeneration(target).retire(); + } + } + + @Test void cancellationAndForgedObservationCannotReleaseTheRealPublicationOwner() throws Exception { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var data = mock(ServerData.class); + var durable = new CompletableFuture(); + when(data.publishShutdownSigns(anyList())).thenReturn(durable); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + signs.getSigns().add(sign(plugin, "0", new Location(world, 1, 2, 3), null)); + var pending = signs.storeSignsForShutdown(); + pending.toCompletableFuture().cancel(true); + pending.toCompletableFuture().complete(null); + signs.storeSignsForShutdown(); + assertEquals(1, signs.workers.size(), "Accepted but unexecuted work retains one owner"); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + signs.workers.getFirst().run(); + assertFalse(pending.toCompletableFuture().isDone(), "Submission is not durability"); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + durable.complete(null); + pending.toCompletableFuture().get(5, TimeUnit.SECONDS); + ServerDataPublicationFence.beginGeneration(target).retire(); + } + + @Test void failedPublicationRetainsPayloadAndRequiresAnExplicitSuccessfulRetry() throws Exception { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var data = mock(ServerData.class); + var attempts = new ArrayList>(); + when(data.publishShutdownSigns(anyList())).thenAnswer(call -> { + attempts.add(call.getArgument(0)); + return attempts.size() == 1 ? CompletableFuture.failedFuture(new IllegalStateException("Synthetic I/O")) + : CompletableFuture.completedFuture(null); + }); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + signs.getSigns().add(sign(plugin, "0", new Location(world, 1, 2, 3), null)); + var first = signs.storeSignsForShutdown(); + signs.workers.getFirst().run(); + assertThrows(java.util.concurrent.CompletionException.class, () -> first.toCompletableFuture().join()); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + signs.getSigns().clear(); + var retry = signs.storeSignsForShutdown(); + signs.workers.get(1).run(); + retry.toCompletableFuture().get(5, TimeUnit.SECONDS); + assertSame(attempts.get(0), attempts.get(1), "Failure must not discard or recapture the accepted batch"); + ServerDataPublicationFence.beginGeneration(target).retire(); + } + + @Test void emptyShutdownDoesNotAdmitAWorkerOrTouchServerData() { + var plugin = mock(VotingPluginMain.class); + var signs = new Admission(plugin); + assertDoesNotThrow(() -> signs.storeSignsForShutdown().toCompletableFuture().join()); + assertEquals(0, signs.workers.size()); + verifyNoInteractions(plugin); + } + + @Test void actualSdkIoFailureKeepsTheGenerationBlocked() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin) { + @Override protected void moveAtomically(Path source, Path destination) throws java.io.IOException { + throw new java.io.IOException("Synthetic failed atomic publication"); + } + }; + backing.setData(new YamlConfiguration()); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = new ServerData(plugin); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + signs.getSigns().add(sign(plugin, "0", new Location(world, 1, 2, 3), null)); + var result = signs.storeSignsForShutdown(); + signs.workers.getFirst().run(); + assertThrows(java.util.concurrent.CompletionException.class, () -> result.toCompletableFuture().join()); + assertFalse(java.nio.file.Files.exists(target)); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target), + "Neither the SDK guard nor sign handoff may call failed I/O durable"); + } + } + + @Test void runtimeSignSavingRemainsSynchronous() throws Exception { + var plugin = mock(VotingPluginMain.class); + var data = mock(ServerData.class); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var location = new Location(mock(World.class), 1, 2, 3); + signs.getSigns().add(sign(plugin, "0", location, null)); + var caller = Thread.currentThread(); + doAnswer(call -> { assertSame(caller, Thread.currentThread()); return null; }) + .when(data).setSign("0", location, null, "SyntheticSign0", 2); + signs.storeSigns(); + verify(data).setSign("0", location, null, "SyntheticSign0", 2); + assertEquals(0, signs.workers.size()); + } +} diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginMainShutdownTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginMainShutdownTest.java index 0ef211f3b..bd7265917 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginMainShutdownTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/VotingPluginMainShutdownTest.java @@ -92,13 +92,13 @@ void integratedUnloadClosesTheCurrentStreakMenuOnce() throws Exception { setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); initializeEarlyLifecycle(plugin); var menu = mock(com.bencodez.votingplugin.streaks.VoteStreaksMenu.class); - var signs = mock(Signs.class); + var signs = shutdownSignsMock(); setField(plugin, "voteStreaksMenu", menu); setField(plugin, "signs", signs); assertDoesNotThrow(plugin::onUnLoad); assertDoesNotThrow(plugin::onUnLoad); verify(menu).close(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } @Test @@ -108,15 +108,15 @@ void failedStreakMenuCloseDoesNotSkipOtherCleanupAndRetriesOnRepeat() throws Exc setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); initializeEarlyLifecycle(plugin); var menu = mock(com.bencodez.votingplugin.streaks.VoteStreaksMenu.class); - var signs = mock(Signs.class); + var signs = shutdownSignsMock(); setField(plugin, "voteStreaksMenu", menu); setField(plugin, "signs", signs); doThrow(new IllegalStateException("synthetic close failure")).doNothing().when(menu).close(); assertDoesNotThrow(plugin::onUnLoad); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); assertDoesNotThrow(plugin::onUnLoad); verify(menu, org.mockito.Mockito.times(2)).close(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } @Test @@ -137,17 +137,17 @@ void failedDiscordCloseDoesNotSkipDisplayRetirementAndRetriesOnlyTheFailedStep() rankings.put(new com.bencodez.votingplugin.topvoter.TopVoterPlayer(java.util.UUID.randomUUID(), "Example", 1L), 10); var display = filter.projectAsync(rankings, true).toCompletableFuture(); org.junit.jupiter.api.Assertions.assertFalse(display.isDone()); - var signs = mock(Signs.class); + var signs = shutdownSignsMock(); setField(plugin, "signs", signs); try (var handlers = org.mockito.Mockito.mockStatic(org.bukkit.event.HandlerList.class)) { assertDoesNotThrow(plugin::onUnLoad); org.junit.jupiter.api.Assertions.assertTrue(display.isDone(), "A failed independent close must not strand pending display evidence"); org.junit.jupiter.api.Assertions.assertEquals(rankings, display.getNow(null)); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); assertDoesNotThrow(plugin::onUnLoad); verify(discord, org.mockito.Mockito.times(2)).close(); verify(discord).retireLeaderboardPublications(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); org.junit.jupiter.api.Assertions.assertEquals(1L, generation.get()); pending.complete(com.bencodez.advancedcore.api.user.presence.PlayerPresence.NEVER_JOINED); org.junit.jupiter.api.Assertions.assertEquals(rankings, display.getNow(null)); @@ -161,17 +161,17 @@ void publicUnloadCleansInitializedQueueAndSignResources() throws Exception { setField(plugin, "unloadInProgress", new java.util.concurrent.atomic.AtomicBoolean()); setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); initializeEarlyLifecycle(plugin); - Signs signs = mock(Signs.class); + Signs signs = shutdownSignsMock(); TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); setField(plugin, "signs", signs); setField(plugin, "timeQueueHandler", timeQueue); plugin.onUnLoad(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); verify(timeQueue).saveForShutdown(); plugin.onUnLoad(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } @Test @@ -181,7 +181,7 @@ void failedCleanupDoesNotSkipLaterResources() throws Exception { setField(plugin, "unloadInProgress", new java.util.concurrent.atomic.AtomicBoolean()); setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); initializeEarlyLifecycle(plugin); - Signs signs = mock(Signs.class); + Signs signs = shutdownSignsMock(); TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); VotingPluginWebhookManager webhook = mock(VotingPluginWebhookManager.class); doThrow(new IllegalStateException("webhook close failed")).when(webhook).shutdown(); @@ -194,7 +194,7 @@ void failedCleanupDoesNotSkipLaterResources() throws Exception { verify(webhook).shutdown(); verify(timeQueue).saveForShutdown(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } @Test @@ -205,7 +205,7 @@ void repeatedUnloadRetriesFailedBackendAndWebhookWithoutRepeatingSuccessfulResou initializeEarlyLifecycle(plugin); var backend = mock(com.bencodez.votingplugin.backendproxy.BackendProxyHandler.class); var webhook = mock(VotingPluginWebhookManager.class); - var signs = mock(Signs.class); + var signs = shutdownSignsMock(); org.mockito.Mockito.doReturn(backend).when(plugin).getBackendProxyHandler(); org.mockito.Mockito.doThrow(new IllegalStateException("synthetic backend close failure")).doNothing() .when(backend).closeForShutdown(); @@ -218,7 +218,7 @@ void repeatedUnloadRetriesFailedBackendAndWebhookWithoutRepeatingSuccessfulResou assertDoesNotThrow(plugin::onUnLoad); verify(backend, org.mockito.Mockito.times(2)).closeForShutdown(); verify(webhook, org.mockito.Mockito.times(2)).shutdown(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); handlers.verify(() -> org.bukkit.event.HandlerList.unregisterAll(plugin)); } } @@ -284,7 +284,7 @@ void repeatedUnloadRetriesOnlyFailedTimedVotePersistence() throws Exception { setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); initializeEarlyLifecycle(plugin); TimeQueueHandler timeQueue = mock(TimeQueueHandler.class); - Signs signs = mock(Signs.class); + Signs signs = shutdownSignsMock(); org.mockito.Mockito.doThrow(new IllegalStateException("disk unavailable")).doNothing() .when(timeQueue).saveForShutdown(); setField(plugin, "timeQueueHandler", timeQueue); @@ -292,7 +292,7 @@ void repeatedUnloadRetriesOnlyFailedTimedVotePersistence() throws Exception { assertDoesNotThrow(plugin::onUnLoad); assertDoesNotThrow(plugin::onUnLoad); verify(timeQueue, org.mockito.Mockito.times(2)).saveForShutdown(); - verify(signs).storeSigns(); + verify(signs).storeSignsForShutdown(); } @Test @@ -410,6 +410,134 @@ void rejectedMysqlStartupRetainsItsCauseAndFullDisableCleansEarlyResources() thr } finally { coreTimer.shutdownNow(); } } + @Test + void signPersistenceDoesNotReblockUnloadAfterATimedVoteCheckpointTimesOut( + @org.junit.jupiter.api.io.TempDir java.nio.file.Path directory) throws Exception { + var entered = new java.util.concurrent.CountDownLatch(1); + var release = new java.util.concurrent.CountDownLatch(1); + var firstMove = new java.util.concurrent.atomic.AtomicBoolean(true); + var caller = java.util.concurrent.Executors.newSingleThreadExecutor(); + var producers = new java.util.concurrent.ScheduledThreadPoolExecutor(1); + producers.setExecuteExistingDelayedTasksAfterShutdownPolicy(false); + try (var schedulers = org.mockito.Mockito.mockConstruction( + com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "unloadInProgress", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + org.mockito.Mockito.doReturn(directory.toFile()).when(plugin).getDataFolder(); + var fence = com.bencodez.votingplugin.util.ServerDataPublicationFence.beginGeneration( + directory.resolve("ServerData.yml")); + setField(plugin, "serverDataPublicationFence", fence); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin) { + @Override protected void moveAtomically(java.nio.file.Path source, java.nio.file.Path target) + throws java.io.IOException { + if (firstMove.compareAndSet(true, false)) { + entered.countDown(); + try { + if (!release.await(10, java.util.concurrent.TimeUnit.SECONDS)) + throw new java.io.IOException("Synthetic stalled writer was not released"); + } catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new java.io.IOException(failure); + } + } + super.moveAtomically(source, target); + } + }; + backing.setData(new org.bukkit.configuration.file.YamlConfiguration()); + var facade = new com.bencodez.votingplugin.data.GenerationFencedServerDataFile(plugin, backing, fence); + org.mockito.Mockito.doReturn(facade).when(plugin).getServerDataFile(); + var data = new com.bencodez.votingplugin.data.ServerData(plugin); + org.mockito.Mockito.doReturn(data).when(plugin).getServerData(); + org.mockito.Mockito.doReturn(producers).when(plugin).getVoteTimer(); + var queue = new TimeQueueHandler(plugin); + queue.addVote("SyntheticVoter", "SyntheticSite"); + producers.shutdown(); + org.junit.jupiter.api.Assertions.assertTrue(producers.awaitTermination(2, + java.util.concurrent.TimeUnit.SECONDS)); + setField(plugin, "timeQueueHandler", queue); + var signs = new Signs(plugin); + var sign = mock(com.bencodez.votingplugin.signs.SignHandler.class, CALLS_REAL_METHODS); + for (var entry : java.util.Map.of("plugin", plugin, "sign", "0", + "data", "SyntheticSign", "position", 1).entrySet()) { + var f = com.bencodez.votingplugin.signs.SignHandler.class.getDeclaredField(entry.getKey()); + f.setAccessible(true); + f.set(sign, entry.getValue()); + } + var world = mock(org.bukkit.World.class); + org.mockito.Mockito.when(world.getName()).thenReturn("SyntheticWorld"); + sign.setLocation(new org.bukkit.Location(world, -1.5, 64, 2.5)); + signs.getSigns().add(sign); + setField(plugin, "signs", signs); + try { + var unload = caller.submit(() -> { + try (var handlers = org.mockito.Mockito.mockStatic(org.bukkit.event.HandlerList.class)) { + plugin.onUnLoad(); + handlers.verify(() -> org.bukkit.event.HandlerList.unregisterAll(plugin)); + } + }); + org.junit.jupiter.api.Assertions.assertTrue(entered.await(5, + java.util.concurrent.TimeUnit.SECONDS), "Actual SDK file writer did not reach disk boundary"); + org.junit.jupiter.api.Assertions.assertDoesNotThrow( + () -> unload.get(3, java.util.concurrent.TimeUnit.SECONDS), + "Sign persistence must not reacquire the stalled YAML monitor on the unload caller"); + org.junit.jupiter.api.Assertions.assertEquals(1, release.getCount(), + "Unload must complete before the stalled writer is released"); + } finally { + release.countDown(); + var pendingSigns = (java.util.concurrent.CompletionStage) field(plugin, "signShutdownPublication"); + if (pendingSigns != null) pendingSigns.toCompletableFuture().get(5, java.util.concurrent.TimeUnit.SECONDS); + queue.saveForShutdown(); + var disk = org.bukkit.configuration.file.YamlConfiguration.loadConfiguration( + directory.resolve("ServerData.yml").toFile()); + org.junit.jupiter.api.Assertions.assertEquals("SyntheticSign", disk.getString("VotingPlugin.Signs.0.Data")); + org.junit.jupiter.api.Assertions.assertEquals(1, data.getTimedVoteCacheKeys().size(), + "Final sign publication must preserve the timed-vote checkpoint"); + caller.shutdown(); + org.junit.jupiter.api.Assertions.assertTrue(caller.awaitTermination(5, + java.util.concurrent.TimeUnit.SECONDS)); + fence.retire(); + } + } finally { + release.countDown(); + caller.shutdownNow(); + producers.shutdownNow(); + } + } + + @Test + void repeatedUnloadRetriesAFailedSignPublicationWithoutRepeatingListenerCleanup() throws Exception { + var plugin = mock(VotingPluginMain.class, CALLS_REAL_METHODS); + setField(plugin, "unloadInProgress", new java.util.concurrent.atomic.AtomicBoolean()); + setField(plugin, "placeholderPlayerPresence", new PlaceholderPlayerPresence()); + initializeEarlyLifecycle(plugin); + var signs = mock(Signs.class); + var pending = new java.util.concurrent.CompletableFuture(); + org.mockito.Mockito.when(signs.storeSignsForShutdown()).thenReturn( + java.util.concurrent.CompletableFuture.failedFuture(new IllegalStateException("Synthetic I/O")), + pending.minimalCompletionStage()); + setField(plugin, "signs", signs); + try (var handlers = org.mockito.Mockito.mockStatic(org.bukkit.event.HandlerList.class)) { + plugin.onUnLoad(); + plugin.onUnLoad(); + plugin.onUnLoad(); + verify(signs, org.mockito.Mockito.times(2)).storeSignsForShutdown(); + verify(signs, never()).storeSigns(); + handlers.verify(() -> org.bukkit.event.HandlerList.unregisterAll(plugin)); + pending.complete(null); + plugin.onUnLoad(); + verify(signs, org.mockito.Mockito.times(2)).storeSignsForShutdown(); + } + } + + private static Signs shutdownSignsMock() { + var signs = mock(Signs.class); + org.mockito.Mockito.when(signs.storeSignsForShutdown()) + .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null)); + return signs; + } + private static Throwable rootCause(Throwable failure) { while (failure.getCause() != null) failure = failure.getCause(); return failure; From 73947ee4cce3f91b4edd8c0b224fa2eb58bec2c8 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 18:08:27 -0600 Subject: [PATCH 8/8] Retain shutdown sign publication ownership across retries Reuse the accepted batch publication lease after failed I/O, coalesce pending retries, and retry display observation without repeating a confirmed write. --- .../data/GenerationFencedServerDataFile.java | 70 +++++++-- .../votingplugin/data/ServerData.java | 34 +++- .../tests/SignShutdownPublicationTest.java | 148 ++++++++++++++++++ 3 files changed, 232 insertions(+), 20 deletions(-) diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/GenerationFencedServerDataFile.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/GenerationFencedServerDataFile.java index ea0102833..cd635d447 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/GenerationFencedServerDataFile.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/GenerationFencedServerDataFile.java @@ -162,21 +162,63 @@ private void finishFinalPublication(ServerDataPublicationFence.Lease lease, catch (Throwable releaseFailure) { result.completeExceptionally(releaseFailure); } } + /** Retain this handle only for retries of the same accepted mutation. */ + FinalPublication prepareFinalPublication(Consumer mutation) { + return new FinalPublication(mutation); + } + + final class FinalPublication { + private final Consumer mutation; + private ServerDataPublicationFence.Lease lease; + private CompletableFuture attempt; + private boolean publicationConfirmed; + + private FinalPublication(Consumer mutation) { this.mutation = mutation; } + + CompletionStage publish() { + CompletableFuture result; + ServerDataPublicationFence.Lease owner; + boolean refreshOnly; + synchronized (this) { + if (attempt != null && !attempt.isCompletedExceptionally()) + return attempt.minimalCompletionStage(); + if (lease == null) { + try { lease = fence.acquire(); } + catch (Throwable failure) { return CompletableFuture.failedFuture(failure); } + } + owner = lease; + result = new CompletableFuture<>(); + attempt = result; + refreshOnly = publicationConfirmed; + } + // No operation lock is held across the actual owner's YAML or I/O. + if (refreshOnly) finish(owner, result, null); + else { + try { + java.util.Objects.requireNonNull(mutation, "mutation"); + java.util.Objects.requireNonNull(delegate.mutateFinalData(data -> applyOwnedMutation(mutation, data))) + .whenComplete((ignored, failure) -> finish(owner, result, failure)); + } catch (Throwable failure) { finish(owner, result, failure); } + } + return result.minimalCompletionStage(); + } + + private void finish(ServerDataPublicationFence.Lease owner, + CompletableFuture result, Throwable failure) { + if (failure == null) { + synchronized (this) { publicationConfirmed = true; } + } + // A retry after a display-observation failure must not repeat a + // confirmed mutation. Failed publication instead retains this one + // lease until this same operation has a confirmed successful retry. + finishFinalPublication(owner, result, failure); + } + } + @Override public CompletionStage mutateFinalData(Consumer mutation) { - var result = new CompletableFuture(); - ServerDataPublicationFence.Lease lease; - try { lease = fence.acquire(); } - catch (Throwable failure) { return CompletableFuture.failedFuture(failure); } - try { - java.util.Objects.requireNonNull(mutation, "mutation"); - // The delegate acquires its monitor. Do not hold a facade/YAML lock - // while observing an asynchronous owner's completion. - java.util.Objects.requireNonNull(delegate.mutateFinalData(data -> applyOwnedMutation(mutation, data))) - .whenComplete((ignored, failure) -> finishFinalPublication(lease, result, failure)); - } catch (Throwable failure) { finishFinalPublication(lease, result, failure); } - // Failure or cancellation cannot prove writer drain; retain the lease. - // A caller may cancel its observation without cancelling the real owner. - return result.minimalCompletionStage(); + // Ordinary one-shot callers retain failed ownership conservatively. + // A caller with an explicit retry contract retains the operation handle. + return prepareFinalPublication(mutation).publish(); } @Override public void setData(String path, Object value) { mutateOrdinaryData(data -> data.set(path, value)); } diff --git a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java index 29070f28f..fe02a32c3 100644 --- a/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java +++ b/VotingPlugin/src/main/java/com/bencodez/votingplugin/data/ServerData.java @@ -108,14 +108,36 @@ public static SignSnapshot captureSignSnapshot(String count, Location location, return new SignSnapshot(count, location, skull, data, position); } - /** Worker-only: publish the whole captured sign batch through the actual durable SDK owner. */ + private record ShutdownSignPublication(List snapshot, + java.util.function.Supplier> publisher) { } + private final Object shutdownSignBinding = new Object(); + private ShutdownSignPublication shutdownSignPublication; + + /** Worker-only: one accepted sign batch, with the same publication owner across retries. */ public java.util.concurrent.CompletionStage publishShutdownSigns(List signs) { var snapshot = List.copyOf(signs); - return getFile().mutateFinalData(candidate -> { - var target = candidate.getConfigurationSection("VotingPlugin"); - if (target == null) target = candidate.createSection("VotingPlugin"); - for (var sign : snapshot) sign.store(target); - }); + ShutdownSignPublication publication; + synchronized (shutdownSignBinding) { + if (shutdownSignPublication == null) { + var file = getFile(); + java.util.function.Consumer mutation = candidate -> { + var target = candidate.getConfigurationSection("VotingPlugin"); + if (target == null) target = candidate.createSection("VotingPlugin"); + for (var sign : snapshot) sign.store(target); + }; + java.util.function.Supplier> publisher; + if (file instanceof GenerationFencedServerDataFile guarded) { + var retained = guarded.prepareFinalPublication(mutation); + publisher = retained::publish; + } else publisher = () -> file.mutateFinalData(mutation); + shutdownSignPublication = new ShutdownSignPublication(snapshot, publisher); + } else if (!shutdownSignPublication.snapshot().equals(snapshot)) { + return java.util.concurrent.CompletableFuture.failedFuture( + new IllegalStateException("Shutdown sign publication already owns a different batch")); + } + publication = shutdownSignPublication; + } + return publication.publisher().get(); } /** Only YAML work belongs here; platform callbacks run outside the file owner. */ diff --git a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java index c33f9b4b3..905ad0b1f 100644 --- a/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java +++ b/VotingPlugin/src/test/java/com/bencodez/votingplugin/tests/SignShutdownPublicationTest.java @@ -174,6 +174,154 @@ private SignHandler sign(VotingPluginMain plugin, String id, Location location, verifyNoInteractions(plugin); } + @Test void actualSdkFailedThenSuccessfulRetryReleasesTheOriginalPublicationOwner() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin) { + @Override protected void moveAtomically(Path source, Path destination) throws java.io.IOException { + if (attempts.incrementAndGet() == 1) + throw new java.io.IOException("Synthetic first publication failure"); + super.moveAtomically(source, destination); + } + }; + backing.setData(new YamlConfiguration()); + backing.getData().set("VotingPlugin.Unrelated", "preserved"); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = new ServerData(plugin); + when(plugin.getServerData()).thenReturn(data); + var signs = new Admission(plugin); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + signs.getSigns().add(sign(plugin, "0", new Location(world, 1, 2, 3), null)); + var failed = signs.storeSignsForShutdown(); + signs.workers.getFirst().run(); + assertThrows(java.util.concurrent.CompletionException.class, () -> failed.toCompletableFuture().join()); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + facade.mutateFinalData(candidate -> candidate.set("Independent.Value", "confirmed")) + .toCompletableFuture().get(5, TimeUnit.SECONDS); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target), + "A different successful operation cannot acknowledge the failed sign batch"); + signs.getSigns().clear(); + var retry = signs.storeSignsForShutdown(); + signs.workers.get(1).run(); + retry.toCompletableFuture().get(5, TimeUnit.SECONDS); + var disk = YamlConfiguration.loadConfiguration(target.toFile()); + assertEquals("SyntheticSign0", disk.getString("VotingPlugin.Signs.0.Data")); + assertEquals("preserved", disk.getString("VotingPlugin.Unrelated")); + assertEquals("confirmed", disk.getString("Independent.Value")); + assertEquals(3, attempts.get()); + signs.storeSignsForShutdown(); + assertEquals(2, signs.workers.size(), "Successful retry is not republished"); + assertDoesNotThrow(() -> ServerDataPublicationFence.beginGeneration(target).retire(), + "Confirmed retry must release both facade and sign-batch ownership"); + } + } + + @Test void actualPendingRetryCoalescesWithoutReplacingItsPayloadOrTrustingObservationCancellation() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var attempts = new java.util.concurrent.atomic.AtomicInteger(); + var retryEntered = new java.util.concurrent.CountDownLatch(1); + var releaseRetry = new java.util.concurrent.CountDownLatch(1); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin) { + @Override protected void moveAtomically(Path source, Path destination) throws java.io.IOException { + if (attempts.incrementAndGet() == 1) + throw new java.io.IOException("Synthetic first publication failure"); + retryEntered.countDown(); + try { + if (!releaseRetry.await(5, TimeUnit.SECONDS)) + throw new java.io.IOException("Synthetic retry barrier expired"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + throw new java.io.IOException(interrupted); + } + super.moveAtomically(source, destination); + } + }; + backing.setData(new YamlConfiguration()); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = new ServerData(plugin); + when(plugin.getServerData()).thenReturn(data); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + var snapshot = List.of(ServerData.captureSignSnapshot("0", new Location(world, 1, 2, 3), + null, "OriginalAcceptedSign", 2)); + var different = List.of(ServerData.captureSignSnapshot("0", new Location(world, 9, 8, 7), + null, "DifferentUnacceptedSign", 4)); + assertThrows(java.util.concurrent.CompletionException.class, + () -> data.publishShutdownSigns(snapshot).toCompletableFuture().join()); + fence.retire(); + var workers = java.util.concurrent.Executors.newFixedThreadPool(2); + try { + var retry = workers.submit(() -> data.publishShutdownSigns(snapshot)); + assertTrue(retryEntered.await(5, TimeUnit.SECONDS)); + var coalesced = workers.submit(() -> data.publishShutdownSigns(snapshot)).get(2, TimeUnit.SECONDS); + coalesced.toCompletableFuture().cancel(true); + coalesced.toCompletableFuture().complete(null); + assertFalse(coalesced.toCompletableFuture().isDone(), "The real publication still owns its lease"); + var rejected = workers.submit(() -> data.publishShutdownSigns(different)).get(2, TimeUnit.SECONDS); + assertThrows(java.util.concurrent.CompletionException.class, () -> rejected.toCompletableFuture().join()); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + releaseRetry.countDown(); + retry.get(5, TimeUnit.SECONDS).toCompletableFuture().get(5, TimeUnit.SECONDS); + coalesced.toCompletableFuture().get(5, TimeUnit.SECONDS); + data.publishShutdownSigns(snapshot).toCompletableFuture().get(5, TimeUnit.SECONDS); + assertEquals(2, attempts.get(), "One failed attempt and one shared durable retry"); + var disk = YamlConfiguration.loadConfiguration(target.toFile()); + assertEquals("OriginalAcceptedSign", disk.getString("VotingPlugin.Signs.0.Data")); + assertEquals(1, disk.getInt("VotingPlugin.Signs.0.X")); + ServerDataPublicationFence.beginGeneration(target).retire(); + } finally { + releaseRetry.countDown(); + workers.shutdownNow(); + assertTrue(workers.awaitTermination(5, TimeUnit.SECONDS)); + } + } + } + + @Test void confirmedPublicationRetriesOnlyFailedDisplayObservationWithoutRepeatingTheWrite() throws Exception { + try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { + var target = directory.resolve("ServerData.yml"); + var fence = ServerDataPublicationFence.beginGeneration(target); + var plugin = plugin(fence); + var writes = new java.util.concurrent.atomic.AtomicInteger(); + var backing = new com.bencodez.advancedcore.data.ServerData(plugin) { + @Override protected void moveAtomically(Path source, Path destination) throws java.io.IOException { + writes.incrementAndGet(); + super.moveAtomically(source, destination); + } + }; + backing.setData(new YamlConfiguration()); + var facade = new GenerationFencedServerDataFile(plugin, backing, fence); + when(plugin.getServerDataFile()).thenReturn(facade); + var data = spy(new ServerData(plugin)); + when(plugin.getServerData()).thenReturn(data); + doThrow(new IllegalStateException("Synthetic observation failure")).doCallRealMethod() + .when(data).refreshVotePartyDisplay(); + var world = mock(World.class); + when(world.getName()).thenReturn("SyntheticWorld"); + var snapshot = List.of(ServerData.captureSignSnapshot("0", new Location(world, 1, 2, 3), + null, "OriginalAcceptedSign", 2)); + assertThrows(java.util.concurrent.CompletionException.class, + () -> data.publishShutdownSigns(snapshot).toCompletableFuture().join()); + assertEquals("OriginalAcceptedSign", YamlConfiguration.loadConfiguration(target.toFile()) + .getString("VotingPlugin.Signs.0.Data"), "The write committed before observation failed"); + fence.retire(); + assertThrows(IllegalStateException.class, () -> ServerDataPublicationFence.beginGeneration(target)); + data.publishShutdownSigns(snapshot).toCompletableFuture().get(5, TimeUnit.SECONDS); + assertEquals(1, writes.get(), "Retry must not replay an already confirmed mutation"); + ServerDataPublicationFence.beginGeneration(target).retire(); + } + } + @Test void actualSdkIoFailureKeepsTheGenerationBlocked() throws Exception { try (var schedulers = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { var target = directory.resolve("ServerData.yml");