From 706102cda2d45a8ba5b10f9187d0b81a38b7f393 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 10:54:12 -0600 Subject: [PATCH 01/28] Preserve storage worker ownership during Spigot shutdown --- .../api/user/usercache/UserDataManager.java | 6 ++ .../user/runtime/BukkitUserCacheOwner.java | 3 +- .../tests/lifecycle/CoreRuntimeTest.java | 19 ++++- .../SharedUserLifecycleRegressionTest.java | 1 + .../user/SpigotShutdownStorageWorkerTest.java | 82 +++++++++++++++++++ docs/shared-sql-user-access.md | 10 +++ 6 files changed, 115 insertions(+), 6 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SpigotShutdownStorageWorkerTest.java diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java index 69da17a1d..b7c48b67d 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java @@ -49,6 +49,8 @@ public class UserDataManager { @Getter private ArrayList booleanColumns; @Getter private AdvancedCorePlugin plugin; @Getter private ScheduledExecutorService timer; + /** Exact private executor identity; Spigot reports every thread as primary after stop. */ + private volatile Thread storageWorker; @Getter private ConcurrentHashMap userDataCache; private volatile Consumer sharedCacheInitializer; @@ -646,6 +648,7 @@ public UserDataManager(AdvancedCorePlugin plugin) { // this component-owned worker daemon prevents that hung call from retaining // the server JVM after the bounded lifecycle watchdog has reported it. worker.setDaemon(true); + storageWorker = worker; return worker; }); loadKeys(); @@ -1360,6 +1363,9 @@ public boolean mustDeferSharedStorageAccess() { * platform-owned execution lanes and must never wait on JDBC/shared-runtime admission. */ public boolean isPlatformOwnedThread() { + // This thread is created by our single-threaded executor, never by Bukkit. + // Do not infer ownership from its name, plugin enabled state, or shutdown state. + if (Thread.currentThread() == storageWorker) return false; Object server = Bukkit.getServer(); if (server == null) return false; try { if (Bukkit.isPrimaryThread()) return true; } diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/user/runtime/BukkitUserCacheOwner.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/user/runtime/BukkitUserCacheOwner.java index 6b561f6e3..9ea7a707a 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/user/runtime/BukkitUserCacheOwner.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/user/runtime/BukkitUserCacheOwner.java @@ -12,7 +12,6 @@ import java.util.function.BiConsumer; import java.util.function.Consumer; -import org.bukkit.Bukkit; import com.bencodez.advancedcore.api.user.UserStorage; import com.bencodez.advancedcore.api.user.usercache.UserDataCache; @@ -156,7 +155,7 @@ private void bind(UserDataCache cache, UUID uuid) { } @Override public void requireBlockingAllowed() { - if ((Bukkit.getServer() != null && Bukkit.isPrimaryThread()) || manager.isPlatformOwnedThread()) { + if (manager.isPlatformOwnedThread()) { throw new IllegalStateException("Shared user storage must run on a worker; use closeAsync for shutdown"); } } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index f47103610..80c6c0640 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -161,11 +161,22 @@ private RuntimePlatform platform() { verify(platform, times(1)).getTimeTimer(); } + private ScheduledExecutorService terminatingExecutor(boolean gracefulTermination) { + ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + var terminated = new java.util.concurrent.atomic.AtomicBoolean(); + when(executor.isTerminated()).thenAnswer(call -> terminated.get()); + doAnswer(call -> { terminated.set(gracefulTermination); return null; }).when(executor).shutdown(); + when(executor.shutdownNow()).thenAnswer(call -> { terminated.set(true); return List.of(); }); + return executor; + } + private ScheduledExecutorService executor(String name, List events) throws Exception { ScheduledExecutorService executor = mock(ScheduledExecutorService.class); + var terminated = new java.util.concurrent.atomic.AtomicBoolean(); + when(executor.isTerminated()).thenAnswer(call -> terminated.get()); doAnswer(call -> { events.add(name + "-stop"); return null; }).when(executor).shutdown(); when(executor.awaitTermination(anyLong(), any())).thenAnswer(call -> { events.add(name + "-wait"); return false; }); - when(executor.shutdownNow()).thenAnswer(call -> { events.add(name + "-force"); return List.of(); }); + when(executor.shutdownNow()).thenAnswer(call -> { events.add(name + "-force"); terminated.set(true); return List.of(); }); return executor; } @@ -222,7 +233,7 @@ private ScheduledExecutorService executor(String name, List events) thro @Test void nonBlockingPlatformLeavesStorageWorkerAliveUntilRetirementCompletes() throws Exception { RuntimePlatform platform = platform(); - ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + ScheduledExecutorService timer = terminatingExecutor(true); CompletableFuture retiring = new CompletableFuture<>(); List events = new java.util.concurrent.CopyOnWriteArrayList<>(); when(platform.beforeExecutorShutdownCompletion()).thenReturn(retiring); @@ -313,7 +324,7 @@ private ScheduledExecutorService executor(String name, List events) thro @Test void deferredRetirementTimeoutForcesStorageWorkerWithoutRepeatingPlatformCleanup() { RuntimePlatform platform = platform(); - ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + ScheduledExecutorService timer = terminatingExecutor(false); CompletableFuture retiring = new CompletableFuture<>(); List events = new java.util.concurrent.CopyOnWriteArrayList<>(); when(platform.beforeExecutorShutdownCompletion()).thenReturn(retiring); @@ -433,7 +444,7 @@ private ScheduledExecutorService executor(String name, List events) thro @Test void failedRetirementTerminatesStorageWorkerAfterReportingTheFailure() { RuntimePlatform platform = platform(); - ScheduledExecutorService timer = mock(ScheduledExecutorService.class); + ScheduledExecutorService timer = terminatingExecutor(false); CompletableFuture retiring = new CompletableFuture<>(); retiring.completeExceptionally(new IllegalStateException("write failed")); when(platform.beforeExecutorShutdownCompletion()).thenReturn(retiring); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserLifecycleRegressionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserLifecycleRegressionTest.java index a6d48904a..339631ed1 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserLifecycleRegressionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserLifecycleRegressionTest.java @@ -468,6 +468,7 @@ private static final class Fixture { final MemoryBackend first = new MemoryBackend(UserStorage.SQLITE); final BukkitUserCacheOwner owner; Fixture() { + when(manager.isPlatformOwnedThread()).thenCallRealMethod(); when(manager.getPlugin()).thenReturn(plugin); when(plugin.getStorageType()).thenReturn(UserStorage.SQLITE); when(manager.getUserDataCache()).thenReturn(caches); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SpigotShutdownStorageWorkerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SpigotShutdownStorageWorkerTest.java new file mode 100644 index 000000000..6c91d844b --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SpigotShutdownStorageWorkerTest.java @@ -0,0 +1,82 @@ +package com.bencodez.advancedcore.tests.user; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +import org.bukkit.Bukkit; +import org.bukkit.Server; +import org.junit.jupiter.api.Test; + +import com.bencodez.advancedcore.AdvancedCorePlugin; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; +import com.bencodez.advancedcore.bukkit.user.runtime.BukkitUserCacheOwner; +import com.bencodez.advancedcore.core.user.runtime.SharedUserDataRuntime; +import com.bencodez.advancedcore.core.user.storage.sql.SqlUserBackend; + +class SpigotShutdownStorageWorkerTest { + @Test void actualStorageWorkerCanRetireWhileSpigotReportsEveryThreadPrimary() throws Exception { + UserDataManager manager = new UserDataManager(mock(AdvancedCorePlugin.class)); + SqlUserBackend backend = mock(SqlUserBackend.class); + Server server = mock(Server.class); + BukkitUserCacheOwner owner = new BukkitUserCacheOwner(manager); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, owner); + try { + manager.getTimer().submit(() -> { + // Mockito's static scope is thread-local: install it on the real storage worker. + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(server); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + assertTrue(Bukkit.isPrimaryThread()); + assertFalse(manager.isPlatformOwnedThread()); + runtime.closeAsync(Runnable::run).toCompletableFuture().join(); + assertTrue(runtime.isClosed()); + } + }).get(5, TimeUnit.SECONDS); + verify(backend).close(); + assertNull(manager.getLastDeferredStorageFailure()); + } finally { + manager.getTimer().shutdownNow(); + } + } + + @Test void primaryCallerStillCannotCloseStorageSynchronously() { + UserDataManager manager = new UserDataManager(mock(AdvancedCorePlugin.class)); + SqlUserBackend backend = mock(SqlUserBackend.class); + Server server = mock(Server.class); + BukkitUserCacheOwner owner = new BukkitUserCacheOwner(manager); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, owner); + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(server); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + assertTrue(manager.isPlatformOwnedThread()); + assertThrows(IllegalStateException.class, runtime::close); + verify(backend, never()).close(); + } finally { + manager.getTimer().shutdownNow(); + } + } + + @Test void matchingWorkerNameDoesNotBypassPlatformGuard() throws Exception { + UserDataManager manager = new UserDataManager(mock(AdvancedCorePlugin.class)); + Server server = mock(Server.class); + BukkitUserCacheOwner owner = new BukkitUserCacheOwner(manager); + String name = manager.getTimer().submit(() -> Thread.currentThread().getName()).get(5, TimeUnit.SECONDS); + var external = Executors.newSingleThreadExecutor(task -> new Thread(task, name)); + try { + external.submit(() -> { + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(server); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + assertTrue(manager.isPlatformOwnedThread()); + assertThrows(IllegalStateException.class, owner::requireBlockingAllowed); + } + }).get(5, TimeUnit.SECONDS); + } finally { + external.shutdownNow(); + manager.getTimer().shutdownNow(); + } + } +} diff --git a/docs/shared-sql-user-access.md b/docs/shared-sql-user-access.md index fd396158e..5031a8475 100644 --- a/docs/shared-sql-user-access.md +++ b/docs/shared-sql-user-access.md @@ -96,3 +96,13 @@ to surface acquisition and query failures with their original cause. A failed re cannot publish a schema-only/default snapshot, and a failed delete cannot remove identity or runtime caches. Existing `getExact`/`getExactQuery` callers retain the legacy read fallback; legacy deletion also leaves caches intact on SQL failure. + +## Spigot shutdown worker ownership + +The user-data manager retains the exact identity of its private storage executor +thread. That worker remains a storage lane during server shutdown, even when +Spigot's primary-thread probe reports every thread as primary after the server +has stopped. Shared-cache retirement still runs asynchronously on this worker. +All other threads retain the existing Bukkit and Folia tick-thread checks; +matching a worker name or disabling a plugin does not bypass them. This needs no +configuration or data migration and does not change the public retirement API. From 427693aac00050891d339080e118bb215aba8733 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 14:42:57 -0600 Subject: [PATCH 02/28] Admit plugin shutdown work before user storage retirement --- AGENTS.md | 2 +- .../advancedcore/AdvancedCorePlugin.java | 21 ++++++ .../api/user/usercache/UserDataManager.java | 4 ++ .../bukkit/runtime/BukkitRuntimePlatform.java | 32 ++++++++- .../tests/lifecycle/CoreRuntimeTest.java | 70 +++++++++++++++++++ .../SharedUserStorageReloadSafetyTest.java | 59 ++++++++++++++++ 6 files changed, 185 insertions(+), 3 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 32fa5e16f..c20415397 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,7 +50,7 @@ For compatibility-sensitive changes, keep regression coverage for established be 2. Do not access Bukkit entities, inventories, worlds, or other thread-confined state from arbitrary async callbacks. 3. Reward execution must not be lost or duplicated across delay, disconnect, reload, retry, or partial failure. Preserve the component's documented delivery contract. 4. Read-only lookups must not create users, mutate caches, or trigger persistence unless that behavior is part of the established API. -5. Keep user identity and UUID resolution consistent across online/offline paths. Coordinate cache updates, database writes, resets, and shutdown flushing. +5. Keep user identity and UUID resolution consistent across online/offline paths. Coordinate cache updates, database writes, resets, and shutdown flushing. Final shared-runtime retirement fences queued notifications before marking storage retiring; callbacks already holding lifecycle admission may finish and re-enter storage. The optional pre-storage shutdown stage must finish before native storage retirement. Platform cleanup and the completion hook may run earlier on deferred shutdown; consumers must retain their own persistence owner until their stage settles. 6. Reload and disable must cancel or retire tasks, listeners, executors, connections, and callbacks without allowing stale work to mutate new state. 7. GUI/editor actions must recheck permissions and ownership and must not duplicate items, lose configuration, or save to the wrong path or user. 8. Optional integrations must be guarded against absent classes and incompatible versions. One platform must not load another platform's API accidentally. diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java index 64a1fbe47..e17db492f 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java @@ -1348,6 +1348,27 @@ public void onDisable() { javaPlugin = null; } + /** + * Gives a plugin one asynchronous admission boundary before native user + * storage starts retiring. The default keeps the established shutdown + * sequence unchanged. Implementations must return promptly and may use the + * returned stage to account for work admitted before storage retirement; + * they must not wait synchronously for that work here. + * + * @return completion of plugin-owned pre-storage-shutdown work + */ + public CompletionStage onBeforeStorageShutdown() { + return CompletableFuture.completedFuture(null); + } + + /** + * Called after platform cleanup, including the final permission-data save. + * Deferred storage retirement may still be running; implementations must + * retain any owner needed by their pre-storage stage until that stage settles. + * The default is a no-op so existing subclasses retain their lifecycle. + */ + public void onShutdownComplete() { } + @Override public void onEnable() { javaPlugin = this; diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java index b7c48b67d..862264a5a 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/usercache/UserDataManager.java @@ -470,6 +470,10 @@ public final CompletionStage closeSharedRuntimeAsyncCompletion(Runnable af // A second shutdown caller must not interpret an in-flight retirement as // "no runtime" and close the native provider underneath its final flush. if (runtime == null) return sharedRuntimeRetiring ? sharedRuntimeRetirement : null; + // Fence queued notifications before closeAsync marks the runtime retiring. + // An already-running callback retains its lifecycle read admission and + // finishes before the runtime's final writer can close storage. + closeSharedUserDataNotifications(); sharedRuntime = null; sharedRuntimeRetiring = true; completion = new CompletableFuture<>(); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 9833e5a89..81ae68abe 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -22,6 +22,7 @@ public final class BukkitRuntimePlatform implements RuntimePlatform { private final AdvancedCorePlugin plugin; private volatile CompletionStage userStorageRetirement = CompletableFuture.completedFuture(null); private CompletionStage startedUserStorageRetirement; + private CompletionStage preStorageShutdownHook; private volatile CompletionStage timeChangeRetirement = CompletableFuture.completedFuture(null); private final Object userStorageRetirementLock = new Object(); private final AtomicBoolean userStorageOwnerClosed = new AtomicBoolean(); @@ -108,7 +109,8 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { }), new Cleanup("dialog service", () -> { if (plugin.getDialogService() != null) plugin.getDialogService().unregister(); - })); + }), + new Cleanup("plugin shutdown completion hook", plugin::onShutdownComplete)); } @Override public void info(String message) { plugin.getLogger().info(message); } @@ -139,12 +141,38 @@ private void closeUserStorageAfterTimeChangeRetirement() { private CompletionStage ensureUserStorageRetirementStarted() { synchronized (userStorageRetirementLock) { if (startedUserStorageRetirement == null) { - startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); + CompletionStage hook = ensurePreStorageShutdownHook(); + if (hook.toCompletableFuture().isDone()) { + try { + hook.toCompletableFuture().join(); + // Preserve the legacy stage identity for the normal completed + // default hook and existing callers that observe it. + startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); + } catch (java.util.concurrent.CompletionException failure) { + startedUserStorageRetirement = hook; + } + } else { + startedUserStorageRetirement = hook + .thenCompose(ignored -> closeUserStorageAfterSharedRetirement()); + } } return startedUserStorageRetirement; } } + private CompletionStage ensurePreStorageShutdownHook() { + synchronized (userStorageRetirementLock) { + if (preStorageShutdownHook != null) return preStorageShutdownHook; + try { + CompletionStage hook = plugin.onBeforeStorageShutdown(); + preStorageShutdownHook = hook == null ? CompletableFuture.completedFuture(null) : hook; + } catch (Throwable failure) { + preStorageShutdownHook = CompletableFuture.failedFuture(failure); + } + return preStorageShutdownHook; + } + } + private CompletionStage closeUserStorageAfterSharedRetirement() { if (!plugin.isLoadUserData()) { return CompletableFuture.completedFuture(null); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index 80c6c0640..cd70f7790 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -546,6 +546,76 @@ private ScheduledExecutorService executor(String name, List events) thro verify(mysql).close(); } + @Test void bukkitAdapterWaitsForPluginAdmissionHookBeforeRetiringStorage() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager dataManager = mock(UserDataManager.class); + CompletableFuture hook = new CompletableFuture<>(); + CompletableFuture retirement = new CompletableFuture<>(); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(dataManager); + when(dataManager.closeSharedRuntimeAsyncCompletion(any(Runnable.class))).thenReturn(retirement); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + + platform.beforeExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + + verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + hook.complete(null); + verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + hook.complete(null); + retirement.complete(null); + assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + } + + @Test void bukkitAdapterReportsAdmissionHookFailureWithoutStartingStorageRetirement() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager dataManager = mock(UserDataManager.class); + IllegalStateException failure = new IllegalStateException("admission hook failed"); + when(plugin.onBeforeStorageShutdown()).thenReturn(CompletableFuture.failedFuture(failure)); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(dataManager); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + + platform.beforeExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + + CompletionException reported = assertThrows(CompletionException.class, + () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + assertSame(failure, reported.getCause()); + verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + } + + @Test void defaultAdmissionHookCompletesForExistingSubclasses() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class, CALLS_REAL_METHODS); + assertDoesNotThrow(() -> plugin.onBeforeStorageShutdown().toCompletableFuture().join()); + } + + @Test void bukkitAdapterRunsShutdownCompletionHookAfterNativeCleanup() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + List order = new ArrayList<>(); + doAnswer(invocation -> { order.add("unload"); return null; }).when(plugin).onUnLoad(); + doAnswer(invocation -> { order.add("complete"); return null; }).when(plugin).onShutdownComplete(); + when(plugin.getSkullCacheHandler()).thenReturn(null); + when(plugin.getHologramHandler()).thenReturn(null); + when(plugin.getPermissionHandler()).thenReturn(null); + when(plugin.getDialogService()).thenReturn(null); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + + List cleanups = platform.afterExecutorShutdown(); + assertEquals("plugin shutdown completion hook", cleanups.get(cleanups.size() - 1).name()); + cleanups.forEach(cleanup -> cleanup.action().run()); + assertEquals(List.of("unload", "complete"), order); + verify(plugin).onShutdownComplete(); + } + @Test void bukkitAdapterDoesNotRetireStorageBeforeAnAdmittedTimeTransitionDrains() { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); TimeChecker checker = mock(TimeChecker.class); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserStorageReloadSafetyTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserStorageReloadSafetyTest.java index 401be45e2..9894d4b95 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserStorageReloadSafetyTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/SharedUserStorageReloadSafetyTest.java @@ -214,6 +214,65 @@ void bulkOperationHoldsLifecycleAdmissionUntilTheOldProviderIsNoLongerInUse() th } } + @Test + void queuedNotificationIsFencedWhenRetirementIsAdmittedBeforeItsWorkerRuns() throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserDataManager manager = new UserDataManager(plugin); + SqlUserBackend backend = mock(SqlUserBackend.class); + when(backend.isOpen()).thenReturn(true); + when(backend.storageType()).thenReturn(UserStorage.SQLITE); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, new RoutingCacheOwner(manager)); + manager.bindSharedRuntime(runtime); + CountDownLatch entered = new CountDownLatch(1), release = new CountDownLatch(1); + var delivered = new java.util.concurrent.atomic.AtomicInteger(); + manager.getTimer().execute(() -> { + entered.countDown(); + try { assertTrue(release.await(5, TimeUnit.SECONDS)); } + catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new RuntimeException(failure); } + }); + try { + assertTrue(entered.await(5, TimeUnit.SECONDS)); + manager.dispatchSharedUserDataNotification(delivered::incrementAndGet); + CompletionStage retirement = manager.closeSharedRuntimeAsyncCompletion(() -> {}); + assertFalse(retirement.toCompletableFuture().isDone()); + release.countDown(); + retirement.toCompletableFuture().get(5, TimeUnit.SECONDS); + manager.getTimer().submit(() -> {}).get(5, TimeUnit.SECONDS); + assertEquals(0, delivered.get()); + assertNull(manager.getLastDeferredStorageFailure()); + verify(backend).close(); + } finally { release.countDown(); manager.getTimer().shutdownNow(); } + } + + @Test + void alreadyAdmittedNotificationFinishesItsNestedStorageAccessBeforeRetirement() throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserDataManager manager = new UserDataManager(plugin); + SqlUserBackend backend = mock(SqlUserBackend.class); + when(backend.isOpen()).thenReturn(true); + when(backend.storageType()).thenReturn(UserStorage.SQLITE); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, new RoutingCacheOwner(manager)); + manager.bindSharedRuntime(runtime); + CountDownLatch entered = new CountDownLatch(1), release = new CountDownLatch(1); + var delivered = new java.util.concurrent.atomic.AtomicInteger(); + try { + manager.dispatchSharedUserDataNotification(() -> { + entered.countDown(); + try { assertTrue(release.await(5, TimeUnit.SECONDS)); } + catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new RuntimeException(failure); } + runtime.withStorageReadAdmission(() -> { delivered.incrementAndGet(); return null; }); + }); + assertTrue(entered.await(5, TimeUnit.SECONDS)); + CompletionStage retirement = manager.closeSharedRuntimeAsyncCompletion(() -> {}); + assertFalse(retirement.toCompletableFuture().isDone()); + release.countDown(); + retirement.toCompletableFuture().get(5, TimeUnit.SECONDS); + assertEquals(1, delivered.get()); + assertNull(manager.getLastDeferredStorageFailure()); + verify(backend).close(); + } finally { release.countDown(); manager.getTimer().shutdownNow(); } + } + @Test void sharedBackendReplacementCompletesOnTheManagerWorkerWithoutBlockingTheCaller() throws Exception { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); From 60984dbd231af23aa5b605bf7807bab612ebe331 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 15:04:11 -0600 Subject: [PATCH 03/28] Retire shared storage after failed shutdown preparation --- .../advancedcore/AdvancedCorePlugin.java | 2 + .../bukkit/runtime/BukkitRuntimePlatform.java | 36 +++++++--- .../tests/lifecycle/CoreRuntimeTest.java | 69 ++++++++++++++----- docs/shared-runtime-foundation.md | 7 ++ 4 files changed, 86 insertions(+), 28 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java index e17db492f..bab31662d 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java @@ -1354,6 +1354,8 @@ public void onDisable() { * sequence unchanged. Implementations must return promptly and may use the * returned stage to account for work admitted before storage retirement; * they must not wait synchronously for that work here. + * Exceptional completion or cancellation is reported after shared storage + * retirement settles; it does not skip flushing accepted cache writes. * * @return completion of plugin-owned pre-storage-shutdown work */ diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 81ae68abe..dbd67ead5 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -143,23 +143,41 @@ private CompletionStage ensureUserStorageRetirementStarted() { if (startedUserStorageRetirement == null) { CompletionStage hook = ensurePreStorageShutdownHook(); if (hook.toCompletableFuture().isDone()) { - try { - hook.toCompletableFuture().join(); - // Preserve the legacy stage identity for the normal completed - // default hook and existing callers that observe it. - startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); - } catch (java.util.concurrent.CompletionException failure) { - startedUserStorageRetirement = hook; + try { hook.toCompletableFuture().join(); } + catch (java.util.concurrent.CompletionException | java.util.concurrent.CancellationException failure) { + startedUserStorageRetirement = retireAfterPreparationFailure(failure); + return startedUserStorageRetirement; } + // Preserve the successful default hook's legacy retirement stage identity. + startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); } else { - startedUserStorageRetirement = hook - .thenCompose(ignored -> closeUserStorageAfterSharedRetirement()); + startedUserStorageRetirement = hook.handle((ignored, failure) -> failure) + .thenCompose(failure -> failure == null ? closeUserStorageAfterSharedRetirement() + : retireAfterPreparationFailure(failure)); } } return startedUserStorageRetirement; } } + /** A failed consumer preparation still has accepted cache writes to retire. */ + private CompletionStage retireAfterPreparationFailure(Throwable preparationFailure) { + Throwable cause = preparationFailure instanceof java.util.concurrent.CompletionException + && preparationFailure.getCause() != null ? preparationFailure.getCause() : preparationFailure; + var reported = new java.util.concurrent.CompletionException(cause); + var result = new CompletableFuture(); + try { + closeUserStorageAfterSharedRetirement().whenComplete((ignored, storageFailure) -> { + if (storageFailure != null && storageFailure != cause) reported.addSuppressed(storageFailure); + result.completeExceptionally(reported); + }); + } catch (Throwable storageFailure) { + if (storageFailure != cause) reported.addSuppressed(storageFailure); + result.completeExceptionally(reported); + } + return result; + } + private CompletionStage ensurePreStorageShutdownHook() { synchronized (userStorageRetirementLock) { if (preStorageShutdownHook != null) return preStorageShutdownHook; diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index cd70f7790..fb6dab14b 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -572,26 +572,57 @@ private ScheduledExecutorService executor(String name, List events) thro assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); } - @Test void bukkitAdapterReportsAdmissionHookFailureWithoutStartingStorageRetirement() { - AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); - UserManager users = mock(UserManager.class); - UserDataManager dataManager = mock(UserDataManager.class); - IllegalStateException failure = new IllegalStateException("admission hook failed"); - when(plugin.onBeforeStorageShutdown()).thenReturn(CompletableFuture.failedFuture(failure)); - when(plugin.isLoadUserData()).thenReturn(true); - when(plugin.getLoadedUserManager()).thenReturn(users); - when(users.getDataManager()).thenReturn(dataManager); - BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); - - platform.beforeExecutorShutdown().stream() - .filter(cleanup -> cleanup.name().equals("user storage")) - .findFirst().orElseThrow().action().run(); + @Test void bukkitAdapterRetiresStorageAfterEachPreparationFailureAndWaitsForItsCompletion() { + for (String mode : List.of("failed", "cancelled", "throwing", "late-failed", "late-cancelled")) { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager dataManager = mock(UserDataManager.class); + var hook = new CompletableFuture(); var retirement = new CompletableFuture(); + var failure = new IllegalStateException("admission hook failed"); + if (mode.equals("failed")) hook.completeExceptionally(failure); + if (mode.equals("cancelled")) hook.cancel(false); + if (mode.equals("throwing")) when(plugin.onBeforeStorageShutdown()).thenThrow(failure); + else when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(dataManager); + when(dataManager.closeSharedRuntimeAsyncCompletion(any(Runnable.class))).thenReturn(retirement); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + Runnable shutdown = platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action(); + assertDoesNotThrow(shutdown::run, mode); + if (mode.startsWith("late")) { + verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + if (mode.equals("late-failed")) hook.completeExceptionally(failure); else hook.cancel(false); + } + verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone(), mode); + retirement.complete(null); + CompletionException reported = assertThrows(CompletionException.class, + () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + if (mode.contains("cancelled")) assertInstanceOf(java.util.concurrent.CancellationException.class, reported.getCause()); + else assertSame(failure, reported.getCause()); + shutdown.run(); verify(dataManager, times(1)).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + verify(plugin, times(1)).onBeforeStorageShutdown(); + } + } - CompletionException reported = assertThrows(CompletionException.class, - () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); - assertSame(failure, reported.getCause()); - verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); - } + @Test void bukkitAdapterRetainsBothPreparationAndStorageFailures() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); UserDataManager dataManager = mock(UserDataManager.class); + var preparation = new IllegalStateException("preparation failed"); var storage = new IllegalStateException("flush failed"); + when(plugin.onBeforeStorageShutdown()).thenReturn(CompletableFuture.failedFuture(preparation)); + when(plugin.isLoadUserData()).thenReturn(true); when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(dataManager); + when(dataManager.closeSharedRuntimeAsyncCompletion(any(Runnable.class))).thenReturn(CompletableFuture.failedFuture(storage)); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + CompletionException reported = assertThrows(CompletionException.class, + () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + assertSame(preparation, reported.getCause()); assertArrayEquals(new Throwable[] {storage}, reported.getSuppressed()); + verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + } @Test void defaultAdmissionHookCompletesForExistingSubclasses() { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class, CALLS_REAL_METHODS); diff --git a/docs/shared-runtime-foundation.md b/docs/shared-runtime-foundation.md index 9db6cbb6c..3a187a606 100644 --- a/docs/shared-runtime-foundation.md +++ b/docs/shared-runtime-foundation.md @@ -30,6 +30,13 @@ parallel runtime with different caches or a second set of background tasks. synchronous; the executor grace periods are not a global bound on all cleanup callbacks. No stronger transactional or idempotency guarantee is claimed. +Consumer `onBeforeStorageShutdown()` stages settle before shared user storage +retirement begins. A thrown exception, exceptional completion, or cancellation +still starts shared-cache retirement and waits for that retirement to settle. +The preparation failure remains the reported cause; a storage failure is retained +as a suppressed failure. Successful default hooks preserve the existing retirement +stage identity. A failing hook does not bypass accepted user-data flushing. + ## What this does not port yet This is the executor/lifecycle slice of the future shared runtime, not a complete From bad9b56c5e3f006ca5de36629a95468346814d9f Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:05:57 -0600 Subject: [PATCH 04/28] Run reload calendar checks on their background owner --- .../advancedcore/AdvancedCorePlugin.java | 2 +- .../advancedcore/api/time/TimeChecker.java | 40 ++++++++- .../time/TimeCheckerRequestedUpdateTest.java | 87 +++++++++++++++++++ .../tests/time/TimeCheckerTest.java | 25 ++++++ README.md | 8 ++ 5 files changed, 160 insertions(+), 2 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java index bab31662d..4c66ee7bf 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java @@ -1674,7 +1674,7 @@ private void reloadAdvancedCoreNow(boolean userStorage) { } private void finishReloadAdvancedCore() { - timeChecker.update(); + timeChecker.requestUpdate(); TabCompleteHandler.getInstance().reload(); TabCompleteHandler.getInstance().loadTabCompleteOptions(); getRewardHandler().checkSubRewards(); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java index 135acfe8a..7eff5c17e 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java @@ -39,6 +39,8 @@ public class TimeChecker implements TimeChangeTransition.Owner { private volatile ActiveTransition activeTransition; private CompletableFuture noActiveTransition = CompletableFuture.completedFuture(null); private CompletableFuture noPendingManualTransitions = CompletableFuture.completedFuture(null); + private CompletableFuture noRequestedUpdate = CompletableFuture.completedFuture(null); + private boolean requestedUpdatePending; private boolean acceptingTransitions = true; private boolean abortingTransitions; @@ -121,7 +123,7 @@ public CompletionStage beginShutdown() { synchronized (transitionLock) { acceptingTransitions = false; transitionLock.notifyAll(); - return CompletableFuture.allOf(noActiveTransition, noPendingManualTransitions); + return CompletableFuture.allOf(noActiveTransition, noPendingManualTransitions, noRequestedUpdate); } } @@ -318,6 +320,42 @@ public void setProcessingEnabled(boolean value) { plugin.debug("Local time change processing " + (value ? "enabled" : "disabled")); } + /** Requests one coalesced calendar check on its existing background timer. + * Reload callers may be Bukkit/Folia owners; asynchronous date events must + * never be dispatched inline there. Shutdown drains an admitted check. + */ + public void requestUpdate() { + final ScheduledExecutorService executor; + final CompletableFuture drain; + synchronized (transitionLock) { + if (!acceptingTransitions || abortingTransitions || requestedUpdatePending || timer == null) return; + executor = timer; + requestedUpdatePending = true; + drain = new CompletableFuture<>(); + noRequestedUpdate = drain; + } + try { + executor.execute(() -> { + try { + synchronized (transitionLock) { + if (!acceptingTransitions || abortingTransitions || timer != executor) return; + } + if (plugin != null && plugin.isEnabled()) runRecurringTask("requested time change check", this::update); + } finally { finishRequestedUpdate(drain); } + }); + } catch (RejectedExecutionException rejected) { + finishRequestedUpdate(drain); + plugin.debug("Ignoring requested time check while the calendar timer is unavailable"); + } + } + + private void finishRequestedUpdate(CompletableFuture drain) { + synchronized (transitionLock) { + if (noRequestedUpdate == drain) requestedUpdatePending = false; + } + drain.complete(null); + } + public void update() { if (plugin == null) return; if (hasTimeOffSet()) plugin.extraDebug("TimeHourOffSet: " + getTime().getHour() + ":" + getTime().getMinute()); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java new file mode 100644 index 000000000..797bf2e4f --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java @@ -0,0 +1,87 @@ +package com.bencodez.advancedcore.api.time; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.*; + +import java.util.ArrayDeque; +import java.util.Queue; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; +import java.util.logging.Logger; + +import org.junit.jupiter.api.Test; +import com.bencodez.advancedcore.AdvancedCorePlugin; + +class TimeCheckerRequestedUpdateTest { + private AdvancedCorePlugin plugin() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.isEnabled()).thenReturn(true); + when(plugin.getLogger()).thenReturn(Logger.getAnonymousLogger()); + return plugin; + } + @Test void repeatedRequestsCoalesceAndNeverRunInline() { + var plugin = plugin(); var checker = spy(new TimeChecker(plugin)); + var timer = mock(ScheduledExecutorService.class); Queue tasks = new ArrayDeque<>(); + doAnswer(call -> { tasks.add(call.getArgument(0)); return null; }).when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); doNothing().when(checker).update(); + for (int i = 0; i < 100; i++) checker.requestUpdate(); + assertEquals(1, tasks.size()); verify(checker, never()).update(); + tasks.remove().run(); verify(checker).update(); + checker.requestUpdate(); assertEquals(1, tasks.size()); + tasks.remove().run(); verify(checker, times(2)).update(); + } + @Test void admittedButNotStartedCheckIsSkippedAndDrainedOnShutdown() { + var checker = spy(new TimeChecker(plugin())); var timer = mock(ScheduledExecutorService.class); + AtomicReference task = new AtomicReference<>(); + doAnswer(call -> { task.set(call.getArgument(0)); return null; }).when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); checker.requestUpdate(); + var stopped = checker.beginShutdown().toCompletableFuture(); assertFalse(stopped.isDone()); + task.get().run(); assertTrue(stopped.isDone()); verify(checker, never()).update(); + checker.requestUpdate(); verify(timer).execute(any(Runnable.class)); + } + @Test void rejectedSubmissionReleasesAdmissionForRetry() { + var checker = spy(new TimeChecker(plugin())); var timer = mock(ScheduledExecutorService.class); + doThrow(new RejectedExecutionException()).doNothing().when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); checker.requestUpdate(); checker.requestUpdate(); + verify(timer, times(2)).execute(any(Runnable.class)); verify(checker, never()).update(); + } + @Test void retiredTimerAndDisabledPluginCannotExecuteChecks() { + for (boolean disabled : new boolean[] { false, true }) { + var plugin = plugin(); var checker = spy(new TimeChecker(plugin)); + var timer = mock(ScheduledExecutorService.class); AtomicReference task = new AtomicReference<>(); + doAnswer(call -> { task.set(call.getArgument(0)); return null; }).when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); checker.requestUpdate(); + if (disabled) when(plugin.isEnabled()).thenReturn(false); else checker.setTimer(mock(ScheduledExecutorService.class)); + task.get().run(); verify(checker, never()).update(); + assertTrue(checker.beginShutdown().toCompletableFuture().isDone()); + } + } + @Test void backgroundExecutionRemainsOwnedUntilRunningCheckReturns() throws Exception { + var plugin = plugin(); var checker = spy(new TimeChecker(plugin)); + var timer = Executors.newSingleThreadScheduledExecutor(); checker.setTimer(timer); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + Thread caller = Thread.currentThread(); AtomicReference worker = new AtomicReference<>(); + doAnswer(call -> { worker.set(Thread.currentThread()); entered.countDown(); + assertTrue(release.await(5, TimeUnit.SECONDS)); return null; }).when(checker).update(); + try { + checker.requestUpdate(); assertTrue(entered.await(5, TimeUnit.SECONDS)); + assertNotSame(caller, worker.get()); + var stopped = checker.beginShutdown().toCompletableFuture(); assertFalse(stopped.isDone()); + release.countDown(); stopped.get(5, TimeUnit.SECONDS); + verify(checker).update(); + } finally { release.countDown(); timer.shutdownNow(); assertTrue(timer.awaitTermination(5, TimeUnit.SECONDS)); } + } + @Test void failureDoesNotStrandShutdownOrFutureRequests() { + var checker = spy(new TimeChecker(plugin())); var timer = mock(ScheduledExecutorService.class); + Queue tasks = new ArrayDeque<>(); + doAnswer(call -> { tasks.add(call.getArgument(0)); return null; }).when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); doThrow(new IllegalStateException("fixture failure")).doNothing().when(checker).update(); + checker.requestUpdate(); tasks.remove().run(); checker.requestUpdate(); tasks.remove().run(); + verify(checker, times(2)).update(); assertTrue(checker.beginShutdown().toCompletableFuture().isDone()); + } +} diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java index 90570c4fb..638ca7a50 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java @@ -869,6 +869,31 @@ public void pendingTransitionIsRecoveredWithItsOriginalIdBeforeClockDetection() verify(serverDataFile).completeTimeChangeTransition(transition); } + @Test + public void reloadRequestDispatchesRealAsyncDateEventsOnCalendarWorker() throws Exception { + TimeChangeTransitionState transition = transitionState(); + PluginManager manager = configureDetectedDay(transition); + when(plugin.isEnabled()).thenReturn(true); + Thread owner = Thread.currentThread(); + java.util.concurrent.atomic.AtomicInteger delivered = new java.util.concurrent.atomic.AtomicInteger(); + Mockito.doAnswer(call -> { + Event event = call.getArgument(0); + if (!event.isAsynchronous() || Thread.currentThread() == owner) + throw new IllegalStateException("Date event dispatched from server owner"); + delivered.incrementAndGet(); return null; + }).when(manager).callEvent(any(Event.class)); + TimeChecker checker = new TimeChecker(plugin, + Clock.fixed(Instant.parse("2025-01-02T12:00:00Z"), ZoneOffset.UTC)); + ScheduledExecutorService timer = Executors.newSingleThreadScheduledExecutor(); checker.setTimer(timer); + try { + checker.requestUpdate(); timer.submit(() -> {}).get(5, TimeUnit.SECONDS); + assertEquals(3, delivered.get()); + verify(serverDataFile).completeTimeChangeTransition(transition); + verify(serverDataFile, Mockito.never()).failTimeChangeTransition(any()); + checker.beginShutdown().toCompletableFuture().get(5, TimeUnit.SECONDS); + } finally { timer.shutdownNow(); assertTrue(timer.awaitTermination(5, TimeUnit.SECONDS)); } + } + private PluginManager configureDetectedDay(TimeChangeTransitionState transition) { Server server = mock(Server.class); PluginManager pluginManager = mock(PluginManager.class); diff --git a/README.md b/README.md index b4bbf561e..cd1a280bc 100644 --- a/README.md +++ b/README.md @@ -32,3 +32,11 @@ API used in the plugins developed by BenCodez, can be used in any project. Versions: LATEST - latest stable release Check out all tags [on the releases tab](https://github.com/BenCodez/AdvancedCore/tags). + + +Calendar checks requested by reload run on the existing background calendar timer, +so asynchronous date events are not called inline from a Bukkit or Folia owner. +Repeated requests coalesce while one check is queued or running. Shutdown waits +for admitted checks; queued callbacks skip a retired checker or replaced timer. +Normal periodic checks remain unchanged. This does not move the remaining reload +file reads off the caller, or make reload completion await all date-event listeners. From f8d261552d14d21453218f35472855df35dddc6a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:19:19 -0600 Subject: [PATCH 05/28] Defer storage retirement after asynchronous preparation callbacks --- .../bukkit/runtime/BukkitRuntimePlatform.java | 40 ++++- .../tests/lifecycle/CoreRuntimeTest.java | 15 ++ .../user/PreStorageHookAdmissionTest.java | 151 ++++++++++++++++++ README.md | 9 ++ 4 files changed, 213 insertions(+), 2 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index dbd67ead5..9111bf32f 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -152,14 +152,50 @@ private CompletionStage ensureUserStorageRetirementStarted() { startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); } else { startedUserStorageRetirement = hook.handle((ignored, failure) -> failure) - .thenCompose(failure -> failure == null ? closeUserStorageAfterSharedRetirement() - : retireAfterPreparationFailure(failure)); + .thenCompose(this::deferUserStorageRetirement); } } return startedUserStorageRetirement; } } + /** A completing consumer callback may still hold shared storage read admission. */ + private CompletionStage deferUserStorageRetirement(Throwable preparationFailure) { + if (!plugin.isLoadUserData() || plugin.getLoadedUserManager() == null) { + return preparationFailure == null ? closeUserStorageAfterSharedRetirement() + : retireAfterPreparationFailure(preparationFailure); + } + var result = new CompletableFuture(); + try { + Objects.requireNonNull(getUserStorageTimer(), "user storage timer").execute(() -> { + try { + CompletionStage retirement = preparationFailure == null + ? closeUserStorageAfterSharedRetirement() + : retireAfterPreparationFailure(preparationFailure); + retirement.whenComplete((ignored, failure) -> { + if (failure == null) result.complete(null); + else result.completeExceptionally(failure); + }); + } catch (Throwable failure) { + result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); + } + }); + } catch (RuntimeException | Error failure) { + // Do not detach the shared runtime or pretend rejected work flushed storage. + result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); + } + return result; + } + + private static Throwable preparationAndStorageFailure(Throwable preparationFailure, Throwable storageFailure) { + if (preparationFailure == null) return storageFailure; + Throwable cause = preparationFailure instanceof java.util.concurrent.CompletionException + && preparationFailure.getCause() != null ? preparationFailure.getCause() : preparationFailure; + var reported = new java.util.concurrent.CompletionException(cause); + if (storageFailure != null && storageFailure != cause) reported.addSuppressed(storageFailure); + return reported; + } + /** A failed consumer preparation still has accepted cache writes to retire. */ private CompletionStage retireAfterPreparationFailure(Throwable preparationFailure) { Throwable cause = preparationFailure instanceof java.util.concurrent.CompletionException diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index fb6dab14b..901c220ef 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -550,6 +550,7 @@ private ScheduledExecutorService executor(String name, List events) thro AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); UserManager users = mock(UserManager.class); UserDataManager dataManager = mock(UserDataManager.class); + var workerTasks = queuedStorageWorker(dataManager); CompletableFuture hook = new CompletableFuture<>(); CompletableFuture retirement = new CompletableFuture<>(); when(plugin.onBeforeStorageShutdown()).thenReturn(hook); @@ -566,6 +567,8 @@ private ScheduledExecutorService executor(String name, List events) thro verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); hook.complete(null); + verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + workerTasks.remove().run(); verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); hook.complete(null); retirement.complete(null); @@ -577,6 +580,7 @@ private ScheduledExecutorService executor(String name, List events) thro AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); UserManager users = mock(UserManager.class); UserDataManager dataManager = mock(UserDataManager.class); + var workerTasks = queuedStorageWorker(dataManager); var hook = new CompletableFuture(); var retirement = new CompletableFuture(); var failure = new IllegalStateException("admission hook failed"); if (mode.equals("failed")) hook.completeExceptionally(failure); @@ -594,6 +598,8 @@ private ScheduledExecutorService executor(String name, List events) thro if (mode.startsWith("late")) { verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); if (mode.equals("late-failed")) hook.completeExceptionally(failure); else hook.cancel(false); + verify(dataManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + workerTasks.remove().run(); } verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone(), mode); @@ -624,6 +630,15 @@ private ScheduledExecutorService executor(String name, List events) thro verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); } + private static java.util.Queue queuedStorageWorker(UserDataManager manager) { + var tasks = new java.util.ArrayDeque(); + ScheduledExecutorService worker = mock(ScheduledExecutorService.class); + when(manager.getTimer()).thenReturn(worker); + doAnswer(call -> { tasks.add(call.getArgument(0, Runnable.class)); return null; }) + .when(worker).execute(any(Runnable.class)); + return tasks; + } + @Test void defaultAdmissionHookCompletesForExistingSubclasses() { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class, CALLS_REAL_METHODS); assertDoesNotThrow(() -> plugin.onBeforeStorageShutdown().toCompletableFuture().join()); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java new file mode 100644 index 000000000..36a0b6be0 --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java @@ -0,0 +1,151 @@ +package com.bencodez.advancedcore.tests.user; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Test; + +import com.bencodez.advancedcore.AdvancedCorePlugin; +import com.bencodez.advancedcore.api.user.UserManager; +import com.bencodez.advancedcore.api.user.UserStorage; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; +import com.bencodez.advancedcore.api.user.userstorage.mysql.MySQL; +import com.bencodez.advancedcore.bukkit.runtime.BukkitRuntimePlatform; +import com.bencodez.advancedcore.core.user.runtime.SharedUserDataRuntime; +import com.bencodez.advancedcore.core.user.runtime.UserCacheOwner; +import com.bencodez.advancedcore.core.user.storage.SqlUserStorage; +import com.bencodez.advancedcore.core.user.storage.sql.SqlUserBackend; + +class PreStorageHookAdmissionTest { + @Test void successfulHookInsideAdmissionFlushesBeforeNativeClose() throws Exception { + completeInsideReadAdmission("success"); + } + @Test void failedHookInsideAdmissionFlushesBeforeReportingFailure() throws Exception { + completeInsideReadAdmission("failure"); + } + @Test void cancelledHookInsideAdmissionStillRetiresStorage() throws Exception { + completeInsideReadAdmission("cancelled"); + } + + private void completeInsideReadAdmission(String mode) throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager manager = new UserDataManager(plugin); + MySQL nativeOwner = mock(MySQL.class); + SqlUserBackend backend = mock(SqlUserBackend.class); + SqlUserStorage storage = mock(SqlUserStorage.class); + UserCacheOwner cache = mock(UserCacheOwner.class); + UUID uuid = UUID.randomUUID(); + when(cache.cachedUsers()).thenReturn(Set.of(uuid)); + when(backend.storageType()).thenReturn(UserStorage.MYSQL); + when(backend.isOpen()).thenReturn(true); + when(backend.user(uuid)).thenReturn(storage); + when(plugin.getNativeUserStorageOwner()).thenReturn( + new AdvancedCorePlugin.UserStorageOwner(UserStorage.MYSQL, nativeOwner, null)); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + var hook = new CompletableFuture(); + var preparationFailure = new IllegalStateException("preparation failed"); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, cache); + manager.bindSharedRuntime(runtime); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + Runnable shutdown = platform.beforeExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("user storage")).findFirst().orElseThrow().action(); + try { + shutdown.run(); + var retirement = platform.beforeExecutorShutdownCompletion().toCompletableFuture(); + manager.getTimer().submit(() -> runtime.withStorageReadAdmission(() -> { + if (mode.equals("success")) hook.complete(null); + else if (mode.equals("failure")) hook.completeExceptionally(preparationFailure); + else hook.cancel(false); + assertFalse(runtime.isRetiring(), "retirement must wait for this callback to unwind"); + assertFalse(retirement.isDone()); + verify(nativeOwner, never()).close(); + // The callback still owns admission and can finish accepted work. + runtime.withStorageReadAdmission(() -> null); + return null; + })).get(5, TimeUnit.SECONDS); + + if (mode.equals("success")) retirement.get(5, TimeUnit.SECONDS); + else { + assertThrows(java.util.concurrent.ExecutionException.class, () -> retirement.get(5, TimeUnit.SECONDS)); + CompletionException reported = assertThrows(CompletionException.class, retirement::join); + if (mode.equals("failure")) assertSame(preparationFailure, reported.getCause()); + else assertInstanceOf(java.util.concurrent.CancellationException.class, reported.getCause()); + } + var order = inOrder(cache, backend, nativeOwner); + order.verify(cache).beginRetirement(); + order.verify(cache).flush(uuid, UserStorage.MYSQL, storage); + order.verify(cache).clearAfterFlush(); + order.verify(cache).shutdown(); + order.verify(backend).close(); + order.verify(nativeOwner).close(); + shutdown.run(); + verify(backend, times(1)).close(); + verify(nativeOwner, times(1)).close(); + assertTrue(runtime.isClosed()); + assertFalse(manager.hasSharedRuntimeLifecycle()); + assertNull(manager.getLastDeferredStorageFailure()); + } finally { manager.getTimer().shutdownNow(); } + } + + @Test void rejectedContinuationDoesNotDetachRuntime() { + rejectContinuation(false); + } + @Test void rejectedContinuationPreservesPreparationFailure() { + rejectContinuation(true); + } + + private void rejectContinuation(boolean failedPreparation) { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager manager = new UserDataManager(plugin); + SqlUserBackend backend = mock(SqlUserBackend.class); + SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, mock(UserCacheOwner.class)); + manager.bindSharedRuntime(runtime); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + var hook = new CompletableFuture(); + var preparationFailure = new IllegalStateException("preparation failed"); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + try { + platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + manager.getTimer().shutdownNow(); + if (failedPreparation) hook.completeExceptionally(preparationFailure); else hook.complete(null); + CompletionException reported = assertThrows(CompletionException.class, + () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + if (failedPreparation) { + assertSame(preparationFailure, reported.getCause()); + assertEquals(1, reported.getSuppressed().length); + assertInstanceOf(RejectedExecutionException.class, reported.getSuppressed()[0]); + } else assertInstanceOf(RejectedExecutionException.class, reported.getCause()); + assertTrue(manager.hasSharedRuntime()); + assertFalse(runtime.isRetiring()); + verify(backend, never()).close(); + } finally { manager.getTimer().shutdownNow(); } + } + + @Test void delayedHookWithNoUserStorageRemainsSupported() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + var hook = new CompletableFuture(); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + hook.complete(null); + assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); + verify(plugin, never()).getLoadedUserManager(); + } +} diff --git a/README.md b/README.md index cd1a280bc..2fd5856e8 100644 --- a/README.md +++ b/README.md @@ -40,3 +40,12 @@ Repeated requests coalesce while one check is queued or running. Shutdown waits for admitted checks; queued callbacks skip a retired checker or replaced timer. Normal periodic checks remain unchanged. This does not move the remaining reload file reads off the caller, or make reload completion await all date-event listeners. + +When an asynchronous `onBeforeStorageShutdown()` stage settles, storage retirement +is queued on the existing user storage worker. The completing callback can finish +its admitted storage work before the shared cache flush and native provider close. +Already completed default hooks retain their existing retirement-stage behavior. +A rejected continuation reports failure and leaves the shared runtime attached; +it does not report a successful flush. Failed or cancelled preparation still +retires accepted writes, with the preparation failure retained. No new executor, +configuration, or storage format is introduced. From f840c4ada2438b070919fe7704be2f0639104003 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 16:30:41 -0600 Subject: [PATCH 06/28] Retain storage admission until consumer shutdown preparation settles --- .../bukkit/runtime/BukkitRuntimePlatform.java | 17 +++- .../core/platform/RuntimePlatform.java | 12 ++- .../core/runtime/AdvancedCoreRuntime.java | 31 ++++++-- .../tests/lifecycle/CoreRuntimeTest.java | 38 +++++++++ .../user/PreStorageHookAdmissionTest.java | 77 ++++++++++++++++++- README.md | 13 ++++ 6 files changed, 178 insertions(+), 10 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 9111bf32f..a6f66632a 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -23,6 +23,7 @@ public final class BukkitRuntimePlatform implements RuntimePlatform { private volatile CompletionStage userStorageRetirement = CompletableFuture.completedFuture(null); private CompletionStage startedUserStorageRetirement; private CompletionStage preStorageShutdownHook; + private CompletableFuture storageRetirementPreparation = CompletableFuture.completedFuture(null); private volatile CompletionStage timeChangeRetirement = CompletableFuture.completedFuture(null); private final Object userStorageRetirementLock = new Object(); private final AtomicBoolean userStorageOwnerClosed = new AtomicBoolean(); @@ -63,6 +64,11 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { } @Override public CompletionStage beforeExecutorShutdownCompletion() { return userStorageRetirement; } + @Override public CompletionStage storageRetirementPreparationCompletion() { + synchronized (userStorageRetirementLock) { + return storageRetirementPreparation; + } + } @Override public boolean holdTimeTimerUntilPreExecutorShutdownCompletion() { return timeChangeRetirement != null && !timeChangeRetirement.toCompletableFuture().isDone(); } @@ -151,8 +157,17 @@ private CompletionStage ensureUserStorageRetirementStarted() { // Preserve the successful default hook's legacy retirement stage identity. startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); } else { + var preparation = new CompletableFuture(); + storageRetirementPreparation = preparation; startedUserStorageRetirement = hook.handle((ignored, failure) -> failure) - .thenCompose(this::deferUserStorageRetirement); + .thenCompose(failure -> { + try { return deferUserStorageRetirement(failure); } + finally { + // Future completion can precede its callbacks. Signal only + // after the final retirement task has been admitted. + preparation.complete(null); + } + }); } } return startedUserStorageRetirement; diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java index fe9db5280..db16a3cff 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java @@ -39,7 +39,17 @@ default CompletionStage beforeExecutorShutdownCompletion() { */ default boolean canBlockForPreExecutorShutdown() { return true; } - /** Maximum time a non-blocking lifecycle waits before forcing its storage worker. */ + /** + * Consumer preparation and final-task admission that must settle before the storage worker can stop. + * A watchdog may report a timeout while this stage is pending, but must retain + * storage admission and the native provider for its eventual final checkpoint. + * The default has no separate preparation phase. + */ + default CompletionStage storageRetirementPreparationCompletion() { + return CompletableFuture.completedFuture(null); + } + + /** Maximum grace for admitted retirement; pending consumer preparation retains its owner. */ default long deferredShutdownTimeoutMillis() { return 5_000; } /** diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java index dd6c737e3..52be2efe7 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java @@ -196,12 +196,7 @@ private void finishDeferredCleanup(CompletionStage completion, String comp finishDeferredStorageTimer(storageTimer, failure != null, failure != null, terminalStorageCleanup); }); long timeoutMillis = Math.max(1, platform.deferredShutdownTimeoutMillis()); - Runnable timeout = () -> { - // Mark a still-draining transition recoverable before racing its final - // lease acknowledgement. Once this hook returns, it cannot advance a - // time marker even if the worker ignores interruption briefly. - if (holdTimeTimer) clean(List.of(new Cleanup("forced time transition shutdown", - platform::beforeForcedTimeTimerShutdown))); + Runnable forceRetirement = () -> { if (!finished.compareAndSet(false, true)) return; platform.cleanupFailed(component, new TimeoutException( "Deferred storage retirement exceeded " + timeoutMillis + " ms")); @@ -212,6 +207,30 @@ private void finishDeferredCleanup(CompletionStage completion, String comp shutdown(storageTimer); finishDeferredStorageTimer(storageTimer, false, true, terminalStorageCleanup); }; + Runnable timeout = () -> { + if (finished.get()) return; + // Mark a still-draining time transition recoverable before racing its + // final lease acknowledgement. This may start consumer preparation. + if (holdTimeTimer) clean(List.of(new Cleanup("forced time transition shutdown", + platform::beforeForcedTimeTimerShutdown))); + CompletionStage preparation = platform.storageRetirementPreparationCompletion(); + if (preparation != null && !preparation.toCompletableFuture().isDone()) { + platform.cleanupFailed(component, new TimeoutException( + "Consumer storage preparation exceeded " + timeoutMillis + + " ms; retaining storage until preparation completes")); + // One continuation, not a polling loop. Keep the daemon storage worker + // open for a checkpoint accepted by a separate consumer worker. Once + // preparation settles, admitted retirement gets its own bounded grace. + preparation.whenComplete((ignored, failure) -> + scheduleStorageWatchdog(timeoutMillis, forceRetirement)); + return; + } + forceRetirement.run(); + }; + scheduleStorageWatchdog(timeoutMillis, timeout); + } + + private void scheduleStorageWatchdog(long timeoutMillis, Runnable timeout) { try { CompletableFuture.delayedExecutor(timeoutMillis, TimeUnit.MILLISECONDS).execute(timeout); } catch (RuntimeException | Error schedulingFailure) { platform.cleanupFailed("deferred storage shutdown watchdog", schedulingFailure); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index 901c220ef..b6dd4f57f 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -347,6 +347,44 @@ private ScheduledExecutorService executor(String name, List events) thro assertEquals(List.of("unload"), events); } + @Test void pendingPreparationRetainsIdleWorkerThenAdmittedRetirementStillTimesOut() throws Exception { + RuntimePlatform platform = platform(); + var timer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(task -> { + Thread worker = new Thread(task, "pending-preparation-test"); + worker.setDaemon(true); + return worker; + }); + var preparation = new CompletableFuture(); + var retirement = new CompletableFuture(); + CountDownLatch reported = new CountDownLatch(1); + CountDownLatch terminal = new CountDownLatch(1); + when(platform.storageRetirementPreparationCompletion()).thenReturn(preparation); + when(platform.beforeExecutorShutdownCompletion()).thenReturn(retirement); + when(platform.canBlockForPreExecutorShutdown()).thenReturn(false); + when(platform.deferredShutdownTimeoutMillis()).thenReturn(20L); + when(platform.getUserStorageTimer()).thenReturn(timer); + when(platform.afterStorageExecutorShutdown()).thenReturn(List.of( + new Cleanup("native storage", terminal::countDown))); + doAnswer(call -> { reported.countDown(); return null; }).when(platform) + .cleanupFailed(eq("pre-executor shutdown"), any(java.util.concurrent.TimeoutException.class)); + try { + new AdvancedCoreRuntime(platform).shutdown(); + assertTrue(reported.await(2, TimeUnit.SECONDS)); + assertFalse(timer.isShutdown(), "idle storage must stay open for a separate consumer checkpoint"); + assertEquals(1, terminal.getCount()); + preparation.complete(null); + assertTrue(terminal.await(2, TimeUnit.SECONDS), "admitted retirement still has a bounded watchdog"); + assertTrue(timer.isTerminated()); + assertFalse(retirement.isDone(), "timeout does not fabricate a successful flush"); + verify(platform, times(2)).cleanupFailed(eq("pre-executor shutdown"), + any(java.util.concurrent.TimeoutException.class)); + } finally { + preparation.complete(null); + retirement.complete(null); + timer.shutdownNow(); + } + } + @Test void watchdogLetsQueuedRetirementFlushBeforeForcingStorageWorker() throws Exception { RuntimePlatform platform = platform(); var timer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java index 36a0b6be0..3c80a01c4 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java @@ -7,6 +7,8 @@ import java.util.UUID; import java.util.concurrent.CompletableFuture; import java.util.concurrent.CompletionException; +import java.util.concurrent.CountDownLatch; +import com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime; import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.TimeUnit; @@ -34,7 +36,21 @@ class PreStorageHookAdmissionTest { completeInsideReadAdmission("cancelled"); } + @Test void successfulHookAfterWatchdogStillFlushesBeforeNativeClose() throws Exception { + completeInsideReadAdmission("success", true); + } + @Test void failedHookAfterWatchdogStillFlushesBeforeReportingFailure() throws Exception { + completeInsideReadAdmission("failure", true); + } + @Test void cancelledHookAfterWatchdogStillRetiresStorage() throws Exception { + completeInsideReadAdmission("cancelled", true); + } + private void completeInsideReadAdmission(String mode) throws Exception { + completeInsideReadAdmission(mode, false); + } + + private void completeInsideReadAdmission(String mode, boolean runWatchdog) throws Exception { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); UserManager users = mock(UserManager.class); UserDataManager manager = new UserDataManager(plugin); @@ -57,11 +73,26 @@ private void completeInsideReadAdmission(String mode) throws Exception { when(plugin.onBeforeStorageShutdown()).thenReturn(hook); SharedUserDataRuntime runtime = new SharedUserDataRuntime(backend, cache); manager.bindSharedRuntime(runtime); - BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + BukkitRuntimePlatform platform = spy(new BukkitRuntimePlatform(plugin)); + var timedOut = new CountDownLatch(1); + if (runWatchdog) { + doReturn(false).when(platform).canBlockForPreExecutorShutdown(); + doReturn(20L).when(platform).deferredShutdownTimeoutMillis(); + doAnswer(call -> { timedOut.countDown(); return null; }) + .when(platform).cleanupFailed(eq("pre-executor shutdown"), + any(java.util.concurrent.TimeoutException.class)); + } Runnable shutdown = platform.beforeExecutorShutdown().stream() .filter(cleanup -> cleanup.name().equals("user storage")).findFirst().orElseThrow().action(); try { - shutdown.run(); + if (runWatchdog) { + new AdvancedCoreRuntime(platform).shutdown(); + assertTrue(timedOut.await(2, TimeUnit.SECONDS)); + assertFalse(manager.getTimer().isShutdown(), "pending consumer preparation retains admission"); + verify(nativeOwner, never()).close(); + assertFalse(runtime.isRetiring()); + } else shutdown.run(); var retirement = platform.beforeExecutorShutdownCompletion().toCompletableFuture(); manager.getTimer().submit(() -> runtime.withStorageReadAdmission(() -> { if (mode.equals("success")) hook.complete(null); @@ -95,6 +126,7 @@ private void completeInsideReadAdmission(String mode) throws Exception { assertTrue(runtime.isClosed()); assertFalse(manager.hasSharedRuntimeLifecycle()); assertNull(manager.getLastDeferredStorageFailure()); + if (runWatchdog) assertTrue(manager.getTimer().awaitTermination(5, TimeUnit.SECONDS)); } finally { manager.getTimer().shutdownNow(); } } @@ -137,6 +169,47 @@ private void rejectContinuation(boolean failedPreparation) { } finally { manager.getTimer().shutdownNow(); } } + @Test void completedHookDoesNotSignalPreparationUntilRetirementIsAdmitted() throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager manager = new UserDataManager(plugin); + var admission = mock(java.util.concurrent.ScheduledExecutorService.class); + var entered = new CountDownLatch(1); + var release = new CountDownLatch(1); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + var hook = new CompletableFuture(); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + BukkitRuntimePlatform platform = spy(new BukkitRuntimePlatform(plugin)); + doReturn(admission).when(platform).getUserStorageTimer(); + doAnswer(call -> { + entered.countDown(); + assertTrue(release.await(2, TimeUnit.SECONDS)); + manager.getTimer().execute(call.getArgument(0, Runnable.class)); + return null; + }).when(admission).execute(any(Runnable.class)); + var callback = java.util.concurrent.Executors.newSingleThreadExecutor(); + try { + platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("user storage")) + .findFirst().orElseThrow().action().run(); + var preparation = platform.storageRetirementPreparationCompletion().toCompletableFuture(); + var completing = callback.submit(() -> hook.complete(null)); + assertTrue(entered.await(2, TimeUnit.SECONDS)); + assertTrue(hook.isDone(), "CompletableFuture becomes done before its callback unwinds"); + assertFalse(preparation.isDone(), "watchdog must retain admission during callback dispatch"); + assertFalse(manager.getTimer().isShutdown()); + release.countDown(); + completing.get(2, TimeUnit.SECONDS); + preparation.get(2, TimeUnit.SECONDS); + platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); + } finally { + release.countDown(); + callback.shutdownNow(); + manager.getTimer().shutdownNow(); + } + } + @Test void delayedHookWithNoUserStorageRemainsSupported() { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); var hook = new CompletableFuture(); diff --git a/README.md b/README.md index 2fd5856e8..db02b5de2 100644 --- a/README.md +++ b/README.md @@ -49,3 +49,16 @@ A rejected continuation reports failure and leaves the shared runtime attached; it does not report a successful flush. Failed or cancelled preparation still retires accepted writes, with the preparation failure retained. No new executor, configuration, or storage format is introduced. + +If consumer preparation exceeds the default five-second shutdown watchdog, the +watchdog reports it and retains the existing daemon storage worker and native +provider. Preparation is acknowledged only after its callback has submitted the +final retirement task. A future becoming complete alone is insufficient because +its callbacks can still be running. After that acknowledgement, a stalled storage +retirement still receives the bounded watchdog and worker grace; terminal native +cleanup waits for the actual storage worker to terminate. + +A consumer hook that never settles retains its storage owner until JVM exit. +This does not block the server thread or keep the JVM alive, but an abrupt exit +cannot guarantee the pending checkpoint was persisted. Consumers must complete, +fail, or cancel their preparation stage. No new executor or polling loop is added. From cffba08614f879700f0f79e7a426be19b36192f1 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sat, 10 Oct 2026 22:11:41 -0600 Subject: [PATCH 07/28] Settle queued calendar checks during deferred shutdown --- .../advancedcore/api/time/TimeChecker.java | 22 +++++++++++++++- .../bukkit/runtime/BukkitRuntimePlatform.java | 5 +++- .../time/TimeCheckerRequestedUpdateTest.java | 26 +++++++++++++++++++ .../tests/lifecycle/CoreRuntimeTest.java | 1 + 4 files changed, 52 insertions(+), 2 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java index 7eff5c17e..75173ba41 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java @@ -41,6 +41,7 @@ public class TimeChecker implements TimeChangeTransition.Owner { private CompletableFuture noPendingManualTransitions = CompletableFuture.completedFuture(null); private CompletableFuture noRequestedUpdate = CompletableFuture.completedFuture(null); private boolean requestedUpdatePending; + private boolean requestedUpdateRunning; private boolean acceptingTransitions = true; private boolean abortingTransitions; @@ -336,12 +337,20 @@ public void requestUpdate() { } try { executor.execute(() -> { + boolean started = false; try { synchronized (transitionLock) { if (!acceptingTransitions || abortingTransitions || timer != executor) return; + requestedUpdateRunning = true; + started = true; } if (plugin != null && plugin.isEnabled()) runRecurringTask("requested time change check", this::update); - } finally { finishRequestedUpdate(drain); } + } finally { + synchronized (transitionLock) { + if (started) requestedUpdateRunning = false; + } + finishRequestedUpdate(drain); + } }); } catch (RejectedExecutionException rejected) { finishRequestedUpdate(drain); @@ -349,6 +358,17 @@ public void requestUpdate() { } } + /** Settles an admitted request that has not started before deferred cleanup. */ + public void cancelQueuedUpdate() { + CompletableFuture drain = null; + synchronized (transitionLock) { + if (!acceptingTransitions && requestedUpdatePending && !requestedUpdateRunning) { + drain = noRequestedUpdate; + } + } + if (drain != null) finishRequestedUpdate(drain); + } + private void finishRequestedUpdate(CompletableFuture drain) { synchronized (transitionLock) { if (noRequestedUpdate == drain) requestedUpdatePending = false; diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index a6f66632a..d26ce84de 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -79,7 +79,10 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { } @Override public void beforeDeferredPlatformCleanup() { TimeChecker checker = plugin.getTimeChecker(); - if (checker != null) checker.cancelActiveTransitionsAndClosePersistence(); + if (checker != null) { + checker.cancelActiveTransitionsAndClosePersistence(); + checker.cancelQueuedUpdate(); + } } @Override public boolean canBlockForPreExecutorShutdown() { return Bukkit.getServer() == null || !Bukkit.isPrimaryThread(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java index 797bf2e4f..0053f79fa 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/api/time/TimeCheckerRequestedUpdateTest.java @@ -44,6 +44,32 @@ private AdvancedCorePlugin plugin() { task.get().run(); assertTrue(stopped.isDone()); verify(checker, never()).update(); checker.requestUpdate(); verify(timer).execute(any(Runnable.class)); } + @Test void deferredCleanupSettlesQueuedRequestBeforeExecutorIsForcedDown() { + var checker = spy(new TimeChecker(plugin())); var timer = mock(ScheduledExecutorService.class); + AtomicReference task = new AtomicReference<>(); + doAnswer(call -> { task.set(call.getArgument(0)); return null; }).when(timer).execute(any(Runnable.class)); + checker.setTimer(timer); checker.requestUpdate(); + var stopped = checker.beginShutdown().toCompletableFuture(); + assertFalse(stopped.isDone()); + checker.cancelQueuedUpdate(); + assertTrue(stopped.isDone()); + task.get().run(); + verify(checker, never()).update(); + } + @Test void deferredCleanupLeavesRunningRequestOwnedUntilItFinishes() throws Exception { + var plugin = plugin(); var checker = spy(new TimeChecker(plugin)); + var timer = Executors.newSingleThreadScheduledExecutor(); checker.setTimer(timer); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + doAnswer(call -> { entered.countDown(); assertTrue(release.await(5, TimeUnit.SECONDS)); return null; }).when(checker).update(); + try { + checker.requestUpdate(); assertTrue(entered.await(5, TimeUnit.SECONDS)); + var stopped = checker.beginShutdown().toCompletableFuture(); + checker.cancelQueuedUpdate(); + assertFalse(stopped.isDone()); + release.countDown(); + stopped.get(5, TimeUnit.SECONDS); + } finally { release.countDown(); timer.shutdownNow(); assertTrue(timer.awaitTermination(5, TimeUnit.SECONDS)); } + } @Test void rejectedSubmissionReleasesAdmissionForRetry() { var checker = spy(new TimeChecker(plugin())); var timer = mock(ScheduledExecutorService.class); doThrow(new RejectedExecutionException()).doNothing().when(timer).execute(any(Runnable.class)); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index b6dd4f57f..c6fccaddf 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -726,6 +726,7 @@ private static java.util.Queue queuedStorageWorker(UserDataManager man assertTrue(platform.holdTimeTimerUntilPreExecutorShutdownCompletion()); platform.beforeDeferredPlatformCleanup(); verify(checker).cancelActiveTransitionsAndClosePersistence(); + verify(checker).cancelQueuedUpdate(); transitionDrain.complete(null); verify(dataManager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); From 03f55245ebce80e9bb0e085ba33d13ecb69117d7 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:11:44 -0600 Subject: [PATCH 08/28] Interrupt fenced calendar work while consumer preparation is pending --- .../core/runtime/AdvancedCoreRuntime.java | 9 ++- .../tests/lifecycle/CoreRuntimeTest.java | 73 +++++++++++++++++++ README.md | 6 ++ 3 files changed, 86 insertions(+), 2 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java index 52be2efe7..cf0726eb7 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java @@ -211,8 +211,13 @@ private void finishDeferredCleanup(CompletionStage completion, String comp if (finished.get()) return; // Mark a still-draining time transition recoverable before racing its // final lease acknowledgement. This may start consumer preparation. - if (holdTimeTimer) clean(List.of(new Cleanup("forced time transition shutdown", - platform::beforeForcedTimeTimerShutdown))); + if (holdTimeTimer) { + clean(List.of(new Cleanup("forced time transition shutdown", + platform::beforeForcedTimeTimerShutdown))); + // Consumer preparation retains only storage ownership. Its completion + // must not delay interruption of the fenced non-daemon calendar worker. + shutdownNow(grace.timeTimer()); + } CompletionStage preparation = platform.storageRetirementPreparationCompletion(); if (preparation != null && !preparation.toCompletableFuture().isDone()) { platform.cleanupFailed(component, new TimeoutException( diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index c6fccaddf..7551d684d 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -385,6 +385,79 @@ private ScheduledExecutorService executor(String name, List events) thro } } + @Test void pendingPreparationDoesNotRetainFencedNonDaemonTimeWorker() throws Exception { + RuntimePlatform platform = platform(); + AtomicReference timeWorker = new AtomicReference<>(); + AtomicReference storageWorker = new AtomicReference<>(); + var timeTimer = Executors.newSingleThreadScheduledExecutor(task -> { + Thread worker = new Thread(task, "held-calendar-test"); + worker.setDaemon(false); + timeWorker.set(worker); + return worker; + }); + var storageTimer = Executors.newSingleThreadScheduledExecutor(task -> { + Thread worker = new Thread(task, "retained-storage-test"); + worker.setDaemon(true); + storageWorker.set(worker); + return worker; + }); + var preparation = new CompletableFuture(); + var retirement = new CompletableFuture(); + var fenced = new java.util.concurrent.atomic.AtomicBoolean(); + var interruptedAfterFence = new java.util.concurrent.atomic.AtomicBoolean(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch reported = new CountDownLatch(1); + CountDownLatch checkpoint = new CountDownLatch(1); + CountDownLatch terminal = new CountDownLatch(1); + when(platform.getTimeTimer()).thenReturn(timeTimer); + when(platform.getTimer()).thenReturn(storageTimer); + when(platform.getUserStorageTimer()).thenReturn(storageTimer); + when(platform.beforeExecutorShutdownCompletion()).thenReturn(retirement); + when(platform.storageRetirementPreparationCompletion()).thenReturn(preparation); + when(platform.canBlockForPreExecutorShutdown()).thenReturn(false); + when(platform.holdTimeTimerUntilPreExecutorShutdownCompletion()).thenReturn(true); + when(platform.deferredShutdownTimeoutMillis()).thenReturn(20L); + doAnswer(call -> { fenced.set(true); return null; }).when(platform).beforeForcedTimeTimerShutdown(); + doAnswer(call -> { reported.countDown(); return null; }).when(platform) + .cleanupFailed(eq("pre-executor shutdown"), any(java.util.concurrent.TimeoutException.class)); + when(platform.afterStorageExecutorShutdown()).thenReturn(List.of( + new Cleanup("native storage", terminal::countDown))); + try { + timeTimer.execute(() -> { + started.countDown(); + try { release.await(); } + catch (InterruptedException interrupted) { + interruptedAfterFence.set(fenced.get()); + Thread.currentThread().interrupt(); + } + }); + assertTrue(started.await(2, TimeUnit.SECONDS)); + assertFalse(timeWorker.get().isDaemon(), "fixture must exercise the calendar's non-daemon owner"); + new AdvancedCoreRuntime(platform).shutdown(); + assertTrue(reported.await(2, TimeUnit.SECONDS)); + assertTrue(timeTimer.awaitTermination(2, TimeUnit.SECONDS), + "first watchdog must interrupt the held calendar without awaiting consumer preparation"); + timeWorker.get().join(1_000L); + assertFalse(timeWorker.get().isAlive(), "no non-daemon calendar worker may remain"); + assertTrue(interruptedAfterFence.get(), "recovery authority must be fenced before interruption"); + assertFalse(preparation.isDone(), "fixture preparation remains permanently incomplete"); + assertFalse(retirement.isDone(), "interruption does not fabricate storage completion"); + assertFalse(storageTimer.isShutdown(), "consumer checkpoint admission must remain open"); + storageTimer.execute(checkpoint::countDown); + assertTrue(checkpoint.await(2, TimeUnit.SECONDS)); + assertTrue(storageWorker.get().isDaemon(), "only the daemon storage owner remains retained"); + assertEquals(1, terminal.getCount(), "native storage cannot close before consumer preparation"); + verify(platform, never()).afterStorageExecutorShutdown(); + } finally { + release.countDown(); + timeTimer.shutdownNow(); + storageTimer.shutdownNow(); + assertTrue(timeTimer.awaitTermination(2, TimeUnit.SECONDS)); + assertTrue(storageTimer.awaitTermination(2, TimeUnit.SECONDS)); + } + } + @Test void watchdogLetsQueuedRetirementFlushBeforeForcingStorageWorker() throws Exception { RuntimePlatform platform = platform(); var timer = java.util.concurrent.Executors.newSingleThreadScheduledExecutor(); diff --git a/README.md b/README.md index db02b5de2..5007eb460 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,12 @@ its callbacks can still be running. After that acknowledgement, a stalled storag retirement still receives the bounded watchdog and worker grace; terminal native cleanup waits for the actual storage worker to terminate. +The first watchdog fences an outstanding calendar transition and immediately +interrupts its held calendar executor, even while consumer preparation remains +pending. Interruptible calendar listeners can therefore retire without waiting +for that hook. Storage admission and its native provider remain available for the +eventual checkpoint; interrupting a listener does not prove its work persisted. + A consumer hook that never settles retains its storage owner until JVM exit. This does not block the server thread or keep the JVM alive, but an abrupt exit cannot guarantee the pending checkpoint was persisted. Consumers must complete, From 89a82c2b137c5399539358cdff5ccd43877637fc Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Sun, 11 Oct 2026 22:37:33 -0600 Subject: [PATCH 09/28] Retain admitted storage through deferred unload --- .../bukkit/runtime/BukkitRuntimePlatform.java | 42 +++++-- .../tests/lifecycle/CoreRuntimeTest.java | 115 ++++++++++++++++++ .../user/PreStorageHookAdmissionTest.java | 6 +- README.md | 6 + 4 files changed, 158 insertions(+), 11 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index d26ce84de..2a4ea157b 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -15,6 +15,7 @@ import com.bencodez.advancedcore.api.time.TimeChecker; import com.bencodez.advancedcore.api.user.UserManager; import com.bencodez.advancedcore.api.user.UserStorage; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; import com.bencodez.advancedcore.core.platform.RuntimePlatform; /** Bukkit services for the shared executor lifecycle; no duplicate owners are created. */ @@ -26,6 +27,8 @@ public final class BukkitRuntimePlatform implements RuntimePlatform { private CompletableFuture storageRetirementPreparation = CompletableFuture.completedFuture(null); private volatile CompletionStage timeChangeRetirement = CompletableFuture.completedFuture(null); private final Object userStorageRetirementLock = new Object(); + private record UserStorageAdmission(boolean enabled, UserManager users, UserDataManager manager) { } + private volatile UserStorageAdmission userStorageAdmission; private final AtomicBoolean userStorageOwnerClosed = new AtomicBoolean(); public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { @@ -34,6 +37,8 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { @Override public ScheduledExecutorService getTimer() { return plugin.getTimer(); } @Override public ScheduledExecutorService getUserStorageTimer() { + UserStorageAdmission admission = userStorageAdmission; + if (admission != null) return admission.manager() == null ? null : admission.manager().getTimer(); UserManager users = plugin.getLoadedUserManager(); return users == null || users.getDataManager() == null ? null : users.getDataManager().getTimer(); } @@ -99,7 +104,7 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { CompletionStage retirement = userStorageRetirement; if (retirement != null && (!retirement.toCompletableFuture().isDone() || retirement.toCompletableFuture().isCompletedExceptionally())) { - closeCurrentUserStorageOwner(plugin.getLoadedUserManager()); + closeCurrentUserStorageOwner(captureUserStorageAdmission()); } })); } @@ -134,6 +139,9 @@ private void closeTimeChangeAdmission() { } private void closeUserStorageAfterTimeChangeRetirement() { + // Deferred platform unload may change configuration or manager references. + // Storage already admitted for this shutdown retains its own retirement identity. + captureUserStorageAdmission(); CompletionStage timeRetirement = timeChangeRetirement; if (timeRetirement.toCompletableFuture().isDone()) { try { @@ -179,7 +187,8 @@ private CompletionStage ensureUserStorageRetirementStarted() { /** A completing consumer callback may still hold shared storage read admission. */ private CompletionStage deferUserStorageRetirement(Throwable preparationFailure) { - if (!plugin.isLoadUserData() || plugin.getLoadedUserManager() == null) { + UserStorageAdmission admission = captureUserStorageAdmission(); + if (!admission.enabled() || admission.users() == null) { return preparationFailure == null ? closeUserStorageAfterSharedRetirement() : retireAfterPreparationFailure(preparationFailure); } @@ -246,19 +255,19 @@ private CompletionStage ensurePreStorageShutdownHook() { } private CompletionStage closeUserStorageAfterSharedRetirement() { - if (!plugin.isLoadUserData()) { + UserStorageAdmission admission = captureUserStorageAdmission(); + if (!admission.enabled()) { return CompletableFuture.completedFuture(null); } - UserManager users = plugin.getLoadedUserManager(); // Resolve this only after shared retirement. A successful replacement may // install a new owner while shutdown is waiting for its worker, and closing // the owner observed before that wait leaks the installed MySQL provider. - Runnable closeMysql = () -> closeCurrentUserStorageOwner(users); - if (users == null) { + Runnable closeMysql = () -> closeCurrentUserStorageOwner(admission); + if (admission.users() == null) { closeMysql.run(); return CompletableFuture.completedFuture(null); } - CompletionStage retirement = users.getDataManager().closeSharedRuntimeAsyncCompletion(closeMysql); + CompletionStage retirement = admission.manager().closeSharedRuntimeAsyncCompletion(closeMysql); if (retirement == null) { closeMysql.run(); return CompletableFuture.completedFuture(null); @@ -266,7 +275,20 @@ private CompletionStage closeUserStorageAfterSharedRetirement() { return retirement; } - private void closeCurrentUserStorageOwner(UserManager users) { + private UserStorageAdmission captureUserStorageAdmission() { + synchronized (userStorageRetirementLock) { + if (userStorageAdmission == null) { + boolean enabled = plugin.isLoadUserData(); + // Even disabled user loading can have an existing worker to retire. + UserManager users = plugin.getLoadedUserManager(); + userStorageAdmission = new UserStorageAdmission(enabled, users, + users == null ? null : users.getDataManager()); + } + return userStorageAdmission; + } + } + + private void closeCurrentUserStorageOwner(UserStorageAdmission admission) { if (!userStorageOwnerClosed.compareAndSet(false, true)) return; try { AdvancedCorePlugin.UserStorageOwner owner = plugin.getNativeUserStorageOwner(); @@ -276,8 +298,8 @@ else if (owner.storageType() == UserStorage.SQLITE && owner.table() != null && owner.table().getSqLite() != null) owner.table().getSqLite().closeConnection(); return; } - boolean ownsMysql = users != null && users.getDataManager().hasSharedSqlBackend() - ? users.getDataManager().usesSharedSqlStorage(UserStorage.MYSQL) + boolean ownsMysql = admission.users() != null && admission.manager().hasSharedSqlBackend() + ? admission.manager().usesSharedSqlStorage(UserStorage.MYSQL) : plugin.getOptions() != null && UserStorage.MYSQL.equals(plugin.getOptions().getStorageType()); if (ownsMysql && plugin.getMysql() != null) plugin.getMysql().close(); } catch (RuntimeException | Error failure) { diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index 7551d684d..124ff6bb0 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -686,6 +686,121 @@ private ScheduledExecutorService executor(String name, List events) thro assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); } + @Test void bukkitAdapterRetainsAdmittedStorageAcrossDeferredUnload() throws Exception { + for (boolean heldCalendar : List.of(false, true)) { + for (String outcome : List.of("success", "failure", "cancelled")) { + assertAdmittedStorageSurvivesUnload(heldCalendar, outcome); + } + } + } + + private void assertAdmittedStorageSurvivesUnload(boolean heldCalendar, String outcome) throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager manager = mock(UserDataManager.class); + UserManager successorUsers = mock(UserManager.class); + UserDataManager successorManager = mock(UserDataManager.class); + when(successorUsers.getDataManager()).thenReturn(successorManager); + AtomicReference currentUsers = new AtomicReference<>(users); + var loadUserData = new java.util.concurrent.atomic.AtomicBoolean(true); + var worker = Executors.newSingleThreadScheduledExecutor(task -> { + Thread thread = new Thread(task, "admitted-storage-test"); + thread.setDaemon(true); + return thread; + }); + var hook = new CompletableFuture(); + var calendar = new CompletableFuture(); + var preparationFailure = new IllegalStateException("consumer preparation failed"); + List events = new java.util.concurrent.CopyOnWriteArrayList<>(); + MySQL mysql = mock(MySQL.class); + when(plugin.getLogger()).thenReturn(mock(java.util.logging.Logger.class)); + when(plugin.isLoadUserData()).thenAnswer(ignored -> loadUserData.get()); + when(plugin.getLoadedUserManager()).thenAnswer(ignored -> currentUsers.get()); + when(users.getDataManager()).thenReturn(manager); + when(manager.getTimer()).thenReturn(worker); + when(plugin.getNativeUserStorageOwner()).thenReturn( + new AdvancedCorePlugin.UserStorageOwner(UserStorage.MYSQL, mysql, null)); + when(plugin.onBeforeStorageShutdown()).thenReturn(hook); + if (heldCalendar) { + TimeChecker checker = mock(TimeChecker.class); + when(plugin.getTimeChecker()).thenReturn(checker); + when(checker.beginShutdown()).thenReturn(calendar); + } + doAnswer(ignored -> { + events.add("unload"); + loadUserData.set(false); + if (heldCalendar) currentUsers.set(successorUsers); + return null; + }).when(plugin).onUnLoad(); + doAnswer(ignored -> { events.add("provider close"); return null; }).when(mysql).close(); + doAnswer(ignored -> { events.add("pending owner cleanup"); return null; }) + .when(plugin).closePendingNativeUserStorageOwners(); + when(manager.closeSharedRuntimeAsyncCompletion(any(Runnable.class))).thenAnswer(call -> { + assertSame(worker, manager.getTimer()); + events.add("shared flush"); + call.getArgument(0, Runnable.class).run(); + return CompletableFuture.completedFuture(null); + }); + BukkitRuntimePlatform platform = spy(new BukkitRuntimePlatform(plugin)); + doReturn(false).when(platform).canBlockForPreExecutorShutdown(); + try { + new AdvancedCoreRuntime(platform).shutdown(); + assertFalse(loadUserData.get(), "actual deferred unload must change the admission flag"); + assertSame(heldCalendar ? successorUsers : users, plugin.getLoadedUserManager()); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + assertFalse(worker.isShutdown(), "admitted storage remains open during preparation"); + verify(manager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + verify(mysql, never()).close(); + if (heldCalendar) calendar.complete(null); + assertSame(worker, platform.getUserStorageTimer(), "unload cannot replace the admitted worker"); + if (outcome.equals("success")) hook.complete(null); + else if (outcome.equals("failure")) hook.completeExceptionally(preparationFailure); + else hook.cancel(false); + var completed = platform.beforeExecutorShutdownCompletion().toCompletableFuture(); + try { completed.get(2, TimeUnit.SECONDS); } + catch (java.util.concurrent.ExecutionException failure) { + if (outcome.equals("success")) throw failure; + } + if (outcome.equals("success")) assertDoesNotThrow(completed::join); + else { + CompletionException failure = assertThrows(CompletionException.class, completed::join); + if (outcome.equals("failure")) assertSame(preparationFailure, failure.getCause()); + else assertInstanceOf(java.util.concurrent.CancellationException.class, failure.getCause()); + } + assertTrue(worker.awaitTermination(2, TimeUnit.SECONDS), "the admitted worker retires after cleanup"); + assertEquals(List.of("unload", "shared flush", "provider close", "pending owner cleanup"), events); + verify(manager).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + verify(successorManager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + verify(mysql, times(1)).close(); + verify(plugin, times(1)).closePendingNativeUserStorageOwners(); + verify(plugin, times(1)).onBeforeStorageShutdown(); + } finally { + worker.shutdownNow(); + assertTrue(worker.awaitTermination(2, TimeUnit.SECONDS)); + } + } + + @Test void disabledUserLoadingStillRetiresItsExistingManagerWorker() throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + UserManager users = mock(UserManager.class); + UserDataManager manager = mock(UserDataManager.class); + var worker = Executors.newSingleThreadScheduledExecutor(); + when(plugin.getLogger()).thenReturn(mock(java.util.logging.Logger.class)); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + when(manager.getTimer()).thenReturn(worker); + try { + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); + new AdvancedCoreRuntime(platform).shutdown(); + assertTrue(worker.isTerminated()); + assertSame(worker, platform.getUserStorageTimer()); + verify(manager, never()).closeSharedRuntimeAsyncCompletion(any(Runnable.class)); + } finally { + worker.shutdownNow(); + assertTrue(worker.awaitTermination(2, TimeUnit.SECONDS)); + } + } + @Test void bukkitAdapterRetiresStorageAfterEachPreparationFailureAndWaitsForItsCompletion() { for (String mode : List.of("failed", "cancelled", "throwing", "late-failed", "late-cancelled")) { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java index 3c80a01c4..fdbe8e8f2 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/PreStorageHookAdmissionTest.java @@ -219,6 +219,10 @@ private void rejectContinuation(boolean failedPreparation) { .findFirst().orElseThrow().action().run(); hook.complete(null); assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().join()); - verify(plugin, never()).getLoadedUserManager(); + // The admitted identity is a non-creating snapshot, even with loading disabled. + verify(plugin).getLoadedUserManager(); + verify(plugin, never()).getUserManager(); + verify(plugin, never()).getNativeUserStorageOwner(); + verify(plugin, never()).getMysql(); } } diff --git a/README.md b/README.md index 5007eb460..c09579e19 100644 --- a/README.md +++ b/README.md @@ -50,6 +50,12 @@ it does not report a successful flush. Failed or cancelled preparation still retires accepted writes, with the preparation failure retained. No new executor, configuration, or storage format is introduced. +Shutdown captures user-storage admission and its manager before waiting for the +calendar or consumer hook. Deferred unload can disable user loading or replace +the public manager reference without skipping that admitted manager's flush and +cleanup. The native provider is still resolved after shared retirement so a +provider replacement completed by admitted work closes its current owner. + If consumer preparation exceeds the default five-second shutdown watchdog, the watchdog reports it and retains the existing daemon storage worker and native provider. Preparation is acknowledged only after its callback has submitted the From fd53fd37a6e4de22a99a6fef0aa7b61d5fa58d1b Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 13 Oct 2026 08:31:28 -0600 Subject: [PATCH 10/28] Preserve final mutations for legacy storage owners --- .../advancedcore/data/ServerData.java | 12 ++- .../ServerDataPublicationOrderingTest.java | 92 +++++++++++++++++++ .../time/ServerDataTimeTransitionTest.java | 2 +- docs/shared-runtime-foundation.md | 13 ++- 4 files changed, 108 insertions(+), 11 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java index fe2dd9560..12744ce1b 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java @@ -28,7 +28,7 @@ */ public class ServerData extends YMLFile { // Selected only while the checked publisher owns this monitor. Unconfirmed - // final mutations must never become input to a following whole-file save. + // native final mutations must never become input to a following whole-file save. private org.bukkit.configuration.file.FileConfiguration finalDataSnapshot; /** * Durable state for one locally detected time transition. The identifier is @@ -97,8 +97,9 @@ public synchronized void saveDataChecked() { * publication by this file's actual owner. Adapters with another persistence * owner must override this method and return that owner's publication stage. * No caller may wait for the stage while holding a YAML owner's monitor. - * Legacy saveData overrides are still invoked, but their void return cannot - * prove durable publication and therefore produces an exceptional stage. + * Legacy saveData overrides still receive the mutation on their live YAML and + * are invoked once, but their void return cannot prove durable publication and + * therefore produces an exceptional stage. */ public java.util.concurrent.CompletionStage mutateFinalData( java.util.function.Consumer mutation) { @@ -107,8 +108,9 @@ public java.util.concurrent.CompletionStage mutateFinalData( java.util.Objects.requireNonNull(mutation, "mutation"); Class saveOwner = getClass().getMethod("saveData").getDeclaringClass(); if (saveOwner != YMLFile.class && saveOwner != ServerData.class) { - // Preserve the legacy callback, but do not expose a mutation whose - // publication this owner cannot confirm. + // Preserve the legacy mutation-and-save contract without treating + // its void return as confirmation of durable publication. + mutation.accept(getData()); saveData(); return java.util.concurrent.CompletableFuture.failedFuture(new UnsupportedOperationException( "Legacy ServerData owner does not confirm final publication")); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java index cbafe4565..e87f0058f 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java @@ -41,6 +41,76 @@ class ServerDataPublicationOrderingTest { verifyPublicationOrder(false); } + @Test void legacyFinalMutationReachesLiveOwnerAndSavedYamlExactlyOnceWithoutClaimingDurability() { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + LegacySavingOwner data = legacySavingOwner(); + FileConfiguration live = data.getData(); + var mutations = new java.util.concurrent.atomic.AtomicInteger(); + var failure = assertThrows(CompletionException.class, () -> data.mutateFinalData(yaml -> { + mutations.incrementAndGet(); + assertSame(live, yaml, "a legacy save override publishes its existing live YAML"); + yaml.set("FullInventory.example.Items.0", "stored-item"); + yaml.set("TimedPermissions.example", java.util.List.of("example.use%line%12345")); + yaml.set("LastUpdated", 2468L); + }).toCompletableFuture().join()); + + assertInstanceOf(UnsupportedOperationException.class, failure.getCause(), + "returning from a void save does not confirm durable publication"); + assertEquals(1, mutations.get()); + assertEquals(1, data.saves); + assertSame(live, data.getData()); + assertSame(live, data.savedData); + assertEquals(2468L, live.getLong("LastUpdated")); + YamlConfiguration saved = YamlConfiguration.loadConfiguration(data.getdFile()); + assertEquals("stored-item", saved.getString("FullInventory.example.Items.0")); + assertEquals(java.util.List.of("example.use%line%12345"), saved.getStringList("TimedPermissions.example")); + assertEquals(2468L, saved.getLong("LastUpdated")); + } + } + + @Test void legacyFinalMutationFailureIsObservableWithoutInvokingSave() { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + LegacySavingOwner data = legacySavingOwner(); + FileConfiguration live = data.getData(); + var mutations = new java.util.concurrent.atomic.AtomicInteger(); + var mutationFailure = new IllegalStateException("legacy mutation failed"); + var failure = assertThrows(CompletionException.class, () -> data.mutateFinalData(yaml -> { + mutations.incrementAndGet(); + yaml.set("LastUpdated", 2468L); + throw mutationFailure; + }).toCompletableFuture().join()); + + assertSame(mutationFailure, failure.getCause()); + assertEquals(1, mutations.get()); + assertEquals(0, data.saves, "a failed legacy mutation must not proceed to save"); + assertSame(live, data.getData()); + assertEquals(2468L, live.getLong("LastUpdated"), "legacy mutation retains its existing live-YAML semantics"); + assertFalse(data.getdFile().exists()); + } + } + + @Test void legacyFinalSaveFailureIsObservableAfterExactlyOneMutationAndOneSave() { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + LegacySavingOwner data = legacySavingOwner(); + FileConfiguration live = data.getData(); + var mutations = new java.util.concurrent.atomic.AtomicInteger(); + data.saveFailure = new IllegalStateException("legacy save failed"); + var failure = assertThrows(CompletionException.class, () -> data.mutateFinalData(yaml -> { + mutations.incrementAndGet(); + yaml.set("LastUpdated", 2468L); + }).toCompletableFuture().join()); + + assertSame(data.saveFailure, failure.getCause()); + assertEquals(1, mutations.get()); + assertEquals(1, data.saves); + assertSame(live, data.getData()); + assertSame(live, data.savedData); + assertEquals(2468L, data.savedData.getLong("LastUpdated")); + assertFalse(data.getdFile().exists()); + } + } + + @Test void inheritedSetValueCannotMutateLiveYamlDuringNativePublication() throws Exception { verifyInheritedMutatorIsPublicationOrdered(data -> data.setValue("Inherited.Value", "new"), "Inherited.Value"); @@ -256,6 +326,28 @@ private static final class CustomOwner extends ServerData { @Override public void saveData() { saved.incrementAndGet(); } } + private LegacySavingOwner legacySavingOwner() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(dataFolder); + LegacySavingOwner data = new LegacySavingOwner(plugin); + data.setData(new YamlConfiguration()); + return data; + } + + private static final class LegacySavingOwner extends ServerData { + int saves; + FileConfiguration savedData; + RuntimeException saveFailure; + LegacySavingOwner(AdvancedCorePlugin plugin) { super(plugin); } + @Override public void saveData() { + saves++; + savedData = getData(); + if (saveFailure != null) throw saveFailure; + try { savedData.save(getdFile()); } + catch (java.io.IOException failure) { throw new java.io.UncheckedIOException(failure); } + } + } + private void verifyPublicationOrder(boolean heartbeat) throws Exception { try (var scheduler = mockConstruction(com.bencodez.simpleapi.scheduler.BukkitScheduler.class)) { verifyNativePublicationOrder(heartbeat); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java index a097d87a4..cece8de90 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java @@ -99,7 +99,7 @@ void legacyVoidSaveOwnerIsInvokedWithoutClaimingConfirmedPublication() { () -> owner.mutateFinalData(yaml -> yaml.set("LastUpdated", 1234L)).toCompletableFuture().join()); assertTrue(failure.getCause() instanceof UnsupportedOperationException); assertEquals(1, owner.saves); - assertFalse(owner.getData().contains("LastUpdated"), "an unconfirmed legacy save cannot expose the final mutation"); + assertEquals(1234L, owner.getData().getLong("LastUpdated"), "legacy owners retain the established live mutation before their void save"); } @Test diff --git a/docs/shared-runtime-foundation.md b/docs/shared-runtime-foundation.md index 350d230b2..653341c54 100644 --- a/docs/shared-runtime-foundation.md +++ b/docs/shared-runtime-foundation.md @@ -40,17 +40,20 @@ standalone callers, while lifecycle shutdown supplies only its remaining budget. Final inventory, server timestamp, and timed-permission mutations are admitted in order on one bounded daemon persistence worker. `ServerData.mutateFinalData` -returns the actual file owner's publication stage. Final mutations are staged in -a private YAML candidate and become live only after confirmed publication. +returns the actual file owner's publication stage. Native and completion-aware +final mutations are staged in a private YAML candidate and become live only after confirmed publication. Native ServerData forces that candidate on the shutdown worker; an ordered adapter serializes under its own YAML monitor and confirms its existing publisher. Later unrelated saves cannot publish a failed final overflow mutation. Ordered snapshots carry only successfully published final changes forward and preserve newer explicit edits, including an edit that writes the same previous value. No caller waits for publication while holding the YAML owner's monitor. Legacy -void `saveData` overrides retain their callback but cannot claim confirmed -publication without overriding the additive completion contract; they report an -unsupported confirmation. An I/O failure remains a failed stage. Inventory persistence gates user-storage +void `saveData` overrides retain the established live-YAML mutation followed by +one save callback, but cannot claim confirmed publication without overriding the +additive completion contract; they report an unsupported confirmation. A legacy +callback failure remains observable, with no second mutation or save invocation. +Its live changes retain the old non-transactional behavior; candidate isolation +remains specific to native and completion-aware owners. An I/O failure remains a failed stage. Inventory persistence gates user-storage retirement and cannot acknowledge reserved replay overflow before its actual save succeeds. The public synchronous handler shutdown methods remain available. The shutdown-completion hook follows submission of final file saves; it is not a From 8faac95263136ab9c89f47b72f2056c4e5f504ab Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Tue, 13 Oct 2026 17:11:40 -0600 Subject: [PATCH 11/28] Retain admitted reward and storage work beyond shutdown caller deadline --- .../api/rewards/RewardHandler.java | 22 +- .../api/user/AdvancedCoreUser.java | 19 + .../bukkit/runtime/BukkitRuntimePlatform.java | 111 +++++- .../core/platform/RuntimePlatform.java | 8 + .../core/runtime/AdvancedCoreRuntime.java | 27 +- .../CoreRuntimeAdmittedRetirementTest.java | 348 ++++++++++++++++++ .../lifecycle/CoreRuntimeDeadlineTest.java | 2 + .../tests/lifecycle/CoreRuntimeTest.java | 2 + .../TimeCheckerShutdownAdmissionTest.java | 38 ++ docs/shared-runtime-foundation.md | 21 +- 10 files changed, 570 insertions(+), 28 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java index 70b87bf4c..9afd68606 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java @@ -8,8 +8,9 @@ import java.util.Set; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CompletionStage; -import java.util.concurrent.Executors; +import java.util.concurrent.CompletableFuture; import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; import org.bukkit.configuration.Configuration; @@ -48,9 +49,13 @@ public class RewardHandler { AdvancedCorePlugin plugin; + private final CompletableFuture delayedTimerTermination = new CompletableFuture<>(); + private final CompletionStage delayedTimerRetirement = delayedTimerTermination.minimalCompletionStage(); @Getter - private ScheduledExecutorService delayedTimer = Executors.newSingleThreadScheduledExecutor( - com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime.daemonThreads("AdvancedCore-DelayedRewards")); + private ScheduledExecutorService delayedTimer = new ScheduledThreadPoolExecutor(1, + com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime.daemonThreads("AdvancedCore-DelayedRewards")) { + @Override protected void terminated() { delayedTimerTermination.complete(null); } + }; public RewardHandler(AdvancedCorePlugin plugin) { this.plugin = plugin; @@ -292,6 +297,17 @@ public void shutdown() { shutdown(10, TimeUnit.SECONDS); } + /** + * Close delayed-task admission without blocking. Completion means the actual + * worker has exited, including an admitted task that ignored interruption. + * Persisted replay callbacks can outlive that worker; the storage lifecycle + * separately drains their existing per-user replay tails before retiring. + */ + public CompletionStage beginShutdown() { + delayedTimer.shutdown(); + return delayedTimerRetirement; + } + /** Lifecycle callers share their remaining disable budget; legacy shutdown keeps its ten-second grace. */ public void shutdown(long timeout, TimeUnit unit) { java.util.Objects.requireNonNull(unit, "unit"); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java index 40a8f6a43..abd1b6ea9 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java @@ -441,6 +441,25 @@ private static final class ReplayClaims { private CompletableFuture serialReplayTail = CompletableFuture.completedFuture(null); } + /** + * Snapshot completion of already-admitted persisted replay, including its + * queue acknowledgement or restore callback. The lifecycle calls this from + * its manager-worker barrier after delayed-task admission and execution end, + * so a check deferred to that worker has appended its tail before the snapshot. + * This creates no new replay owner and does not admit or execute rewards. + */ + public static CompletionStage persistedReplayCompletion(AdvancedCorePlugin plugin) { + java.util.Objects.requireNonNull(plugin, "plugin"); + synchronized (plugin) { + synchronized (REPLAY_CLAIMS_LOCK) { + HashMap users = REPLAY_CLAIMS.get(plugin); + if (users == null || users.isEmpty()) return CompletableFuture.completedFuture(null); + return CompletableFuture.allOf(users.values().stream() + .map(claims -> claims.serialReplayTail).toArray(CompletableFuture[]::new)); + } + } + } + /** * User data fetch mode for this user. * diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 0d877b7bd..e575c1472 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -19,7 +19,9 @@ import com.bencodez.advancedcore.AdvancedCorePlugin; import com.bencodez.advancedcore.api.item.FullInventoryHandler; import com.bencodez.advancedcore.api.javascript.JavascriptEngineHandler; +import com.bencodez.advancedcore.api.rewards.RewardHandler; import com.bencodez.advancedcore.api.time.TimeChecker; +import com.bencodez.advancedcore.api.user.AdvancedCoreUser; import com.bencodez.advancedcore.api.user.UserManager; import com.bencodez.advancedcore.api.user.UserStorage; import com.bencodez.advancedcore.api.user.usercache.UserDataManager; @@ -30,6 +32,7 @@ /** Bukkit services for the shared executor lifecycle; no duplicate owners are created. */ public final class BukkitRuntimePlatform implements RuntimePlatform { private final AdvancedCorePlugin plugin; + private final RewardHandler rewardHandler; private volatile LongSupplier shutdownRemainingNanos; private final CompletableFuture persistenceWorkerTermination = new CompletableFuture<>(); // The hook can be admitted later, after a retained time transition drains. @@ -48,6 +51,8 @@ public final class BukkitRuntimePlatform implements RuntimePlatform { private volatile CompletionStage permissionPersistence = CompletableFuture.completedFuture(null); private volatile CompletionStage userStorageRetirement = CompletableFuture.completedFuture(null); private CompletionStage startedUserStorageRetirement; + private volatile CompletionStage admittedUserStorageRetirement; + private volatile CompletionStage delayedRewardRetirement = CompletableFuture.completedFuture(null); private CompletionStage preStorageShutdownHook; private CompletableFuture storageRetirementPreparation = CompletableFuture.completedFuture(null); private volatile CompletionStage timeChangeRetirement = CompletableFuture.completedFuture(null); @@ -60,6 +65,7 @@ private record UserStorageAdmission(boolean enabled, UserManager users, UserData public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { this.plugin = Objects.requireNonNull(plugin, "plugin"); + rewardHandler = plugin.getRewardHandler(); // Capture identities before consumer unload changes handler/manager fields. var owners = new IdentityHashMap(); addExecutor(owners, plugin.getTimer()); @@ -70,6 +76,7 @@ public BukkitRuntimePlatform(AdvancedCorePlugin plugin) { FullInventoryHandler inventory = plugin.getFullInventoryHandler(); if (inventory != null) addExecutor(owners, inventory.getTimer()); if (plugin.getPermissionHandler() != null) addExecutor(owners, plugin.getPermissionHandler().getTimer()); + if (rewardHandler != null) addExecutor(owners, rewardHandler.getDelayedTimer()); UserManager users = plugin.getLoadedUserManager(); if (users != null && users.getDataManager() != null) addExecutor(owners, users.getDataManager().getTimer()); // Keep the same non-creating snapshot for retirement that was used to @@ -120,6 +127,7 @@ private static void addExecutor(IdentityHashMap { @@ -128,6 +136,9 @@ private static void addExecutor(IdentityHashMap beforeExecutorShutdownCompletion() { return userStorageRetirement; } + @Override public CompletionStage admittedStorageRetirementCompletion() { + return admittedUserStorageRetirement; + } @Override public CompletionStage storageRetirementPreparationCompletion() { synchronized (userStorageRetirementLock) { return storageRetirementPreparation; @@ -154,10 +165,10 @@ private static void addExecutor(IdentityHashMap afterExecutorGrace() { return List.of(new Cleanup("reward handler", () -> { - if (plugin.getRewardHandler() != null) { + if (rewardHandler != null) { LongSupplier budget = shutdownRemainingNanos; - if (budget == null) plugin.getRewardHandler().shutdown(); - else plugin.getRewardHandler().shutdown(Math.min(TimeUnit.SECONDS.toNanos(10), + if (budget == null) rewardHandler.shutdown(); + else rewardHandler.shutdown(Math.min(TimeUnit.SECONDS.toNanos(10), Math.max(0L, budget.getAsLong())), TimeUnit.NANOSECONDS); } })); @@ -227,6 +238,12 @@ private void closeTimeChangeAdmission() { } catch (Throwable failure) { timeChangeRetirement = CompletableFuture.failedFuture(failure); } } + private void closeDelayedRewardAdmission() { + if (rewardHandler == null) return; + CompletionStage retirement = rewardHandler.beginShutdown(); + delayedRewardRetirement = retirement == null ? CompletableFuture.completedFuture(null) : retirement; + } + private void closeUserStorageAfterTimeChangeRetirement() { // Deferred platform unload may change configuration or manager references. // Storage already admitted for this shutdown retains its own retirement identity. @@ -237,11 +254,14 @@ private void closeUserStorageAfterTimeChangeRetirement() { timeRetirement.toCompletableFuture().join(); userStorageRetirement = ensureUserStorageRetirementStarted(); } catch (java.util.concurrent.CompletionException | java.util.concurrent.CancellationException failure) { - userStorageRetirement = CompletableFuture.failedFuture(failure); + userStorageRetirement = preservePreparationFailure(ensureUserStorageRetirementStarted(), failure); } return; } - userStorageRetirement = timeRetirement.thenCompose(ignored -> ensureUserStorageRetirementStarted()); + userStorageRetirement = timeRetirement.handle((ignored, failure) -> failure).thenCompose(failure -> { + CompletionStage retirement = ensureUserStorageRetirementStarted(); + return failure == null ? retirement : preservePreparationFailure(retirement, failure); + }); } private CompletionStage ensureUserStorageRetirementStarted() { @@ -251,11 +271,17 @@ private CompletionStage ensureUserStorageRetirementStarted() { if (hook.toCompletableFuture().isDone()) { try { hook.toCompletableFuture().join(); } catch (java.util.concurrent.CompletionException | java.util.concurrent.CancellationException failure) { - startedUserStorageRetirement = retireAfterPreparationFailure(failure); + startedUserStorageRetirement = rewardHandler == null + ? retireAfterPreparationFailure(failure) : deferUserStorageRetirement(failure); + if (captureUserStorageAdmission().enabled()) admittedUserStorageRetirement = startedUserStorageRetirement; return startedUserStorageRetirement; } - // Preserve the successful default hook's legacy retirement stage identity. - startedUserStorageRetirement = closeUserStorageAfterSharedRetirement(); + // A delayed check may have returned after deferring itself to + // the manager. Its worker barrier must precede the replay-tail + // snapshot even when the consumer hook completed immediately. + startedUserStorageRetirement = rewardHandler == null + ? closeUserStorageAfterSharedRetirement() : deferUserStorageRetirement(null); + if (captureUserStorageAdmission().enabled()) admittedUserStorageRetirement = startedUserStorageRetirement; } else { var preparation = new CompletableFuture(); storageRetirementPreparation = preparation; @@ -278,24 +304,41 @@ private CompletionStage ensureUserStorageRetirementStarted() { private CompletionStage deferUserStorageRetirement(Throwable preparationFailure) { UserStorageAdmission admission = captureUserStorageAdmission(); if (!admission.enabled() || admission.users() == null) { - return preparationFailure == null ? closeUserStorageAfterSharedRetirement() + CompletionStage retirement = preparationFailure == null ? closeUserStorageAfterSharedRetirement() : retireAfterPreparationFailure(preparationFailure); + if (admission.enabled()) admittedUserStorageRetirement = retirement; + return retirement; } var result = new CompletableFuture(); try { Objects.requireNonNull(getUserStorageTimer(), "user storage timer").execute(() -> { try { - CompletionStage retirement = preparationFailure == null - ? closeUserStorageAfterSharedRetirement() - : retireAfterPreparationFailure(preparationFailure); - retirement.whenComplete((ignored, failure) -> { - if (failure == null) result.complete(null); - else result.completeExceptionally(failure); - }); + CompletionStage replay = AdvancedCoreUser.persistedReplayCompletion(plugin); + if (replay.toCompletableFuture().isDone()) { + replay.toCompletableFuture().join(); + completeDeferredUserStorageRetirement(result, preparationFailure); + } else { + // Queue after acknowledgement/restore finishes. The + // completing callback may still hold shared read admission. + replay.whenComplete((ignored, failure) -> { + if (failure != null) { + result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); + return; + } + try { + getUserStorageTimer().execute(() -> completeDeferredUserStorageRetirement(result, + preparationFailure)); + } catch (RuntimeException | Error admissionFailure) { + result.completeExceptionally(preparationAndStorageFailure(preparationFailure, + admissionFailure)); + } + }); + } } catch (Throwable failure) { result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); } }); + admittedUserStorageRetirement = result; } catch (RuntimeException | Error failure) { // Do not detach the shared runtime or pretend rejected work flushed storage. result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); @@ -303,6 +346,19 @@ private CompletionStage deferUserStorageRetirement(Throwable preparationFa return result; } + private void completeDeferredUserStorageRetirement(CompletableFuture result, Throwable preparationFailure) { + try { + CompletionStage retirement = preparationFailure == null + ? closeUserStorageAfterSharedRetirement() : retireAfterPreparationFailure(preparationFailure); + retirement.whenComplete((ignored, failure) -> { + if (failure == null) result.complete(null); + else result.completeExceptionally(failure); + }); + } catch (Throwable failure) { + result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); + } + } + private static Throwable preparationAndStorageFailure(Throwable preparationFailure, Throwable storageFailure) { if (preparationFailure == null) return storageFailure; Throwable cause = preparationFailure instanceof java.util.concurrent.CompletionException @@ -314,12 +370,19 @@ private static Throwable preparationAndStorageFailure(Throwable preparationFailu /** A failed consumer preparation still has accepted cache writes to retire. */ private CompletionStage retireAfterPreparationFailure(Throwable preparationFailure) { + try { return preservePreparationFailure(closeUserStorageAfterSharedRetirement(), preparationFailure); } + catch (Throwable storageFailure) { + return CompletableFuture.failedFuture(preparationAndStorageFailure(preparationFailure, storageFailure)); + } + } + + private CompletionStage preservePreparationFailure(CompletionStage retirement, Throwable preparationFailure) { Throwable cause = preparationFailure instanceof java.util.concurrent.CompletionException && preparationFailure.getCause() != null ? preparationFailure.getCause() : preparationFailure; var reported = new java.util.concurrent.CompletionException(cause); var result = new CompletableFuture(); try { - closeUserStorageAfterSharedRetirement().whenComplete((ignored, storageFailure) -> { + retirement.whenComplete((ignored, storageFailure) -> { if (storageFailure != null && storageFailure != cause) reported.addSuppressed(storageFailure); result.completeExceptionally(reported); }); @@ -344,6 +407,11 @@ private CompletionStage ensurePreStorageShutdownHook() { preStorageShutdownHook = CompletableFuture.allOf(inventory, preStorageShutdownHook.toCompletableFuture()); } + CompletableFuture rewards = delayedRewardRetirement.toCompletableFuture(); + if (!rewards.isDone() || rewards.isCompletedExceptionally()) { + preStorageShutdownHook = CompletableFuture.allOf(rewards, + preStorageShutdownHook.toCompletableFuture()); + } preStorageShutdownHook.whenComplete((ignored, failure) -> { if (failure == null) consumerPreparationCompletion.complete(null); else consumerPreparationCompletion.completeExceptionally(failure); @@ -469,7 +537,14 @@ private CompletionStage closeUserStorageOwnerAsync(UserStorageAdmission ad if (asynchronousOwnerClose.compareAndSet(null, completion)) break; } Thread worker = AdvancedCoreRuntime.daemonThreads("AdvancedCore-Storage-Owner-Close").newThread(() -> { - try { closeCurrentUserStorageOwner(admission); completion.complete(null); } + try { + // Without a manager there is no worker barrier to run the + // snapshot. This existing daemon owner-close worker can drain + // callbacks without waiting on the lifecycle caller. + AdvancedCoreUser.persistedReplayCompletion(plugin).toCompletableFuture().join(); + closeCurrentUserStorageOwner(admission); + completion.complete(null); + } catch (Throwable failure) { asynchronousOwnerClose.compareAndSet(completion, null); completion.completeExceptionally(failure); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java index ba96453f1..16cfc6059 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/platform/RuntimePlatform.java @@ -53,6 +53,14 @@ default CompletionStage storageRetirementPreparationCompletion() { return CompletableFuture.completedFuture(null); } + /** + * Actual accepted native-storage retirement, or {@code null} when no such task + * has been admitted. An incomplete stage retains its daemon storage worker and + * provider until it settles, even after the caller's deadline. A pending time + * transition or consumer hook alone is not native-task admission. + */ + default CompletionStage admittedStorageRetirementCompletion() { return null; } + /** Watchdog grace capped by the disable deadline; pending preparation retains its durable owner. */ default long deferredShutdownTimeoutMillis() { return 5_000; } diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java index f8ce7fa31..da6415ec9 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/core/runtime/AdvancedCoreRuntime.java @@ -12,6 +12,7 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.LongSupplier; +import java.util.function.BiConsumer; import com.bencodez.advancedcore.core.platform.RuntimePlatform; import com.bencodez.advancedcore.core.platform.RuntimePlatform.Cleanup; @@ -191,29 +192,45 @@ private void finishDeferredCleanup(CompletionStage completion, String comp final ScheduledExecutorService storageTimer = timer; AtomicBoolean finished = new AtomicBoolean(); AtomicBoolean terminalStorageCleanup = new AtomicBoolean(); + AtomicBoolean retirementFailureReported = new AtomicBoolean(); if (holdTimeTimer) clean(List.of(new Cleanup("time change cancellation", platform::beforeDeferredPlatformCleanup))); // Thread-confined Bukkit cleanup must remain on the lifecycle thread. An // active transition is first cancelled above and its lease still gates // storage retirement through completion. finishDeferredPlatformCleanup(grace, storageTimer, holdTimeTimer); - completion.whenComplete((ignored, failure) -> { + BiConsumer retirementFinished = (ignored, failure) -> { + if (failure != null && retirementFailureReported.compareAndSet(false, true)) { + Throwable cause = failure instanceof CompletionException && failure.getCause() != null + ? failure.getCause() : failure; + platform.cleanupFailed(component, cause); + } if (!finished.compareAndSet(false, true)) { if (failure != null) finishDeferredStorageTimer(storageTimer, true, true, terminalStorageCleanup); return; } if (failure == null) shutdown(storageTimer); else { - Throwable cause = failure instanceof CompletionException && failure.getCause() != null - ? failure.getCause() : failure; - platform.cleanupFailed(component, cause); shutdownNow(storageTimer); } if (holdTimeTimer) shutdownNow(grace.timeTimer()); finishDeferredStorageTimer(storageTimer, failure != null, failure != null, terminalStorageCleanup); - }); + }; + completion.whenComplete(retirementFinished); long timeoutMillis = Math.max(1, platform.deferredShutdownTimeoutMillis()); Runnable forceRetirement = () -> { + if (finished.get()) return; + CompletionStage admitted = platform.admittedStorageRetirementCompletion(); + if (admitted != null) { + platform.cleanupFailed(component, new TimeoutException( + "Deferred storage retirement exceeded " + timeoutMillis + + " ms; retaining accepted retirement until completion")); + // The accepted barrier/final flush owns this daemon worker. A + // deadline or an interruption request cannot prove its completion, + // and shutting the worker down can discard a queued final task. + admitted.whenComplete(retirementFinished); + return; + } if (!finished.compareAndSet(false, true)) return; platform.cleanupFailed(component, new TimeoutException( "Deferred storage retirement exceeded " + timeoutMillis + " ms")); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java new file mode 100644 index 000000000..a356cc428 --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java @@ -0,0 +1,348 @@ +package com.bencodez.advancedcore.tests.lifecycle; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.ArgumentMatchers.*; +import static org.mockito.Mockito.*; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Logger; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; +import org.bukkit.Bukkit; +import org.bukkit.Server; + +import com.bencodez.advancedcore.AdvancedCoreConfigOptions; +import com.bencodez.advancedcore.AdvancedCorePlugin; +import com.bencodez.advancedcore.api.item.FullInventoryHandler; +import com.bencodez.advancedcore.api.rewards.RewardHandler; +import com.bencodez.advancedcore.api.rewards.RewardOptions; +import com.bencodez.advancedcore.api.time.TimeChecker; +import com.bencodez.advancedcore.api.user.AdvancedCoreUser; +import com.bencodez.advancedcore.api.user.PersistedQueueReference; +import com.bencodez.advancedcore.api.user.UserManager; +import com.bencodez.advancedcore.api.user.UserStorage; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; +import com.bencodez.advancedcore.api.user.UserData; +import com.bencodez.advancedcore.api.user.UserDataFetchMode; +import com.bencodez.advancedcore.api.user.userstorage.mysql.MySQL; +import com.bencodez.advancedcore.bukkit.runtime.BukkitRuntimePlatform; +import com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime; +import com.bencodez.advancedcore.core.user.runtime.SharedUserDataRuntime; +import com.bencodez.advancedcore.core.user.runtime.UserCacheOwner; +import com.bencodez.advancedcore.core.user.storage.SqlUserStorage; +import com.bencodez.advancedcore.core.user.storage.sql.SqlUserBackend; + +class CoreRuntimeAdmittedRetirementTest { + @TempDir File directory; + + @Test void preparationJustBeforeDeadlineCannotDiscardAcceptedFinalRetirement() throws Exception { + Fixture fixture = new Fixture(); + AtomicLong clock = new AtomicLong(TimeUnit.SECONDS.toNanos(14)); + CompletableFuture preparation = new CompletableFuture<>(); + FullInventoryHandler inventory = mock(FullInventoryHandler.class); + when(fixture.plugin.getFullInventoryHandler()).thenReturn(inventory); + when(inventory.shutdownAsync(any())).thenReturn(preparation); + CountDownLatch occupied = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + CountDownLatch preparationTimeout = new CountDownLatch(1); + CountDownLatch retirementTimeout = new CountDownLatch(1); + fixture.manager.getTimer().execute(() -> { + assertTrue(Thread.currentThread().isDaemon()); + occupied.countDown(); + awaitIgnoringInterrupt(release); + }); + assertTrue(occupied.await(2, TimeUnit.SECONDS)); + BukkitRuntimePlatform platform = fixture.platform(); + doReturn(1L).when(platform).deferredShutdownTimeoutMillis(); + doAnswer(call -> { + Throwable failure = call.getArgument(1); + if (failure instanceof TimeoutException) { + if (failure.getMessage().startsWith("Consumer storage preparation")) preparationTimeout.countDown(); + if (failure.getMessage().startsWith("Deferred storage retirement")) retirementTimeout.countDown(); + } + return null; + }).when(platform).cleanupFailed(anyString(), any()); + try { + new AdvancedCoreRuntime(platform, clock::get).shutdown(0L); + assertTrue(preparationTimeout.await(2, TimeUnit.SECONDS)); + assertNull(platform.admittedStorageRetirementCompletion(), "pending preparation is not native-task admission"); + clock.set(TimeUnit.SECONDS.toNanos(15) - 1L); + preparation.complete(null); + assertNotNull(platform.admittedStorageRetirementCompletion()); + assertTrue(retirementTimeout.await(2, TimeUnit.SECONDS)); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + assertFalse(fixture.shared.isClosed()); + verify(fixture.mysql, never()).close(); + release.countDown(); + assertDoesNotThrow(() -> platform.beforeExecutorShutdownCompletion().toCompletableFuture() + .get(2, TimeUnit.SECONDS), "accepted final retirement must survive the exhausted watchdog grace"); + assertTrue(fixture.manager.getTimer().awaitTermination(2, TimeUnit.SECONDS)); + verify(fixture.cache, times(1)).flush(fixture.uuid, UserStorage.MYSQL, fixture.storage); + verify(fixture.backend, times(1)).close(); + verify(fixture.mysql, times(1)).close(); + assertTrue(fixture.shared.isClosed()); + assertEquals(TimeUnit.SECONDS.toNanos(15) - 1L, clock.get()); + } finally { + release.countDown(); + fixture.manager.getTimer().shutdownNow(); + } + } + + @Test void admittedTimedRewardFinishesBeforeNativeRetirementEvenAfterCallerDeadline() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("normal"); + } + + @Test void failedPreparationStillWaitsForAdmittedTimedReward() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("failed"); + } + + @Test void cancelledPreparationStillWaitsForAdmittedTimedReward() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("cancelled"); + } + + @Test void failedTimeRetirementStillWaitsForAdmittedTimedReward() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("time-failed"); + } + + @Test void cancelledTimeRetirementStillWaitsForAdmittedTimedReward() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("time-cancelled"); + } + + private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String preparationOutcome) + throws Exception { + Fixture fixture = new Fixture(); + AtomicLong clock = new AtomicLong(TimeUnit.SECONDS.toNanos(15)); + RewardHandler rewards = spy(new RewardHandler(fixture.plugin)); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "shutdown-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + CountDownLatch reading = new CountDownLatch(1); + CountDownLatch interrupted = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean nativeClosed = new AtomicBoolean(); + doAnswer(call -> { nativeClosed.set(true); return null; }).when(fixture.mysql).close(); + ArrayList persisted = new ArrayList<>(List.of("ShutdownReward%ExecutionTime/%1")); + when(data.getStringList("TimedRewards", UserDataFetchMode.DEFAULT)).thenAnswer(call -> { + reading.countDown(); + while (release.getCount() != 0L) { + try { release.await(); } + catch (InterruptedException ignored) { interrupted.countDown(); } + } + assertFalse(nativeClosed.get(), "admitted timed reward cannot read after its native owner closes"); + return new ArrayList<>(persisted); + }); + doAnswer(call -> { + assertFalse(nativeClosed.get(), "claim/checkpoint/acknowledgement retain native storage ownership"); + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }).when(data).setStringList(eq("TimedRewards"), any(), anyBoolean()); + doAnswer(call -> { + assertFalse(nativeClosed.get(), "timed acknowledgement retains native storage ownership"); + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }).when(data).setStringList(eq("TimedRewards"), any()); + doReturn(CompletableFuture.completedFuture(null)).when(rewards) + .givePersistedQueueRewardAsync(eq(user), any(PersistedQueueReference.class), any(RewardOptions.class)); + CompletableFuture hook = new CompletableFuture<>(); + if (preparationOutcome.equals("normal") || preparationOutcome.startsWith("time-")) hook.complete(null); + else if (preparationOutcome.equals("failed")) hook.completeExceptionally(new IllegalStateException("consumer failed")); + else hook.cancel(false); + when(fixture.plugin.onBeforeStorageShutdown()).thenReturn(hook); + if (preparationOutcome.startsWith("time-")) { + TimeChecker checker = mock(TimeChecker.class); + CompletableFuture timeRetirement = new CompletableFuture<>(); + if (preparationOutcome.equals("time-failed")) { + timeRetirement.completeExceptionally(new IllegalStateException("time retirement failed")); + } else timeRetirement.cancel(false); + when(fixture.plugin.getTimeChecker()).thenReturn(checker); + when(checker.beginShutdown()).thenReturn(timeRetirement); + } + var execution = rewards.getDelayedTimer().submit(user::checkDelayedTimedRewards); + assertTrue(reading.await(2, TimeUnit.SECONDS)); + BukkitRuntimePlatform platform = fixture.platform(); + try { + new AdvancedCoreRuntime(platform, clock::get).shutdown(0L); + assertTrue(interrupted.await(2, TimeUnit.SECONDS), "zero caller budget still cancels queued timer work"); + // A real manager-worker barrier removes the race with the old native + // retirement task. The fixed path keeps admission open for the held reward. + fixture.manager.getTimer().submit(() -> null).get(2, TimeUnit.SECONDS); + assertFalse(fixture.shared.isClosed()); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + verify(fixture.mysql, never()).close(); + release.countDown(); + execution.get(2, TimeUnit.SECONDS); + if (preparationOutcome.equals("normal")) { + platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); + } else { + assertThrows(java.util.concurrent.ExecutionException.class, + () -> platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS)); + } + assertTrue(fixture.manager.getTimer().awaitTermination(2, TimeUnit.SECONDS)); + verify(rewards, times(1)).givePersistedQueueRewardAsync(eq(user), + any(PersistedQueueReference.class), any(RewardOptions.class)); + assertTrue(persisted.isEmpty(), "the actual timed queue acknowledgement finishes before final flush"); + verify(fixture.cache, times(1)).flush(fixture.uuid, UserStorage.MYSQL, fixture.storage); + verify(fixture.mysql, times(1)).close(); + assertEquals(TimeUnit.SECONDS.toNanos(15), clock.get(), "accepted work does not renew caller waits"); + } finally { + release.countDown(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + + private static void awaitIgnoringInterrupt(CountDownLatch release) { + while (release.getCount() != 0L) { + try { release.await(); } + catch (InterruptedException ignored) { /* Model an admitted noncooperative storage call. */ } + } + } + + @Test void detachedTimedAcknowledgementRetainsStorageAfterDelayedWorkerExits() throws Exception { + assertDetachedTimedAcknowledgementRetainsStorage(false, false); + } + + @Test void spigotDeferredTimedCheckRetainsItsDetachedAcknowledgement() throws Exception { + assertDetachedTimedAcknowledgementRetainsStorage(true, false); + } + + @Test void failedDetachedTimedDispatchRestoresItsQueueBeforeNativeRetirement() throws Exception { + assertDetachedTimedAcknowledgementRetainsStorage(false, true); + } + + private void assertDetachedTimedAcknowledgementRetainsStorage(boolean deferredCheck, boolean failedDispatch) + throws Exception { + Fixture fixture = new Fixture(); + AtomicLong clock = new AtomicLong(TimeUnit.SECONDS.toNanos(15)); + RewardHandler rewards = spy(new RewardHandler(fixture.plugin)); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "async-shutdown-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + ArrayList persisted = new ArrayList<>(List.of("ShutdownReward%ExecutionTime/%1")); + AtomicBoolean nativeClosed = new AtomicBoolean(); + doAnswer(call -> { nativeClosed.set(true); return null; }).when(fixture.mysql).close(); + when(data.getStringList("TimedRewards", UserDataFetchMode.DEFAULT)) + .thenAnswer(call -> new ArrayList<>(persisted)); + CountDownLatch acknowledging = new CountDownLatch(1); + CountDownLatch releaseAcknowledgement = new CountDownLatch(1); + doAnswer(call -> { + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }).when(data).setStringList(eq("TimedRewards"), any(), anyBoolean()); + doAnswer(call -> { + acknowledging.countDown(); + awaitIgnoringInterrupt(releaseAcknowledgement); + assertFalse(nativeClosed.get(), "detached timed callback retains its native storage owner"); + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }).when(data).setStringList(eq("TimedRewards"), any()); + CompletableFuture dispatch = new CompletableFuture<>(); + CountDownLatch dispatched = new CountDownLatch(1); + doAnswer(call -> { dispatched.countDown(); return dispatch; }).when(rewards) + .givePersistedQueueRewardAsync(eq(user), any(PersistedQueueReference.class), any(RewardOptions.class)); + var checking = rewards.getDelayedTimer().submit(() -> { + if (deferredCheck) { + // Exercise Spigot's caller classification on the actual delayed + // worker; the manager's own worker remains the storage owner. + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + assertTrue(fixture.manager.mustDeferSharedStorageAccess()); + user.checkDelayedTimedRewards(); + } + } else user.checkDelayedTimedRewards(); + }); + checking.get(2, TimeUnit.SECONDS); + assertTrue(dispatched.await(2, TimeUnit.SECONDS)); + BukkitRuntimePlatform platform = fixture.platform(); + var callback = java.util.concurrent.Executors.newSingleThreadExecutor( + AdvancedCoreRuntime.daemonThreads("held-timed-acknowledgement")); + try { + new AdvancedCoreRuntime(platform, clock::get).shutdown(0L); + fixture.manager.getTimer().submit(() -> null).get(2, TimeUnit.SECONDS); + assertTrue(rewards.getDelayedTimer().isTerminated(), "worker exit alone does not finish replay"); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + verify(fixture.mysql, never()).close(); + var acknowledgingCallback = callback.submit(() -> { + if (failedDispatch) dispatch.completeExceptionally(new IllegalStateException("dispatch failed")); + else dispatch.complete(null); + }); + assertTrue(acknowledging.await(2, TimeUnit.SECONDS)); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone(), + "completed dispatch future is not a completed queue acknowledgement"); + verify(fixture.mysql, never()).close(); + releaseAcknowledgement.countDown(); + acknowledgingCallback.get(2, TimeUnit.SECONDS); + platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); + assertTrue(fixture.manager.getTimer().awaitTermination(2, TimeUnit.SECONDS)); + if (failedDispatch) { + assertEquals(1, persisted.size(), "failed dispatch restores a durable retry instead of acknowledging"); + assertTrue(persisted.get(0).contains("%asyncretry%")); + } else assertTrue(persisted.isEmpty()); + verify(fixture.cache, times(1)).flush(fixture.uuid, UserStorage.MYSQL, fixture.storage); + verify(fixture.mysql, times(1)).close(); + assertEquals(TimeUnit.SECONDS.toNanos(15), clock.get()); + } finally { + releaseAcknowledgement.countDown(); + callback.shutdownNow(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + + private final class Fixture { + final AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + final UserManager users = mock(UserManager.class); + final UserDataManager manager = new UserDataManager(plugin); + final SqlUserBackend backend = mock(SqlUserBackend.class); + final SqlUserStorage storage = mock(SqlUserStorage.class); + final UserCacheOwner cache = mock(UserCacheOwner.class); + final MySQL mysql = mock(MySQL.class); + final UUID uuid = UUID.randomUUID(); + final SharedUserDataRuntime shared; + + Fixture() { + when(plugin.getLogger()).thenReturn(mock(Logger.class)); + when(plugin.getDataFolder()).thenReturn(directory); + when(plugin.getOptions()).thenReturn(mock(AdvancedCoreConfigOptions.class)); + when(plugin.getOptions().isOnlineMode()).thenReturn(true); + when(plugin.isLoadUserData()).thenReturn(true); + when(plugin.getLoadedUserManager()).thenReturn(users); + when(plugin.getUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + when(plugin.getNativeUserStorageOwner()).thenReturn( + new AdvancedCorePlugin.UserStorageOwner(UserStorage.MYSQL, mysql, null)); + when(cache.cachedUsers()).thenReturn(java.util.Set.of(uuid)); + when(backend.storageType()).thenReturn(UserStorage.MYSQL); + when(backend.user(uuid)).thenReturn(storage); + shared = new SharedUserDataRuntime(backend, cache); + manager.bindSharedRuntime(shared); + manager.bindSharedSqlBackend(backend, operation -> shared.withStorageReadAdmission(() -> { + operation.run(); + return null; + })); + } + + BukkitRuntimePlatform platform() { + BukkitRuntimePlatform platform = spy(new BukkitRuntimePlatform(plugin)); + doReturn(false).when(platform).canBlockForPreExecutorShutdown(); + return platform; + } + } +} diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java index be15fcf2f..8a35c6fdf 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java @@ -256,6 +256,8 @@ class CoreRuntimeDeadlineTest { private RuntimePlatform platform() { RuntimePlatform platform = mock(RuntimePlatform.class); + // This optional native-admission signal defaults to null on real platforms. + when(platform.admittedStorageRetirementCompletion()).thenReturn(null); when(platform.beforeExecutorShutdown()).thenReturn(List.of()); when(platform.afterExecutorGrace()).thenReturn(List.of()); when(platform.afterExecutorShutdown()).thenReturn(List.of()); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index 5e3bde9d9..d7e51fb45 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -89,6 +89,8 @@ class CoreRuntimeTest { private RuntimePlatform platform() { RuntimePlatform platform = mock(RuntimePlatform.class); + // This optional native-admission signal defaults to null on real platforms. + when(platform.admittedStorageRetirementCompletion()).thenReturn(null); when(platform.beforeExecutorShutdown()).thenReturn(List.of()); when(platform.afterExecutorGrace()).thenReturn(List.of()); when(platform.afterStorageExecutorShutdown()).thenReturn(List.of()); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java index 03244f92b..86265df45 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java @@ -78,6 +78,7 @@ class TimeCheckerShutdownAdmissionTest { @Test void expiredLifecycleDeadlineReturnsBehindAnIndependentSaveAndRetainsStorage() throws Exception { try (Fixture fixture = new Fixture(directory)) { fixture.enableUserStorage(); + fixture.data.blockTimestampPublication.set(true); fixture.startCalendarCheck(); CompletableFuture unrelatedSave = fixture.startIndependentSave(); fixture.allowValidationBehindSave(); @@ -87,6 +88,7 @@ class TimeCheckerShutdownAdmissionTest { AdvancedCoreRuntime runtime = new AdvancedCoreRuntime(platform, () -> TimeUnit.SECONDS.toNanos(15)); + fixture.expectFinalFilePersistence = true; fixture.caller.submit(() -> runtime.shutdown(0L)).get(2, TimeUnit.SECONDS); assertFalse(unrelatedSave.isDone(), "caller return is not an I/O acknowledgement"); @@ -112,6 +114,11 @@ class TimeCheckerShutdownAdmissionTest { verify(fixture.nativeOwner, times(1)).close(); verify(fixture.backend, times(1)).close(); assertEquals(0, fixture.events.get(), "late validation cannot dispatch after platform cleanup"); + assertTrue(fixture.data.timestampPublicationEntered.await(5, TimeUnit.SECONDS)); + assertFalse(fixture.finalFilePersistence.toCompletableFuture().isDone(), + "storage retirement does not confirm final file publication"); + fixture.data.releaseTimestampPublication.countDown(); + fixture.awaitFinalFilePersistence(); assertTrue(YamlConfiguration.loadConfiguration(fixture.data.getdFile()) .getBoolean("TimeTransitions.DAY.Pending"), "cancelled work remains recoverable"); } @@ -183,6 +190,8 @@ private static final class Fixture implements AutoCloseable { UserDataManager manager; MySQL nativeOwner; SqlUserBackend backend; + final CompletableFuture finalFilePersistence = new CompletableFuture<>(); + boolean expectFinalFilePersistence; Fixture(File directory) { Server server = mock(Server.class); @@ -194,6 +203,14 @@ private static final class Fixture implements AutoCloseable { when(plugin.getLogger()).thenReturn(Logger.getAnonymousLogger()); when(plugin.isEnabled()).thenReturn(true); when(plugin.getOptions()).thenReturn(mock(AdvancedCoreConfigOptions.class)); + doAnswer(call -> { + CompletionStage finalFilesStage = call.getArgument(0); + finalFilesStage.whenComplete((ignored, failure) -> { + if (failure == null) finalFilePersistence.complete(null); + else finalFilePersistence.completeExceptionally(failure); + }); + return null; + }).when(plugin).onShutdownComplete(any()); doAnswer(call -> { Event event = call.getArgument(0); assertTrue(event.isAsynchronous()); @@ -254,9 +271,14 @@ CompletionStage beginShutdownPromptly() throws Exception { return caller.submit(() -> { return checker.beginShutdown(); }).get(2, TimeUnit.SECONDS); } + void awaitFinalFilePersistence() throws Exception { + if (expectFinalFilePersistence) finalFilePersistence.get(5, TimeUnit.SECONDS); + } + @Override public void close() throws Exception { data.allowValidation.countDown(); data.releaseCheckedSave.countDown(); + data.releaseTimestampPublication.countDown(); calendar.shutdownNow(); persistence.shutdownNow(); caller.shutdownNow(); @@ -265,6 +287,7 @@ CompletionStage beginShutdownPromptly() throws Exception { assertTrue(calendar.awaitTermination(5, TimeUnit.SECONDS)); assertTrue(persistence.awaitTermination(5, TimeUnit.SECONDS)); assertTrue(caller.awaitTermination(5, TimeUnit.SECONDS)); + awaitFinalFilePersistence(); } } @@ -274,8 +297,12 @@ private static final class GatedServerData extends ServerData { final CountDownLatch validationAttempted = new CountDownLatch(1); final CountDownLatch checkedSaveEntered = new CountDownLatch(1); final CountDownLatch releaseCheckedSave = new CountDownLatch(1); + final CountDownLatch timestampPublicationEntered = new CountDownLatch(1); + final CountDownLatch releaseTimestampPublication = new CountDownLatch(1); final AtomicBoolean blockNextSave = new AtomicBoolean(); + final AtomicBoolean blockTimestampPublication = new AtomicBoolean(); final AtomicBoolean failValidation = new AtomicBoolean(); + private final AtomicInteger finalMutations = new AtomicInteger(); private final AtomicBoolean firstRead = new AtomicBoolean(true); private final AtomicReference calendarThread; @@ -301,6 +328,17 @@ private static final class GatedServerData extends ServerData { return super.isPendingTimeChangeTransition(transition); } + @Override public CompletionStage mutateFinalData( + java.util.function.Consumer mutation) { + // The platform submits inventory first, then timestamp. Gate the latter + // outside the YAML monitor so storage retirement can finish independently. + if (finalMutations.incrementAndGet() == 2 && blockTimestampPublication.get()) { + timestampPublicationEntered.countDown(); + await(releaseTimestampPublication); + } + return super.mutateFinalData(mutation); + } + @Override protected synchronized void saveTimeChangeTransitionData() { if (blockNextSave.compareAndSet(true, false)) { checkedSaveEntered.countDown(); diff --git a/docs/shared-runtime-foundation.md b/docs/shared-runtime-foundation.md index 653341c54..af13a4acc 100644 --- a/docs/shared-runtime-foundation.md +++ b/docs/shared-runtime-foundation.md @@ -69,8 +69,11 @@ interrupt-ignoring JDBC call or filesystem operation cannot retain the JVM. A timeout is reported while the actual stage stays pending. The native storage owner remains open until its executing storage worker ends; accepted final file saves are not discarded to meet the deadline. Preparation that settles after -the deadline still admits its final checkpoint and retains the daemon worker -until that retirement actually settles, without renewing any caller wait. +or just before the deadline still admits its final checkpoint and retains the +daemon worker until that accepted retirement actually settles, without renewing +any caller wait. The runtime distinguishes an accepted native retirement from a +pending time transition or consumer hook; its watchdog cannot discard that +retirement's queued final task. Consequently, durable completion can outlive plugin disable, and a JVM exit before that completion can lose the unfinished work. A timeout never means that persistence succeeded. @@ -89,6 +92,20 @@ The preparation failure remains the reported cause; a storage failure is retaine as a suppressed failure. Successful default hooks preserve the existing retirement stage identity. A failing hook does not bypass accepted user-data flushing. +Delayed-reward admission closes before storage preparation. The additive +`RewardHandler.beginShutdown()` stage reports actual delayed-worker termination; +the existing bounded shutdown methods still cancel queued timers and interrupt +the worker when their wait ends. An interruption request does not release native +storage while an admitted check continues running. After that worker exits, a +manager-worker barrier includes checks it deferred under Spigot/Folia thread +classification and snapshots the existing per-user persisted replay tails. +Those tails include claim/checkpoint acknowledgement or restore callbacks that +can outlive timer execution. Final storage retirement is queued only after those +callbacks finish, so callback-held storage admission can unwind before the final +writer runs. Failed or cancelled preparation and time retirement retain this +ordering and still report their failure. This uses the existing replay owner; +it adds no reward engine or universal delivery guarantee. + ## Native ServerData successor admission Enable acquires native `ServerData.yml` admission before `onPreLoad()` and the From 9dcfa67d502e63217b6f301cb503762bbc459270 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 14 Oct 2026 00:08:35 -0600 Subject: [PATCH 12/28] Separate shutdown lifecycle admission from replay persistence locking --- .../advancedcore/AdvancedCorePlugin.java | 48 ++++++++++-------- .../NativeServerDataAdmissionTest.java | 50 +++++++++++++++++++ 2 files changed, 77 insertions(+), 21 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java index 805110889..1bd7ed0a9 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/AdvancedCorePlugin.java @@ -92,38 +92,44 @@ public abstract class AdvancedCorePlugin extends JavaPlugin { private static AdvancedCorePlugin javaPlugin; + /** Lifecycle admission is independent from consumer replay/checkpoint synchronization. */ + private final Object lifecycleLock = new Object(); private com.bencodez.advancedcore.lifecycle.ServerDataOwnership serverDataOwnership; private boolean advancedCoreShutdownStarted; private boolean advancedCoreEnableStarted; private boolean serverDataStartupRejected; /** Native file admission also covers consumer hooks that read ServerData early. */ - private synchronized void acquireServerDataOwnership() { - if (serverDataOwnership != null && !serverDataOwnership.isReleased()) { - if (!advancedCoreShutdownStarted) return; - throw new IllegalStateException("Previous ServerData shutdown has not finished"); - } - try { - serverDataOwnership = com.bencodez.advancedcore.lifecycle.ServerDataOwnership.acquire( - new File(getDataFolder(), "ServerData.yml").toPath()); - advancedCoreShutdownStarted = false; - advancedCoreEnableStarted = false; - serverDataStartupRejected = false; - } catch (RuntimeException | Error failure) { - serverDataStartupRejected = true; - throw failure; + private void acquireServerDataOwnership() { + synchronized (lifecycleLock) { + if (serverDataOwnership != null && !serverDataOwnership.isReleased()) { + if (!advancedCoreShutdownStarted) return; + throw new IllegalStateException("Previous ServerData shutdown has not finished"); + } + try { + serverDataOwnership = com.bencodez.advancedcore.lifecycle.ServerDataOwnership.acquire( + new File(getDataFolder(), "ServerData.yml").toPath()); + advancedCoreShutdownStarted = false; + advancedCoreEnableStarted = false; + serverDataStartupRejected = false; + } catch (RuntimeException | Error failure) { + serverDataStartupRejected = true; + throw failure; + } } } /** Internal lifecycle fence, independent of consumer shutdown-hook overrides. */ - public final synchronized boolean hasServerDataOwnership() { - return serverDataOwnership != null && !serverDataOwnership.isReleased(); + public final boolean hasServerDataOwnership() { + synchronized (lifecycleLock) { + return serverDataOwnership != null && !serverDataOwnership.isReleased(); + } } /** Internal lifecycle fence, independent of consumer shutdown-hook overrides. */ public final void releaseServerDataOwnershipAfter(CompletionStage completion) { com.bencodez.advancedcore.lifecycle.ServerDataOwnership ownership; - synchronized (this) { ownership = serverDataOwnership; } + synchronized (lifecycleLock) { ownership = serverDataOwnership; } if (ownership != null) ownership.releaseAfter(completion); } @@ -1396,7 +1402,7 @@ private void loadVersionFile() { @Override public void onDisable() { long startedNanos = System.nanoTime(); - synchronized (this) { + synchronized (lifecycleLock) { if (advancedCoreShutdownStarted || serverDataStartupRejected) return; advancedCoreShutdownStarted = true; } @@ -1420,8 +1426,8 @@ public CompletionStage onBeforeStorageShutdown() { } /** - * Called after platform cleanup and submission of the final permission-data save. - * Deferred storage retirement and final file saves may still be running; implementations must + * Called after platform cleanup and submission of the final permission-data save. + * Deferred storage retirement and final file saves may still be running; implementations must * retain any owner needed by their pre-storage stage until that stage settles. * The default is a no-op so existing subclasses retain their lifecycle. */ @@ -1440,7 +1446,7 @@ public void onShutdownComplete(CompletionStage finalFilesStage) { @Override public void onEnable() { - synchronized (this) { + synchronized (lifecycleLock) { acquireServerDataOwnership(); if (advancedCoreEnableStarted) throw new IllegalStateException("AdvancedCore is already enabled"); advancedCoreEnableStarted = true; diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java index b38363344..7b1e9369a 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java @@ -175,6 +175,45 @@ class NativeServerDataAdmissionTest { assertEquals(6, workers.size(), "reenable keeps the established fresh shared executor creation"); } + @Test void shutdownAdmissionDoesNotWaitForConsumerReplayMonitor() throws Exception { + Fixture fixture = fixture(); + CountDownLatch monitorHeld = new CountDownLatch(1); + CountDownLatch releaseMonitor = new CountDownLatch(1); + Thread holder = new Thread(() -> { + synchronized (fixture.plugin) { + monitorHeld.countDown(); + try { + assertTrue(releaseMonitor.await(4, TimeUnit.SECONDS)); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } + } + }, "replay-monitor-holder"); + holder.start(); + assertTrue(monitorHeld.await(1, TimeUnit.SECONDS)); + Thread watchdog = new Thread(() -> { + try { + Thread.sleep(2_000); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); + } finally { + releaseMonitor.countDown(); + } + }, "shutdown-admission-watchdog"); + watchdog.start(); + try { + fixture.plugin.onDisable(); + assertEquals(1, releaseMonitor.getCount(), + "shutdown admission must not wait for a replay/checkpoint monitor"); + } finally { + releaseMonitor.countDown(); + watchdog.interrupt(); + watchdog.join(2_000); + holder.join(2_000); + } + assertFalse(holder.isAlive(), "monitor holder must be released after the test"); + } + @Test void admittedPreparationRetainsAdmissionUntilItsActualSuccess() throws Exception { Fixture predecessor = fixture(); var preparation = new CompletableFuture(); @@ -344,6 +383,7 @@ else assertThrows(java.util.concurrent.ExecutionException.class, private Fixture fixture(Path dataDirectory) { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + initializeLifecycleLock(plugin); Server server = mock(Server.class); when(plugin.getDataFolder()).thenReturn(dataDirectory.toFile()); when(plugin.getName()).thenReturn("native-admission-fixture"); @@ -376,6 +416,16 @@ private Fixture fixture(Path dataDirectory) { return new Fixture(plugin, finalFiles, releaseFence); } + private static void initializeLifecycleLock(AdvancedCorePlugin plugin) { + try { + var lock = AdvancedCorePlugin.class.getDeclaredField("lifecycleLock"); + lock.setAccessible(true); + lock.set(plugin, new Object()); + } catch (ReflectiveOperationException failure) { + throw new AssertionError("Unable to initialize lifecycle test fixture", failure); + } + } + private record Fixture(AdvancedCorePlugin plugin, AtomicReference> finalFiles, AtomicReference> releaseFence) { void awaitRelease() throws Exception { From 425805f6866a2843db6ff02cb4023d0b94d43009 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:09:04 -0600 Subject: [PATCH 13/28] Preserve visible ServerData publications through write failures --- .../advancedcore/api/time/TimeChecker.java | 25 +- .../advancedcore/data/ServerData.java | 180 +++++++-- .../ServerDataPublicationOrderingTest.java | 269 ++++++++++++++ .../time/ServerDataTimeTransitionTest.java | 350 ++++++++++++++++++ .../tests/time/TimeCheckerTest.java | 2 +- 5 files changed, 791 insertions(+), 35 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java index fc51e885d..6fb9983eb 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java @@ -385,7 +385,7 @@ public void update() { TimeChangeTransitionState pending = plugin.getServerDataFile().getPendingTimeChangeTransition(type); if (pending != null) { plugin.getLogger().info("Recovering pending " + type + " time change " + pending.id()); - startDetectedTransition(type, pending); + startDetectedTransition(type, pending, true); return; } } @@ -417,12 +417,12 @@ private void startDetectedTransition(TimeType type) { LocalDateTime current = getTime(); TimeChangeTransitionState persisted = plugin.getServerDataFile().beginTimeChangeTransition(type, periodKey(type, current), markerValue(type, current)); - startDetectedTransition(type, persisted); + startDetectedTransition(type, persisted, false); } - private void startDetectedTransition(TimeType type, TimeChangeTransitionState persisted) { + private void startDetectedTransition(TimeType type, TimeChangeTransitionState persisted, boolean recovered) { TimeChangeTransition transition = new TimeChangeTransition(this, persisted.id(), persisted.periodKey(), type); - startTransition(type, false, true, true, new DurableTransition(transition, persisted)); + startTransition(type, false, true, true, new DurableTransition(transition, persisted, recovered)); } private void startTransition(TimeType type, boolean fake, boolean preDate, boolean postDate, DurableTransition durable) { @@ -483,10 +483,21 @@ private void startTransition(TimeType type, boolean fake, boolean preDate, boole // still acquire transitionLock to close admission and fence event dispatch. boolean validated = false; try { - if (!plugin.getServerDataFile().isPendingTimeChangeTransition(durable.persisted)) { + var serverData = plugin.getServerDataFile(); + if (!serverData.isPendingTimeChangeTransition(durable.persisted)) { plugin.debug("Ignoring stale time change transition " + durable.persisted.id()); return; } + if (durable.recovered) { + // The read is observational. Settle publication on the existing + // checked owner only after reserving the shutdown drain. Recheck + // under that owner's monitor so reload cannot revive a stale slot. + synchronized (serverData) { + if (!serverData.isPendingTimeChangeTransition(durable.persisted)) return; + serverData.beginTimeChangeTransition(type, durable.persisted.periodKey(), + durable.persisted.markerValue()); + } + } validated = true; } finally { // No event or lease has been published yet. Stale/failed validation @@ -721,10 +732,12 @@ private Lease(ActiveTransition active) { private static final class DurableTransition { private final TimeChangeTransition transition; private final TimeChangeTransitionState persisted; + private final boolean recovered; - private DurableTransition(TimeChangeTransition transition, TimeChangeTransitionState persisted) { + private DurableTransition(TimeChangeTransition transition, TimeChangeTransitionState persisted, boolean recovered) { this.transition = transition; this.persisted = persisted; + this.recovered = recovered; } } diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java index 12744ce1b..aa81e174a 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java @@ -27,9 +27,13 @@ * The Class ServerData. */ public class ServerData extends YMLFile { - // Selected only while the checked publisher owns this monitor. Unconfirmed - // native final mutations must never become input to a following whole-file save. + // Selected only while the checked publisher owns this monitor. Publication + // visibility and its durable acknowledgement are separate outcomes. private org.bukkit.configuration.file.FileConfiguration finalDataSnapshot; + private long visiblePublication; + private boolean publicationSettlementPending; + private boolean replacementCleanupPending; + private boolean replacementUncertain; /** * Durable state for one locally detected time transition. The identifier is * deliberately stable while a transition is pending so consumers can store @@ -45,6 +49,7 @@ public ServerData(AdvancedCorePlugin plugin) { @Override public synchronized void saveData() { + prepareSnapshotPublication(); super.saveData(); } @@ -61,6 +66,7 @@ public synchronized void setup() { /** Writes transition state as one forced snapshot and reports failures. */ protected synchronized void saveTimeChangeTransitionData() { + prepareSnapshotPublication(); Path target = getdFile().toPath().toAbsolutePath(); Path parent = target.getParent(); Path temporary = null; @@ -124,17 +130,21 @@ public java.util.concurrent.CompletionStage mutateFinalData( } mutation.accept(candidate); finalDataSnapshot = candidate; + long previousPublication = visiblePublication; try { saveDataChecked(); } - finally { finalDataSnapshot = null; } - // Publication succeeded; retain the existing live YAML identity. - var paths = new java.util.HashSet<>(before.keySet()); - paths.addAll(candidate.getKeys(false)); - for (String path : paths) { - Object next = finalValue(candidate.get(path)); - if (!java.util.Objects.equals(before.get(path), next) - && java.util.Objects.equals(before.get(path), finalValue(live.get(path)))) - setFinalValue(live, path, next); + catch (Throwable failure) { + // A failed directory force/cleanup does not undo a visible move. + if (visiblePublication != previousPublication) { + try { reconcileFinalData(live, candidate, before); } + catch (Throwable reconciliationFailure) { + replacementUncertain = true; + if (failure != reconciliationFailure) failure.addSuppressed(reconciliationFailure); + } + } + throw failure; } + finally { finalDataSnapshot = null; } + reconcileFinalData(live, candidate, before); } return java.util.concurrent.CompletableFuture.completedFuture(null); } catch (Throwable failure) { @@ -142,6 +152,58 @@ public java.util.concurrent.CompletionStage mutateFinalData( } } + private void reconcileFinalData(org.bukkit.configuration.file.FileConfiguration live, + org.bukkit.configuration.file.FileConfiguration candidate, java.util.Map before) { + // Retain the live YAML identity and any concurrent direct live updates. + try { + var paths = new java.util.HashSet<>(before.keySet()); + paths.addAll(candidate.getKeys(false)); + for (String path : paths) { + reconcileFinalValue(live, path, before.get(path), finalValue(candidate.get(path))); + } + } catch (RuntimeException | Error failure) { + replacementUncertain = true; + throw failure; + } + } + + private static void reconcileFinalValue(org.bukkit.configuration.ConfigurationSection live, String path, + Object before, Object next) { + if (java.util.Objects.equals(before, next)) return; + Object current = live.get(path); + if ((current instanceof org.bukkit.configuration.ConfigurationSection || current instanceof java.util.Map) + && (before == null || before instanceof java.util.Map) + && (next == null || next instanceof java.util.Map)) { + // Sections are containers, not indivisible CAS values. A change to one + // child must not discard a published change to its independent sibling. + if (current instanceof java.util.Map) setFinalValue(live, path, finalValue(current)); + var section = live.getConfigurationSection(path); + var previousChildren = finalSectionChildren(before); + var nextChildren = finalSectionChildren(next); + var paths = new java.util.HashSet<>(previousChildren.keySet()); + paths.addAll(nextChildren.keySet()); + for (String child : paths) { + reconcileFinalValue(section, child, previousChildren.get(child), nextChildren.get(child)); + } + // Removing a container may remove unchanged children, but must retain + // concurrent child changes/additions rather than delete their parent. + if (next == null && section.getKeys(false).isEmpty()) live.set(path, null); + } else if (java.util.Objects.equals(before, finalValue(current))) { + // Scalars, lists and incompatible type changes still compete at this + // exact field; a newer live value wins over the published candidate. + setFinalValue(live, path, next); + } + } + + private static java.util.Map finalSectionChildren(Object value) { + // Only section child names use string paths. Values remain untouched so + // maps inside lists retain their original key types and distinct entries. + var children = new java.util.LinkedHashMap(); + if (value instanceof java.util.Map map) + map.forEach((key, child) -> children.put(key.toString(), child)); + return children; + } + private static Object finalValue(Object value) { if (value instanceof org.bukkit.configuration.ConfigurationSection section) { var copy = new java.util.LinkedHashMap(); @@ -175,9 +237,11 @@ protected void forceDirectory(Path directory) throws IOException { } private void replaceSnapshot(Path temporary, Path target) throws IOException { + String intendedHash = snapshotHash(temporary); + String previousHash = Files.isRegularFile(target) ? snapshotHash(target) : null; + boolean previouslyAbsent = Files.notExists(target); try { moveAtomically(temporary, target); - forceDirectory(target.getParent()); } catch (AtomicMoveNotSupportedException unsupported) { Path backup = backupPath(target); Path replacementMarker = replacementMarkerPath(target); @@ -188,7 +252,10 @@ private void replaceSnapshot(Path temporary, Path target) throws IOException { } forceDirectory(target.getParent()); } - Files.writeString(replacementMarker, snapshotHash(temporary), StandardOpenOption.CREATE, + // Even a failed marker write can leave recovery state that must be + // removed before a later save changes the target's hash. + replacementCleanupPending = true; + Files.writeString(replacementMarker, intendedHash, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING, StandardOpenOption.WRITE); try (FileChannel channel = FileChannel.open(replacementMarker, StandardOpenOption.WRITE)) { channel.force(true); @@ -196,19 +263,61 @@ private void replaceSnapshot(Path temporary, Path target) throws IOException { forceDirectory(target.getParent()); try { Files.move(temporary, target, StandardCopyOption.REPLACE_EXISTING); - } catch (IOException replacementFailure) { - if (Files.exists(backup)) { - Files.copy(backup, target, StandardCopyOption.REPLACE_EXISTING); - DurableFiles.forceFile(target); - forceDirectory(target.getParent()); - } + } catch (IOException | RuntimeException | Error replacementFailure) { + inspectFailedMove(target, intendedHash, previousHash, previouslyAbsent, replacementFailure); throw replacementFailure; } + visiblePublication++; + publicationSettlementPending = true; // Keep the recovery marker until the replacement's directory entry is // durable. If this force fails, startup can identify the committed target // by its recorded hash instead of rolling it back as an incomplete move. forceDirectory(target.getParent()); + publicationSettlementPending = false; clearReplacementState(replacementMarker, backup); + replacementCleanupPending = false; + return; + } catch (IOException | RuntimeException | Error replacementFailure) { + inspectFailedMove(target, intendedHash, previousHash, previouslyAbsent, replacementFailure); + throw replacementFailure; + } + visiblePublication++; + publicationSettlementPending = true; + forceDirectory(target.getParent()); + publicationSettlementPending = false; + } + + private void inspectFailedMove(Path target, String intendedHash, String previousHash, + boolean previouslyAbsent, Throwable failure) { + try { + if (Files.isRegularFile(target)) { + String actualHash = snapshotHash(target); + if (intendedHash.equals(actualHash)) { + visiblePublication++; + publicationSettlementPending = true; + return; + } + if (actualHash.equals(previousHash)) return; + } else if (previouslyAbsent && Files.notExists(target)) return; + } catch (IOException | RuntimeException inspectionFailure) { + if (failure != inspectionFailure) failure.addSuppressed(inspectionFailure); + } + // A throwing move may already have changed its target. Never overwrite + // an unknown outcome with stale live YAML or a predecessor backup. + replacementUncertain = true; + } + + private void prepareSnapshotPublication() { + if (replacementUncertain) + throw new IllegalStateException("ServerData replacement outcome is unknown; restart required before saving"); + if (!publicationSettlementPending && !replacementCleanupPending) return; + Path target = getdFile().toPath().toAbsolutePath(); + try { forceDirectory(target.getParent()); } + catch (IOException failure) { throw new UncheckedIOException("Failed to settle ServerData replacement", failure); } + publicationSettlementPending = false; + if (replacementCleanupPending) { + clearReplacementState(replacementMarkerPath(target), backupPath(target)); + replacementCleanupPending = false; } } @@ -265,9 +374,11 @@ private void restoreBackup(Path backup, Path target) { private void clearReplacementState(Path marker, Path backup) { try { - Files.deleteIfExists(backup); + // The target is settled before cleanup. Remove recovery admission + // first; an orphan backup is safe, a marker without its backup is not. + Files.deleteIfExists(marker); forceDirectory(marker.getParent()); - Files.delete(marker); + Files.deleteIfExists(backup); forceDirectory(marker.getParent()); } catch (IOException failure) { throw new UncheckedIOException("Failed to clear interrupted replacement marker", failure); @@ -429,6 +540,7 @@ public synchronized TimeChangeTransitionState beginTimeChangeTransition(TimeType String existingId = getData().getString(path + ".Id", ""); boolean pending = getData().getBoolean(path + ".Pending", false); if (pending && periodKey.equals(existingPeriod) && !existingId.isEmpty()) { + prepareSnapshotPublication(); return new TimeChangeTransitionState(type, existingId, existingPeriod, getData().getString(path + ".Marker", markerValue), true); } @@ -438,6 +550,7 @@ public synchronized TimeChangeTransitionState beginTimeChangeTransition(TimeType Object previousPeriod = getData().get(path + ".Period"); Object previousMarker = getData().get(path + ".Marker"); Object previousPending = getData().get(path + ".Pending"); + long previousPublication = visiblePublication; try { getData().set(path + ".Id", id); getData().set(path + ".Period", periodKey); @@ -445,19 +558,27 @@ public synchronized TimeChangeTransitionState beginTimeChangeTransition(TimeType getData().set(path + ".Pending", true); saveTimeChangeTransitionData(); } catch (RuntimeException | Error failure) { - getData().set(path + ".Id", previousId); - getData().set(path + ".Period", previousPeriod); - getData().set(path + ".Marker", previousMarker); - getData().set(path + ".Pending", previousPending); + if (visiblePublication == previousPublication) { + getData().set(path + ".Id", previousId); + getData().set(path + ".Period", previousPeriod); + getData().set(path + ".Marker", previousMarker); + getData().set(path + ".Pending", previousPending); + } throw failure; } return new TimeChangeTransitionState(type, id, periodKey, markerValue, true); } - /** Returns an unfinished transition even when the current clock has moved on. */ + /** + * Returns an unfinished transition even when the current clock has moved on. + * This observes known visible state without confirming durable publication; + * checked transition admission must settle it before event dispatch. + */ public synchronized TimeChangeTransitionState getPendingTimeChangeTransition(TimeType type) { String path = transitionPath(type); if (!getData().getBoolean(path + ".Pending", false)) return null; + if (replacementUncertain) + throw new IllegalStateException("ServerData replacement outcome is unknown; restart required before processing a pending transition"); String id = getData().getString(path + ".Id", ""); String period = getData().getString(path + ".Period", ""); String marker = getData().getString(path + ".Marker", ""); @@ -503,6 +624,7 @@ public synchronized void completeTimeChangeTransition(TimeChangeTransitionState }; Object previousMarker = getData().get(markerPath); String pendingPath = transitionPath(transition.type()) + ".Pending"; + long previousPublication = visiblePublication; try { switch (transition.type()) { case DAY, WEEK -> getData().set(markerPath, Integer.parseInt(transition.markerValue())); @@ -511,8 +633,10 @@ public synchronized void completeTimeChangeTransition(TimeChangeTransitionState getData().set(pendingPath, false); saveTimeChangeTransitionData(); } catch (RuntimeException | Error failure) { - getData().set(markerPath, previousMarker); - getData().set(pendingPath, true); + if (visiblePublication == previousPublication) { + getData().set(markerPath, previousMarker); + getData().set(pendingPath, true); + } throw failure; } } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java index e87f0058f..2ded44a89 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java @@ -41,6 +41,255 @@ class ServerDataPublicationOrderingTest { verifyPublicationOrder(false); } + @Test void failedDirectoryForceReconcilesCommittedFieldsWithoutOverwritingConcurrentLiveUpdates() + throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(dataFolder); + FailingDirectoryForceServerData data = new FailingDirectoryForceServerData(plugin); + data.setData(new YamlConfiguration()); + data.getData().set("TimedPermissions.example", java.util.List.of("previous")); + data.getData().set("Unrelated.Value", "previous"); + FileConfiguration live = data.getData(); + var result = new CompletableFuture(); + Thread writer = new Thread(() -> data.mutateFinalData(yaml -> { + yaml.set("FullInventory.committed.Time", 456L); + yaml.set("TimedPermissions.example", java.util.List.of("candidate")); + yaml.set("Unrelated.Value", "candidate"); + }).whenComplete((ignored, failure) -> { + if (failure == null) result.complete(null); else result.completeExceptionally(failure); + }), "failing-final-server-data-publication"); + try { + writer.start(); + assertTrue(data.forceStarted.await(2, TimeUnit.SECONDS)); + live.set("TimedPermissions.example", java.util.List.of("concurrent")); + live.set("Unrelated.Value", "concurrent"); + } finally { + data.releaseForce.countDown(); + writer.join(2000); + if (writer.isAlive()) writer.interrupt(); + } + assertFalse(writer.isAlive()); + ExecutionException failure = assertThrows(ExecutionException.class, () -> result.get(2, TimeUnit.SECONDS)); + assertInstanceOf(java.io.UncheckedIOException.class, failure.getCause()); + assertSame(data.forceFailure, failure.getCause().getCause()); + assertSame(live, data.getData()); + assertEquals(456L, live.getLong("FullInventory.committed.Time")); + assertEquals(java.util.List.of("concurrent"), live.getStringList("TimedPermissions.example")); + assertEquals("concurrent", live.getString("Unrelated.Value")); + assertEquals("candidate", YamlConfiguration.loadConfiguration(data.getdFile()).getString("Unrelated.Value")); + data.failing = false; + data.setLastUpdated(); + var restarted = YamlConfiguration.loadConfiguration(data.getdFile()); + assertEquals(456L, restarted.getLong("FullInventory.committed.Time")); + assertEquals(java.util.List.of("concurrent"), restarted.getStringList("TimedPermissions.example")); + assertEquals("concurrent", restarted.getString("Unrelated.Value")); + } + } + + @Test void failedDirectoryForceMergesNestedCommittedSiblingsAndPreservesCompetingChildren() throws Exception { + verifyNestedFinalReconciliation(true); + } + + @Test void successfulPublicationMergesNestedCommittedSiblingsAndPreservesCompetingChildren() throws Exception { + verifyNestedFinalReconciliation(false); + } + + @Test void successfulPublicationPreservesTypedMapKeysInsideUnchangedLists() throws Exception { + verifyUnchangedListPayload(true); + } + + @Test void failedDirectoryForcePreservesTypedMapKeysInsideUnchangedLists() throws Exception { + verifyUnchangedListPayload(false); + } + + private void verifyUnchangedListPayload(boolean successful) throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + Files.writeString(dataFolder.toPath().resolve("ServerData.yml"), """ + Rows: + - 1: first + '1': second + Nested: + 2: nested-integer + '2': nested-string + Values: + - 3: list-integer + '3': list-string + - - 4: deepest-integer + '4': deepest-string + Unrelated: previous + """); + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(dataFolder); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + FailingDirectoryForceServerData data = new FailingDirectoryForceServerData(plugin); + data.setup(); + FileConfiguration live = data.getData(); + java.util.List expectedRows = live.getList("Rows"); + assertPreservedRows(live, expectedRows); + data.failAfterBarrier = !successful; + data.releaseForce.countDown(); + var result = data.mutateFinalData(yaml -> yaml.set("Unrelated", "candidate")).toCompletableFuture(); + if (successful) result.get(2, TimeUnit.SECONDS); + else { + CompletionException failure = assertThrows(CompletionException.class, result::join); + assertInstanceOf(java.io.UncheckedIOException.class, failure.getCause()); + assertSame(data.forceFailure, failure.getCause().getCause()); + } + assertSame(live, data.getData()); + assertEquals("candidate", live.getString("Unrelated")); + assertPreservedRows(live, expectedRows); + var published = YamlConfiguration.loadConfiguration(data.getdFile()); + assertEquals("candidate", published.getString("Unrelated")); + assertPreservedRows(published, expectedRows); + data.failing = false; + data.setLastUpdated(); + data.reloadData(); + assertPreservedRows(data.getData(), expectedRows); + assertPreservedRows(YamlConfiguration.loadConfiguration(data.getdFile()), expectedRows); + assertEquals(!successful, result.isCompletedExceptionally()); + } + } + + private static void assertPreservedRows(FileConfiguration yaml, java.util.List expectedRows) { + assertNotNull(expectedRows); + assertEquals(expectedRows, yaml.getList("Rows")); + Map row = (Map) yaml.getList("Rows").get(0); + assertEquals("first", row.get(1)); + assertEquals("second", row.get("1")); + Map nested = (Map) row.get("Nested"); + assertEquals("nested-integer", nested.get(2)); + assertEquals("nested-string", nested.get("2")); + java.util.List values = (java.util.List) nested.get("Values"); + Map listMap = (Map) values.get(0); + assertEquals("list-integer", listMap.get(3)); + assertEquals("list-string", listMap.get("3")); + Map deepest = (Map) ((java.util.List) values.get(1)).get(0); + assertEquals("deepest-integer", deepest.get(4)); + assertEquals("deepest-string", deepest.get("4")); + } + + private void verifyNestedFinalReconciliation(boolean failForce) throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(dataFolder); + FailingDirectoryForceServerData data = new FailingDirectoryForceServerData(plugin); + data.failAfterBarrier = failForce; + data.setData(new YamlConfiguration()); + FileConfiguration live = data.getData(); + live.set("TimedPermissions.existing", java.util.List.of("previous")); + live.set("Nested.branch.existing", "previous"); + live.set("Nested.branch.removed", "previous"); + live.set("Nested.branch.competingRemoval", "previous"); + live.set("Nested.branch.competingValue", "previous"); + live.set("Nested.removedTree.unchanged", "previous"); + live.set("Nested.removedTree.competing", "previous"); + live.set("Nested.scalarToSection", "previous"); + live.set("Nested.competingScalarToSection", "previous"); + live.set("Nested.sectionToScalar.child", "previous"); + live.set("Nested.competingSectionToScalar.child", "previous"); + live.set("Nested.concurrentlyRemoved.child", "previous"); + live.set("Nested.concurrentlyReplaced.child", "previous"); + live.set("RawMap", Map.of("existing", "previous", "removed", "previous", + "competing", "previous", "numeric", Map.of(1, "previous", 2, "previous"))); + var result = new CompletableFuture(); + Thread writer = new Thread(() -> data.mutateFinalData(yaml -> { + yaml.set("TimedPermissions.committed", java.util.List.of("candidate")); + yaml.set("Nested.branch.committed", "candidate"); + yaml.set("Nested.branch.removed", null); + yaml.set("Nested.branch.competingRemoval", null); + yaml.set("Nested.branch.competingValue", "candidate"); + yaml.set("Nested.removedTree", null); + yaml.set("Nested.addedTree.committed", "candidate"); + yaml.set("Nested.addedTree.competing", "candidate"); + yaml.set("Nested.scalarToSection", null); + yaml.set("Nested.scalarToSection.child", "candidate"); + yaml.set("Nested.competingScalarToSection", null); + yaml.set("Nested.competingScalarToSection.child", "candidate"); + yaml.set("Nested.sectionToScalar", "candidate"); + yaml.set("Nested.competingSectionToScalar", "candidate"); + yaml.set("Nested.concurrentlyRemoved.committed", "candidate"); + yaml.set("Nested.concurrentlyReplaced.committed", "candidate"); + yaml.set("RawMap.committed", "candidate"); + yaml.set("RawMap.removed", null); + yaml.set("RawMap.competing", "candidate"); + yaml.set("RawMap.numeric.1", "candidate"); + }).whenComplete((ignored, failure) -> { + if (failure == null) result.complete(null); else result.completeExceptionally(failure); + }), "nested-final-server-data-publication"); + try { + writer.start(); + assertTrue(data.forceStarted.await(2, TimeUnit.SECONDS)); + live.set("TimedPermissions.existing", java.util.List.of("concurrent")); + live.set("Nested.branch.existing", "concurrent"); + live.set("Nested.branch.competingRemoval", "concurrent"); + live.set("Nested.branch.competingValue", "concurrent"); + live.set("Nested.removedTree.competing", "concurrent"); + live.set("Nested.removedTree.added", "concurrent"); + live.set("Nested.addedTree.added", "concurrent"); + live.set("Nested.addedTree.competing", "concurrent"); + live.set("Nested.competingScalarToSection", "concurrent"); + live.set("Nested.competingSectionToScalar.child", "concurrent"); + live.set("Nested.concurrentlyRemoved", null); + live.set("Nested.concurrentlyReplaced", "concurrent"); + live.set("RawMap", Map.of("existing", "concurrent", "removed", "previous", + "competing", "concurrent", "added", "concurrent", + "numeric", Map.of(1, "previous", 2, "concurrent", 3, "concurrent"))); + } finally { + data.releaseForce.countDown(); + writer.join(2000); + if (writer.isAlive()) writer.interrupt(); + } + assertFalse(writer.isAlive()); + if (failForce) { + ExecutionException failure = assertThrows(ExecutionException.class, + () -> result.get(2, TimeUnit.SECONDS)); + assertInstanceOf(java.io.UncheckedIOException.class, failure.getCause()); + assertSame(data.forceFailure, failure.getCause().getCause()); + } else result.get(2, TimeUnit.SECONDS); + assertSame(live, data.getData()); + var published = YamlConfiguration.loadConfiguration(data.getdFile()); + assertEquals(java.util.List.of("candidate"), published.getStringList("TimedPermissions.committed")); + assertEquals(java.util.List.of("previous"), published.getStringList("TimedPermissions.existing")); + assertEquals("candidate", published.getString("Nested.branch.committed")); + assertNestedFinalValues(live); + data.failing = false; + data.setLastUpdated(); + assertNestedFinalValues(YamlConfiguration.loadConfiguration(data.getdFile())); + assertEquals(failForce, result.isCompletedExceptionally()); + } + } + + private static void assertNestedFinalValues(FileConfiguration yaml) { + assertEquals(java.util.List.of("candidate"), yaml.getStringList("TimedPermissions.committed")); + assertEquals(java.util.List.of("concurrent"), yaml.getStringList("TimedPermissions.existing")); + assertEquals("candidate", yaml.getString("Nested.branch.committed")); + assertEquals("concurrent", yaml.getString("Nested.branch.existing")); + assertFalse(yaml.contains("Nested.branch.removed")); + assertEquals("concurrent", yaml.getString("Nested.branch.competingRemoval")); + assertEquals("concurrent", yaml.getString("Nested.branch.competingValue")); + assertFalse(yaml.contains("Nested.removedTree.unchanged")); + assertEquals("concurrent", yaml.getString("Nested.removedTree.competing")); + assertEquals("concurrent", yaml.getString("Nested.removedTree.added")); + assertEquals("candidate", yaml.getString("Nested.addedTree.committed")); + assertEquals("concurrent", yaml.getString("Nested.addedTree.competing")); + assertEquals("concurrent", yaml.getString("Nested.addedTree.added")); + assertEquals("candidate", yaml.getString("Nested.scalarToSection.child")); + assertEquals("concurrent", yaml.getString("Nested.competingScalarToSection")); + assertEquals("candidate", yaml.getString("Nested.sectionToScalar")); + assertEquals("concurrent", yaml.getString("Nested.competingSectionToScalar.child")); + assertFalse(yaml.contains("Nested.concurrentlyRemoved")); + assertEquals("concurrent", yaml.getString("Nested.concurrentlyReplaced")); + assertEquals("candidate", yaml.getString("RawMap.committed")); + assertEquals("concurrent", yaml.getString("RawMap.existing")); + assertEquals("concurrent", yaml.getString("RawMap.competing")); + assertEquals("concurrent", yaml.getString("RawMap.added")); + assertFalse(yaml.contains("RawMap.removed")); + assertEquals("candidate", yaml.getString("RawMap.numeric.1")); + assertEquals("concurrent", yaml.getString("RawMap.numeric.2")); + assertEquals("concurrent", yaml.getString("RawMap.numeric.3")); + } + @Test void legacyFinalMutationReachesLiveOwnerAndSavedYamlExactlyOnceWithoutClaimingDurability() { try (var scheduler = mockConstruction(BukkitScheduler.class)) { LegacySavingOwner data = legacySavingOwner(); @@ -477,4 +726,24 @@ void install() { }); } } + + private static final class FailingDirectoryForceServerData extends ServerData { + final CountDownLatch forceStarted = new CountDownLatch(1); + final CountDownLatch releaseForce = new CountDownLatch(1); + final java.io.IOException forceFailure = new java.io.IOException("synthetic directory force failure"); + boolean failing = true; + boolean failAfterBarrier = true; + FailingDirectoryForceServerData(AdvancedCorePlugin plugin) { super(plugin); } + @Override protected void forceDirectory(java.nio.file.Path directory) throws java.io.IOException { + if (!failing) { super.forceDirectory(directory); return; } + forceStarted.countDown(); + try { + if (!releaseForce.await(5, TimeUnit.SECONDS)) throw new java.io.IOException("force barrier timed out"); + } catch (InterruptedException interrupted) { + Thread.currentThread().interrupt(); throw new java.io.IOException(interrupted); + } + if (failAfterBarrier) throw forceFailure; + super.forceDirectory(directory); + } + } } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java index cece8de90..f1a8dc283 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java @@ -3,6 +3,8 @@ import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertArrayEquals; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import static org.mockito.Mockito.mock; @@ -118,12 +120,360 @@ void failedFinalMutationCannotLeakPartialChangesIntoTheNextWholeFileSave() { assertEquals(789L, restarted.getLong("LastUpdated")); } + @Test + void atomicVisibleFinalMutationSurvivesDirectoryForceFailureAndLaterSaves() throws Exception { + verifyVisibleFinalMutationSurvivesFailure(false, 1); + } + + @Test + void fallbackVisibleFinalMutationSurvivesDirectoryForceFailureAndLaterSaves() throws Exception { + verifyVisibleFinalMutationSurvivesFailure(true, 3); + } + + @Test + void fallbackVisibleFinalMutationSurvivesMarkerCleanupForceFailureAndLaterSaves() throws Exception { + verifyVisibleFinalMutationSurvivesFailure(true, 4); + } + + @Test + void fallbackVisibleFinalMutationSurvivesBackupCleanupForceFailureAndLaterSaves() throws Exception { + verifyVisibleFinalMutationSurvivesFailure(true, 5); + } + + private void verifyVisibleFinalMutationSurvivesFailure(boolean fallback, int failingForce) throws Exception { + PublicationFailureServerData data = new PublicationFailureServerData(plugin(), fallback); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + var live = data.getData(); + data.directoryForces = 0; + data.failingForce = failingForce; + var result = data.mutateFinalData(yaml -> { + yaml.set("FullInventory", null); + yaml.set("FullInventory.committed.Time", 456L); + yaml.set("FullInventory.committed.Items.0", "stored-item"); + }).toCompletableFuture(); + var failure = assertThrows(java.util.concurrent.CompletionException.class, result::join); + assertTrue(failure.getCause() instanceof UncheckedIOException); + assertEquals(data.forceFailure, failure.getCause().getCause(), "the original durability failure remains observable"); + assertSame(live, data.getData()); + assertEquals(456L, live.getLong("FullInventory.committed.Time")); + assertFalse(live.contains("FullInventory.previous")); + assertEquals("stored-item", YamlConfiguration.loadConfiguration(data.getdFile()) + .getString("FullInventory.committed.Items.0")); + + data.failingForce = 0; + data.setLastUpdated(); + data.setData("TimedPermissions.example", java.util.List.of("example.use%line%1234")); + data.mutateFinalData(yaml -> yaml.set("IgnoreTime", true)).toCompletableFuture().join(); + assertTrue(result.isCompletedExceptionally(), "later saves cannot acknowledge the failed stage"); + TestServerData restarted = serverData(); + restarted.setup(); + assertEquals(456L, restarted.getData().getLong("FullInventory.committed.Time")); + assertEquals("stored-item", restarted.getData().getString("FullInventory.committed.Items.0")); + assertFalse(restarted.getData().contains("FullInventory.previous")); + assertTrue(restarted.getLastUpdated() > 0L); + assertEquals(java.util.List.of("example.use%line%1234"), + restarted.getData().getStringList("TimedPermissions.example")); + assertTrue(restarted.isIgnoreTime()); + Path target = data.getdFile().toPath(); + assertFalse(Files.exists(target.resolveSibling("ServerData.yml.replacement-pending"))); + assertFalse(Files.exists(target.resolveSibling("ServerData.yml.backup"))); + } + + @Test + void failedFallbackCleanupFencesOrdinaryAndCheckedWritesUntilItSettles() throws Exception { + PublicationFailureServerData data = new PublicationFailureServerData(plugin(), true); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + data.directoryForces = 0; + data.failingForce = 3; + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("FullInventory.committed.Time", 456L)).toCompletableFuture().join()); + byte[] visibleSnapshot = Files.readAllBytes(data.getdFile().toPath()); + data.failEveryForce = true; + assertThrows(UncheckedIOException.class, data::setLastUpdated); + assertThrows(UncheckedIOException.class, data::saveData); + assertThrows(UncheckedIOException.class, data::saveDataChecked); + assertArrayEquals(visibleSnapshot, Files.readAllBytes(data.getdFile().toPath())); + data.failEveryForce = false; + data.failingForce = 0; + data.saveData(); + TestServerData restarted = serverData(); + restarted.setup(); + assertEquals(456L, restarted.getData().getLong("FullInventory.committed.Time")); + assertTrue(restarted.getLastUpdated() > 0L); + } + + @Test + void failedFallbackBeforeMoveDiscardsCandidateAndAllowsLaterSave() throws Exception { + PublicationFailureServerData data = new PublicationFailureServerData(plugin(), true); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + data.directoryForces = 0; + data.failingForce = 2; + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("FullInventory.unpublished.Time", 456L)).toCompletableFuture().join()); + assertFalse(data.getData().contains("FullInventory.unpublished")); + data.failingForce = 0; + data.setLastUpdated(); + TestServerData restarted = serverData(); + restarted.setup(); + assertEquals(123L, restarted.getData().getLong("FullInventory.previous.Time")); + assertFalse(restarted.getData().contains("FullInventory.unpublished")); + assertTrue(restarted.getLastUpdated() > 0L); + } + + @Test + void moveThatThrowsAfterReplacingReconcilesVisibleCandidateWithoutAcknowledgingIt() throws Exception { + MoveFailureServerData data = new MoveFailureServerData(plugin()); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + data.moveFailure = 1; + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("FullInventory.committed.Time", 456L)).toCompletableFuture().join()); + assertEquals(456L, data.getData().getLong("FullInventory.committed.Time")); + data.moveFailure = 0; + data.setLastUpdated(); + assertEquals(456L, YamlConfiguration.loadConfiguration(data.getdFile()).getLong("FullInventory.committed.Time")); + } + + @Test + void moveThatFailsBeforeReplacingDiscardsCandidateAndAllowsLaterSave() throws Exception { + MoveFailureServerData data = new MoveFailureServerData(plugin()); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + data.moveFailure = 2; + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("FullInventory.unpublished.Time", 456L)).toCompletableFuture().join()); + assertFalse(data.getData().contains("FullInventory.unpublished")); + data.moveFailure = 0; + data.setLastUpdated(); + var saved = YamlConfiguration.loadConfiguration(data.getdFile()); + assertEquals(123L, saved.getLong("FullInventory.previous.Time")); + assertFalse(saved.contains("FullInventory.unpublished")); + } + + @Test + void ambiguousMoveOutcomeCannotBeOverwrittenByLaterWholeFileSaves() throws Exception { + MoveFailureServerData data = new MoveFailureServerData(plugin()); + data.setup(); + data.getData().set("FullInventory.previous.Time", 123L); + data.saveDataChecked(); + data.moveFailure = 3; + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("FullInventory.unpublished.Time", 456L)).toCompletableFuture().join()); + byte[] unknownSnapshot = Files.readAllBytes(data.getdFile().toPath()); + data.moveFailure = 0; + assertThrows(IllegalStateException.class, data::setLastUpdated); + assertThrows(IllegalStateException.class, data::saveData); + assertThrows(IllegalStateException.class, data::saveDataChecked); + assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( + yaml -> yaml.set("IgnoreTime", true)).toCompletableFuture().join()); + assertArrayEquals(unknownSnapshot, Files.readAllBytes(data.getdFile().toPath())); + } + + @Test + void visibleTransitionInitializationIsRetainedAfterDirectoryForceFailure() throws Exception { + PublicationFailureServerData data = new PublicationFailureServerData(plugin(), false); + data.setup(); + data.failingForce = 1; + assertThrows(UncheckedIOException.class, + () -> data.beginTimeChangeTransition(TimeType.DAY, "2026-09-21", "21")); + assertTrue(data.getData().getBoolean("TimeTransitions.DAY.Pending")); + data.failEveryForce = true; + int forcesBeforeRead = data.directoryForces; + assertEquals("DAY:2026-09-21", data.getPendingTimeChangeTransition(TimeType.DAY).id()); + assertEquals(forcesBeforeRead, data.directoryForces, "pending observation remains memory-only after a save failure"); + assertThrows(UncheckedIOException.class, + () -> data.beginTimeChangeTransition(TimeType.DAY, "2026-09-21", "21")); + data.failEveryForce = false; + data.failingForce = 0; + assertEquals("DAY:2026-09-21", data.beginTimeChangeTransition(TimeType.DAY, "2026-09-21", "21").id()); + assertEquals("DAY:2026-09-21", data.getPendingTimeChangeTransition(TimeType.DAY).id()); + data.setLastUpdated(); + TestServerData restarted = serverData(); + restarted.setup(); + assertEquals("DAY:2026-09-21", restarted.getPendingTimeChangeTransition(TimeType.DAY).id()); + } + + @Test + void visibleTransitionCompletionIsRetainedAfterDirectoryForceFailure() throws Exception { + PublicationFailureServerData data = new PublicationFailureServerData(plugin(), false); + data.setup(); + data.getData().set("PrevDay", 20); + var transition = data.beginTimeChangeTransition(TimeType.DAY, "2026-09-21", "21"); + data.directoryForces = 0; + data.failingForce = 1; + assertThrows(UncheckedIOException.class, () -> data.completeTimeChangeTransition(transition)); + assertEquals(21, data.getPrevDay()); + assertFalse(data.getData().getBoolean("TimeTransitions.DAY.Pending")); + data.failingForce = 0; + data.setLastUpdated(); + TestServerData restarted = serverData(); + restarted.setup(); + assertEquals(21, restarted.getPrevDay()); + assertNull(restarted.getPendingTimeChangeTransition(TimeType.DAY)); + } + + @Test + void calendarWorkerRetriesVisiblePendingPublicationBeforeDispatchWithoutAnUnrelatedSave() throws Exception { + try (CalendarRetryFixture fixture = new CalendarRetryFixture()) { + fixture.data.failingForce = 1; + assertThrows(UncheckedIOException.class, + () -> fixture.data.beginTimeChangeTransition(TimeType.DAY, "2025-01-02", "2")); + fixture.data.failEveryForce = true; + fixture.requestCheck(); + assertTrue(fixture.dayIds.isEmpty(), "failed settlement cannot dispatch the retained pending record"); + assertFalse(fixture.checker.isActiveProcessing(), "failed admission must release its drain reservation"); + assertEquals("DAY:2025-01-02", fixture.data.getPendingTimeChangeTransition(TimeType.DAY).id()); + fixture.data.failEveryForce = false; + fixture.data.failingForce = 0; + fixture.requestCheck(); + assertEquals(java.util.List.of("DAY:2025-01-02"), fixture.dayIds); + assertEquals(java.util.List.of(3), fixture.dayPublicationForces, + "the worker settles the failed force before dispatching the original identity"); + assertNull(fixture.data.getPendingTimeChangeTransition(TimeType.DAY)); + assertEquals(2, fixture.data.getPrevDay()); + } + } + + @Test + void calendarWorkerSettlesFailedVisibleCompletionBeforeDispatchingTheNextPeriod() throws Exception { + try (CalendarRetryFixture fixture = new CalendarRetryFixture()) { + fixture.data.failingForce = 2; + fixture.requestCheck(); + assertEquals(java.util.List.of("DAY:2025-01-02"), fixture.dayIds); + assertEquals(2, fixture.data.getPrevDay()); + assertNull(fixture.data.getPendingTimeChangeTransition(TimeType.DAY)); + assertFalse(YamlConfiguration.loadConfiguration(fixture.data.getdFile()) + .getBoolean("TimeTransitions.DAY.Pending")); + fixture.now.set(java.time.Instant.parse("2025-01-03T12:00:00Z")); + fixture.data.failEveryForce = true; + fixture.requestCheck(); + assertEquals(java.util.List.of("DAY:2025-01-02"), fixture.dayIds, + "the next period cannot dispatch while the prior completion remains unsettled"); + assertEquals("DAY:2025-01-02", fixture.data.getData().getString("TimeTransitions.DAY.Id")); + fixture.data.failEveryForce = false; + fixture.data.failingForce = 0; + fixture.requestCheck(); + assertEquals(java.util.List.of("DAY:2025-01-02", "DAY:2025-01-03"), fixture.dayIds, + "recovery must not replay the already completed period"); + assertEquals(java.util.List.of(1, 5), fixture.dayPublicationForces); + assertEquals(3, YamlConfiguration.loadConfiguration(fixture.data.getdFile()).getInt("PrevDay")); + } + } + + private final class CalendarRetryFixture implements AutoCloseable { + private final java.util.concurrent.atomic.AtomicReference calendarThread = + new java.util.concurrent.atomic.AtomicReference<>(); + private final java.util.concurrent.atomic.AtomicReference now = + new java.util.concurrent.atomic.AtomicReference<>(java.time.Instant.parse("2025-01-02T12:00:00Z")); + private final java.util.concurrent.ScheduledExecutorService calendar = + java.util.concurrent.Executors.newSingleThreadScheduledExecutor(task -> { + Thread worker = com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime + .daemonThreads("calendar-publication-retry-test").newThread(task); + calendarThread.set(worker); + return worker; + }); + private final PublicationFailureServerData data; + private final com.bencodez.advancedcore.api.time.TimeChecker checker; + private final java.util.List dayIds = new java.util.ArrayList<>(); + private final java.util.List dayPublicationForces = new java.util.ArrayList<>(); + + private CalendarRetryFixture() { + AdvancedCorePlugin plugin = plugin(); + var options = mock(com.bencodez.advancedcore.AdvancedCoreConfigOptions.class); + when(options.getTimeZone()).thenReturn("UTC"); + when(options.isTimeChangeFailSafeBypass()).thenReturn(true); + when(plugin.getOptions()).thenReturn(options); + when(plugin.isEnabled()).thenReturn(true); + var eventManager = mock(org.bukkit.plugin.PluginManager.class); + when(plugin.getServer().getPluginManager()).thenReturn(eventManager); + data = new PublicationFailureServerData(plugin, false); + data.setup(); + when(plugin.getServerDataFile()).thenReturn(data); + data.getData().set("Month", "JANUARY"); + data.getData().set("PrevWeek", com.bencodez.advancedcore.api.time.TimeCalculation.weekNumber( + java.time.LocalDateTime.of(2025, 1, 2, 12, 0), 0, java.util.Locale.getDefault())); + data.getData().set("PrevDay", 1); + data.saveDataChecked(); + data.directoryForces = 0; + org.mockito.Mockito.doAnswer(call -> { + org.bukkit.event.Event event = call.getArgument(0); + assertTrue(event.isAsynchronous()); + assertSame(calendarThread.get(), Thread.currentThread()); + if (event instanceof com.bencodez.advancedcore.api.time.events.DayChangeEvent day) { + dayIds.add(day.getTransition().getId()); + dayPublicationForces.add(data.directoryForces); + } + return null; + }).when(eventManager).callEvent(org.mockito.ArgumentMatchers.any(org.bukkit.event.Event.class)); + checker = new com.bencodez.advancedcore.api.time.TimeChecker(plugin, new java.time.Clock() { + @Override public java.time.ZoneId getZone() { return java.time.ZoneOffset.UTC; } + @Override public java.time.Clock withZone(java.time.ZoneId zone) { return java.time.Clock.fixed(now.get(), zone); } + @Override public java.time.Instant instant() { return now.get(); } + }); + checker.setTimer(calendar); + } + + private void requestCheck() throws Exception { + checker.requestUpdate(); + calendar.submit(() -> { }).get(5, java.util.concurrent.TimeUnit.SECONDS); + } + + @Override public void close() throws Exception { + try { checker.beginShutdown().toCompletableFuture().get(5, java.util.concurrent.TimeUnit.SECONDS); } + finally { + calendar.shutdownNow(); + assertTrue(calendar.awaitTermination(5, java.util.concurrent.TimeUnit.SECONDS)); + } + } + } + private static final class LegacySaveOwner extends ServerData { private int saves; private LegacySaveOwner(AdvancedCorePlugin plugin) { super(plugin); } @Override public void saveData() { saves++; } } + private static final class PublicationFailureServerData extends ServerData { + private final boolean fallback; + private final IOException forceFailure = new IOException("synthetic directory force failure"); + private int directoryForces; + private int failingForce; + private boolean failEveryForce; + + private PublicationFailureServerData(AdvancedCorePlugin plugin, boolean fallback) { + super(plugin); + this.fallback = fallback; + } + + @Override protected void moveAtomically(Path source, Path target) throws IOException { + if (fallback) throw new AtomicMoveNotSupportedException(source.toString(), target.toString(), "test provider"); + super.moveAtomically(source, target); + } + + @Override protected void forceDirectory(Path directory) throws IOException { + if (++directoryForces == failingForce || failEveryForce) throw forceFailure; + super.forceDirectory(directory); + } + } + + private static final class MoveFailureServerData extends ServerData { + private int moveFailure; + private MoveFailureServerData(AdvancedCorePlugin plugin) { super(plugin); } + @Override protected void moveAtomically(Path source, Path target) throws IOException { + if (moveFailure == 0 || moveFailure == 1) super.moveAtomically(source, target); + if (moveFailure == 3) Files.writeString(target, "Unknown: outcome\n"); + if (moveFailure != 0) throw new IOException("synthetic move failure"); + } + } + @Test void atomicServerDataSavePropagatesReplacementFailure() throws Exception { TestServerData data = serverData(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java index 638ca7a50..af5964632 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java @@ -865,7 +865,7 @@ public void pendingTransitionIsRecoveredWithItsOriginalIdBeforeClockDetection() assertNotNull(observed.get()); assertEquals("DAY:2025-01-02", observed.get().getId()); - verify(serverDataFile, Mockito.never()).beginTimeChangeTransition(any(), anyString(), anyString()); + verify(serverDataFile).beginTimeChangeTransition(TimeType.DAY, transition.periodKey(), transition.markerValue()); verify(serverDataFile).completeTimeChangeTransition(transition); } From ca7a4461f4547e3fe9ad1dbcdbac11de5205d04c Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Wed, 14 Oct 2026 02:28:01 -0600 Subject: [PATCH 14/28] Publish storage retirement admission before worker scheduling --- .../bukkit/runtime/BukkitRuntimePlatform.java | 9 +++- .../NativeServerDataAdmissionTest.java | 52 +++++++++++++++++++ 2 files changed, 60 insertions(+), 1 deletion(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index e575c1472..5af12a14f 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -310,6 +310,14 @@ private CompletionStage deferUserStorageRetirement(Throwable preparationFa return retirement; } var result = new CompletableFuture(); + // Publish the admission before handing the task to the worker. The + // lifecycle watchdog can run re-entrantly from an executor's execute + // implementation (and can otherwise race the return from execute). + // Reserving the completion first ensures it retains the worker and + // native owner even while the task is being accepted. If acceptance + // is rejected, the same stage is completed exceptionally below, so a + // rejection cannot masquerade as a successful retirement. + admittedUserStorageRetirement = result; try { Objects.requireNonNull(getUserStorageTimer(), "user storage timer").execute(() -> { try { @@ -338,7 +346,6 @@ private CompletionStage deferUserStorageRetirement(Throwable preparationFa result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); } }); - admittedUserStorageRetirement = result; } catch (RuntimeException | Error failure) { // Do not detach the shared runtime or pretend rejected work flushed storage. result.completeExceptionally(preparationAndStorageFailure(preparationFailure, failure)); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java index 7b1e9369a..eda7550c6 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java @@ -13,6 +13,7 @@ import java.util.concurrent.CompletionStage; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; @@ -34,6 +35,8 @@ import com.bencodez.advancedcore.api.permissions.PermissionHandler; import com.bencodez.advancedcore.api.rewards.RewardHandler; import com.bencodez.advancedcore.api.time.TimeChecker; +import com.bencodez.advancedcore.api.user.UserManager; +import com.bencodez.advancedcore.api.user.usercache.UserDataManager; import com.bencodez.advancedcore.bukkit.runtime.BukkitRuntimePlatform; import com.bencodez.advancedcore.command.CommandLoader; import com.bencodez.advancedcore.core.runtime.AdvancedCoreRuntime; @@ -379,6 +382,55 @@ else assertThrows(java.util.concurrent.ExecutionException.class, } } + @Test void storageRetirementAdmissionIsVisibleBeforeWorkerAcceptReturns() throws Exception { + Fixture fixture = fixture(); + UserManager users = mock(UserManager.class); + UserDataManager manager = mock(UserDataManager.class); + ScheduledExecutorService worker = mock(ScheduledExecutorService.class); + when(fixture.plugin.isLoadUserData()).thenReturn(true); + when(fixture.plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + when(manager.getTimer()).thenReturn(worker); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(fixture.plugin); + AtomicReference> observed = new AtomicReference<>(); + doAnswer(invocation -> { + observed.set(platform.admittedStorageRetirementCompletion()); + return null; + }).when(worker).execute(any(Runnable.class)); + + CompletionStage returned = invokeDeferredRetirement(platform); + + assertSame(returned, observed.get(), + "the watchdog-visible admission must be reserved before execute returns"); + assertSame(returned, platform.admittedStorageRetirementCompletion()); + } + + @Test void rejectedStorageRetirementAdmissionRemainsFailedAndObservable() throws Exception { + Fixture fixture = fixture(); + UserManager users = mock(UserManager.class); + UserDataManager manager = mock(UserDataManager.class); + ScheduledExecutorService worker = mock(ScheduledExecutorService.class); + when(fixture.plugin.isLoadUserData()).thenReturn(true); + when(fixture.plugin.getLoadedUserManager()).thenReturn(users); + when(users.getDataManager()).thenReturn(manager); + when(manager.getTimer()).thenReturn(worker); + doThrow(new RejectedExecutionException("retirement worker stopped")).when(worker).execute(any(Runnable.class)); + BukkitRuntimePlatform platform = new BukkitRuntimePlatform(fixture.plugin); + + CompletionStage returned = invokeDeferredRetirement(platform); + + assertSame(returned, platform.admittedStorageRetirementCompletion()); + assertTrue(returned.toCompletableFuture().isCompletedExceptionally(), + "worker rejection must not leave an apparently pending successful admission"); + } + + private static CompletionStage invokeDeferredRetirement(BukkitRuntimePlatform platform) + throws Exception { + var method = BukkitRuntimePlatform.class.getDeclaredMethod("deferUserStorageRetirement", Throwable.class); + method.setAccessible(true); + return (CompletionStage) method.invoke(platform, new Object[] { null }); + } + private Fixture fixture() { return fixture(directory); } private Fixture fixture(Path dataDirectory) { From 778800f0c4b355320f1d67875567a43c0d4ed36a Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 15 Oct 2026 19:51:12 -0600 Subject: [PATCH 15/28] Fence persisted replay checks before shutdown snapshots --- .../api/rewards/RewardHandler.java | 50 +++++++++- .../api/user/AdvancedCoreUser.java | 16 +++- .../CoreRuntimeAdmittedRetirementTest.java | 92 ++++++++++++++++++- .../lifecycle/CoreRuntimeDeadlineTest.java | 10 +- .../tests/user/AdvancedCoreUserTest.java | 3 +- 5 files changed, 159 insertions(+), 12 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java index 9afd68606..3e457fa59 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/rewards/RewardHandler.java @@ -49,6 +49,40 @@ public class RewardHandler { AdvancedCorePlugin plugin; + private final Object replayAdmissionLock = new Object(); + private final CompletableFuture replayChecksDrained = new CompletableFuture<>(); + private boolean replayAdmissionClosed; + private int activeReplayChecks; + + /** + * Admit the synchronous claim/registration part of a persisted queue check. + * Closing its token means every async replay tail has been registered, not + * that its reward or durable acknowledgement has completed. + */ + public PersistedReplayCheck admitPersistedReplayCheck() { + synchronized (replayAdmissionLock) { + if (replayAdmissionClosed) return null; + activeReplayChecks++; + return new PersistedReplayCheck(); + } + } + + public final class PersistedReplayCheck implements AutoCloseable { + private boolean closed; + private PersistedReplayCheck() { } + @Override public void close() { + boolean drained; + synchronized (replayAdmissionLock) { + if (closed) return; + closed = true; + drained = --activeReplayChecks == 0 && replayAdmissionClosed; + } + // Completion callbacks may enter the manager. Never run them while + // holding the admission lock or doing blocking storage work. + if (drained) replayChecksDrained.complete(null); + } + } + private final CompletableFuture delayedTimerTermination = new CompletableFuture<>(); private final CompletionStage delayedTimerRetirement = delayedTimerTermination.minimalCompletionStage(); @Getter @@ -298,20 +332,28 @@ public void shutdown() { } /** - * Close delayed-task admission without blocking. Completion means the actual - * worker has exited, including an admitted task that ignored interruption. + * Close delayed-task and persisted-replay admission without blocking. + * Completion means the actual worker and admitted synchronous replay checks + * have exited, including tasks that ignored interruption. * Persisted replay callbacks can outlive that worker; the storage lifecycle * separately drains their existing per-user replay tails before retiring. */ public CompletionStage beginShutdown() { + boolean drained; + synchronized (replayAdmissionLock) { + replayAdmissionClosed = true; + drained = activeReplayChecks == 0; + } + if (drained) replayChecksDrained.complete(null); delayedTimer.shutdown(); - return delayedTimerRetirement; + return CompletableFuture.allOf(delayedTimerRetirement.toCompletableFuture(), replayChecksDrained) + .minimalCompletionStage(); } /** Lifecycle callers share their remaining disable budget; legacy shutdown keeps its ten-second grace. */ public void shutdown(long timeout, TimeUnit unit) { java.util.Objects.requireNonNull(unit, "unit"); - delayedTimer.shutdown(); + beginShutdown(); try { if (timeout > 0) delayedTimer.awaitTermination(timeout, unit); } catch (InterruptedException e) { diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java index abd1b6ea9..164f5d26f 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java @@ -444,7 +444,7 @@ private static final class ReplayClaims { /** * Snapshot completion of already-admitted persisted replay, including its * queue acknowledgement or restore callback. The lifecycle calls this from - * its manager-worker barrier after delayed-task admission and execution end, + * its manager-worker barrier after delayed-task and replay-check admission end, * so a check deferred to that worker has appended its tail before the snapshot. * This creates no new replay owner and does not admit or execute rewards. */ @@ -1092,6 +1092,13 @@ public void checkDelayedTimedRewards() { } private void checkDelayedTimedRewards(ReplayPlayerState capturedState) { + try (RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck()) { + if (admission == null) return; + checkAdmittedDelayedTimedRewards(capturedState); + } + } + + private void checkAdmittedDelayedTimedRewards(ReplayPlayerState capturedState) { UserDataManager manager = sharedReplayDataManager(); if (capturedState == null && manager != null && manager.mustDeferSharedStorageAccess()) { ReplayPlayerState state = captureReplayPlayerState(); @@ -1187,6 +1194,13 @@ private void checkOfflineRewards(ReplayPlayerState capturedState) { } private CompletionStage dispatchOfflineRewards(boolean force, ReplayPlayerState capturedState) { + try (RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck()) { + if (admission == null) return CompletableFuture.completedFuture(null); + return dispatchAdmittedOfflineRewards(force, capturedState); + } + } + + private CompletionStage dispatchAdmittedOfflineRewards(boolean force, ReplayPlayerState capturedState) { ArrayList rewards = new ArrayList<>(getOfflineRewards()); ArrayList> completions = new ArrayList<>(); for (String rewardEntry : rewards) { diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java index a356cc428..6e55b5b50 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java @@ -118,6 +118,10 @@ class CoreRuntimeAdmittedRetirementTest { assertAdmittedTimedRewardFinishesBeforeNativeRetirement("time-cancelled"); } + @Test void admittedExternalTimedCheckDrainsBeforeReplaySnapshot() throws Exception { + assertAdmittedTimedRewardFinishesBeforeNativeRetirement("external"); + } + private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String preparationOutcome) throws Exception { Fixture fixture = new Fixture(); @@ -157,7 +161,7 @@ private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String prep doReturn(CompletableFuture.completedFuture(null)).when(rewards) .givePersistedQueueRewardAsync(eq(user), any(PersistedQueueReference.class), any(RewardOptions.class)); CompletableFuture hook = new CompletableFuture<>(); - if (preparationOutcome.equals("normal") || preparationOutcome.startsWith("time-")) hook.complete(null); + if (preparationOutcome.equals("normal") || preparationOutcome.equals("external") || preparationOutcome.startsWith("time-")) hook.complete(null); else if (preparationOutcome.equals("failed")) hook.completeExceptionally(new IllegalStateException("consumer failed")); else hook.cancel(false); when(fixture.plugin.onBeforeStorageShutdown()).thenReturn(hook); @@ -170,12 +174,18 @@ private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String prep when(fixture.plugin.getTimeChecker()).thenReturn(checker); when(checker.beginShutdown()).thenReturn(timeRetirement); } - var execution = rewards.getDelayedTimer().submit(user::checkDelayedTimedRewards); + var external = java.util.concurrent.Executors.newSingleThreadExecutor( + AdvancedCoreRuntime.daemonThreads("external-admitted-replay")); + var execution = preparationOutcome.equals("external") + ? external.submit(user::checkDelayedTimedRewards) + : rewards.getDelayedTimer().submit(user::checkDelayedTimedRewards); assertTrue(reading.await(2, TimeUnit.SECONDS)); BukkitRuntimePlatform platform = fixture.platform(); try { new AdvancedCoreRuntime(platform, clock::get).shutdown(0L); - assertTrue(interrupted.await(2, TimeUnit.SECONDS), "zero caller budget still cancels queued timer work"); + if (!preparationOutcome.equals("external")) { + assertTrue(interrupted.await(2, TimeUnit.SECONDS), "zero caller budget still cancels queued timer work"); + } // A real manager-worker barrier removes the race with the old native // retirement task. The fixed path keeps admission open for the held reward. fixture.manager.getTimer().submit(() -> null).get(2, TimeUnit.SECONDS); @@ -184,7 +194,7 @@ private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String prep verify(fixture.mysql, never()).close(); release.countDown(); execution.get(2, TimeUnit.SECONDS); - if (preparationOutcome.equals("normal")) { + if (preparationOutcome.equals("normal") || preparationOutcome.equals("external")) { platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); } else { assertThrows(java.util.concurrent.ExecutionException.class, @@ -199,6 +209,7 @@ private void assertAdmittedTimedRewardFinishesBeforeNativeRetirement(String prep assertEquals(TimeUnit.SECONDS.toNanos(15), clock.get(), "accepted work does not renew caller waits"); } finally { release.countDown(); + external.shutdownNow(); rewards.shutdown(0, TimeUnit.NANOSECONDS); fixture.manager.getTimer().shutdownNow(); } @@ -306,6 +317,79 @@ private void assertDetachedTimedAcknowledgementRetainsStorage(boolean deferredCh } } + @Test void consumerCallbackCannotStartNewTimedReplayAfterShutdownFence() throws Exception { + assertConsumerCannotStartLateReplay("timed"); + } + + @Test void consumerCallbackCannotStartNewOfflineReplayAfterShutdownFence() throws Exception { + assertConsumerCannotStartLateReplay("offline"); + } + + @Test void consumerCallbackCannotForceNewOfflineReplayAfterShutdownFence() throws Exception { + assertConsumerCannotStartLateReplay("forced"); + } + + private void assertConsumerCannotStartLateReplay(String kind) throws Exception { + Fixture fixture = new Fixture(); + when(fixture.backend.isOpen()).thenReturn(true); + RewardHandler rewards = spy(new RewardHandler(fixture.plugin)); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "late-replay-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + when(fixture.plugin.getOptions().isProcessRewards()).thenReturn(true); + when(fixture.users.getOfflineRewardsPath()).thenReturn("OfflineRewards"); + String key = kind.equals("timed") ? "TimedRewards" : "OfflineRewards"; + String entry = kind.equals("timed") ? "ShutdownReward%ExecutionTime/%1" : "ShutdownReward"; + ArrayList persisted = new ArrayList<>(List.of(entry)); + when(data.getStringList(key, UserDataFetchMode.DEFAULT)) + .thenAnswer(call -> new ArrayList<>(persisted)); + doAnswer(call -> { + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }).when(data).setStringList(eq(key), any(), anyBoolean()); + CompletableFuture dispatch = new CompletableFuture<>(); + doReturn(dispatch).when(rewards).givePersistedQueueRewardAsync(eq(user), + any(PersistedQueueReference.class), any(RewardOptions.class)); + CompletableFuture hook = new CompletableFuture<>(); + when(fixture.plugin.onBeforeStorageShutdown()).thenReturn(hook); + CountDownLatch admitted = new CountDownLatch(1); + CountDownLatch finishHook = new CountDownLatch(1); + var callback = java.util.concurrent.Executors.newSingleThreadExecutor( + AdvancedCoreRuntime.daemonThreads("late-replay-consumer")); + BukkitRuntimePlatform platform = fixture.platform(); + try { + var execution = callback.submit(() -> fixture.shared.withStorageReadAdmission(() -> { + admitted.countDown(); + awaitIgnoringInterrupt(finishHook); + hook.complete(null); + // The completion callback is still admitted to storage, but + // shutdown has already fenced admission of fresh replay work. + if (kind.equals("timed")) user.checkDelayedTimedRewards(); + else if (kind.equals("offline")) user.checkOfflineRewards(); + else user.forceRunOfflineRewardsAsync().toCompletableFuture().join(); + return null; + })); + assertTrue(admitted.await(2, TimeUnit.SECONDS)); + new AdvancedCoreRuntime(platform, () -> TimeUnit.SECONDS.toNanos(15)).shutdown(0L); + finishHook.countDown(); + execution.get(2, TimeUnit.SECONDS); + verify(rewards, never()).givePersistedQueueRewardAsync(eq(user), + any(PersistedQueueReference.class), any(RewardOptions.class)); + assertEquals(List.of(entry), persisted, + "late replay remains durable for the next enable instead of extending a snapshotted tail"); + platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); + verify(fixture.mysql, times(1)).close(); + } finally { + finishHook.countDown(); + dispatch.complete(null); + callback.shutdownNow(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + private final class Fixture { final AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); final UserManager users = mock(UserManager.class); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java index 8a35c6fdf..d6d6bcd0d 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeDeadlineTest.java @@ -33,6 +33,12 @@ class CoreRuntimeDeadlineTest { @org.junit.jupiter.api.io.TempDir java.io.File directory; + private AdvancedCorePlugin rewardPlugin() { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(directory); + return plugin; + } + @Test void overlappingExecutorAndActualRewardWaitsShareOneDisableDeadline() throws Exception { AtomicLong clock = new AtomicLong(); RuntimePlatform platform = platform(); @@ -42,7 +48,7 @@ class CoreRuntimeDeadlineTest { ScheduledExecutorService inventory = timedExecutor(clock); ScheduledExecutorService separateStorage = timedExecutor(clock); ScheduledExecutorService delayedRewards = timedExecutor(clock); - RewardHandler rewards = mock(RewardHandler.class, CALLS_REAL_METHODS); + RewardHandler rewards = mock(RewardHandler.class, withSettings().useConstructor(rewardPlugin()).defaultAnswer(CALLS_REAL_METHODS)); var timerField = RewardHandler.class.getDeclaredField("delayedTimer"); timerField.setAccessible(true); timerField.set(rewards, delayedRewards); @@ -182,7 +188,7 @@ class CoreRuntimeDeadlineTest { } @Test void standaloneRewardShutdownKeepsItsExistingTenSecondGrace() throws Exception { - RewardHandler rewards = mock(RewardHandler.class, CALLS_REAL_METHODS); + RewardHandler rewards = mock(RewardHandler.class, withSettings().useConstructor(rewardPlugin()).defaultAnswer(CALLS_REAL_METHODS)); ScheduledExecutorService timer = mock(ScheduledExecutorService.class); var timerField = RewardHandler.class.getDeclaredField("delayedTimer"); timerField.setAccessible(true); timerField.set(rewards, timer); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/AdvancedCoreUserTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/AdvancedCoreUserTest.java index 79f4466d8..61153cb5a 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/AdvancedCoreUserTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/user/AdvancedCoreUserTest.java @@ -98,7 +98,8 @@ public void setUp() { userManager = mock(UserManager.class); dataManager = mock(UserDataManager.class); data = mock(UserData.class); - rewardHandler = mock(RewardHandler.class); + rewardHandler = mock(RewardHandler.class, org.mockito.Mockito.withSettings().useConstructor(plugin)); + org.mockito.Mockito.doCallRealMethod().when(rewardHandler).admitPersistedReplayCheck(); delayedTimer = mock(ScheduledExecutorService.class); MySQL mysql = mock(MySQL.class); From 79bd96704f040db4e455ee36fbe2d3d9fbb1e950 Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 15 Oct 2026 21:37:54 -0600 Subject: [PATCH 16/28] Settle final snapshot publications in order --- AGENTS.md | 2 +- .../bukkit/runtime/BukkitRuntimePlatform.java | 44 ++++++--- .../advancedcore/data/ServerData.java | 32 ++++++- .../FullInventoryFinalPublicationTest.java | 42 +++++++++ .../tests/lifecycle/CoreRuntimeTest.java | 2 +- .../FinalPublicationSequenceTest.java | 92 +++++++++++++++++++ .../ServerDataPublicationOrderingTest.java | 36 ++++++-- .../time/ServerDataTimeTransitionTest.java | 65 ++++++++----- docs/shared-runtime-foundation.md | 19 +++- 9 files changed, 285 insertions(+), 49 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java diff --git a/AGENTS.md b/AGENTS.md index 006e107f4..fb094a421 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -51,7 +51,7 @@ For compatibility-sensitive changes, keep regression coverage for established be 3. Reward execution must not be lost or duplicated across delay, disconnect, reload, retry, or partial failure. Preserve the component's documented delivery contract. 4. Read-only lookups must not create users, mutate caches, or trigger persistence unless that behavior is part of the established API. 5. Keep user identity and UUID resolution consistent across online/offline paths. Coordinate cache updates, database writes, resets, and shutdown flushing. Final shared-runtime retirement fences queued notifications before marking storage retiring; callbacks already holding lifecycle admission may finish and re-enter storage. The optional pre-storage shutdown stage must finish before native storage retirement. Platform cleanup and the completion hook may run earlier on deferred shutdown; consumers must retain their own persistence owner until their stage settles. -6. Reload and disable must cancel or retire tasks, listeners, executors, connections, and callbacks without allowing stale work to mutate new state. Native ServerData admission precedes early enable hooks and reads and is shared across shaded classloaders through bounded opaque JVM Properties tokens. Release only after successful publication/preparation and actual captured native executor termination; failed or cancelled stages retain admission until JVM restart because settlement alone cannot prove drain. Retain the originating Properties object for release. Repeated disable must not submit stale final writes after release. +6. Reload and disable must cancel or retire tasks, listeners, executors, connections, and callbacks without allowing stale work to mutate new state. Native ServerData admission precedes early enable hooks and reads and is shared across shaded classloaders through bounded opaque JVM Properties tokens. Release only after successful publication/preparation and actual captured native executor termination; failed or cancelled stages retain admission until JVM restart because settlement alone cannot prove drain. Retain the originating Properties object for release. Repeated disable must not submit stale final writes after release. Final whole-file publications settle in order even for asynchronous owners. A visible native final replacement gets one bounded settlement retry; if unconfirmed, fence that owner against subsequent writes/reload/setup so another save cannot commit its unacknowledged values. 7. GUI/editor actions must recheck permissions and ownership and must not duplicate items, lose configuration, or save to the wrong path or user. 8. Optional integrations must be guarded against absent classes and incompatible versions. One platform must not load another platform's API accidentally. diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 5af12a14f..92bbd20b1 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -12,7 +12,8 @@ import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.function.LongSupplier; -import java.util.function.Supplier; +import java.util.function.Function; +import java.util.concurrent.Executor; import org.bukkit.Bukkit; @@ -49,6 +50,7 @@ public final class BukkitRuntimePlatform implements RuntimePlatform { private volatile CompletionStage inventoryPersistence = CompletableFuture.completedFuture(null); private volatile CompletionStage timestampPersistence = CompletableFuture.completedFuture(null); private volatile CompletionStage permissionPersistence = CompletableFuture.completedFuture(null); + private CompletionStage finalPublicationOrder = CompletableFuture.completedFuture(null); private volatile CompletionStage userStorageRetirement = CompletableFuture.completedFuture(null); private CompletionStage startedUserStorageRetirement; private volatile CompletionStage admittedUserStorageRetirement; @@ -115,10 +117,10 @@ private static void addExecutor(IdentityHashMap beforeExecutorShutdown() { return List.of( new Cleanup("full inventory handler", () -> { - inventoryPersistence = finalPersistence("full inventory snapshot", () -> { + inventoryPersistence = finalPersistence("full inventory snapshot", executor -> { FullInventoryHandler handler = plugin.getFullInventoryHandler(); return handler == null ? CompletableFuture.completedFuture(null) - : Objects.requireNonNull(handler.shutdownAsync(persistenceWorker), "inventory persistence"); + : Objects.requireNonNull(handler.shutdownAsync(executor), "inventory persistence"); }); }), new Cleanup("Javascript engine", () -> { @@ -202,9 +204,9 @@ private static void addExecutor(IdentityHashMap { - permissionPersistence = finalPersistence("timed permission snapshot", () -> + permissionPersistence = finalPersistence("timed permission snapshot", executor -> plugin.getPermissionHandler() == null ? CompletableFuture.completedFuture(null) - : Objects.requireNonNull(plugin.getPermissionHandler().shutDownAsync(persistenceWorker), + : Objects.requireNonNull(plugin.getPermissionHandler().shutDownAsync(executor), "permission persistence")); }), new Cleanup("dialog service", () -> { @@ -434,20 +436,36 @@ private CompletionStage observePersistence(String component, CompletionSta return completion; } - private CompletionStage finalPersistence(String component, Supplier> submission) { - CompletionStage completion; - try { completion = submission.get(); } - catch (Throwable failure) { completion = CompletableFuture.failedFuture(failure); } + private CompletionStage finalPersistence(String component, + Function> submission) { + var completion = new CompletableFuture(); + CompletionStage predecessor = finalPublicationOrder; + finalPublicationOrder = completion; + Executor ordered = task -> predecessor.whenComplete((ignored, priorFailure) -> { + // Failure is still reported by its original stage; independent + // cleanup may proceed only after that owner has actually settled. + try { persistenceWorker.execute(task); } + catch (Throwable failure) { completion.completeExceptionally(failure); } + }); + try { + // Handler shutdown fences admission now, while its actual snapshot + // waits on the preceding publication through the supplied executor. + Objects.requireNonNull(submission.apply(ordered), "final persistence") + .whenComplete((ignored, failure) -> { + if (failure == null) completion.complete(null); + else completion.completeExceptionally(failure); + }); + } catch (Throwable failure) { completion.completeExceptionally(failure); } return observePersistence(component, completion); } - private CompletionStage submitTimestampPersistence() { + private CompletionStage submitTimestampPersistence(Executor executor) { ServerData owner = plugin.getServerDataFile(); if (owner == null) return CompletableFuture.completedFuture(null); long lastUpdated = System.currentTimeMillis(); var completion = new CompletableFuture(); try { - persistenceWorker.execute(() -> { + executor.execute(() -> { try { owner.mutateFinalData(data -> data.set("LastUpdated", lastUpdated)) .whenComplete((ignored, failure) -> { @@ -505,8 +523,10 @@ private CompletableFuture finalFilePersistence() { private void finishPersistenceSubmission() { // Do not discard accepted snapshots on timeout or close the native owner // while inventory delivery can still advance its durable replay receipt. - persistenceWorker.shutdown(); CompletableFuture completion = finalFilePersistence(); + // Asynchronous owners can settle after this lifecycle callback returns. + // Keep the existing worker available for their ordered successors. + completion.whenComplete((ignored, failure) -> persistenceWorker.shutdown()); if (completion.isDone()) return; LongSupplier budget = shutdownRemainingNanos; long remaining = budget == null ? TimeUnit.SECONDS.toNanos(15) : Math.max(0L, budget.getAsLong()); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java index aa81e174a..e25a96103 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java @@ -34,6 +34,7 @@ public class ServerData extends YMLFile { private boolean publicationSettlementPending; private boolean replacementCleanupPending; private boolean replacementUncertain; + private boolean finalPublicationIndeterminate; /** * Durable state for one locally detected time transition. The identifier is * deliberately stable while a transition is pending so consumers can store @@ -55,6 +56,7 @@ public synchronized void saveData() { @Override public synchronized void setup() { + requireFinalPublicationSettled(); Path target = getdFile().toPath().toAbsolutePath(); Path backup = backupPath(target); Path replacementMarker = replacementMarkerPath(target); @@ -111,6 +113,7 @@ public java.util.concurrent.CompletionStage mutateFinalData( java.util.function.Consumer mutation) { try { synchronized (this) { + requireFinalPublicationSettled(); java.util.Objects.requireNonNull(mutation, "mutation"); Class saveOwner = getClass().getMethod("saveData").getDeclaringClass(); if (saveOwner != YMLFile.class && saveOwner != ServerData.class) { @@ -140,8 +143,23 @@ public java.util.concurrent.CompletionStage mutateFinalData( replacementUncertain = true; if (failure != reconciliationFailure) failure.addSuppressed(reconciliationFailure); } + // Retry only the retained commit's durability/cleanup, never its + // mutation or file replacement. A confirmed commit must settle + // this original stage before a later save can publish its values. + if (!replacementUncertain && (publicationSettlementPending || replacementCleanupPending)) { + try { prepareSnapshotPublication(); } + catch (Throwable settlementFailure) { + finalPublicationIndeterminate = true; + if (failure != settlementFailure) failure.addSuppressed(settlementFailure); + throw failure; + } + } else { + finalPublicationIndeterminate = true; + throw failure; + } + } else { + throw failure; } - throw failure; } finally { finalDataSnapshot = null; } reconcileFinalData(live, candidate, before); @@ -292,12 +310,14 @@ private void inspectFailedMove(Path target, String intendedHash, String previous try { if (Files.isRegularFile(target)) { String actualHash = snapshotHash(target); + // Equal predecessor/candidate bytes cannot prove that the forced + // temporary file replaced an older, potentially unforced target. + if (actualHash.equals(previousHash)) return; if (intendedHash.equals(actualHash)) { visiblePublication++; publicationSettlementPending = true; return; } - if (actualHash.equals(previousHash)) return; } else if (previouslyAbsent && Files.notExists(target)) return; } catch (IOException | RuntimeException inspectionFailure) { if (failure != inspectionFailure) failure.addSuppressed(inspectionFailure); @@ -308,6 +328,7 @@ private void inspectFailedMove(Path target, String intendedHash, String previous } private void prepareSnapshotPublication() { + requireFinalPublicationSettled(); if (replacementUncertain) throw new IllegalStateException("ServerData replacement outcome is unknown; restart required before saving"); if (!publicationSettlementPending && !replacementCleanupPending) return; @@ -321,6 +342,11 @@ private void prepareSnapshotPublication() { } } + private void requireFinalPublicationSettled() { + if (finalPublicationIndeterminate) + throw new IllegalStateException("Final ServerData publication is unconfirmed; restart required before reuse"); + } + private Path backupPath(Path target) { return target.resolveSibling(target.getFileName().toString() + ".backup"); } @@ -438,7 +464,7 @@ private void withNativePublicationOrdering(Runnable operation) { try { Class owner = getClass().getMethod("saveData").getDeclaringClass(); if (owner == ServerData.class || owner == YMLFile.class) { - synchronized (this) { operation.run(); } + synchronized (this) { requireFinalPublicationSettled(); operation.run(); } } else { // Existing adapters retain their own nonblocking mutation/publication model. operation.run(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryFinalPublicationTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryFinalPublicationTest.java index 82ea6fcdf..37932c57f 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryFinalPublicationTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryFinalPublicationTest.java @@ -49,7 +49,33 @@ void successfulFinalPublicationKeepsOverflowAndReceiptConsistentAfterRestart() t laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(false); } + @Test void transientDirectoryFailureSettlesOriginalInventoryReceiptBeforeLaterSaves() throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(false, true, false); + } + + @Test void fallbackDirectoryFailureSettlesOriginalInventoryReceiptBeforeLaterSaves() throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(false, true, true); + } + + @Test void fallbackMarkerCleanupFailureSettlesOriginalInventoryReceiptBeforeLaterSaves() throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(false, true, true, 2); + } + + @Test void fallbackBackupCleanupFailureSettlesOriginalInventoryReceiptBeforeLaterSaves() throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(false, true, true, 3); + } + private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(boolean failInventory) throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(failInventory, false, false); + } + + private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(boolean failInventory, + boolean failVisibleDirectory, boolean fallback) throws Exception { + laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(failInventory, failVisibleDirectory, fallback, 1); + } + + private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent(boolean failInventory, + boolean failVisibleDirectory, boolean fallback, int failingVisibleForce) throws Exception { ConfigurationSerialization.registerClass(PersistedItem.class, "ACFinalOverflowTestItem"); var plugin = mock(AdvancedCorePlugin.class); var options = mock(AdvancedCoreConfigOptions.class); @@ -65,6 +91,8 @@ private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent( .when(checkpointExecutor).execute(any(Runnable.class)); when(plugin.getTimer()).thenReturn(checkpointExecutor); var owner = new FailingPublicationOwner(plugin); owner.setup(); + owner.fallback = fallback; + owner.failingVisibleForce = failingVisibleForce; when(plugin.getServerDataFile()).thenReturn(owner); var handler = new FullInventoryHandler(plugin); when(plugin.getFullInventoryHandler()).thenReturn(handler); @@ -99,6 +127,7 @@ private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent( var replay = user.endAsyncActionCollection(actions).toCompletableFuture(); ownerTask.get().run(); assertFalse(replay.isDone()); owner.failNext.set(failInventory); + owner.failVisibleDirectory.set(failVisibleDirectory); var finalFiles = handler.shutdownAsync(Runnable::run).toCompletableFuture(); if (failInventory) assertThrows(java.util.concurrent.CompletionException.class, finalFiles::join); else finalFiles.get(2, TimeUnit.SECONDS); @@ -139,11 +168,24 @@ private void laterWholeFileSaveAndRestartKeepOverflowAndReplayReceiptConsistent( private static final class FailingPublicationOwner extends ServerData { private final AtomicBoolean failNext = new AtomicBoolean(); + private final AtomicBoolean failVisibleDirectory = new AtomicBoolean(); + private boolean fallback; + private int failingVisibleForce = 1; + private int visibleForces; private FailingPublicationOwner(AdvancedCorePlugin plugin) { super(plugin); } @Override protected void moveAtomically(Path source, Path target) throws IOException { if (failNext.compareAndSet(true, false)) throw new IOException("inventory publication rejected"); + if (fallback) throw new java.nio.file.AtomicMoveNotSupportedException(source.toString(), target.toString(), "fixture"); super.moveAtomically(source, target); } + @Override protected void forceDirectory(Path directory) throws IOException { + if (java.nio.file.Files.exists(getdFile().toPath()) + && java.nio.file.Files.readString(getdFile().toPath()).contains("FullInventory:") + && failVisibleDirectory.get() && ++visibleForces == failingVisibleForce + && failVisibleDirectory.compareAndSet(true, false)) + throw new IOException("synthetic transient directory force failure after visible inventory"); + super.forceDirectory(directory); + } } /** A headless item serializer; delivery still uses the production inventory path. */ diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index d7e51fb45..50e8dca96 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -961,12 +961,12 @@ private static java.util.Queue queuedStorageWorker(UserDataManager man BukkitRuntimePlatform platform = new BukkitRuntimePlatform(plugin); platform.beforeExecutorShutdown().stream().filter(cleanup -> cleanup.name().equals("full inventory handler") || cleanup.name().equals("server data timestamp")).forEach(cleanup -> cleanup.action().run()); + inventorySave.complete(null); assertTrue(admitted.await(2, TimeUnit.SECONDS)); platform.afterExecutorShutdown().forEach(cleanup -> cleanup.action().run()); assertNotNull(files.get()); assertFalse(files.get().toCompletableFuture().isDone()); var failedPermission = new IllegalStateException("permission publication failed"); permissionSave.completeExceptionally(failedPermission); - inventorySave.complete(null); assertFalse(files.get().toCompletableFuture().isDone(), "aggregate waits for every owner settlement"); timestampSave.complete(null); assertSame(failedPermission, assertThrows(CompletionException.class, diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java new file mode 100644 index 000000000..c40003c0e --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java @@ -0,0 +1,92 @@ +package com.bencodez.advancedcore.tests.lifecycle; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.util.List; +import java.util.concurrent.*; +import java.util.concurrent.CopyOnWriteArrayList; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.jupiter.api.Test; + +import com.bencodez.advancedcore.AdvancedCorePlugin; +import com.bencodez.advancedcore.api.item.FullInventoryHandler; +import com.bencodez.advancedcore.api.permissions.PermissionHandler; +import com.bencodez.advancedcore.bukkit.runtime.BukkitRuntimePlatform; +import com.bencodez.advancedcore.data.ServerData; + +class FinalPublicationSequenceTest { + @Test void asynchronousOwnersPublishEachFinalSnapshotBeforeStartingTheNext() throws Exception { + verifySequence(false); + } + + @Test void failedInventoryPublicationStillOrdersIndependentFinalCleanup() throws Exception { + verifySequence(true); + } + + private void verifySequence(boolean failInventory) throws Exception { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + FullInventoryHandler inventory = mock(FullInventoryHandler.class); + PermissionHandler permissions = mock(PermissionHandler.class); + ServerData owner = mock(ServerData.class); + when(plugin.getFullInventoryHandler()).thenReturn(inventory); + when(plugin.getPermissionHandler()).thenReturn(permissions); + when(plugin.getServerDataFile()).thenReturn(owner); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); + var inventorySave = new CompletableFuture(); + var timestampSave = new CompletableFuture(); + var permissionSave = new CompletableFuture(); + List started = new CopyOnWriteArrayList<>(); + when(inventory.shutdownAsync(any())).thenAnswer(call -> { + call.getArgument(0, Executor.class).execute(() -> started.add("inventory")); + return inventorySave; + }); + when(owner.mutateFinalData(any())).thenAnswer(call -> { + started.add("timestamp"); + return timestampSave; + }); + when(permissions.shutDownAsync(any())).thenAnswer(call -> { + call.getArgument(0, Executor.class).execute(() -> started.add("permissions")); + return permissionSave; + }); + AtomicReference> finalFiles = new AtomicReference<>(); + doAnswer(call -> { finalFiles.set(call.getArgument(0)); return null; }) + .when(plugin).onShutdownComplete(any(CompletionStage.class)); + var platform = new BukkitRuntimePlatform(plugin); + var workerField = BukkitRuntimePlatform.class.getDeclaredField("persistenceWorker"); + workerField.setAccessible(true); + var worker = (ThreadPoolExecutor) workerField.get(platform); + try { + platform.beforeExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("full inventory handler") + || cleanup.name().equals("server data timestamp")) + .forEach(cleanup -> cleanup.action().run()); + platform.afterExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("permission handler")) + .forEach(cleanup -> cleanup.action().run()); + worker.submit(() -> { }).get(2, TimeUnit.SECONDS); + assertEquals(List.of("inventory"), started, "returning from the worker task is not publication"); + if (failInventory) inventorySave.completeExceptionally(new IllegalStateException("synthetic failure")); + else inventorySave.complete(null); + worker.submit(() -> { }).get(2, TimeUnit.SECONDS); + assertEquals(List.of("inventory", "timestamp"), started); + platform.afterExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("shutdown persistence worker") + || cleanup.name().equals("plugin shutdown completion hook")) + .forEach(cleanup -> cleanup.action().run()); + timestampSave.complete(null); + // Completion remains observable after submission closes; the final + // permission task must still be accepted by the retained worker. + permissionSave.complete(null); + if (failInventory) assertThrows(ExecutionException.class, + () -> finalFiles.get().toCompletableFuture().get(2, TimeUnit.SECONDS)); + else finalFiles.get().toCompletableFuture().get(2, TimeUnit.SECONDS); + assertTrue(worker.awaitTermination(2, TimeUnit.SECONDS)); + assertEquals(List.of("inventory", "timestamp", "permissions"), started); + } finally { + inventorySave.complete(null); timestampSave.complete(null); permissionSave.complete(null); + worker.shutdownNow(); + } + } +} diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java index 2ded44a89..cb2a42817 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java @@ -79,11 +79,11 @@ class ServerDataPublicationOrderingTest { assertEquals("concurrent", live.getString("Unrelated.Value")); assertEquals("candidate", YamlConfiguration.loadConfiguration(data.getdFile()).getString("Unrelated.Value")); data.failing = false; - data.setLastUpdated(); + assertQuarantinedAfterUnconfirmedFinalPublication(data); var restarted = YamlConfiguration.loadConfiguration(data.getdFile()); assertEquals(456L, restarted.getLong("FullInventory.committed.Time")); - assertEquals(java.util.List.of("concurrent"), restarted.getStringList("TimedPermissions.example")); - assertEquals("concurrent", restarted.getString("Unrelated.Value")); + assertEquals(java.util.List.of("candidate"), restarted.getStringList("TimedPermissions.example")); + assertEquals("candidate", restarted.getString("Unrelated.Value")); } } @@ -143,8 +143,8 @@ private void verifyUnchangedListPayload(boolean successful) throws Exception { assertEquals("candidate", published.getString("Unrelated")); assertPreservedRows(published, expectedRows); data.failing = false; - data.setLastUpdated(); - data.reloadData(); + if (successful) { data.setLastUpdated(); data.reloadData(); } + else assertQuarantinedAfterUnconfirmedFinalPublication(data); assertPreservedRows(data.getData(), expectedRows); assertPreservedRows(YamlConfiguration.loadConfiguration(data.getdFile()), expectedRows); assertEquals(!successful, result.isCompletedExceptionally()); @@ -254,12 +254,31 @@ private void verifyNestedFinalReconciliation(boolean failForce) throws Exception assertEquals("candidate", published.getString("Nested.branch.committed")); assertNestedFinalValues(live); data.failing = false; - data.setLastUpdated(); - assertNestedFinalValues(YamlConfiguration.loadConfiguration(data.getdFile())); + if (failForce) assertQuarantinedAfterUnconfirmedFinalPublication(data); + else { + data.setLastUpdated(); + assertNestedFinalValues(YamlConfiguration.loadConfiguration(data.getdFile())); + } assertEquals(failForce, result.isCompletedExceptionally()); } } + private static void assertQuarantinedAfterUnconfirmedFinalPublication(FailingDirectoryForceServerData data) + throws Exception { + byte[] visible = Files.readAllBytes(data.getdFile().toPath()); + assertEquals(2, data.failedForces, "only one settlement retry is allowed"); + assertThrows(IllegalStateException.class, data::setLastUpdated); + assertThrows(IllegalStateException.class, data::saveData); + assertThrows(IllegalStateException.class, data::saveDataChecked); + assertThrows(IllegalStateException.class, data::reloadData); + assertThrows(IllegalStateException.class, data::setup); + assertThrows(CompletionException.class, + () -> data.mutateFinalData(yaml -> yaml.set("UnacknowledgedSuccessor", true)).toCompletableFuture().join()); + assertFalse(data.getData().contains("UnacknowledgedSuccessor")); + assertArrayEquals(visible, Files.readAllBytes(data.getdFile().toPath()), + "later operations cannot silently make an unacknowledged final candidate durable"); + } + private static void assertNestedFinalValues(FileConfiguration yaml) { assertEquals(java.util.List.of("candidate"), yaml.getStringList("TimedPermissions.committed")); assertEquals(java.util.List.of("concurrent"), yaml.getStringList("TimedPermissions.existing")); @@ -733,6 +752,7 @@ private static final class FailingDirectoryForceServerData extends ServerData { final java.io.IOException forceFailure = new java.io.IOException("synthetic directory force failure"); boolean failing = true; boolean failAfterBarrier = true; + int failedForces; FailingDirectoryForceServerData(AdvancedCorePlugin plugin) { super(plugin); } @Override protected void forceDirectory(java.nio.file.Path directory) throws java.io.IOException { if (!failing) { super.forceDirectory(directory); return; } @@ -742,7 +762,7 @@ private static final class FailingDirectoryForceServerData extends ServerData { } catch (InterruptedException interrupted) { Thread.currentThread().interrupt(); throw new java.io.IOException(interrupted); } - if (failAfterBarrier) throw forceFailure; + if (failAfterBarrier) { failedForces++; throw forceFailure; } super.forceDirectory(directory); } } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java index f1a8dc283..de74f67bf 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java @@ -153,9 +153,7 @@ private void verifyVisibleFinalMutationSurvivesFailure(boolean fallback, int fai yaml.set("FullInventory.committed.Time", 456L); yaml.set("FullInventory.committed.Items.0", "stored-item"); }).toCompletableFuture(); - var failure = assertThrows(java.util.concurrent.CompletionException.class, result::join); - assertTrue(failure.getCause() instanceof UncheckedIOException); - assertEquals(data.forceFailure, failure.getCause().getCause(), "the original durability failure remains observable"); + result.join(); assertSame(live, data.getData()); assertEquals(456L, live.getLong("FullInventory.committed.Time")); assertFalse(live.contains("FullInventory.previous")); @@ -166,7 +164,7 @@ private void verifyVisibleFinalMutationSurvivesFailure(boolean fallback, int fai data.setLastUpdated(); data.setData("TimedPermissions.example", java.util.List.of("example.use%line%1234")); data.mutateFinalData(yaml -> yaml.set("IgnoreTime", true)).toCompletableFuture().join(); - assertTrue(result.isCompletedExceptionally(), "later saves cannot acknowledge the failed stage"); + assertFalse(result.isCompletedExceptionally(), "the original stage confirms its own settled publication"); TestServerData restarted = serverData(); restarted.setup(); assertEquals(456L, restarted.getData().getLong("FullInventory.committed.Time")); @@ -182,28 +180,24 @@ private void verifyVisibleFinalMutationSurvivesFailure(boolean fallback, int fai } @Test - void failedFallbackCleanupFencesOrdinaryAndCheckedWritesUntilItSettles() throws Exception { + void failedFallbackCleanupQuarantinesOwnerAfterOneSettlementRetry() throws Exception { PublicationFailureServerData data = new PublicationFailureServerData(plugin(), true); data.setup(); data.getData().set("FullInventory.previous.Time", 123L); data.saveDataChecked(); data.directoryForces = 0; - data.failingForce = 3; + data.failFromForce = 3; assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( yaml -> yaml.set("FullInventory.committed.Time", 456L)).toCompletableFuture().join()); byte[] visibleSnapshot = Files.readAllBytes(data.getdFile().toPath()); - data.failEveryForce = true; - assertThrows(UncheckedIOException.class, data::setLastUpdated); - assertThrows(UncheckedIOException.class, data::saveData); - assertThrows(UncheckedIOException.class, data::saveDataChecked); + assertEquals(4, data.directoryForces, "two preparation forces, failed target force and one retry"); + data.failFromForce = 0; + assertThrows(IllegalStateException.class, data::setLastUpdated); + assertThrows(IllegalStateException.class, data::saveData); + assertThrows(IllegalStateException.class, data::saveDataChecked); + assertThrows(IllegalStateException.class, data::reloadData); + assertThrows(IllegalStateException.class, data::setup); assertArrayEquals(visibleSnapshot, Files.readAllBytes(data.getdFile().toPath())); - data.failEveryForce = false; - data.failingForce = 0; - data.saveData(); - TestServerData restarted = serverData(); - restarted.setup(); - assertEquals(456L, restarted.getData().getLong("FullInventory.committed.Time")); - assertTrue(restarted.getLastUpdated() > 0L); } @Test @@ -227,20 +221,47 @@ void failedFallbackBeforeMoveDiscardsCandidateAndAllowsLaterSave() throws Except } @Test - void moveThatThrowsAfterReplacingReconcilesVisibleCandidateWithoutAcknowledgingIt() throws Exception { + void moveThatThrowsAfterReplacingSettlesAndAcknowledgesTheProvenCandidate() throws Exception { MoveFailureServerData data = new MoveFailureServerData(plugin()); data.setup(); data.getData().set("FullInventory.previous.Time", 123L); data.saveDataChecked(); data.moveFailure = 1; - assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( - yaml -> yaml.set("FullInventory.committed.Time", 456L)).toCompletableFuture().join()); + data.mutateFinalData(yaml -> yaml.set("FullInventory.committed.Time", 456L)).toCompletableFuture().join(); assertEquals(456L, data.getData().getLong("FullInventory.committed.Time")); data.moveFailure = 0; data.setLastUpdated(); assertEquals(456L, YamlConfiguration.loadConfiguration(data.getdFile()).getLong("FullInventory.committed.Time")); } + @Test + void unchangedCandidateCannotAcknowledgeMoveFailureBeforeReplacement() throws Exception { + verifyUnchangedCandidateMoveFailure(2); + } + + @Test + void unchangedCandidateCannotInferReplacementFromEqualHashes() throws Exception { + verifyUnchangedCandidateMoveFailure(1); + } + + private void verifyUnchangedCandidateMoveFailure(int moveFailure) throws Exception { + MoveFailureServerData data = new MoveFailureServerData(plugin()); + data.setup(); + data.getData().set("Stored", 123L); + // An ordinary visible file is not evidence that its contents were forced. + Files.writeString(data.getdFile().toPath(), data.getData().saveToString()); + byte[] original = Files.readAllBytes(data.getdFile().toPath()); + data.moveFailure = moveFailure; + assertThrows(java.util.concurrent.CompletionException.class, + () -> data.mutateFinalData(yaml -> { }).toCompletableFuture().join()); + assertArrayEquals(original, Files.readAllBytes(data.getdFile().toPath())); + // A subsequent confirmed replacement still succeeds; no no-op failure + // can manufacture a visibility transition or poison ordinary recovery. + data.moveFailure = 0; + data.mutateFinalData(yaml -> { }).toCompletableFuture().join(); + assertEquals(123L, YamlConfiguration.loadConfiguration(data.getdFile()).getLong("Stored")); + } + @Test void moveThatFailsBeforeReplacingDiscardsCandidateAndAllowsLaterSave() throws Exception { MoveFailureServerData data = new MoveFailureServerData(plugin()); @@ -447,6 +468,7 @@ private static final class PublicationFailureServerData extends ServerData { private int directoryForces; private int failingForce; private boolean failEveryForce; + private int failFromForce; private PublicationFailureServerData(AdvancedCorePlugin plugin, boolean fallback) { super(plugin); @@ -459,7 +481,8 @@ private PublicationFailureServerData(AdvancedCorePlugin plugin, boolean fallback } @Override protected void forceDirectory(Path directory) throws IOException { - if (++directoryForces == failingForce || failEveryForce) throw forceFailure; + if (++directoryForces == failingForce || failEveryForce + || failFromForce > 0 && directoryForces >= failFromForce) throw forceFailure; super.forceDirectory(directory); } } diff --git a/docs/shared-runtime-foundation.md b/docs/shared-runtime-foundation.md index d4985cc87..1abb0c76d 100644 --- a/docs/shared-runtime-foundation.md +++ b/docs/shared-runtime-foundation.md @@ -39,9 +39,14 @@ period. The public `RewardHandler.shutdown()` keeps its ten-second maximum for standalone callers, while lifecycle shutdown supplies only its remaining budget. Final inventory, server timestamp, and timed-permission mutations are admitted in -order on one bounded daemon persistence worker. `ServerData.mutateFinalData` +order on one bounded daemon persistence worker. Each publication stage settles +before its successor snapshot starts, including asynchronous adapters. Handler +admission is fenced immediately, and the worker remains available for ordered +successors until all final stages settle. `ServerData.mutateFinalData` returns the actual file owner's publication stage. Native and completion-aware -final mutations are staged in a private YAML candidate and become live only after confirmed publication. +final mutations are staged in a private YAML candidate. Native live YAML retains +a replacement that has become visible even if its directory sync then fails; +visibility alone does not acknowledge its reward receipt. Native ServerData forces that candidate on the shutdown worker; an ordered adapter serializes under its own YAML monitor and confirms its existing publisher. Later unrelated saves cannot publish a failed final overflow mutation. Ordered @@ -53,7 +58,15 @@ one save callback, but cannot claim confirmed publication without overriding the additive completion contract; they report an unsupported confirmation. A legacy callback failure remains observable, with no second mutation or save invocation. Its live changes retain the old non-transactional behavior; candidate isolation -remains specific to native and completion-aware owners. An I/O failure remains a failed stage. Inventory persistence gates user-storage +remains specific to native and completion-aware owners. For a known visible +native replacement, one immediate retry may settle only its retained directory +sync and cleanup, without repeating the mutation or move. Confirmation settles +the original publication successfully. If that retry fails, its stage fails and +the owner rejects subsequent saves, final mutations, reload and setup; later +cleanup cannot silently commit unacknowledged overflow. This fence does not +remove the separate file/receipt crash window or promise replay safety after an +uncertain restart. Failures before replacement retain their existing recovery +behavior. Inventory persistence gates user-storage retirement and cannot acknowledge reserved replay overflow before its actual save succeeds. The public synchronous handler shutdown methods remain available. The shutdown-completion hook follows submission of final file saves; it is not a From df034e63b1a5f7a270b845b9f8b0b1972cf1adfc Mon Sep 17 00:00:00 2001 From: BenCodez <17074231+BenCodez@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:00:32 -0600 Subject: [PATCH 17/28] Await delayed worker termination in shutdown fixture --- .../tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java index 6e55b5b50..1cb54dce3 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java @@ -287,7 +287,9 @@ private void assertDetachedTimedAcknowledgementRetainsStorage(boolean deferredCh try { new AdvancedCoreRuntime(platform, clock::get).shutdown(0L); fixture.manager.getTimer().submit(() -> null).get(2, TimeUnit.SECONDS); - assertTrue(rewards.getDelayedTimer().isTerminated(), "worker exit alone does not finish replay"); + // The storage barrier does not order the delayed worker's final exit. + assertTrue(rewards.getDelayedTimer().awaitTermination(2, TimeUnit.SECONDS), + "worker exit alone does not finish replay"); assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); verify(fixture.mysql, never()).close(); var acknowledgingCallback = callback.submit(() -> { From 6719a7c9534a2c378feda388d6bf5f693adfc22e Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 16 Oct 2026 16:00:24 -0600 Subject: [PATCH 18/28] Use actual publication monitors for live ServerData transactions --- .../api/item/FullInventoryHandler.java | 2 +- .../advancedcore/api/time/TimeChecker.java | 2 +- .../advancedcore/data/ServerData.java | 14 ++ .../tests/item/FullInventoryHandlerTest.java | 1 + .../FullInventoryPublicationMonitorTest.java | 139 ++++++++++++++++++ .../TimeCheckerShutdownAdmissionTest.java | 43 ++++++ .../tests/time/TimeCheckerTest.java | 1 + docs/shared-runtime-foundation.md | 11 ++ 8 files changed, 211 insertions(+), 2 deletions(-) create mode 100644 AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java index 078220ae9..128a6a737 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java @@ -477,7 +477,7 @@ private void completeReplayOverflow(String reservationId, CompletableFuture reservedOverflows, ServerData serverData) { if (serverData == null) return false; - synchronized (serverData) { + synchronized (serverData.publicationMonitor()) { return savePendingItemsUnderDataLock(reservedOverflows, serverData); } } diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java index 6fb9983eb..2844294bf 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/time/TimeChecker.java @@ -492,7 +492,7 @@ private void startTransition(TimeType type, boolean fake, boolean preDate, boole // The read is observational. Settle publication on the existing // checked owner only after reserving the shutdown drain. Recheck // under that owner's monitor so reload cannot revive a stale slot. - synchronized (serverData) { + synchronized (serverData.publicationMonitor()) { if (!serverData.isPendingTimeChangeTransition(durable.persisted)) return; serverData.beginTimeChangeTransition(type, durable.persisted.periodKey(), durable.persisted.markerValue()); diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java index e25a96103..57d40983e 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java @@ -48,6 +48,20 @@ public ServerData(AdvancedCorePlugin plugin) { super(plugin, new File(plugin.getDataFolder(), "ServerData.yml")); } + /** + * The stable monitor protecting this owner's live YAML and whole-file snapshots. + * Callers performing a compound live read/mutate/save must use this monitor, + * rather than the exposed facade. Adapters must return their actual backing + * owner's monitor and use it consistently for their delegated mutations and + * publications. The native owner uses itself. This does not acknowledge queued + * persistence: never wait for a publication stage while holding this monitor. + * + * @return the non-null, stable publication monitor + */ + public Object publicationMonitor() { + return this; + } + @Override public synchronized void saveData() { prepareSnapshotPublication(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index b776615cb..1c6971ede 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -821,6 +821,7 @@ private Fixture createFixture(long itemDeliveryTimeoutMillis) { ScheduledExecutorService sharedInventoryTimer = mock(ScheduledExecutorService.class); BukkitScheduler bukkitScheduler = mock(BukkitScheduler.class); ServerData serverData = mock(ServerData.class); + when(serverData.publicationMonitor()).thenCallRealMethod(); org.mockito.Mockito.doCallRealMethod().when(serverData).mutateFinalData(any()); YamlConfiguration data = new YamlConfiguration(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java new file mode 100644 index 000000000..b4dd6fd12 --- /dev/null +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java @@ -0,0 +1,139 @@ +package com.bencodez.advancedcore.tests.item; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.*; + +import java.lang.management.ManagementFactory; +import java.nio.file.Path; +import java.util.UUID; +import java.util.concurrent.*; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.logging.Logger; + +import org.bukkit.Server; +import org.bukkit.configuration.file.FileConfiguration; +import org.bukkit.configuration.file.YamlConfiguration; +import org.bukkit.configuration.serialization.ConfigurationSerialization; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import com.bencodez.advancedcore.AdvancedCorePlugin; +import com.bencodez.advancedcore.api.item.FullInventoryHandler; +import com.bencodez.advancedcore.data.ServerData; + +class FullInventoryPublicationMonitorTest { + @TempDir Path directory; + + @Test void ordinaryNativeOwnerCannotPublishAnIncompleteOverflowSection() throws Exception { + verifyPublication(false, false); + } + + @Test void ordinaryFacadeOwnerCannotPublishAnIncompleteOverflowSection() throws Exception { + verifyPublication(true, false); + } + + @Test void finalNativeOwnerCannotPublishAnIncompleteOverflowSection() throws Exception { + verifyPublication(false, true); + } + + @Test void finalFacadeOwnerCannotPublishAnIncompleteOverflowSection() throws Exception { + verifyPublication(true, true); + } + + private void verifyPublication(boolean wrapped, boolean finalPublication) throws Exception { + ConfigurationSerialization.registerClass(FullInventoryFinalPublicationTest.PersistedItem.class, + "ACFinalOverflowTestItem"); + var plugin = mock(AdvancedCorePlugin.class); + var server = mock(Server.class); + when(plugin.getServer()).thenReturn(server); + when(server.getScheduler()).thenReturn(mock(org.bukkit.scheduler.BukkitScheduler.class)); + when(plugin.getDataFolder()).thenReturn(directory.toFile()); + when(plugin.getLogger()).thenReturn(Logger.getAnonymousLogger()); + var owner = new ServerData(plugin); + var yaml = new GatedYaml(); + owner.setData(yaml); + owner.saveDataChecked(); + ServerData exposed = wrapped ? new Facade(plugin, owner) : owner; + when(plugin.getServerDataFile()).thenReturn(exposed); + var handler = new FullInventoryHandler(plugin); + UUID player = UUID.randomUUID(); + handler.add(player, new FullInventoryFinalPublicationTest.PersistedItem()); + assertTrue(handler.saveDurably()); + assertNotNull(YamlConfiguration.loadConfiguration(owner.getdFile()) + .getItemStack("FullInventory." + player + ".Items.0"), + "the regression starts with real durable recovery data"); + var writer = Executors.newSingleThreadExecutor(); + var checkpoint = Executors.newSingleThreadExecutor(); + var writerThread = new java.util.concurrent.atomic.AtomicReference(); + var checkpointThread = new java.util.concurrent.atomic.AtomicReference(); + yaml.armed.set(true); + Future inventory = writer.submit(() -> { + writerThread.set(Thread.currentThread()); + return handler.saveDurably(); + }); + try { + assertTrue(yaml.cleared.await(3, TimeUnit.SECONDS)); + Future other = checkpoint.submit(() -> { + checkpointThread.set(Thread.currentThread()); + if (finalPublication) owner.mutateFinalData(data -> data.set("VoteParty.Votes", 7)) + .toCompletableFuture().join(); + else owner.setValue("VoteParty.Votes", 7); + }); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3); + boolean blockedOnActualOwner = false; + while (!other.isDone() && System.nanoTime() < deadline) { + Thread thread = checkpointThread.get(); + var info = thread == null ? null : ManagementFactory.getThreadMXBean().getThreadInfo(thread.threadId()); + if (info != null && info.getThreadState() == Thread.State.BLOCKED + && info.getLockOwnerId() == writerThread.get().threadId() + && info.getLockInfo().getIdentityHashCode() == System.identityHashCode(owner)) { + blockedOnActualOwner = true; + break; + } + Thread.yield(); + } + if (other.isDone()) { + other.get(3, TimeUnit.SECONDS); + assertTrue(YamlConfiguration.loadConfiguration(owner.getdFile()).contains("FullInventory"), + "another real whole-file publication persisted the temporary section removal"); + } + assertTrue(blockedOnActualOwner, "publication must wait for the actual YAML owner's transaction"); + yaml.release.countDown(); + assertTrue(inventory.get(3, TimeUnit.SECONDS)); + other.get(3, TimeUnit.SECONDS); + var stored = YamlConfiguration.loadConfiguration(owner.getdFile()); + assertNotNull(stored.getItemStack("FullInventory." + player + ".Items.0")); + assertEquals(7, stored.getInt("VoteParty.Votes")); + } finally { + yaml.release.countDown(); + writer.shutdownNow(); + checkpoint.shutdownNow(); + handler.getTimer().shutdownNow(); + assertTrue(writer.awaitTermination(5, TimeUnit.SECONDS)); + assertTrue(checkpoint.awaitTermination(5, TimeUnit.SECONDS)); + ConfigurationSerialization.unregisterClass(FullInventoryFinalPublicationTest.PersistedItem.class); + } + } + + private static final class GatedYaml extends YamlConfiguration { + final CountDownLatch cleared = new CountDownLatch(1), release = new CountDownLatch(1); + final AtomicBoolean armed = new AtomicBoolean(); + + @Override public void set(String path, Object value) { + super.set(path, value); + if (path.equals("FullInventory") && value == null && armed.compareAndSet(true, false)) { + cleared.countDown(); + try { assertTrue(release.await(10, TimeUnit.SECONDS)); } + catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new AssertionError(failure); } + } + } + } + + private static final class Facade extends ServerData { + private final ServerData owner; + Facade(AdvancedCorePlugin plugin, ServerData owner) { super(plugin); this.owner = owner; } + @Override public Object publicationMonitor() { return owner.publicationMonitor(); } + @Override public FileConfiguration getData() { return owner.getData(); } + @Override public void saveData() { owner.saveData(); } + } +} diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java index 86265df45..2125a0141 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerShutdownAdmissionTest.java @@ -46,6 +46,49 @@ class TimeCheckerShutdownAdmissionTest { @TempDir File directory; + @Test void recoveredFacadeValidationAndBeginShareTheActualPublicationOwner() throws Exception { + try (Fixture fixture = new Fixture(directory)) { + var facade = new PublicationFacade(fixture.plugin, fixture.data); + when(fixture.plugin.getServerDataFile()).thenReturn(facade); + fixture.startCalendarCheck(); + fixture.data.allowValidation.countDown(); + fixture.calendarCheck.get(5, TimeUnit.SECONDS); + assertEquals(2, facade.validations.get()); + assertTrue(facade.recoveryValidatedUnderOwner.get(), + "recovered validation must remain atomic with begin against native publication"); + assertTrue(facade.recoveryBeganUnderOwner.get()); + assertEquals(3, fixture.events.get()); + assertNull(fixture.data.getPendingTimeChangeTransition(TimeType.DAY)); + } + } + + private static final class PublicationFacade extends ServerData { + private final ServerData owner; + final AtomicInteger validations = new AtomicInteger(); + final AtomicBoolean recoveryValidatedUnderOwner = new AtomicBoolean(); + final AtomicBoolean recoveryBeganUnderOwner = new AtomicBoolean(); + PublicationFacade(AdvancedCorePlugin plugin, ServerData owner) { super(plugin); this.owner = owner; } + @Override public Object publicationMonitor() { return owner.publicationMonitor(); } + @Override public TimeChangeTransitionState getPendingTimeChangeTransition(TimeType type) { + return owner.getPendingTimeChangeTransition(type); + } + @Override public boolean isPendingTimeChangeTransition(TimeChangeTransitionState state) { + if (validations.incrementAndGet() == 2) + recoveryValidatedUnderOwner.set(Thread.holdsLock(owner)); + return owner.isPendingTimeChangeTransition(state); + } + @Override public TimeChangeTransitionState beginTimeChangeTransition(TimeType type, String period, String marker) { + recoveryBeganUnderOwner.set(Thread.holdsLock(owner)); + return owner.beginTimeChangeTransition(type, period, marker); + } + @Override public void completeTimeChangeTransition(TimeChangeTransitionState state) { + owner.completeTimeChangeTransition(state); + } + @Override public void failTimeChangeTransition(TimeChangeTransitionState state) { + owner.failTimeChangeTransition(state); + } + } + @Test void inventorySaveDoesNotBlockShutdownAdmissionOrCompleteItsDrainEarly() throws Exception { try (Fixture fixture = new Fixture(directory)) { fixture.startCalendarCheck(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java index af5964632..302916037 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/TimeCheckerTest.java @@ -63,6 +63,7 @@ public void setUp() { plugin = mock(AdvancedCorePlugin.class); options = mock(AdvancedCoreConfigOptions.class); serverDataFile = mock(ServerData.class); + when(serverDataFile.publicationMonitor()).thenCallRealMethod(); // 3) stub plugin getters when(plugin.getOptions()).thenReturn(options); diff --git a/docs/shared-runtime-foundation.md b/docs/shared-runtime-foundation.md index 1abb0c76d..b864ef613 100644 --- a/docs/shared-runtime-foundation.md +++ b/docs/shared-runtime-foundation.md @@ -53,6 +53,17 @@ Later unrelated saves cannot publish a failed final overflow mutation. Ordered snapshots carry only successfully published final changes forward and preserve newer explicit edits, including an edit that writes the same previous value. No caller waits for publication while holding the YAML owner's monitor. Legacy +live read/mutate/save transactions use `ServerData.publicationMonitor()`, whose +native default is the file object itself. A facade must return its backing +owner's stable monitor and delegate YAML access, mutations and publication to +that same owner; overriding the monitor alone does not reroute inherited native +methods. Overflow-section replacement and recovered time-transition validation +use this contract so ordinary or final whole-file snapshots cannot interleave +with those compound live operations. This lock does not turn a queued or void +save into a durable acknowledgement and must never be held while awaiting an +asynchronous publication stage. + +Legacy void `saveData` overrides retain the established live-YAML mutation followed by one save callback, but cannot claim confirmed publication without overriding the additive completion contract; they report an unsupported confirmation. A legacy From c5fba2c5041979e675909e2ac3e9340c5ad18e3b Mon Sep 17 00:00:00 2001 From: Ben Date: Fri, 16 Oct 2026 20:47:48 -0600 Subject: [PATCH 19/28] Retain admitted offline replay across queued capture and storage handoffs --- .../api/user/AdvancedCoreUser.java | 145 ++++---- .../CoreRuntimeAdmittedRetirementTest.java | 318 ++++++++++++++++++ 2 files changed, 404 insertions(+), 59 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java index fa04e8b7b..ad41b062f 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/user/AdvancedCoreUser.java @@ -1304,32 +1304,44 @@ public void checkOfflineRewards() { * this pass; a concurrent replay may already own other pending occurrences. */ public CompletionStage checkOfflineRewardsAsync() { - return Reward.continueOnServerThread(plugin, this, - () -> CompletableFuture.completedFuture(captureReplayPlayerState())).thenCompose(state -> { - CompletableFuture> handoff = new CompletableFuture<>(); - Runnable work = () -> { - CompletableFuture outcome = new CompletableFuture<>(); - if (!handoff.complete(outcome)) return; + RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck(); + if (admission == null) return CompletableFuture.completedFuture(null); + try { + CompletionStage captured = Reward.continueOnServerThread(plugin, this, + () -> CompletableFuture.completedFuture(captureReplayPlayerState())); + // Observe the owned handoffs, not the caller's cancellable dependent stage. + captured.whenComplete((state, failure) -> { if (failure != null) admission.close(); }); + CompletionStage owned = captured.thenCompose(state -> { + CompletableFuture> handoff = new CompletableFuture<>(); + handoff.whenComplete((stage, failure) -> { if (failure != null) admission.close(); }); + Runnable work = () -> { + CompletableFuture outcome = new CompletableFuture<>(); + if (!handoff.complete(outcome)) return; + try (admission) { + if (!plugin.isEnabled()) throw new IllegalStateException("Plugin disabled before offline replay"); + checkAdmittedOfflineRewards(false, state).whenComplete((ignored, failure) -> { + if (failure == null) outcome.complete(null); else outcome.completeExceptionally(failure); + }); + } catch (Throwable failure) { outcome.completeExceptionally(failure); } + }; try { - if (!plugin.isEnabled()) throw new IllegalStateException("Plugin disabled before offline replay"); - if (!plugin.getOptions().isProcessRewards()) { outcome.complete(null); return; } - if (isCheckWorld()) setCheckWorld(false); - dispatchOfflineRewards(false, state).whenComplete((ignored, failure) -> { - if (failure == null) outcome.complete(null); else outcome.completeExceptionally(failure); + // Shared-runtime admission and queue storage stay off the owner. + plugin.getTimer().execute(() -> { + try { + UserDataManager manager = sharedReplayDataManager(); + if (manager == null || !manager.deferSharedStorageWork(work)) work.run(); + } catch (Throwable failure) { handoff.completeExceptionally(failure); } }); - } catch (Throwable failure) { outcome.completeExceptionally(failure); } - }; - try { - // Runtime admission and queue storage must also stay off the owner. - plugin.getTimer().execute(() -> { - try { - UserDataManager manager = sharedReplayDataManager(); - if (manager == null || !manager.deferSharedStorageWork(work)) work.run(); - } catch (Throwable failure) { handoff.completeExceptionally(failure); } - }); - } catch (Throwable failure) { handoff.completeExceptionally(failure); } - return handoff.orTimeout(30, TimeUnit.SECONDS).thenCompose(stage -> stage); - }); + } catch (Throwable failure) { handoff.completeExceptionally(failure); } + return handoff.orTimeout(30, TimeUnit.SECONDS).thenCompose(stage -> stage); + }); + // Cancelling the public result must not suppress a successful owner capture's + // continuation: that continuation still owns admission and durable replay. + return owned.thenApply(ignored -> null); + } catch (RuntimeException | Error failure) { + admission.close(); + return CompletableFuture.failedFuture(failure); + } } /** Snapshot of local replay ownership only; never reads user storage. */ @@ -1366,22 +1378,31 @@ private void checkOfflineRewards(ReplayPlayerState capturedState) { plugin.debug("Processing rewards is disabled"); return; } - UserDataManager manager = sharedReplayDataManager(); - if (capturedState == null && manager != null && manager.mustDeferSharedStorageAccess()) { - ReplayPlayerState state = captureReplayPlayerState(); - if (manager.deferSharedStorageWork(() -> checkOfflineRewards(state))) return; - } - if (isCheckWorld()) { - setCheckWorld(false); + RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck(); + if (admission == null) return; + boolean deferred = false; + try { + UserDataManager manager = sharedReplayDataManager(); + if (capturedState == null && manager != null && manager.mustDeferSharedStorageAccess()) { + ReplayPlayerState state = captureReplayPlayerState(); + deferred = manager.deferSharedStorageWork(() -> { + try (admission) { checkAdmittedOfflineRewards(false, state); } + }); + if (deferred) return; + } + checkAdmittedOfflineRewards(false, capturedState); + } finally { + if (!deferred) admission.close(); } - dispatchOfflineRewards(false, capturedState); } - private CompletionStage dispatchOfflineRewards(boolean force, ReplayPlayerState capturedState) { - try (RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck()) { - if (admission == null) return CompletableFuture.completedFuture(null); - return dispatchAdmittedOfflineRewards(force, capturedState); + private CompletionStage checkAdmittedOfflineRewards(boolean force, ReplayPlayerState capturedState) { + if (!plugin.getOptions().isProcessRewards()) { + plugin.debug("Processing rewards is disabled"); + return CompletableFuture.completedFuture(null); } + if (force || isCheckWorld()) setCheckWorld(false); + return dispatchAdmittedOfflineRewards(force, capturedState); } private CompletionStage dispatchAdmittedOfflineRewards(boolean force, ReplayPlayerState capturedState) { @@ -1744,32 +1765,38 @@ private CompletionStage forceRunOfflineRewardsAsync(ReplayPlayerState capt plugin.debug("Processing rewards is disabled"); return CompletableFuture.completedFuture(null); } - UserDataManager manager = sharedReplayDataManager(); - if (capturedState == null && manager != null && manager.mustDeferSharedStorageAccess()) { - ReplayPlayerState state = captureReplayPlayerState(); - CompletableFuture completion = new CompletableFuture<>(); + RewardHandler.PersistedReplayCheck admission = plugin.getRewardHandler().admitPersistedReplayCheck(); + if (admission == null) return CompletableFuture.completedFuture(null); + boolean deferred = false; + try { + UserDataManager manager = sharedReplayDataManager(); + if (capturedState == null && manager != null && manager.mustDeferSharedStorageAccess()) { + ReplayPlayerState state = captureReplayPlayerState(); + CompletableFuture completion = new CompletableFuture<>(); + try { + deferred = manager.deferSharedStorageWork(() -> { + try (admission) { + checkAdmittedOfflineRewards(true, state).whenComplete((ignored, failure) -> { + if (failure == null) completion.complete(null); + else completion.completeExceptionally(failure); + }); + } catch (Throwable failure) { + completion.completeExceptionally(failure); + } + }); + if (deferred) return completion; + } catch (RuntimeException | Error failure) { + completion.completeExceptionally(failure); + return completion; + } + } try { - if (manager.deferSharedStorageWork(() -> { - try { - forceRunOfflineRewardsAsync(state).whenComplete((ignored, failure) -> { - if (failure == null) completion.complete(null); - else completion.completeExceptionally(failure); - }); - } catch (Throwable failure) { - completion.completeExceptionally(failure); - } - })) return completion; + return checkAdmittedOfflineRewards(true, capturedState); } catch (RuntimeException | Error failure) { - completion.completeExceptionally(failure); - return completion; + return CompletableFuture.failedFuture(failure); } - } - - setCheckWorld(false); - try { - return dispatchOfflineRewards(true, capturedState); - } catch (RuntimeException | Error failure) { - return CompletableFuture.failedFuture(failure); + } finally { + if (!deferred) admission.close(); } } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java index e2234d88b..cefc8c773 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeAdmittedRetirementTest.java @@ -250,6 +250,324 @@ private static void awaitIgnoringInterrupt(CountDownLatch release) { assertFailedTimedStorageHandoffReleasesReplayAdmission(true); } + @Test void queuedOfflineStorageCheckRetainsAdmissionAcrossShutdownFence() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL, false, false); + } + + @Test void queuedForcedOfflineCheckRetainsAdmissionAcrossShutdownFence() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.FORCED_ASYNC, false, false); + } + + @Test void queuedVoidForcedOfflineCheckRetainsAdmissionAcrossShutdownFence() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.FORCED_VOID, false, false); + } + + @Test void queuedAsyncOfflineCheckRetainsAdmissionAcrossShutdownFence() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL_ASYNC, false, false); + } + + @Test void failedQueuedOfflineCheckRestoresBeforeNativeRetirement() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL, true, false); + } + + @Test void failedQueuedForcedOfflineCheckRestoresBeforeNativeRetirement() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.FORCED_ASYNC, true, false); + } + + @Test void failedQueuedAsyncOfflineCheckRestoresBeforeNativeRetirement() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL_ASYNC, true, false); + } + + @Test void cancelledForcedOfflineCallerCannotRetireAcceptedQueuedWork() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.FORCED_ASYNC, false, true); + } + + @Test void cancelledAsyncOfflineCallerCannotRetireAcceptedQueuedWork() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL_ASYNC, false, true); + } + + private enum OfflineEntry { NORMAL, NORMAL_ASYNC, FORCED_ASYNC, FORCED_VOID } + + @Test void disabledOfflineProcessingDoesNotRetainReplayAdmission() throws Exception { + assertOfflineNoOpReleasesAdmission(false); + } + + @Test void emptyOfflineQueuesDoNotRetainReplayAdmission() throws Exception { + assertOfflineNoOpReleasesAdmission(true); + } + + @Test void offlineChecksAfterShutdownFenceDoNotReadQueues() throws Exception { + assertOfflineNoOpReleasesAdmission(true, true); + } + + private void assertOfflineNoOpReleasesAdmission(boolean process) throws Exception { + assertOfflineNoOpReleasesAdmission(process, false); + } + + private void assertOfflineNoOpReleasesAdmission(boolean process, boolean closed) throws Exception { + for (OfflineEntry entry : OfflineEntry.values()) { + Fixture fixture = new Fixture(); + when(fixture.plugin.isEnabled()).thenReturn(true); + when(fixture.plugin.getOptions().isProcessRewards()).thenReturn(process); + when(fixture.users.getOfflineRewardsPath()).thenReturn("OfflineRewards"); + var apiWorker = java.util.concurrent.Executors.newSingleThreadScheduledExecutor( + AdvancedCoreRuntime.daemonThreads("empty-offline-api-handoff")); + when(fixture.plugin.getTimer()).thenReturn(apiWorker); + var scheduler = mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(fixture.plugin.getBukkitScheduler()).thenReturn(scheduler); + doAnswer(call -> { call.getArgument(1, Runnable.class).run(); return null; }).when(scheduler) + .executeOrScheduleSync(eq(fixture.plugin), any(Runnable.class)); + RewardHandler rewards = new RewardHandler(fixture.plugin); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "empty-offline-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + when(data.getStringList("OfflineRewards", UserDataFetchMode.DEFAULT)).thenReturn(new ArrayList<>()); + try { + if (closed) rewards.beginShutdown().toCompletableFuture().get(2, TimeUnit.SECONDS); + java.util.concurrent.CompletionStage completion; + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + completion = invokeOffline(user, entry); + } + if (completion != null) completion.toCompletableFuture().get(2, TimeUnit.SECONDS); + fixture.manager.getTimer().submit(() -> null).get(2, TimeUnit.SECONDS); + rewards.beginShutdown().toCompletableFuture().get(2, TimeUnit.SECONDS); + assertFalse(user.hasPendingRewardReplay(), entry.name()); + verify(data, times(process && !closed ? 1 : 0)).getStringList("OfflineRewards", UserDataFetchMode.DEFAULT); + } finally { + apiWorker.shutdownNow(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + } + + @Test void rejectedOfflineStorageHandoffReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.NORMAL, false); + } + + @Test void failedOfflinePlayerCaptureReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.NORMAL, true); + } + + @Test void rejectedForcedOfflineStorageHandoffReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.FORCED_ASYNC, false); + } + + @Test void failedForcedOfflinePlayerCaptureReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.FORCED_ASYNC, true); + } + + @Test void rejectedAsyncOfflineWorkerHandoffReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.NORMAL_ASYNC, false); + } + + @Test void failedAsyncOfflinePlayerCaptureReleasesAdmission() throws Exception { + assertFailedOfflineHandoffReleasesAdmission(OfflineEntry.NORMAL_ASYNC, true); + } + + private void assertFailedOfflineHandoffReleasesAdmission(OfflineEntry entry, boolean failCapture) throws Exception { + Fixture fixture = new Fixture(); + when(fixture.plugin.isEnabled()).thenReturn(true); + when(fixture.plugin.getOptions().isProcessRewards()).thenReturn(true); + when(fixture.users.getOfflineRewardsPath()).thenReturn("OfflineRewards"); + var apiWorker = java.util.concurrent.Executors.newSingleThreadScheduledExecutor( + AdvancedCoreRuntime.daemonThreads("rejected-offline-api-handoff")); + when(fixture.plugin.getTimer()).thenReturn(apiWorker); + var scheduler = mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(fixture.plugin.getBukkitScheduler()).thenReturn(scheduler); + doAnswer(call -> { call.getArgument(1, Runnable.class).run(); return null; }).when(scheduler) + .executeOrScheduleSync(eq(fixture.plugin), any(Runnable.class)); + RewardHandler rewards = new RewardHandler(fixture.plugin); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "rejected-offline-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + if (failCapture) when(fixture.plugin.getOptions().isTreatVanishAsOffline()) + .thenThrow(new IllegalStateException("capture failed")); + else if (entry == OfflineEntry.NORMAL_ASYNC) apiWorker.shutdown(); + else fixture.manager.getTimer().shutdown(); + try { + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + Class type = failCapture ? IllegalStateException.class + : java.util.concurrent.RejectedExecutionException.class; + if (entry == OfflineEntry.NORMAL || (failCapture && entry != OfflineEntry.NORMAL_ASYNC)) { + assertThrows(type, () -> invokeOffline(user, entry)); + } else { + var completion = invokeOffline(user, entry); + Throwable failure = assertThrows(java.util.concurrent.ExecutionException.class, + () -> completion.toCompletableFuture().get(2, TimeUnit.SECONDS)).getCause(); + assertInstanceOf(type, failure); + } + } + rewards.beginShutdown().toCompletableFuture().get(2, TimeUnit.SECONDS); + verify(data, never()).getStringList(eq("OfflineRewards"), any()); + } finally { + apiWorker.shutdownNow(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + + @Test void cancelledAsyncOfflineCallerStillReleasesFailedOwnerCaptureAdmission() throws Exception { + Fixture fixture = new Fixture(); + when(fixture.plugin.isEnabled()).thenReturn(true); + when(fixture.plugin.getOptions().isProcessRewards()).thenReturn(true); + when(fixture.plugin.getOptions().isTreatVanishAsOffline()).thenThrow(new IllegalStateException("capture failed")); + var scheduler = mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(fixture.plugin.getBukkitScheduler()).thenReturn(scheduler); + var capture = new java.util.concurrent.atomic.AtomicReference(); + doAnswer(call -> { capture.set(call.getArgument(1)); return null; }).when(scheduler) + .executeOrScheduleSync(eq(fixture.plugin), any(Runnable.class)); + RewardHandler rewards = new RewardHandler(fixture.plugin); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "cancelled-capture-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + try { + var completion = user.checkOfflineRewardsAsync().toCompletableFuture(); + assertNotNull(capture.get()); + assertTrue(completion.cancel(false)); + var drain = rewards.beginShutdown().toCompletableFuture(); + assertFalse(drain.isDone(), "caller cancellation cannot discard the owned capture"); + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + capture.get().run(); + } + drain.get(2, TimeUnit.SECONDS); + verify(data, never()).getStringList(anyString(), any()); + } finally { + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + + private java.util.concurrent.CompletionStage invokeOffline(AdvancedCoreUser user, OfflineEntry entry) { + return switch (entry) { + case NORMAL -> { user.checkOfflineRewards(); yield null; } + case NORMAL_ASYNC -> user.checkOfflineRewardsAsync(); + case FORCED_ASYNC -> user.forceRunOfflineRewardsAsync(); + case FORCED_VOID -> { user.forceRunOfflineRewards(); yield null; } + }; + } + + @Test void cancelledAsyncCallerBeforeSuccessfulCaptureRetainsReplayAndRetirement() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL_ASYNC, false, true, true); + } + + @Test void cancelledAsyncCallerBeforeSuccessfulCaptureRetainsFailedReplayForRetry() throws Exception { + assertQueuedOfflineCheckRetainsStorage(OfflineEntry.NORMAL_ASYNC, true, true, true); + } + + private void assertQueuedOfflineCheckRetainsStorage(OfflineEntry entry, boolean failedDispatch, + boolean cancelCaller) throws Exception { + assertQueuedOfflineCheckRetainsStorage(entry, failedDispatch, cancelCaller, false); + } + + private void assertQueuedOfflineCheckRetainsStorage(OfflineEntry entry, boolean failedDispatch, + boolean cancelCaller, boolean deferCapture) throws Exception { + Fixture fixture = new Fixture(); + when(fixture.plugin.isEnabled()).thenReturn(true); + when(fixture.plugin.getOptions().isProcessRewards()).thenReturn(true); + when(fixture.users.getOfflineRewardsPath()).thenReturn("OfflineRewards"); + var apiWorker = java.util.concurrent.Executors.newSingleThreadScheduledExecutor( + AdvancedCoreRuntime.daemonThreads("offline-api-handoff")); + when(fixture.plugin.getTimer()).thenReturn(apiWorker); + var scheduler = mock(com.bencodez.simpleapi.scheduler.BukkitScheduler.class); + when(fixture.plugin.getBukkitScheduler()).thenReturn(scheduler); + var capture = new java.util.concurrent.atomic.AtomicReference(); + doAnswer(call -> { + Runnable task = call.getArgument(1); + if (deferCapture) capture.set(task); else task.run(); + return null; + }).when(scheduler) + .executeOrScheduleSync(eq(fixture.plugin), any(Runnable.class)); + RewardHandler rewards = spy(new RewardHandler(fixture.plugin)); + when(fixture.plugin.getRewardHandler()).thenReturn(rewards); + AdvancedCoreUser user = new AdvancedCoreUser(fixture.plugin, fixture.uuid, "offline-handoff-fixture"); + UserData data = mock(UserData.class); + user.setData(data); + ArrayList persisted = new ArrayList<>(List.of("ShutdownReward")); + AtomicBoolean nativeClosed = new AtomicBoolean(); + doAnswer(call -> { nativeClosed.set(true); return null; }).when(fixture.mysql).close(); + when(data.getStringList("OfflineRewards", UserDataFetchMode.DEFAULT)) + .thenAnswer(call -> new ArrayList<>(persisted)); + org.mockito.stubbing.Answer write = call -> { + assertFalse(nativeClosed.get(), "queue handling must retain its native owner"); + persisted.clear(); + persisted.addAll(call.getArgument(1)); + return null; + }; + doAnswer(write).when(data).setStringList(eq("OfflineRewards"), any(), anyBoolean()); + doAnswer(write).when(data).setStringList(eq("OfflineRewards"), any()); + CompletableFuture dispatch = new CompletableFuture<>(); + CountDownLatch dispatched = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicReference dispatchedEntry = new java.util.concurrent.atomic.AtomicReference<>(); + doAnswer(call -> { + dispatchedEntry.set(persisted.get(0)); + dispatched.countDown(); + return dispatch; + }).when(rewards) + .givePersistedQueueRewardAsync(eq(user), any(PersistedQueueReference.class), any(RewardOptions.class)); + CountDownLatch occupied = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + var handoffWorker = entry == OfflineEntry.NORMAL_ASYNC ? apiWorker : fixture.manager.getTimer(); + handoffWorker.execute(() -> { occupied.countDown(); awaitIgnoringInterrupt(release); }); + assertTrue(occupied.await(2, TimeUnit.SECONDS)); + try { + java.util.concurrent.CompletionStage completion; + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + completion = invokeOffline(user, entry); + } + verify(data, never()).getStringList(eq("OfflineRewards"), any()); + if (cancelCaller) assertTrue(completion.toCompletableFuture().cancel(false)); + var checksDrained = rewards.beginShutdown().toCompletableFuture(); + assertFalse(checksDrained.isDone(), "accepted offline work must own admission across its queued handoff"); + if (deferCapture) { + assertNotNull(capture.get()); + try (var bukkit = mockStatic(Bukkit.class)) { + bukkit.when(Bukkit::getServer).thenReturn(mock(Server.class)); + bukkit.when(Bukkit::isPrimaryThread).thenReturn(true); + capture.get().run(); + } + assertFalse(checksDrained.isDone(), "successful capture must retain the queued worker's admission"); + verify(data, never()).getStringList(eq("OfflineRewards"), any()); + } + release.countDown(); + assertTrue(dispatched.await(2, TimeUnit.SECONDS), "admitted work must dispatch after the fence closes"); + checksDrained.get(2, TimeUnit.SECONDS); + BukkitRuntimePlatform platform = fixture.platform(); + new AdvancedCoreRuntime(platform).shutdown(0L); + assertFalse(platform.beforeExecutorShutdownCompletion().toCompletableFuture().isDone()); + verify(fixture.mysql, never()).close(); + if (completion != null && !cancelCaller) assertFalse(completion.toCompletableFuture().isDone()); + if (failedDispatch) dispatch.completeExceptionally(new IllegalStateException("dispatch failed")); + else dispatch.complete(null); + if (completion != null && !cancelCaller) completion.toCompletableFuture().get(2, TimeUnit.SECONDS); + platform.beforeExecutorShutdownCompletion().toCompletableFuture().get(2, TimeUnit.SECONDS); + if (failedDispatch) { + assertEquals(1, persisted.size()); + assertEquals(dispatchedEntry.get(), persisted.get(0), + "failed offline replay must preserve the exact occurrence and payload"); + } else assertTrue(persisted.isEmpty()); + verify(fixture.cache, times(1)).flush(fixture.uuid, UserStorage.MYSQL, fixture.storage); + verify(fixture.mysql, times(1)).close(); + } finally { + release.countDown(); + dispatch.complete(null); + apiWorker.shutdownNow(); + rewards.shutdown(0, TimeUnit.NANOSECONDS); + fixture.manager.getTimer().shutdownNow(); + } + } + private void assertFailedTimedStorageHandoffReleasesReplayAdmission(boolean failCapture) throws Exception { Fixture fixture = new Fixture(); RewardHandler rewards = new RewardHandler(fixture.plugin); From ad72cc614e3623ecf54486c8be5ede4992ec33f2 Mon Sep 17 00:00:00 2001 From: Ben Date: Sat, 17 Oct 2026 19:28:00 -0600 Subject: [PATCH 20/28] Select published scheduler correction in the shared SDK --- AdvancedCore/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AdvancedCore/pom.xml b/AdvancedCore/pom.xml index 3dd2990fa..7cd4fd59c 100644 --- a/AdvancedCore/pom.xml +++ b/AdvancedCore/pom.xml @@ -238,8 +238,8 @@ com.bencodez simpleapi - - 1.0.2-20261011.161601-40 + + 1.0.2-20261018.002045-42 compile From 052daff39d59972aac7226ed519fc6d6f26bb735 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 18 Oct 2026 17:02:29 -0600 Subject: [PATCH 21/28] Preserve shutdown cleanup when ServerData is disabled --- .../api/item/FullInventoryHandler.java | 7 +++ .../api/permissions/PermissionHandler.java | 6 +++ .../bukkit/runtime/BukkitRuntimePlatform.java | 4 +- .../tests/lifecycle/CoreRuntimeTest.java | 1 + .../FinalPublicationSequenceTest.java | 1 + .../NativeServerDataAdmissionTest.java | 54 +++++++++++++++++++ .../ShutdownProcessPersistenceTest.java | 1 + 7 files changed, 73 insertions(+), 1 deletion(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java index 7583d3e5b..c46b470c7 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java @@ -306,6 +306,13 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { ScheduledFuture task = checkTask; if (task != null) task.cancel(false); ServerData serverData = plugin.getServerDataFile(); + if (!plugin.isLoadServerData() && (serverData == null || serverData.getData() == null)) { + // No native file was loaded. Still fence delivery and retire its timer, + // but do not snapshot or acknowledge any unpublished replay overflow. + if (timer != null) timer.shutdownNow(); + completion.complete(null); + return completion; + } try { persistenceExecutor.execute(() -> { try { diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/permissions/PermissionHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/permissions/PermissionHandler.java index 19b5cb5bc..31d577069 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/permissions/PermissionHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/permissions/PermissionHandler.java @@ -401,6 +401,12 @@ public CompletionStage shutDownAsync(Executor persistenceExecutor) { acceptingExpirations.set(false); timer.shutdownNow(); ServerData serverData = plugin.getServerDataFile(); + if (!plugin.isLoadServerData() && (serverData == null || serverData.getData() == null)) { + // Expiration admission and its timer are already stopped; intentionally + // unloaded ServerData has no final permission file to publish. + completion.complete(null); + return completion; + } try { persistenceExecutor.execute(() -> { try { diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java index 92bbd20b1..c5b9ccd62 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/bukkit/runtime/BukkitRuntimePlatform.java @@ -461,7 +461,9 @@ private CompletionStage finalPersistence(String component, private CompletionStage submitTimestampPersistence(Executor executor) { ServerData owner = plugin.getServerDataFile(); - if (owner == null) return CompletableFuture.completedFuture(null); + if (owner == null || (!plugin.isLoadServerData() && owner.getData() == null)) { + return CompletableFuture.completedFuture(null); + } long lastUpdated = System.currentTimeMillis(); var completion = new CompletableFuture(); try { diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java index 50e8dca96..01bcebe6c 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/CoreRuntimeTest.java @@ -940,6 +940,7 @@ private static java.util.Queue queuedStorageWorker(UserDataManager man @Test void shutdownHookReceivesActualFinalStagesAndRetainsExceptionalSettlement() throws Exception { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.isLoadServerData()).thenReturn(true); var inventory = mock(FullInventoryHandler.class); var permissions = mock(com.bencodez.advancedcore.api.permissions.PermissionHandler.class); var data = mock(com.bencodez.advancedcore.data.ServerData.class); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java index c40003c0e..dec44c62c 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/FinalPublicationSequenceTest.java @@ -27,6 +27,7 @@ class FinalPublicationSequenceTest { private void verifySequence(boolean failInventory) throws Exception { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.isLoadServerData()).thenReturn(true); FullInventoryHandler inventory = mock(FullInventoryHandler.class); PermissionHandler permissions = mock(PermissionHandler.class); ServerData owner = mock(ServerData.class); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java index a80991de2..fdbbd361d 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/NativeServerDataAdmissionTest.java @@ -33,6 +33,7 @@ import com.bencodez.advancedcore.AdvancedCorePlugin; import com.bencodez.advancedcore.api.permissions.PermissionHandler; +import com.bencodez.advancedcore.api.item.FullInventoryHandler; import com.bencodez.advancedcore.api.rewards.RewardHandler; import com.bencodez.advancedcore.api.rewards.RewardEligibility; import com.bencodez.advancedcore.api.time.TimeChecker; @@ -159,6 +160,58 @@ class NativeServerDataAdmissionTest { successor.awaitRelease(); } + @Test void disabledUninitializedServerDataSkipsAllFinalWritesAndAllowsSameJvmReenable() throws Exception { + Fixture fixture = fixture(); + when(fixture.plugin.isLoadServerData()).thenReturn(false); + ServerData owner = spy(new ServerData(fixture.plugin)); + when(fixture.plugin.getServerDataFile()).thenReturn(owner); + for (int generation = 0; generation < 2; generation++) { + FullInventoryHandler inventory = new FullInventoryHandler(fixture.plugin); + PermissionHandler permissions = new PermissionHandler(fixture.plugin); + workers.add(inventory.getTimer()); + workers.add(permissions.getTimer()); + when(fixture.plugin.getFullInventoryHandler()).thenReturn(inventory); + when(fixture.plugin.getPermissionHandler()).thenReturn(permissions); + fixture.plugin.onEnable(); + fixture.plugin.onDisable(); + fixture.finalFiles.get().toCompletableFuture().get(2, TimeUnit.SECONDS); + fixture.awaitRelease(); + assertTrue(inventory.getTimer().isTerminated()); + assertTrue(permissions.getTimer().isTerminated()); + assertFalse(fixture.plugin.hasServerDataOwnership()); + } + verify(owner, never()).mutateFinalData(any()); + assertNull(owner.getData()); + assertFalse(Files.exists(directory.resolve("ServerData.yml"))); + verify(fixture.plugin, times(2)).onUnLoad(); + } + + @Test void enabledUninitializedServerDataStillFailsAndRetainsNativeAdmission() throws Exception { + Fixture fixture = fixture(); + ServerData owner = new ServerData(fixture.plugin); + when(fixture.plugin.getServerDataFile()).thenReturn(owner); + fixture.plugin.onEnable(); + fixture.plugin.onDisable(); + assertThrows(java.util.concurrent.ExecutionException.class, + () -> fixture.finalFiles.get().toCompletableFuture().get(2, TimeUnit.SECONDS)); + assertThrows(java.util.concurrent.ExecutionException.class, + () -> fixture.releaseFence.get().get(2, TimeUnit.SECONDS)); + assertTrue(fixture.plugin.hasServerDataOwnership(), "enabled but failed persistence cannot release ownership"); + } + + @Test void initializedServerDataStillPublishesWhenLoadFlagChangesBeforeShutdown() throws Exception { + Fixture fixture = fixture(); + ServerData owner = new ServerData(fixture.plugin); + owner.setup(); + when(fixture.plugin.getServerDataFile()).thenReturn(owner); + fixture.plugin.onEnable(); + when(fixture.plugin.isLoadServerData()).thenReturn(false); + fixture.plugin.onDisable(); + fixture.awaitRelease(); + assertTrue(YamlConfiguration.loadConfiguration(directory.resolve("ServerData.yml").toFile()) + .getLong("LastUpdated") > 0); + } + @Test void successfulShutdownAllowsSameInstanceReenableAndDuplicateDisableStaysInert() throws Exception { Fixture plugin = fixture(); ServerData owner = new ServerData(plugin.plugin); @@ -437,6 +490,7 @@ private static CompletionStage invokeDeferredRetirement(BukkitRuntimePlatf private Fixture fixture(Path dataDirectory) { AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); initializeLifecycleLock(plugin); + when(plugin.isLoadServerData()).thenReturn(true); Server server = mock(Server.class); when(plugin.getDataFolder()).thenReturn(dataDirectory.toFile()); when(plugin.getName()).thenReturn("native-admission-fixture"); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ShutdownProcessPersistenceTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ShutdownProcessPersistenceTest.java index 5cc9116ce..ad34fcd2c 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ShutdownProcessPersistenceTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ShutdownProcessPersistenceTest.java @@ -61,6 +61,7 @@ public static void main(String[] args) throws Exception { Path directory = Path.of(args[0]); boolean blocked = args[1].equals("blocked"); var plugin = mock(AdvancedCorePlugin.class); + when(plugin.isLoadServerData()).thenReturn(true); var owner = mock(ServerData.class); when(plugin.getLogger()).thenReturn(Logger.getLogger("shutdown-fixture")); when(plugin.getServerDataFile()).thenReturn(owner); From 9491420f56a9a676110eda3d8ab54622d8948e87 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 18 Oct 2026 22:04:28 -0600 Subject: [PATCH 22/28] Confirm replay overflow publication before acknowledging delivery --- .../api/item/FullInventoryHandler.java | 147 ++++++++---------- .../tests/item/FullInventoryHandlerTest.java | 99 +++++++++++- .../FullInventoryPublicationMonitorTest.java | 9 +- 3 files changed, 161 insertions(+), 94 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java index c46b470c7..00f48339d 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java @@ -192,6 +192,8 @@ private void scheduleItemDelivery(Player player, ItemStack[] item, CompletableFu if (pendingOverflow) { persistReservedOverflowAsync(reservationId).whenComplete((ignored, failure) -> { if (failure == null) completeReplayOverflow(reservationId, completion); + else if (failure instanceof UnconfirmedOverflowPublication) + scheduleReservedOverflowPersistenceRetry(reservationId, completion); else completeStartedOverflowWithFallback(player, deliveryPlayerId, reservationId, completion); }); } else { @@ -316,7 +318,7 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { try { persistenceExecutor.execute(() -> { try { - finalSnapshot(serverData).whenComplete((ignored, failure) -> { + finalSnapshot(serverData, false).whenComplete((ignored, failure) -> { if (failure == null) completion.complete(null); else completion.completeExceptionally(failure); }); @@ -333,24 +335,34 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { return completion; } - private CompletionStage finalSnapshot(ServerData serverData) { + private final java.util.Map overflowPublications = new HashMap<>(); + + private CompletionStage finalSnapshot(ServerData serverData, boolean requireRunning) { HashMap reservations; CompletionStage publication; deliveryLock.writeLock().lock(); try { + if (requireRunning && shuttingDown.get()) return CompletableFuture.failedFuture( + new IllegalStateException("Final inventory publication owns pending overflow")); reservations = new HashMap<>(replayOverflowReservations); YamlConfiguration snapshot = pendingItemsSnapshot(reservations.values()); YamlConfiguration replacement = copySection(snapshot, "FullInventory"); - publication = serverData.mutateFinalData(data -> replaceSection(data, "FullInventory", replacement)); + reservations.keySet().forEach(id -> overflowPublications.merge(id, 1, Integer::sum)); + try { publication = serverData.mutateFinalData(data -> replaceSection(data, "FullInventory", replacement)); } + catch (Throwable failure) { publication = CompletableFuture.failedFuture(failure); } + if (publication == null) publication = CompletableFuture.failedFuture( + new IllegalStateException("ServerData owner returned no publication stage")); } finally { deliveryLock.writeLock().unlock(); } - // Publication may belong to a consumer's different worker. Never join while - // holding delivery/YAML admission; that worker may finish earlier accepted work. - return publication.thenRun(() -> { + // Publication may belong to a different worker. Never join while holding + // delivery/YAML admission. Settle promotion and fallback ownership together. + return publication.whenComplete((ignored, failure) -> { deliveryLock.writeLock().lock(); try { - reservations.forEach(this::promoteReservedOverflowLocked); + reservations.keySet().forEach(id -> overflowPublications.computeIfPresent(id, + (key, count) -> count == 1 ? null : count - 1)); + if (failure == null) reservations.forEach(this::promoteReservedOverflowLocked); } finally { deliveryLock.writeLock().unlock(); } - reservations.keySet().forEach(id -> completeReplayOverflow(id, null)); + if (failure == null) reservations.keySet().forEach(id -> completeReplayOverflow(id, null)); }); } @@ -389,54 +401,58 @@ public boolean saveDurably() { } private boolean saveDurably(ServerData serverData) { - HashMap reservations; - boolean saved; - deliveryLock.writeLock().lock(); try { - reservations = new HashMap<>(replayOverflowReservations); - if (!savePendingItemsLocked(reservations.values(), serverData)) return false; - for (Entry entry : reservations.entrySet()) { - promoteReservedOverflowLocked(entry.getKey(), entry.getValue()); - } - saved = true; - } finally { - deliveryLock.writeLock().unlock(); - } - if (saved) { - for (String reservationId : reservations.keySet()) completeReplayOverflow(reservationId, null); + // Native owners complete synchronously. A facade's queued save is not + // durable yet; do not block the caller or claim that it has completed. + var publication = finalSnapshot(serverData, false).toCompletableFuture(); + if (!publication.isDone()) return false; + publication.join(); + return true; + } catch (Throwable failure) { + plugin.getLogger().log(Level.WARNING, "Failed to confirm pending full-inventory items", failure); + return false; } - return true; } - /** Persists one replay reservation before making it visible to ordinary pending checks. */ + /** A failed owner stage may already have replaced disk bytes; never drop/replay it. */ + private static final class UnconfirmedOverflowPublication extends RuntimeException { + private static final long serialVersionUID = 1L; + private UnconfirmedOverflowPublication(Throwable cause) { super("Overflow publication is unconfirmed", cause); } + } + + /** Persists replay overflow through the actual owner's acknowledged publication. */ private CompletionStage persistReservedOverflowAsync(String reservationId) { CompletableFuture persisted = new CompletableFuture<>(); try { timer.execute(() -> { - if (persistReservedOverflow(reservationId)) persisted.complete(null); - else persisted.completeExceptionally( - new IllegalStateException("Unable to persist full-inventory reward overflow")); + try { + persistReservedOverflow(reservationId).whenComplete((ignored, failure) -> { + if (failure == null) persisted.complete(null); + else persisted.completeExceptionally(new UnconfirmedOverflowPublication(failure)); + }); + } catch (Throwable failure) { persisted.completeExceptionally(failure); } }); - } catch (Throwable failure) { - persisted.completeExceptionally(failure); - } + } catch (Throwable failure) { persisted.completeExceptionally(failure); } return persisted; } - private boolean persistReservedOverflow(String reservationId) { + private CompletionStage persistReservedOverflow(String reservationId) { deliveryLock.writeLock().lock(); try { ReservedOverflow reservation = replayOverflowReservations.get(reservationId); - if (reservation == null || reservation.items.isEmpty()) return true; - // The captured final snapshot owns shutdown acknowledgement. Do not race it - // with a void legacy save from a previously queued timer task. - if (shuttingDown.get()) return false; - if (!savePendingItemsLocked(java.util.List.of(reservation), plugin.getServerDataFile())) return false; - promoteReservedOverflowLocked(reservationId, reservation); - return true; - } finally { - deliveryLock.writeLock().unlock(); - } + if (reservation == null || reservation.items.isEmpty()) + return CompletableFuture.completedFuture(null); + if (shuttingDown.get()) return CompletableFuture.failedFuture( + new IllegalStateException("Final inventory publication owns pending overflow")); + } finally { deliveryLock.writeLock().unlock(); } + // A disabled/uninitialized native file has not admitted any publication. + // Keep its established safe overflow fallback instead of treating it as uncertain. + ServerData owner = plugin.getServerDataFile(); + if (owner == null || owner.getData() == null) + throw new IllegalStateException("Full-inventory persistence owner is uninitialized"); + // Include every unpublished reservation. Invoke completion callbacks only + // after releasing this caller's admission, including a synchronous owner. + return finalSnapshot(owner, true); } /** @@ -471,7 +487,7 @@ private boolean dropReservedOverflowOwnedPlayer(Player player, UUID playerId, St if (reservation == null) return true; // A pending final publication already owns these items. An older queued // fallback cannot drop them while the owner's snapshot is publishing. - if (shuttingDown.get()) return false; + if (shuttingDown.get() || overflowPublications.containsKey(reservationId)) return false; if (!playerId.equals(reservation.playerId)) return false; ArrayList remaining = new ArrayList<>(); for (ItemStack item : reservation.items) { @@ -496,8 +512,12 @@ private void scheduleReservedOverflowPersistenceRetry(String reservationId, Comp } try { timer.schedule(() -> { - if (persistReservedOverflow(reservationId)) completeReplayOverflow(reservationId, completion); - else scheduleReservedOverflowPersistenceRetry(reservationId, completion); + try { + persistReservedOverflow(reservationId).whenComplete((ignored, failure) -> { + if (failure == null) completeReplayOverflow(reservationId, completion); + else scheduleReservedOverflowPersistenceRetry(reservationId, completion); + }); + } catch (Throwable failure) { scheduleReservedOverflowPersistenceRetry(reservationId, completion); } }, 30, TimeUnit.SECONDS); } catch (Throwable ignored) { // Shutdown may reject this retry. Leave the completion pending: acknowledging @@ -511,45 +531,6 @@ private void completeReplayOverflow(String reservationId, CompletableFuture reservedOverflows, - ServerData serverData) { - if (serverData == null) return false; - synchronized (serverData.publicationMonitor()) { - return savePendingItemsUnderDataLock(reservedOverflows, serverData); - } - } - - private boolean savePendingItemsUnderDataLock(Collection reservedOverflows, - ServerData serverData) { - FileConfiguration data = null; - YamlConfiguration previous = null; - try { - if (serverData == null || serverData.getData() == null) return false; - - YamlConfiguration snapshot = pendingItemsSnapshot(reservedOverflows); - - data = serverData.getData(); - previous = copySection(data, "FullInventory"); - data.set("FullInventory", null); - ConfigurationSection replacement = snapshot.getConfigurationSection("FullInventory"); - if (replacement != null) { - for (String path : replacement.getKeys(true)) { - if (!replacement.isConfigurationSection(path)) data.set("FullInventory." + path, replacement.get(path)); - } - } - serverData.saveData(); - return true; - } catch (Exception e) { - if (data != null && previous != null) replaceSection(data, "FullInventory", previous); - try { - plugin.getLogger().log(Level.WARNING, "Failed to save pending full-inventory items", e); - } catch (Throwable ignored) { - // A partially initialized plugin can still retain the reservation for retry. - } - return false; - } - } private static YamlConfiguration copySection(ConfigurationSection source, String path) { YamlConfiguration copy = new YamlConfiguration(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index 6c5af7241..5690ed86f 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -415,7 +415,7 @@ public void replayDropFailurePersistsOnlyTheUndroppedOverflow() throws Exception } @Test - public void replayOverflowSaveFailureDropsOnlyReservedOverflowWithoutReplayingPartialInsert() throws Exception { + public void replayOverflowRejectedBeforePublicationDropsOnlyReservedOverflowWithoutReplayingPartialInsert() throws Exception { Fixture fixture = createFixture(); UUID uuid = UUID.randomUUID(); Player player = mock(Player.class); @@ -433,9 +433,7 @@ public void replayOverflowSaveFailureDropsOnlyReservedOverflowWithoutReplayingPa when(inventory.addItem(item)).thenReturn(new HashMap<>(java.util.Map.of(0, excess))); when(world.dropItem(location, excess)).thenReturn(mock(org.bukkit.entity.Item.class)); fixture.handler.getLastMessageTime().put(uuid, System.currentTimeMillis()); - doAnswer(invocation -> { - throw new IllegalStateException("disk unavailable"); - }).when(fixture.serverData).saveData(); + fixture.handler.getTimer().shutdownNow(); // Rejected before any owner publication. AtomicInteger scheduled = new AtomicInteger(); AtomicReference initialDelivery = new AtomicReference<>(); @@ -487,9 +485,7 @@ public void replayOverflowDoesNotAcknowledgeAnUnavailableFallback() throws Excep when(player.getInventory()).thenReturn(inventory); when(inventory.addItem(item)).thenReturn(new HashMap<>(java.util.Map.of(0, excess))); fixture.handler.getLastMessageTime().put(uuid, System.currentTimeMillis()); - doAnswer(invocation -> { - throw new IllegalStateException("disk unavailable"); - }).when(fixture.serverData).saveData(); + fixture.handler.getTimer().shutdownNow(); // Rejected before any owner publication. AtomicReference initialDelivery = new AtomicReference<>(); AtomicReference fallback = new AtomicReference<>(); @@ -613,7 +609,7 @@ public void pendingFinalPublicationFencesPreviouslyQueuedOverflowDropFallback() if (!first.compareAndSet(null, task)) { fallback.set(task); fallbackScheduled.countDown(); } return null; }).when(fixture.bukkitScheduler).runTask(eq(fixture.plugin), any(Runnable.class), eq(player)); - doAnswer(call -> { throw new IllegalStateException("ordinary snapshot failed"); }).when(fixture.serverData).saveData(); + fixture.handler.getTimer().shutdownNow(); // Queue the fallback before any owner save. CompletionStage delivery = fixture.handler.giveItemAsync(player, item); try (MockedStatic bukkit = mockStatic(Bukkit.class)) { bukkit.when(() -> Bukkit.getPlayer(uuid)).thenReturn(player); @@ -905,6 +901,90 @@ public void saveWaitsForInflightDeliveryBeforeSnapshotting() throws Exception { assertEquals(excessItem, root.getItemStack(uuid + ".Items.0")); } + @Test void disabledUninitializedServerDataPreservesSafeOverflowFallback() throws Exception { + Fixture f = createFixture(); + when(f.plugin.isLoadServerData()).thenReturn(false); when(f.serverData.getData()).thenReturn(null); + UUID owner = UUID.randomUUID(); var player = mock(Player.class); var slots = mock(PlayerInventory.class); + var input = mock(ItemStack.class); var overflow = mock(ItemStack.class); + var world = mock(World.class); var location = mock(Location.class); + when(player.getUniqueId()).thenReturn(owner); when(player.isOnline()).thenReturn(true); + when(player.getInventory()).thenReturn(slots); when(player.getWorld()).thenReturn(world); when(player.getLocation()).thenReturn(location); + when(slots.addItem(input)).thenReturn(new HashMap<>(java.util.Map.of(0, overflow))); + when(world.dropItem(location, overflow)).thenReturn(mock(org.bukkit.entity.Item.class)); + f.handler.getLastMessageTime().put(owner, System.currentTimeMillis()); + var tasks = new java.util.concurrent.LinkedBlockingQueue(); + doAnswer(call -> { tasks.add(call.getArgument(1)); return null; }) + .when(f.bukkitScheduler).runTask(eq(f.plugin), any(Runnable.class), eq(player)); + try (MockedStatic bukkit = mockStatic(Bukkit.class)) { + bukkit.when(() -> Bukkit.getPlayer(owner)).thenReturn(player); + var receipt = f.handler.giveItemAsync(player, input).toCompletableFuture(); + tasks.poll(3, TimeUnit.SECONDS).run(); f.handler.getTimer().submit(() -> {}).get(3, TimeUnit.SECONDS); + Runnable fallback = tasks.poll(3, TimeUnit.SECONDS); assertNotNull(fallback); assertFalse(receipt.isDone()); + fallback.run(); receipt.get(3, TimeUnit.SECONDS); + verify(slots, times(1)).addItem(input); verify(world, times(1)).dropItem(location, overflow); + verify(f.serverData, never()).mutateFinalData(any()); assertFalse(f.handler.getItems().containsKey(owner)); + } + } + + @Test void overlappingSuccessfulAsyncOwnerSavesRetainEveryUnpublishedOverflow() throws Exception { + overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(false); + } + @Test void failedEarlierSaveCannotDropOverflowOwnedByAnotherPendingPublication() throws Exception { + overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(true); + } + private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean failFirst) throws Exception { + Fixture f = createFixture(); + var submissions = new java.util.concurrent.LinkedBlockingQueue>>(); + doAnswer(call -> { + var candidate = new YamlConfiguration(); + call.getArgument(0, java.util.function.Consumer.class).accept(candidate); + var confirmed = new java.util.concurrent.CompletableFuture(); + submissions.add(java.util.Map.entry(candidate, confirmed)); + return confirmed.thenRun(() -> { + synchronized (f.serverData.publicationMonitor()) { + f.data.set("FullInventory", null); + for (String path : candidate.getKeys(true)) + if (!candidate.isConfigurationSection(path)) f.data.set(path, candidate.get(path)); + } + }); + }).when(f.serverData).mutateFinalData(any()); + var ownerTasks = new java.util.concurrent.LinkedBlockingQueue(); + doAnswer(call -> { ownerTasks.add(call.getArgument(1)); return null; }) + .when(f.bukkitScheduler).runTask(eq(f.plugin), any(Runnable.class), any(Player.class)); + UUID first = UUID.randomUUID(), second = UUID.randomUUID(); + var a = mock(Player.class); var b = mock(Player.class); + var slots = mock(PlayerInventory.class); var item = mock(ItemStack.class); + when(a.getUniqueId()).thenReturn(first); when(b.getUniqueId()).thenReturn(second); + for (Player player : List.of(a, b)) { when(player.isOnline()).thenReturn(true); when(player.getInventory()).thenReturn(slots); } + when(slots.addItem(item)).thenReturn(new HashMap<>(java.util.Map.of(0, item))); + f.handler.getLastMessageTime().put(first, System.currentTimeMillis()); + f.handler.getLastMessageTime().put(second, System.currentTimeMillis()); + try (MockedStatic bukkit = mockStatic(Bukkit.class)) { + bukkit.when(() -> Bukkit.getPlayer(first)).thenReturn(a); bukkit.when(() -> Bukkit.getPlayer(second)).thenReturn(b); + var receiptA = f.handler.giveItemAsync(a, item).toCompletableFuture(); + ownerTasks.poll(3, TimeUnit.SECONDS).run(); var publicationA = submissions.poll(3, TimeUnit.SECONDS); + var receiptB = f.handler.giveItemAsync(b, item).toCompletableFuture(); + ownerTasks.poll(3, TimeUnit.SECONDS).run(); var publicationB = submissions.poll(3, TimeUnit.SECONDS); + assertNotNull(publicationA); assertNotNull(publicationB); + assertFalse(receiptA.isDone()); assertFalse(receiptB.isDone()); assertTrue(f.handler.getItems().isEmpty()); + assertTrue(publicationB.getKey().contains("FullInventory." + first)); + assertTrue(publicationB.getKey().contains("FullInventory." + second)); + if (failFirst) { + publicationA.getValue().completeExceptionally(new IllegalStateException("first publication rejected")); + // An unconfirmed owner save can already be visible on disk. + // It must remain reserved for confirmed retry, never fall back to a drop. + f.handler.getTimer().submit(() -> {}).get(3, TimeUnit.SECONDS); + assertTrue(ownerTasks.isEmpty()); verify(a, never()).getWorld(); assertFalse(receiptA.isDone()); + } else publicationA.getValue().complete(null); + publicationB.getValue().complete(null); + receiptA.get(3, TimeUnit.SECONDS); receiptB.get(3, TimeUnit.SECONDS); + assertEquals(List.of(item), f.handler.getItems().get(first)); + assertEquals(List.of(item), f.handler.getItems().get(second)); + assertEquals(item, f.data.getItemStack("FullInventory." + first + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + second + ".Items.0")); + } + } + private Fixture createFixture() { return createFixture(TimeUnit.SECONDS.toMillis(30)); } @@ -917,8 +997,11 @@ private Fixture createFixture(long itemDeliveryTimeoutMillis) { ServerData serverData = mock(ServerData.class); when(serverData.publicationMonitor()).thenCallRealMethod(); org.mockito.Mockito.doCallRealMethod().when(serverData).mutateFinalData(any()); + // The fixture's synchronous publisher is the existing controllable saveData stub. + doAnswer(call -> { serverData.saveData(); return null; }).when(serverData).saveDataChecked(); YamlConfiguration data = new YamlConfiguration(); + when(plugin.getLogger()).thenReturn(java.util.logging.Logger.getAnonymousLogger()); when(plugin.getInventoryTimer()).thenReturn(sharedInventoryTimer); when(plugin.getOptions()).thenReturn(options); when(plugin.isEnabled()).thenReturn(true); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java index b4dd6fd12..cd5030d94 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryPublicationMonitorTest.java @@ -119,13 +119,13 @@ private static final class GatedYaml extends YamlConfiguration { final CountDownLatch cleared = new CountDownLatch(1), release = new CountDownLatch(1); final AtomicBoolean armed = new AtomicBoolean(); - @Override public void set(String path, Object value) { - super.set(path, value); - if (path.equals("FullInventory") && value == null && armed.compareAndSet(true, false)) { + @Override public java.util.Set getKeys(boolean deep) { + if (!deep && armed.compareAndSet(true, false)) { cleared.countDown(); try { assertTrue(release.await(10, TimeUnit.SECONDS)); } catch (InterruptedException failure) { Thread.currentThread().interrupt(); throw new AssertionError(failure); } } + return super.getKeys(deep); } } @@ -135,5 +135,8 @@ private static final class Facade extends ServerData { @Override public Object publicationMonitor() { return owner.publicationMonitor(); } @Override public FileConfiguration getData() { return owner.getData(); } @Override public void saveData() { owner.saveData(); } + @Override public CompletionStage mutateFinalData(java.util.function.Consumer mutation) { + return owner.mutateFinalData(mutation); + } } } From 4f41f34f453e40f87c2a01b28c7df5d5c91273b3 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 18 Oct 2026 22:20:45 -0600 Subject: [PATCH 23/28] Derive reward scheduler fixture from the declared API type --- .../tests/api/rewards/RetiringOwnerScheduler.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/api/rewards/RetiringOwnerScheduler.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/api/rewards/RetiringOwnerScheduler.java index 74125dbdd..348ad5f30 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/api/rewards/RetiringOwnerScheduler.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/api/rewards/RetiringOwnerScheduler.java @@ -34,11 +34,10 @@ private enum Admission { NORMAL, IMMEDIATE_RETIREMENT, LATER_RETIREMENT, REJECTE Consumer retirementQueue) { this.ownerQueue = ownerQueue; this.retirementQueue = retirementQueue; - FoliaLib folia = mock(FoliaLib.class); - ServerImplementation implementation = mock(ServerImplementation.class); + FoliaLib folia = mock(FoliaLib.class, RETURNS_DEEP_STUBS); + ServerImplementation implementation = folia.getImpl(); when(scheduler.getFoliaLib()).thenReturn(folia); when(folia.isFolia()).thenReturn(true); - when(folia.getImpl()).thenReturn(implementation); // Exercise the real wrapper that discards runAtEntity's returned future. doCallRealMethod().when(scheduler).runTask(eq(plugin), any(Runnable.class), any(Entity.class)); when(implementation.runAtEntity(any(Entity.class), any())).thenAnswer(call -> From 5da4c2b74c06d2d3b5c0fb32bff9ca3e71d8f71f Mon Sep 17 00:00:00 2001 From: Ben Date: Thu, 22 Oct 2026 18:09:21 -0600 Subject: [PATCH 24/28] Expose ordered ordinary YAML mutations for live projections Reject uncertain replacements before callbacks or reloads can change live state. --- .../advancedcore/data/ServerData.java | 22 ++++- .../ServerDataPublicationOrderingTest.java | 87 ++++++++++++++++++- .../time/ServerDataTimeTransitionTest.java | 18 ++++ 3 files changed, 122 insertions(+), 5 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java index 57d40983e..1f2e6c4aa 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/data/ServerData.java @@ -343,8 +343,6 @@ private void inspectFailedMove(Path target, String intendedHash, String previous private void prepareSnapshotPublication() { requireFinalPublicationSettled(); - if (replacementUncertain) - throw new IllegalStateException("ServerData replacement outcome is unknown; restart required before saving"); if (!publicationSettlementPending && !replacementCleanupPending) return; Path target = getdFile().toPath().toAbsolutePath(); try { forceDirectory(target.getParent()); } @@ -359,6 +357,8 @@ private void prepareSnapshotPublication() { private void requireFinalPublicationSettled() { if (finalPublicationIndeterminate) throw new IllegalStateException("Final ServerData publication is unconfirmed; restart required before reuse"); + if (replacementUncertain) + throw new IllegalStateException("ServerData replacement outcome is unknown; restart required before saving"); } private Path backupPath(Path target) { @@ -488,7 +488,23 @@ private void withNativePublicationOrdering(Runnable operation) { } } - private void mutateOrdinaryData(java.util.function.Consumer mutation) { + /** + * Mutates the live YAML and then invokes the ordinary {@link #saveData()} path. + * The callback runs synchronously on the caller before saving, so an adapter + * can update its in-memory projection before disk I/O begins. Native owners + * check final-publication settlement and hold their publication monitor across + * both operations; the callback receives the existing live YAML, not a copy. + * Adapters with another owner must delegate this operation to that owner and + * compose their projection update inside its callback. + * + * This is an ordinary save, not a durable publication acknowledgement. A callback + * failure propagates without saving or rolling back any live changes. Callbacks + * must not wait for asynchronous work or perform another save/reload operation. + * Existing custom save owners retain their own synchronization and save behavior. + * + * @param mutation the synchronous mutation and optional in-memory projection update + */ + public void mutateOrdinaryData(java.util.function.Consumer mutation) { withNativePublicationOrdering(() -> { mutation.accept(getData()); saveData(); }); } diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java index cb2a42817..c8742ed3c 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/lifecycle/ServerDataPublicationOrderingTest.java @@ -33,6 +33,76 @@ class ServerDataPublicationOrderingTest { @TempDir File dataFolder; + @Test void ordinaryCallbackPublishesLiveProjectionBeforeBlockedNativeDiskIo() throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + AdvancedCorePlugin plugin = mock(AdvancedCorePlugin.class); + when(plugin.getDataFolder()).thenReturn(dataFolder); + BarrierServerData data = new BarrierServerData(plugin); + data.install(); + FileConfiguration live = data.getData(); + live.set("VoteParty.Total", 7); + var display = new java.util.concurrent.atomic.AtomicInteger(7); + var finished = new CompletableFuture(); + Thread writer = new Thread(() -> { + try { + data.mutateOrdinaryData(yaml -> { + assertSame(live, yaml); + assertTrue(Thread.holdsLock(data.publicationMonitor())); + yaml.set("VoteParty.Total", 17); + display.set(yaml.getInt("VoteParty.Total")); + }); + finished.complete(null); + } catch (Throwable failure) { finished.completeExceptionally(failure); } + }, "ordinary-live-projection"); + try { + writer.start(); + assertTrue(data.ordinaryWriteStarted.await(2, TimeUnit.SECONDS)); + assertFalse(finished.isDone(), "disk I/O is still held"); + assertEquals(17, display.get(), "display must not wait for disk I/O"); + assertSame(live, data.getData()); + } finally { + data.releaseOrdinaryWrite.countDown(); + writer.join(2000); + if (writer.isAlive()) writer.interrupt(); + } + assertFalse(writer.isAlive()); + finished.get(2, TimeUnit.SECONDS); + assertEquals(17, YamlConfiguration.loadConfiguration(data.getdFile()).getInt("VoteParty.Total")); + } + } + + @Test void ordinaryCallbackFailurePreservesLiveIdentityWithoutSavingOrClaimingRollback() throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + LegacySavingOwner data = legacySavingOwner(); + FileConfiguration live = data.getData(); + var failure = new IllegalArgumentException("synthetic projection failure"); + assertSame(failure, assertThrows(IllegalArgumentException.class, () -> data.mutateOrdinaryData(yaml -> { + assertSame(live, yaml); + yaml.set("VoteParty.Total", 17); + throw failure; + }))); + assertEquals(0, data.saves); + assertSame(live, data.getData()); + assertEquals(17, live.getInt("VoteParty.Total")); + assertFalse(data.getdFile().exists()); + } + } + + @Test void ordinaryCallbackRetainsTheCustomOwnersSaveFailure() throws Exception { + try (var scheduler = mockConstruction(BukkitScheduler.class)) { + LegacySavingOwner data = legacySavingOwner(); + data.saveFailure = new IllegalStateException("synthetic owner failure"); + var display = new java.util.concurrent.atomic.AtomicInteger(); + assertSame(data.saveFailure, assertThrows(IllegalStateException.class, () -> data.mutateOrdinaryData(yaml -> { + yaml.set("VoteParty.Total", 17); + display.set(yaml.getInt("VoteParty.Total")); + }))); + assertEquals(1, data.saves); + assertEquals(17, display.get()); + assertSame(data.getData(), data.savedData); + } + } + @Test void heartbeatCannotOverwriteAcknowledgedFinalPublication() throws Exception { verifyPublicationOrder(true); } @@ -272,6 +342,9 @@ private static void assertQuarantinedAfterUnconfirmedFinalPublication(FailingDir assertThrows(IllegalStateException.class, data::saveDataChecked); assertThrows(IllegalStateException.class, data::reloadData); assertThrows(IllegalStateException.class, data::setup); + var callbackCalled = new java.util.concurrent.atomic.AtomicBoolean(); + assertThrows(IllegalStateException.class, () -> data.mutateOrdinaryData(yaml -> callbackCalled.set(true))); + assertFalse(callbackCalled.get(), "an unconfirmed final publication fences the callback itself"); assertThrows(CompletionException.class, () -> data.mutateFinalData(yaml -> yaml.set("UnacknowledgedSuccessor", true)).toCompletableFuture().join()); assertFalse(data.getData().contains("UnacknowledgedSuccessor")); @@ -384,6 +457,13 @@ private static void assertNestedFinalValues(FileConfiguration yaml) { "Inherited.Value"); } + @Test void ordinaryCallbackCannotMutateLiveYamlDuringNativeFinalPublication() throws Exception { + verifyInheritedMutatorIsPublicationOrdered(data -> data.mutateOrdinaryData(yaml -> { + assertTrue(Thread.holdsLock(data.publicationMonitor())); + yaml.set("Ordinary.Value", "new"); + }), "Ordinary.Value"); + } + @Test void inheritedCreateSectionCannotMutateLiveYamlDuringNativePublication() throws Exception { verifyInheritedMutatorIsPublicationOrdered(data -> data.createSection("Inherited.Section"), "Inherited.Section"); @@ -572,13 +652,16 @@ private static void assertNestedFinalValues(FileConfiguration yaml) { when(source.getName()).thenReturn("ExamplePlugin"); when(source.getDescription()).thenReturn(new org.bukkit.plugin.PluginDescriptionFile( "ExamplePlugin", "1.0", "example.Main")); - data.setPluginVersion(source); written.complete(null); + data.setPluginVersion(source); + data.mutateOrdinaryData(yaml -> yaml.set("Custom.Callback", "value")); + written.complete(null); } catch (Throwable failure) { written.completeExceptionally(failure); } }, "custom-ordinary-mutation"); try { holder.start(); assertTrue(ownerHeld.await(2, TimeUnit.SECONDS)); writer.start(); written.get(2, TimeUnit.SECONDS); - assertEquals(7, data.saved.get()); + assertEquals(8, data.saved.get()); + assertEquals("value", data.getData().getString("Custom.Callback")); assertEquals(3, data.getData().getInt("PrevDay")); } finally { releaseOwner.countDown(); holder.join(2000); writer.join(2000); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java index de74f67bf..9e5e09910 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/time/ServerDataTimeTransitionTest.java @@ -290,12 +290,30 @@ void ambiguousMoveOutcomeCannotBeOverwrittenByLaterWholeFileSaves() throws Excep yaml -> yaml.set("FullInventory.unpublished.Time", 456L)).toCompletableFuture().join()); byte[] unknownSnapshot = Files.readAllBytes(data.getdFile().toPath()); data.moveFailure = 0; + var live = data.getData(); + String liveBefore = live.saveToString(); + var callbackCalled = new java.util.concurrent.atomic.AtomicBoolean(); + assertThrows(IllegalStateException.class, () -> data.mutateOrdinaryData(yaml -> { + callbackCalled.set(true); + yaml.set("Ordinary.Unconfirmed", 27); + })); + assertFalse(callbackCalled.get(), "unknown replacement must reject before invoking a live callback"); assertThrows(IllegalStateException.class, data::setLastUpdated); + assertThrows(IllegalStateException.class, () -> data.setData("Ordinary.Value", 27)); + assertThrows(IllegalStateException.class, () -> data.setValue("Ordinary.Value", 27)); + assertThrows(IllegalStateException.class, () -> data.createSection("Ordinary.Section")); + var bulk = new YamlConfiguration(); + bulk.set("Ordinary.Bulk", 27); + assertThrows(IllegalStateException.class, () -> data.setData(bulk)); + assertThrows(IllegalStateException.class, data::reloadData); + assertThrows(IllegalStateException.class, data::setup); assertThrows(IllegalStateException.class, data::saveData); assertThrows(IllegalStateException.class, data::saveDataChecked); assertThrows(java.util.concurrent.CompletionException.class, () -> data.mutateFinalData( yaml -> yaml.set("IgnoreTime", true)).toCompletableFuture().join()); assertArrayEquals(unknownSnapshot, Files.readAllBytes(data.getdFile().toPath())); + assertSame(live, data.getData()); + assertEquals(liveBefore, live.saveToString(), "rejected operations must not change live YAML"); } @Test From f596f4bc181fe3018959792376139b47655b7f1a Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 00:26:30 -0600 Subject: [PATCH 25/28] Serialize overflow snapshots through owner settlement --- .../api/item/FullInventoryHandler.java | 60 ++++-- .../tests/item/FullInventoryHandlerTest.java | 181 +++++++++++++++++- 2 files changed, 222 insertions(+), 19 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java index 00f48339d..c6000a94b 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java @@ -282,18 +282,12 @@ public synchronized void loadTimer() { } public synchronized void shutdown() { - shuttingDown.set(true); - if (checkTask != null) { - checkTask.cancel(false); - checkTask = null; - } - // Flush accepted replay reservations while their completion stages can still - // advance the outer reward checkpoint. Stopping the executor first can discard - // an accepted persistence task and later replay the already-inserted items. - saveDurably(); - if (timer != null) { - timer.shutdownNow(); - } + // Reuse the same final publication for legacy and asynchronous shutdown. + // A second shutdown must not retire a worker needed by an earlier save. + shutdownAsync(Runnable::run).whenComplete((ignored, failure) -> { + if (failure != null) plugin.getLogger().log(Level.WARNING, + "Failed to confirm pending full-inventory items", failure); + }); } /** @@ -307,6 +301,8 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { shuttingDown.set(true); ScheduledFuture task = checkTask; if (task != null) task.cancel(false); + // Reject new timer work immediately, but drain already accepted continuations. + if (timer != null) timer.shutdown(); ServerData serverData = plugin.getServerDataFile(); if (!plugin.isLoadServerData() && (serverData == null || serverData.getData() == null)) { // No native file was loaded. Still fence delivery and retire its timer, @@ -318,14 +314,14 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { try { persistenceExecutor.execute(() -> { try { - finalSnapshot(serverData, false).whenComplete((ignored, failure) -> { + finalSnapshot(serverData, false, persistenceExecutor).whenComplete((ignored, failure) -> { + if (timer != null) timer.shutdownNow(); if (failure == null) completion.complete(null); else completion.completeExceptionally(failure); }); } catch (Throwable failure) { - completion.completeExceptionally(failure); - } finally { if (timer != null) timer.shutdownNow(); + completion.completeExceptionally(failure); } }); } catch (Throwable failure) { @@ -337,7 +333,41 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { private final java.util.Map overflowPublications = new HashMap<>(); + private final Object overflowPublicationOrder = new Object(); + private CompletableFuture overflowPublicationTail = CompletableFuture.completedFuture(null); + private CompletionStage finalSnapshot(ServerData serverData, boolean requireRunning) { + return finalSnapshot(serverData, requireRunning, timer); + } + + private CompletionStage finalSnapshot(ServerData serverData, boolean requireRunning, Executor executor) { + CompletableFuture settled = new CompletableFuture<>(); + CompletableFuture previous; + synchronized (overflowPublicationOrder) { + previous = overflowPublicationTail; + overflowPublicationTail = settled; + } + Runnable publish = () -> { + try { + publishSnapshot(serverData, requireRunning).whenComplete((ignored, failure) -> { + if (failure == null) settled.complete(null); + else settled.completeExceptionally(failure); + }); + } catch (Throwable failure) { settled.completeExceptionally(failure); } + }; + // Claim order before invoking any owner or callback. Snapshot capture belongs + // to that order too: an older asynchronous replacement must not publish last. + if (previous.isDone()) publish.run(); + else previous.whenComplete((ignored, failure) -> { + // Owner completion may run on a platform thread; never start disk work there. + try { executor.execute(publish); } + catch (Throwable rejected) { settled.completeExceptionally(rejected); } + }); + // A caller cancelling its view must not release the actual publication queue. + return settled.copy(); + } + + private CompletionStage publishSnapshot(ServerData serverData, boolean requireRunning) { HashMap reservations; CompletionStage publication; deliveryLock.writeLock().lock(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index 5690ed86f..646629015 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -926,13 +926,180 @@ public void saveWaitsForInflightDeliveryBeforeSnapshotting() throws Exception { } } + @Test void reversedOwnerCompletionsCannotEraseLaterSavedItems() throws Exception { + Fixture f = createFixture(); + var submissions = deferredPublications(f); + UUID first = UUID.randomUUID(), second = UUID.randomUUID(); + var item = mock(ItemStack.class); + f.handler.add(first, item); assertFalse(f.handler.saveDurably()); + var publicationA = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationA); + f.handler.add(second, item); assertFalse(f.handler.saveDurably()); + var publicationB = submissions.poll(); + if (publicationB != null) { + // An owner is allowed to finish concurrent submissions in reverse order. + publicationB.getValue().complete(null); publicationA.getValue().complete(null); + } else { + publicationA.getValue().complete(null); + publicationB = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationB); + publicationB.getValue().complete(null); + } + assertEquals(item, f.data.getItemStack("FullInventory." + first + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + second + ".Items.0"), + "an older snapshot must never erase a later completed save"); + } + + @Test void laterSaveWaitsForEarlierOwnerAndCapturesLatestItems() throws Exception { + Fixture f = createFixture(); + var submissions = deferredPublications(f); + UUID first = UUID.randomUUID(), second = UUID.randomUUID(), latest = UUID.randomUUID(); + var item = mock(ItemStack.class); + f.handler.add(first, item); + assertFalse(f.handler.saveDurably()); + var publicationA = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationA); + f.handler.add(second, item); + assertFalse(f.handler.saveDurably()); + // This addition must be captured when B can actually publish, not when B queues. + f.handler.add(latest, item); + assertTrue(submissions.isEmpty(), "a later owner stage must not overtake the pending stage"); + publicationA.getValue().complete(null); + var publicationB = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationB); + assertTrue(publicationB.getKey().contains("FullInventory." + latest)); + publicationB.getValue().complete(null); + assertEquals(item, f.data.getItemStack("FullInventory." + first + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + second + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + latest + ".Items.0")); + } + + @Test void shutdownWaitsForPendingOrdinaryOwnerSave() throws Exception { + Fixture f = createFixture(); + var submissions = deferredPublications(f); + UUID first = UUID.randomUUID(), second = UUID.randomUUID(); + var item = mock(ItemStack.class); + f.handler.add(first, item); assertFalse(f.handler.saveDurably()); + var publicationA = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationA); + f.handler.add(second, item); + var shutdown = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + assertFalse(shutdown.isDone()); + assertTrue(submissions.isEmpty(), "shutdown must wait for the outstanding owner stage"); + publicationA.getValue().complete(null); + var finalPublication = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(finalPublication); + finalPublication.getValue().complete(null); shutdown.get(3, TimeUnit.SECONDS); + assertEquals(item, f.data.getItemStack("FullInventory." + first + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + second + ".Items.0")); + assertTrue(f.handler.getTimer().isShutdown()); + } + + @Test void repeatedLegacyShutdownReusesPendingAndCompletedPublication() throws Exception { + Fixture f = createFixture(); var submissions = deferredPublications(f); + f.handler.add(UUID.randomUUID(), mock(ItemStack.class)); + f.handler.shutdown(); var last = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(last); + f.handler.shutdown(); + var finalStage = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + last.getValue().complete(null); + finalStage.get(3, TimeUnit.SECONDS); + assertTrue(f.handler.getTimer().isShutdown()); + f.handler.shutdown(); + assertTrue(submissions.isEmpty(), "repeated shutdown must not submit stale final writes"); + verify(f.serverData, times(1)).mutateFinalData(any()); + } + + @Test void shutdownDrainsAcceptedSnapshotContinuationBeforeFinalPublication() throws Exception { + Fixture f = createFixture(); var submissions = deferredPublications(f); + UUID first = UUID.randomUUID(), second = UUID.randomUUID(); var item = mock(ItemStack.class); + f.handler.add(first, item); assertFalse(f.handler.saveDurably()); + var a = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(a); + var entered = new CountDownLatch(1); var release = new CountDownLatch(1); + var interrupted = new AtomicBoolean(); + f.handler.getTimer().execute(() -> { + entered.countDown(); + try { release.await(5, TimeUnit.SECONDS); } + catch (InterruptedException failure) { interrupted.set(true); } + }); + assertTrue(entered.await(3, TimeUnit.SECONDS)); + try { + f.handler.add(second, item); assertFalse(f.handler.saveDurably()); + // B is accepted by the timer but cannot run until the blocked task drains. + a.getValue().complete(null); + var shutdown = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + assertTrue(f.handler.getTimer().isShutdown()); assertFalse(shutdown.isDone()); + release.countDown(); + var b = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(b); + assertFalse(interrupted.get()); assertFalse(shutdown.isDone()); + b.getValue().complete(null); + var last = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(last); + assertFalse(shutdown.isDone()); last.getValue().complete(null); shutdown.get(3, TimeUnit.SECONDS); + assertEquals(item, f.data.getItemStack("FullInventory." + first + ".Items.0")); + assertEquals(item, f.data.getItemStack("FullInventory." + second + ".Items.0")); + } finally { release.countDown(); } + } + + @Test void cancellingShutdownViewCannotReleasePendingOwner() throws Exception { + Fixture f = createFixture(); var submissions = deferredPublications(f); + UUID owner = UUID.randomUUID(); var item = mock(ItemStack.class); + f.handler.add(owner, item); assertFalse(f.handler.saveDurably()); + var a = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(a); + var shutdown = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + assertTrue(shutdown.cancel(false)); assertTrue(submissions.isEmpty()); + assertTrue(f.handler.getTimer().isShutdown()); + a.getValue().complete(null); + var last = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(last); + last.getValue().complete(null); + assertEquals(item, f.data.getItemStack("FullInventory." + owner + ".Items.0")); + } + + @Test void deferredSnapshotRunsOnWorkerInsteadOfOwnerCompletionThread() throws Exception { + Fixture f = createFixture(); + var calls = new java.util.concurrent.LinkedBlockingQueue(); + var submissions = deferredPublications(f, calls); + f.handler.add(UUID.randomUUID(), mock(ItemStack.class)); assertFalse(f.handler.saveDurably()); + var a = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(a); calls.take(); + assertFalse(f.handler.saveDurably()); a.getValue().complete(null); + var b = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(b); + Thread worker = calls.poll(3, TimeUnit.SECONDS); assertNotNull(worker); + assertNotSame(Thread.currentThread(), worker); assertTrue(worker.isDaemon()); + b.getValue().complete(null); + } + + private java.util.concurrent.LinkedBlockingQueue>> deferredPublications(Fixture f) { + return deferredPublications(f, null); + } + + private java.util.concurrent.LinkedBlockingQueue>> deferredPublications(Fixture f, + java.util.Queue threads) { + var submissions = new java.util.concurrent.LinkedBlockingQueue>>(); + doAnswer(call -> { + if (threads != null) threads.add(Thread.currentThread()); + var candidate = new YamlConfiguration(); + call.getArgument(0, java.util.function.Consumer.class).accept(candidate); + var confirmed = new java.util.concurrent.CompletableFuture(); + submissions.add(java.util.Map.entry(candidate, confirmed)); + return confirmed.thenRun(() -> { + synchronized (f.serverData.publicationMonitor()) { + f.data.set("FullInventory", null); + for (String path : candidate.getKeys(true)) + if (!candidate.isConfigurationSection(path)) f.data.set(path, candidate.get(path)); + } + }); + }).when(f.serverData).mutateFinalData(any()); + return submissions; + } + @Test void overlappingSuccessfulAsyncOwnerSavesRetainEveryUnpublishedOverflow() throws Exception { overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(false); } @Test void failedEarlierSaveCannotDropOverflowOwnedByAnotherPendingPublication() throws Exception { overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(true); } + @Test void shutdownConfirmsQueuedOverflowOnlyAfterItsOrderedFinalSave() throws Exception { + overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(false, true); + } private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean failFirst) throws Exception { + overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(failFirst, false); + } + private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean failFirst, boolean shutdown) throws Exception { Fixture f = createFixture(); var submissions = new java.util.concurrent.LinkedBlockingQueue>>(); doAnswer(call -> { @@ -964,11 +1131,12 @@ private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean fa var receiptA = f.handler.giveItemAsync(a, item).toCompletableFuture(); ownerTasks.poll(3, TimeUnit.SECONDS).run(); var publicationA = submissions.poll(3, TimeUnit.SECONDS); var receiptB = f.handler.giveItemAsync(b, item).toCompletableFuture(); - ownerTasks.poll(3, TimeUnit.SECONDS).run(); var publicationB = submissions.poll(3, TimeUnit.SECONDS); - assertNotNull(publicationA); assertNotNull(publicationB); + ownerTasks.poll(3, TimeUnit.SECONDS).run(); + f.handler.getTimer().submit(() -> {}).get(3, TimeUnit.SECONDS); + assertNotNull(publicationA); + assertTrue(submissions.isEmpty(), "the second snapshot must await the first owner"); assertFalse(receiptA.isDone()); assertFalse(receiptB.isDone()); assertTrue(f.handler.getItems().isEmpty()); - assertTrue(publicationB.getKey().contains("FullInventory." + first)); - assertTrue(publicationB.getKey().contains("FullInventory." + second)); + var finalFiles = shutdown ? f.handler.shutdownAsync(Runnable::run).toCompletableFuture() : null; if (failFirst) { publicationA.getValue().completeExceptionally(new IllegalStateException("first publication rejected")); // An unconfirmed owner save can already be visible on disk. @@ -976,7 +1144,12 @@ private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean fa f.handler.getTimer().submit(() -> {}).get(3, TimeUnit.SECONDS); assertTrue(ownerTasks.isEmpty()); verify(a, never()).getWorld(); assertFalse(receiptA.isDone()); } else publicationA.getValue().complete(null); + var publicationB = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publicationB); + assertTrue(publicationB.getKey().contains("FullInventory." + first)); + assertTrue(publicationB.getKey().contains("FullInventory." + second)); + assertFalse(receiptB.isDone()); publicationB.getValue().complete(null); + if (finalFiles != null) finalFiles.get(3, TimeUnit.SECONDS); receiptA.get(3, TimeUnit.SECONDS); receiptB.get(3, TimeUnit.SECONDS); assertEquals(List.of(item), f.handler.getItems().get(first)); assertEquals(List.of(item), f.handler.getItems().get(second)); From 41080b460f004e2c20f19706ac790c9c103376f3 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 11:04:31 -0600 Subject: [PATCH 26/28] Register test owner callbacks before exposing completion --- .../tests/item/FullInventoryHandlerTest.java | 64 ++++++++++++++----- 1 file changed, 47 insertions(+), 17 deletions(-) diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index 646629015..9c7d482b8 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -926,6 +926,40 @@ public void saveWaitsForInflightDeliveryBeforeSnapshotting() throws Exception { } } + @Test void deferredFixturePublishesControlOnlyAfterOwnerCallbackRegistration() throws Exception { + Fixture f = createFixture(); + var registering = new CountDownLatch(1); + var allowRegistration = new CountDownLatch(1); + var control = new java.util.concurrent.CompletableFuture() { + @Override public java.util.concurrent.CompletableFuture thenRun(Runnable action) { + registering.countDown(); + try { assertTrue(allowRegistration.await(3, TimeUnit.SECONDS)); } + catch (InterruptedException failure) { + Thread.currentThread().interrupt(); + throw new AssertionError(failure); + } + return super.thenRun(action); + } + }; + var submissions = deferredPublications(f, null, () -> control); + UUID owner = UUID.randomUUID(); var item = mock(ItemStack.class); + f.handler.add(owner, item); + var worker = java.util.concurrent.Executors.newSingleThreadExecutor(); + var save = worker.submit(f.handler::saveDurably); + try { + assertTrue(registering.await(3, TimeUnit.SECONDS)); + assertTrue(submissions.isEmpty(), "raw completion must not precede its owner installation callback"); + allowRegistration.countDown(); + assertFalse(save.get(3, TimeUnit.SECONDS)); + var publication = submissions.poll(3, TimeUnit.SECONDS); assertNotNull(publication); + publication.getValue().complete(null); + assertEquals(item, f.data.getItemStack("FullInventory." + owner + ".Items.0")); + } finally { + allowRegistration.countDown(); control.complete(null); + worker.shutdown(); assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + } + } + @Test void reversedOwnerCompletionsCannotEraseLaterSavedItems() throws Exception { Fixture f = createFixture(); var submissions = deferredPublications(f); @@ -1068,21 +1102,30 @@ java.util.concurrent.CompletableFuture>> deferredPublications(Fixture f) { private java.util.concurrent.LinkedBlockingQueue>> deferredPublications(Fixture f, java.util.Queue threads) { + return deferredPublications(f, threads, java.util.concurrent.CompletableFuture::new); + } + + private java.util.concurrent.LinkedBlockingQueue>> deferredPublications(Fixture f, + java.util.Queue threads, + java.util.function.Supplier> controls) { var submissions = new java.util.concurrent.LinkedBlockingQueue>>(); doAnswer(call -> { if (threads != null) threads.add(Thread.currentThread()); var candidate = new YamlConfiguration(); call.getArgument(0, java.util.function.Consumer.class).accept(candidate); - var confirmed = new java.util.concurrent.CompletableFuture(); - submissions.add(java.util.Map.entry(candidate, confirmed)); - return confirmed.thenRun(() -> { + var confirmed = controls.get(); + var publication = confirmed.thenRun(() -> { synchronized (f.serverData.publicationMonitor()) { f.data.set("FullInventory", null); for (String path : candidate.getKeys(true)) if (!candidate.isConfigurationSection(path)) f.data.set(path, candidate.get(path)); } }); + // Completion is observable only after its installation callback exists. + submissions.add(java.util.Map.entry(candidate, confirmed)); + return publication; }).when(f.serverData).mutateFinalData(any()); return submissions; } @@ -1101,20 +1144,7 @@ private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean fa } private void overlappingAsyncOwnerSavesRetainEveryUnpublishedOverflow(boolean failFirst, boolean shutdown) throws Exception { Fixture f = createFixture(); - var submissions = new java.util.concurrent.LinkedBlockingQueue>>(); - doAnswer(call -> { - var candidate = new YamlConfiguration(); - call.getArgument(0, java.util.function.Consumer.class).accept(candidate); - var confirmed = new java.util.concurrent.CompletableFuture(); - submissions.add(java.util.Map.entry(candidate, confirmed)); - return confirmed.thenRun(() -> { - synchronized (f.serverData.publicationMonitor()) { - f.data.set("FullInventory", null); - for (String path : candidate.getKeys(true)) - if (!candidate.isConfigurationSection(path)) f.data.set(path, candidate.get(path)); - } - }); - }).when(f.serverData).mutateFinalData(any()); + var submissions = deferredPublications(f); var ownerTasks = new java.util.concurrent.LinkedBlockingQueue(); doAnswer(call -> { ownerTasks.add(call.getArgument(1)); return null; }) .when(f.bukkitScheduler).runTask(eq(f.plugin), any(Runnable.class), any(Player.class)); From 5537d94c62c7656e8f18798e49b887b22dcd2c5a Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 11:51:00 -0600 Subject: [PATCH 27/28] Isolate shutdown observations from persistence completion --- .../api/item/FullInventoryHandler.java | 10 +- .../tests/item/FullInventoryHandlerTest.java | 100 ++++++++++++++++++ 2 files changed, 106 insertions(+), 4 deletions(-) diff --git a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java index c6000a94b..33ff20ea4 100644 --- a/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java +++ b/AdvancedCore/src/main/java/com/bencodez/advancedcore/api/item/FullInventoryHandler.java @@ -56,7 +56,7 @@ public class FullInventoryHandler { private final AdvancedCorePlugin plugin; private final ReentrantReadWriteLock deliveryLock = new ReentrantReadWriteLock(true); private final AtomicBoolean shuttingDown = new AtomicBoolean(); - private final java.util.concurrent.atomic.AtomicReference> shutdownPersistence = + private final java.util.concurrent.atomic.AtomicReference> shutdownPersistence = new java.util.concurrent.atomic.AtomicReference<>(); @Getter @@ -297,7 +297,9 @@ public synchronized void shutdown() { */ public CompletionStage shutdownAsync(Executor persistenceExecutor) { CompletableFuture completion = new CompletableFuture<>(); - if (!shutdownPersistence.compareAndSet(null, completion)) return shutdownPersistence.get(); + // Keep one shared observation stage; each CompletableFuture view is isolated. + CompletionStage observation = completion.minimalCompletionStage(); + if (!shutdownPersistence.compareAndSet(null, observation)) return shutdownPersistence.get(); shuttingDown.set(true); ScheduledFuture task = checkTask; if (task != null) task.cancel(false); @@ -309,7 +311,7 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { // but do not snapshot or acknowledge any unpublished replay overflow. if (timer != null) timer.shutdownNow(); completion.complete(null); - return completion; + return observation; } try { persistenceExecutor.execute(() -> { @@ -328,7 +330,7 @@ public CompletionStage shutdownAsync(Executor persistenceExecutor) { completion.completeExceptionally(failure); if (timer != null) timer.shutdownNow(); } - return completion; + return observation; } private final java.util.Map overflowPublications = new HashMap<>(); diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index 9c7d482b8..93cc6f9f2 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -1067,6 +1067,106 @@ public void saveWaitsForInflightDeliveryBeforeSnapshotting() throws Exception { } finally { release.countDown(); } } + @Test void cancelledShutdownObservationCannotAdvanceActualLifecyclePublications() throws Exception { + verifyShutdownObservationCannotAdvanceActualLifecyclePublications("cancel"); + } + @Test void completedShutdownObservationCannotAdvanceActualLifecyclePublications() throws Exception { + verifyShutdownObservationCannotAdvanceActualLifecyclePublications("complete"); + } + @Test void failedShutdownObservationCannotAdvanceActualLifecyclePublications() throws Exception { + verifyShutdownObservationCannotAdvanceActualLifecyclePublications("fail"); + } + @Test void untouchedShutdownObservationRetainsActualLifecycleOrder() throws Exception { + verifyShutdownObservationCannotAdvanceActualLifecyclePublications("untouched"); + } + private void verifyShutdownObservationCannotAdvanceActualLifecyclePublications(String action) throws Exception { + Fixture f = createFixture(); + when(f.plugin.isLoadServerData()).thenReturn(true); + when(f.plugin.getFullInventoryHandler()).thenReturn(f.handler); + var permissions = mock(com.bencodez.advancedcore.api.permissions.PermissionHandler.class); + when(f.plugin.getPermissionHandler()).thenReturn(permissions); + var publications = new java.util.concurrent.LinkedBlockingQueue>>(); + var controls = new java.util.concurrent.CopyOnWriteArrayList>(); + doAnswer(call -> { + var snapshot = new YamlConfiguration(); + synchronized (f.serverData.publicationMonitor()) { + for (String path : f.data.getKeys(true)) + if (!f.data.isConfigurationSection(path)) snapshot.set(path, f.data.get(path)); + call.getArgument(0, java.util.function.Consumer.class).accept(snapshot); + } + var control = new java.util.concurrent.CompletableFuture(); + controls.add(control); + var installed = control.thenRun(() -> { + synchronized (f.serverData.publicationMonitor()) { + for (String key : new ArrayList<>(f.data.getKeys(false))) f.data.set(key, null); + for (String path : snapshot.getKeys(true)) + if (!snapshot.isConfigurationSection(path)) f.data.set(path, snapshot.get(path)); + } + }); + publications.add(java.util.Map.entry(snapshot, control)); + return installed; + }).when(f.serverData).mutateFinalData(any()); + var permissionsStarted = new AtomicBoolean(); + var permissionSave = new java.util.concurrent.CompletableFuture(); + when(permissions.shutDownAsync(any())).thenAnswer(call -> { + call.getArgument(0, java.util.concurrent.Executor.class).execute(() -> permissionsStarted.set(true)); + return permissionSave; + }); + var finalFiles = new AtomicReference>(); + doAnswer(call -> { finalFiles.set(call.getArgument(0)); return null; }) + .when(f.plugin).onShutdownComplete(any(CompletionStage.class)); + UUID owner = UUID.randomUUID(); var item = mock(ItemStack.class); + f.handler.add(owner, item); + var caller = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + var inventory = publications.poll(3, TimeUnit.SECONDS); assertNotNull(inventory); + switch (action) { + case "cancel" -> assertTrue(caller.cancel(false)); + case "complete" -> assertTrue(caller.complete(null)); + case "fail" -> assertTrue(caller.completeExceptionally(new IllegalStateException("synthetic caller failure"))); + default -> assertFalse(caller.isDone()); + } + var platform = new com.bencodez.advancedcore.bukkit.runtime.BukkitRuntimePlatform(f.plugin); + var workerField = platform.getClass().getDeclaredField("persistenceWorker"); workerField.setAccessible(true); + var worker = (java.util.concurrent.ThreadPoolExecutor) workerField.get(platform); + try { + platform.beforeExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("full inventory handler") + || cleanup.name().equals("server data timestamp")) + .forEach(cleanup -> cleanup.action().run()); + platform.afterExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("permission handler")) + .forEach(cleanup -> cleanup.action().run()); + worker.submit(() -> {}).get(3, TimeUnit.SECONDS); + assertTrue(publications.isEmpty(), "timestamp must await the actual inventory owner after caller completion"); + assertFalse(permissionsStarted.get()); + var lifecycleObservation = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); + assertFalse(lifecycleObservation.isDone()); + inventory.getValue().complete(null); + lifecycleObservation.get(3, TimeUnit.SECONDS); + var timestamp = publications.poll(3, TimeUnit.SECONDS); assertNotNull(timestamp); + assertTrue(timestamp.getKey().contains("LastUpdated")); + assertEquals(item, timestamp.getKey().getItemStack("FullInventory." + owner + ".Items.0")); + assertFalse(permissionsStarted.get()); + timestamp.getValue().complete(null); + worker.submit(() -> {}).get(3, TimeUnit.SECONDS); + assertTrue(permissionsStarted.get()); + platform.afterExecutorShutdown().stream() + .filter(cleanup -> cleanup.name().equals("shutdown persistence worker") + || cleanup.name().equals("plugin shutdown completion hook")) + .forEach(cleanup -> cleanup.action().run()); + assertFalse(finalFiles.get().toCompletableFuture().isDone()); + permissionSave.complete(null); + finalFiles.get().toCompletableFuture().get(3, TimeUnit.SECONDS); + assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + assertEquals(item, f.data.getItemStack("FullInventory." + owner + ".Items.0")); + assertTrue(f.data.contains("LastUpdated")); + } finally { + controls.forEach(control -> control.complete(null)); permissionSave.complete(null); + worker.shutdownNow(); assertTrue(worker.awaitTermination(3, TimeUnit.SECONDS)); + } + } + @Test void cancellingShutdownViewCannotReleasePendingOwner() throws Exception { Fixture f = createFixture(); var submissions = deferredPublications(f); UUID owner = UUID.randomUUID(); var item = mock(ItemStack.class); From 7b46a93c8da19f3671daa6c47ad9903f7caa4977 Mon Sep 17 00:00:00 2001 From: Ben Date: Sun, 25 Oct 2026 12:14:38 -0600 Subject: [PATCH 28/28] Await the actual permission-start event in lifecycle tests --- .../tests/item/FullInventoryHandlerTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java index 93cc6f9f2..7fd6915a6 100644 --- a/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java +++ b/AdvancedCore/src/test/java/com/bencodez/advancedcore/tests/item/FullInventoryHandlerTest.java @@ -1107,10 +1107,10 @@ private void verifyShutdownObservationCannotAdvanceActualLifecyclePublications(S publications.add(java.util.Map.entry(snapshot, control)); return installed; }).when(f.serverData).mutateFinalData(any()); - var permissionsStarted = new AtomicBoolean(); + var permissionsStarted = new CountDownLatch(1); var permissionSave = new java.util.concurrent.CompletableFuture(); when(permissions.shutDownAsync(any())).thenAnswer(call -> { - call.getArgument(0, java.util.concurrent.Executor.class).execute(() -> permissionsStarted.set(true)); + call.getArgument(0, java.util.concurrent.Executor.class).execute(permissionsStarted::countDown); return permissionSave; }); var finalFiles = new AtomicReference>(); @@ -1139,7 +1139,7 @@ private void verifyShutdownObservationCannotAdvanceActualLifecyclePublications(S .forEach(cleanup -> cleanup.action().run()); worker.submit(() -> {}).get(3, TimeUnit.SECONDS); assertTrue(publications.isEmpty(), "timestamp must await the actual inventory owner after caller completion"); - assertFalse(permissionsStarted.get()); + assertEquals(1L, permissionsStarted.getCount()); var lifecycleObservation = f.handler.shutdownAsync(Runnable::run).toCompletableFuture(); assertFalse(lifecycleObservation.isDone()); inventory.getValue().complete(null); @@ -1147,10 +1147,10 @@ private void verifyShutdownObservationCannotAdvanceActualLifecyclePublications(S var timestamp = publications.poll(3, TimeUnit.SECONDS); assertNotNull(timestamp); assertTrue(timestamp.getKey().contains("LastUpdated")); assertEquals(item, timestamp.getKey().getItemStack("FullInventory." + owner + ".Items.0")); - assertFalse(permissionsStarted.get()); + assertEquals(1L, permissionsStarted.getCount()); timestamp.getValue().complete(null); - worker.submit(() -> {}).get(3, TimeUnit.SECONDS); - assertTrue(permissionsStarted.get()); + assertTrue(permissionsStarted.await(3, TimeUnit.SECONDS), + "permission publication must start after the timestamp owner settles"); platform.afterExecutorShutdown().stream() .filter(cleanup -> cleanup.name().equals("shutdown persistence worker") || cleanup.name().equals("plugin shutdown completion hook"))